X86CodeEmitter.cpp revision 205218 
1//===-- X86/X86CodeEmitter.cpp - Convert X86 code to machine code ---------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file contains the pass that transforms the X86 machine instructions into
11// relocatable machine code.
12//
13//===----------------------------------------------------------------------===//
14
15#define DEBUG_TYPE "x86-emitter"
16#include "X86InstrInfo.h"
17#include "X86JITInfo.h"
18#include "X86Subtarget.h"
19#include "X86TargetMachine.h"
20#include "X86Relocations.h"
21#include "X86.h"
22#include "llvm/LLVMContext.h"
23#include "llvm/PassManager.h"
24#include "llvm/CodeGen/JITCodeEmitter.h"
25#include "llvm/CodeGen/MachineFunctionPass.h"
26#include "llvm/CodeGen/MachineInstr.h"
27#include "llvm/CodeGen/MachineModuleInfo.h"
28#include "llvm/CodeGen/Passes.h"
29#include "llvm/Function.h"
30#include "llvm/ADT/Statistic.h"
31#include "llvm/MC/MCCodeEmitter.h"
32#include "llvm/MC/MCExpr.h"
33#include "llvm/MC/MCInst.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/raw_ostream.h"
37#include "llvm/Target/TargetOptions.h"
38using namespace llvm;
39
40STATISTIC(NumEmitted, "Number of machine instructions emitted");
41
42namespace {
43  template<class CodeEmitter>
44  class Emitter : public MachineFunctionPass {
45    const X86InstrInfo  *II;
46    const TargetData    *TD;
47    X86TargetMachine    &TM;
48    CodeEmitter         &MCE;
49    MachineModuleInfo   *MMI;
50    intptr_t PICBaseOffset;
51    bool Is64BitMode;
52    bool IsPIC;
53  public:
54    static char ID;
55    explicit Emitter(X86TargetMachine &tm, CodeEmitter &mce)
56      : MachineFunctionPass(&ID), II(0), TD(0), TM(tm),
57      MCE(mce), PICBaseOffset(0), Is64BitMode(false),
58      IsPIC(TM.getRelocationModel() == Reloc::PIC_) {}
59    Emitter(X86TargetMachine &tm, CodeEmitter &mce,
60            const X86InstrInfo &ii, const TargetData &td, bool is64)
61      : MachineFunctionPass(&ID), II(&ii), TD(&td), TM(tm),
62      MCE(mce), PICBaseOffset(0), Is64BitMode(is64),
63      IsPIC(TM.getRelocationModel() == Reloc::PIC_) {}
64
65    bool runOnMachineFunction(MachineFunction &MF);
66
67    virtual const char *getPassName() const {
68      return "X86 Machine Code Emitter";
69    }
70
71    void emitInstruction(const MachineInstr &MI,
72                         const TargetInstrDesc *Desc);
73
74    void getAnalysisUsage(AnalysisUsage &AU) const {
75      AU.setPreservesAll();
76      AU.addRequired<MachineModuleInfo>();
77      MachineFunctionPass::getAnalysisUsage(AU);
78    }
79
80  private:
81    void emitPCRelativeBlockAddress(MachineBasicBlock *MBB);
82    void emitGlobalAddress(GlobalValue *GV, unsigned Reloc,
83                           intptr_t Disp = 0, intptr_t PCAdj = 0,
84                           bool Indirect = false);
85    void emitExternalSymbolAddress(const char *ES, unsigned Reloc);
86    void emitConstPoolAddress(unsigned CPI, unsigned Reloc, intptr_t Disp = 0,
87                              intptr_t PCAdj = 0);
88    void emitJumpTableAddress(unsigned JTI, unsigned Reloc,
89                              intptr_t PCAdj = 0);
90
91    void emitDisplacementField(const MachineOperand *RelocOp, int DispVal,
92                               intptr_t Adj = 0, bool IsPCRel = true);
93
94    void emitRegModRMByte(unsigned ModRMReg, unsigned RegOpcodeField);
95    void emitRegModRMByte(unsigned RegOpcodeField);
96    void emitSIBByte(unsigned SS, unsigned Index, unsigned Base);
97    void emitConstant(uint64_t Val, unsigned Size);
98
99    void emitMemModRMByte(const MachineInstr &MI,
100                          unsigned Op, unsigned RegOpcodeField,
101                          intptr_t PCAdj = 0);
102
103    unsigned getX86RegNum(unsigned RegNo) const;
104  };
105
106template<class CodeEmitter>
107  char Emitter<CodeEmitter>::ID = 0;
108} // end anonymous namespace.
109
110/// createX86CodeEmitterPass - Return a pass that emits the collected X86 code
111/// to the specified templated MachineCodeEmitter object.
112FunctionPass *llvm::createX86JITCodeEmitterPass(X86TargetMachine &TM,
113                                                JITCodeEmitter &JCE) {
114  return new Emitter<JITCodeEmitter>(TM, JCE);
115}
116
117template<class CodeEmitter>
118bool Emitter<CodeEmitter>::runOnMachineFunction(MachineFunction &MF) {
119  MMI = &getAnalysis<MachineModuleInfo>();
120  MCE.setModuleInfo(MMI);
121
122  II = TM.getInstrInfo();
123  TD = TM.getTargetData();
124  Is64BitMode = TM.getSubtarget<X86Subtarget>().is64Bit();
125  IsPIC = TM.getRelocationModel() == Reloc::PIC_;
126
127  do {
128    DEBUG(dbgs() << "JITTing function '"
129          << MF.getFunction()->getName() << "'\n");
130    MCE.startFunction(MF);
131    for (MachineFunction::iterator MBB = MF.begin(), E = MF.end();
132         MBB != E; ++MBB) {
133      MCE.StartMachineBasicBlock(MBB);
134      for (MachineBasicBlock::const_iterator I = MBB->begin(), E = MBB->end();
135           I != E; ++I) {
136        const TargetInstrDesc &Desc = I->getDesc();
137        emitInstruction(*I, &Desc);
138        // MOVPC32r is basically a call plus a pop instruction.
139        if (Desc.getOpcode() == X86::MOVPC32r)
140          emitInstruction(*I, &II->get(X86::POP32r));
141        NumEmitted++;  // Keep track of the # of mi's emitted
142      }
143    }
144  } while (MCE.finishFunction(MF));
145
146  return false;
147}
148
149/// emitPCRelativeBlockAddress - This method keeps track of the information
150/// necessary to resolve the address of this block later and emits a dummy
151/// value.
152///
153template<class CodeEmitter>
154void Emitter<CodeEmitter>::emitPCRelativeBlockAddress(MachineBasicBlock *MBB) {
155  // Remember where this reference was and where it is to so we can
156  // deal with it later.
157  MCE.addRelocation(MachineRelocation::getBB(MCE.getCurrentPCOffset(),
158                                             X86::reloc_pcrel_word, MBB));
159  MCE.emitWordLE(0);
160}
161
162/// emitGlobalAddress - Emit the specified address to the code stream assuming
163/// this is part of a "take the address of a global" instruction.
164///
165template<class CodeEmitter>
166void Emitter<CodeEmitter>::emitGlobalAddress(GlobalValue *GV, unsigned Reloc,
167                                intptr_t Disp /* = 0 */,
168                                intptr_t PCAdj /* = 0 */,
169                                bool Indirect /* = false */) {
170  intptr_t RelocCST = Disp;
171  if (Reloc == X86::reloc_picrel_word)
172    RelocCST = PICBaseOffset;
173  else if (Reloc == X86::reloc_pcrel_word)
174    RelocCST = PCAdj;
175  MachineRelocation MR = Indirect
176    ? MachineRelocation::getIndirectSymbol(MCE.getCurrentPCOffset(), Reloc,
177                                           GV, RelocCST, false)
178    : MachineRelocation::getGV(MCE.getCurrentPCOffset(), Reloc,
179                               GV, RelocCST, false);
180  MCE.addRelocation(MR);
181  // The relocated value will be added to the displacement
182  if (Reloc == X86::reloc_absolute_dword)
183    MCE.emitDWordLE(Disp);
184  else
185    MCE.emitWordLE((int32_t)Disp);
186}
187
188/// emitExternalSymbolAddress - Arrange for the address of an external symbol to
189/// be emitted to the current location in the function, and allow it to be PC
190/// relative.
191template<class CodeEmitter>
192void Emitter<CodeEmitter>::emitExternalSymbolAddress(const char *ES,
193                                                     unsigned Reloc) {
194  intptr_t RelocCST = (Reloc == X86::reloc_picrel_word) ? PICBaseOffset : 0;
195
196  // X86 never needs stubs because instruction selection will always pick
197  // an instruction sequence that is large enough to hold any address
198  // to a symbol.
199  // (see X86ISelLowering.cpp, near 2039: X86TargetLowering::LowerCall)
200  bool NeedStub = false;
201  MCE.addRelocation(MachineRelocation::getExtSym(MCE.getCurrentPCOffset(),
202                                                 Reloc, ES, RelocCST,
203                                                 0, NeedStub));
204  if (Reloc == X86::reloc_absolute_dword)
205    MCE.emitDWordLE(0);
206  else
207    MCE.emitWordLE(0);
208}
209
210/// emitConstPoolAddress - Arrange for the address of an constant pool
211/// to be emitted to the current location in the function, and allow it to be PC
212/// relative.
213template<class CodeEmitter>
214void Emitter<CodeEmitter>::emitConstPoolAddress(unsigned CPI, unsigned Reloc,
215                                   intptr_t Disp /* = 0 */,
216                                   intptr_t PCAdj /* = 0 */) {
217  intptr_t RelocCST = 0;
218  if (Reloc == X86::reloc_picrel_word)
219    RelocCST = PICBaseOffset;
220  else if (Reloc == X86::reloc_pcrel_word)
221    RelocCST = PCAdj;
222  MCE.addRelocation(MachineRelocation::getConstPool(MCE.getCurrentPCOffset(),
223                                                    Reloc, CPI, RelocCST));
224  // The relocated value will be added to the displacement
225  if (Reloc == X86::reloc_absolute_dword)
226    MCE.emitDWordLE(Disp);
227  else
228    MCE.emitWordLE((int32_t)Disp);
229}
230
231/// emitJumpTableAddress - Arrange for the address of a jump table to
232/// be emitted to the current location in the function, and allow it to be PC
233/// relative.
234template<class CodeEmitter>
235void Emitter<CodeEmitter>::emitJumpTableAddress(unsigned JTI, unsigned Reloc,
236                                   intptr_t PCAdj /* = 0 */) {
237  intptr_t RelocCST = 0;
238  if (Reloc == X86::reloc_picrel_word)
239    RelocCST = PICBaseOffset;
240  else if (Reloc == X86::reloc_pcrel_word)
241    RelocCST = PCAdj;
242  MCE.addRelocation(MachineRelocation::getJumpTable(MCE.getCurrentPCOffset(),
243                                                    Reloc, JTI, RelocCST));
244  // The relocated value will be added to the displacement
245  if (Reloc == X86::reloc_absolute_dword)
246    MCE.emitDWordLE(0);
247  else
248    MCE.emitWordLE(0);
249}
250
251template<class CodeEmitter>
252unsigned Emitter<CodeEmitter>::getX86RegNum(unsigned RegNo) const {
253  return X86RegisterInfo::getX86RegNum(RegNo);
254}
255
256inline static unsigned char ModRMByte(unsigned Mod, unsigned RegOpcode,
257                                      unsigned RM) {
258  assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!");
259  return RM | (RegOpcode << 3) | (Mod << 6);
260}
261
262template<class CodeEmitter>
263void Emitter<CodeEmitter>::emitRegModRMByte(unsigned ModRMReg,
264                                            unsigned RegOpcodeFld){
265  MCE.emitByte(ModRMByte(3, RegOpcodeFld, getX86RegNum(ModRMReg)));
266}
267
268template<class CodeEmitter>
269void Emitter<CodeEmitter>::emitRegModRMByte(unsigned RegOpcodeFld) {
270  MCE.emitByte(ModRMByte(3, RegOpcodeFld, 0));
271}
272
273template<class CodeEmitter>
274void Emitter<CodeEmitter>::emitSIBByte(unsigned SS,
275                                       unsigned Index,
276                                       unsigned Base) {
277  // SIB byte is in the same format as the ModRMByte...
278  MCE.emitByte(ModRMByte(SS, Index, Base));
279}
280
281template<class CodeEmitter>
282void Emitter<CodeEmitter>::emitConstant(uint64_t Val, unsigned Size) {
283  // Output the constant in little endian byte order...
284  for (unsigned i = 0; i != Size; ++i) {
285    MCE.emitByte(Val & 255);
286    Val >>= 8;
287  }
288}
289
290/// isDisp8 - Return true if this signed displacement fits in a 8-bit
291/// sign-extended field.
292static bool isDisp8(int Value) {
293  return Value == (signed char)Value;
294}
295
296static bool gvNeedsNonLazyPtr(const MachineOperand &GVOp,
297                              const TargetMachine &TM) {
298  // For Darwin-64, simulate the linktime GOT by using the same non-lazy-pointer
299  // mechanism as 32-bit mode.
300  if (TM.getSubtarget<X86Subtarget>().is64Bit() &&
301      !TM.getSubtarget<X86Subtarget>().isTargetDarwin())
302    return false;
303
304  // Return true if this is a reference to a stub containing the address of the
305  // global, not the global itself.
306  return isGlobalStubReference(GVOp.getTargetFlags());
307}
308
309template<class CodeEmitter>
310void Emitter<CodeEmitter>::emitDisplacementField(const MachineOperand *RelocOp,
311                                                 int DispVal,
312                                                 intptr_t Adj /* = 0 */,
313                                                 bool IsPCRel /* = true */) {
314  // If this is a simple integer displacement that doesn't require a relocation,
315  // emit it now.
316  if (!RelocOp) {
317    emitConstant(DispVal, 4);
318    return;
319  }
320
321  // Otherwise, this is something that requires a relocation.  Emit it as such
322  // now.
323  unsigned RelocType = Is64BitMode ?
324    (IsPCRel ? X86::reloc_pcrel_word : X86::reloc_absolute_word_sext)
325    : (IsPIC ? X86::reloc_picrel_word : X86::reloc_absolute_word);
326  if (RelocOp->isGlobal()) {
327    // In 64-bit static small code model, we could potentially emit absolute.
328    // But it's probably not beneficial. If the MCE supports using RIP directly
329    // do it, otherwise fallback to absolute (this is determined by IsPCRel).
330    //  89 05 00 00 00 00     mov    %eax,0(%rip)  # PC-relative
331    //  89 04 25 00 00 00 00  mov    %eax,0x0      # Absolute
332    bool Indirect = gvNeedsNonLazyPtr(*RelocOp, TM);
333    emitGlobalAddress(RelocOp->getGlobal(), RelocType, RelocOp->getOffset(),
334                      Adj, Indirect);
335  } else if (RelocOp->isSymbol()) {
336    emitExternalSymbolAddress(RelocOp->getSymbolName(), RelocType);
337  } else if (RelocOp->isCPI()) {
338    emitConstPoolAddress(RelocOp->getIndex(), RelocType,
339                         RelocOp->getOffset(), Adj);
340  } else {
341    assert(RelocOp->isJTI() && "Unexpected machine operand!");
342    emitJumpTableAddress(RelocOp->getIndex(), RelocType, Adj);
343  }
344}
345
346template<class CodeEmitter>
347void Emitter<CodeEmitter>::emitMemModRMByte(const MachineInstr &MI,
348                                            unsigned Op,unsigned RegOpcodeField,
349                                            intptr_t PCAdj) {
350  const MachineOperand &Op3 = MI.getOperand(Op+3);
351  int DispVal = 0;
352  const MachineOperand *DispForReloc = 0;
353
354  // Figure out what sort of displacement we have to handle here.
355  if (Op3.isGlobal()) {
356    DispForReloc = &Op3;
357  } else if (Op3.isSymbol()) {
358    DispForReloc = &Op3;
359  } else if (Op3.isCPI()) {
360    if (!MCE.earlyResolveAddresses() || Is64BitMode || IsPIC) {
361      DispForReloc = &Op3;
362    } else {
363      DispVal += MCE.getConstantPoolEntryAddress(Op3.getIndex());
364      DispVal += Op3.getOffset();
365    }
366  } else if (Op3.isJTI()) {
367    if (!MCE.earlyResolveAddresses() || Is64BitMode || IsPIC) {
368      DispForReloc = &Op3;
369    } else {
370      DispVal += MCE.getJumpTableEntryAddress(Op3.getIndex());
371    }
372  } else {
373    DispVal = Op3.getImm();
374  }
375
376  const MachineOperand &Base     = MI.getOperand(Op);
377  const MachineOperand &Scale    = MI.getOperand(Op+1);
378  const MachineOperand &IndexReg = MI.getOperand(Op+2);
379
380  unsigned BaseReg = Base.getReg();
381
382  // Indicate that the displacement will use an pcrel or absolute reference
383  // by default. MCEs able to resolve addresses on-the-fly use pcrel by default
384  // while others, unless explicit asked to use RIP, use absolute references.
385  bool IsPCRel = MCE.earlyResolveAddresses() ? true : false;
386
387  // Is a SIB byte needed?
388  // If no BaseReg, issue a RIP relative instruction only if the MCE can
389  // resolve addresses on-the-fly, otherwise use SIB (Intel Manual 2A, table
390  // 2-7) and absolute references.
391  unsigned BaseRegNo = -1U;
392  if (BaseReg != 0 && BaseReg != X86::RIP)
393    BaseRegNo = getX86RegNum(BaseReg);
394
395  if (// The SIB byte must be used if there is an index register.
396      IndexReg.getReg() == 0 &&
397      // The SIB byte must be used if the base is ESP/RSP/R12, all of which
398      // encode to an R/M value of 4, which indicates that a SIB byte is
399      // present.
400      BaseRegNo != N86::ESP &&
401      // If there is no base register and we're in 64-bit mode, we need a SIB
402      // byte to emit an addr that is just 'disp32' (the non-RIP relative form).
403      (!Is64BitMode || BaseReg != 0)) {
404    if (BaseReg == 0 ||          // [disp32]     in X86-32 mode
405        BaseReg == X86::RIP) {   // [disp32+RIP] in X86-64 mode
406      MCE.emitByte(ModRMByte(0, RegOpcodeField, 5));
407      emitDisplacementField(DispForReloc, DispVal, PCAdj, true);
408      return;
409    }
410
411    // If the base is not EBP/ESP and there is no displacement, use simple
412    // indirect register encoding, this handles addresses like [EAX].  The
413    // encoding for [EBP] with no displacement means [disp32] so we handle it
414    // by emitting a displacement of 0 below.
415    if (!DispForReloc && DispVal == 0 && BaseRegNo != N86::EBP) {
416      MCE.emitByte(ModRMByte(0, RegOpcodeField, BaseRegNo));
417      return;
418    }
419
420    // Otherwise, if the displacement fits in a byte, encode as [REG+disp8].
421    if (!DispForReloc && isDisp8(DispVal)) {
422      MCE.emitByte(ModRMByte(1, RegOpcodeField, BaseRegNo));
423      emitConstant(DispVal, 1);
424      return;
425    }
426
427    // Otherwise, emit the most general non-SIB encoding: [REG+disp32]
428    MCE.emitByte(ModRMByte(2, RegOpcodeField, BaseRegNo));
429    emitDisplacementField(DispForReloc, DispVal, PCAdj, IsPCRel);
430    return;
431  }
432
433  // Otherwise we need a SIB byte, so start by outputting the ModR/M byte first.
434  assert(IndexReg.getReg() != X86::ESP &&
435         IndexReg.getReg() != X86::RSP && "Cannot use ESP as index reg!");
436
437  bool ForceDisp32 = false;
438  bool ForceDisp8  = false;
439  if (BaseReg == 0) {
440    // If there is no base register, we emit the special case SIB byte with
441    // MOD=0, BASE=4, to JUST get the index, scale, and displacement.
442    MCE.emitByte(ModRMByte(0, RegOpcodeField, 4));
443    ForceDisp32 = true;
444  } else if (DispForReloc) {
445    // Emit the normal disp32 encoding.
446    MCE.emitByte(ModRMByte(2, RegOpcodeField, 4));
447    ForceDisp32 = true;
448  } else if (DispVal == 0 && getX86RegNum(BaseReg) != N86::EBP) {
449    // Emit no displacement ModR/M byte
450    MCE.emitByte(ModRMByte(0, RegOpcodeField, 4));
451  } else if (isDisp8(DispVal)) {
452    // Emit the disp8 encoding...
453    MCE.emitByte(ModRMByte(1, RegOpcodeField, 4));
454    ForceDisp8 = true;           // Make sure to force 8 bit disp if Base=EBP
455  } else {
456    // Emit the normal disp32 encoding...
457    MCE.emitByte(ModRMByte(2, RegOpcodeField, 4));
458  }
459
460  // Calculate what the SS field value should be...
461  static const unsigned SSTable[] = { ~0, 0, 1, ~0, 2, ~0, ~0, ~0, 3 };
462  unsigned SS = SSTable[Scale.getImm()];
463
464  if (BaseReg == 0) {
465    // Handle the SIB byte for the case where there is no base, see Intel
466    // Manual 2A, table 2-7. The displacement has already been output.
467    unsigned IndexRegNo;
468    if (IndexReg.getReg())
469      IndexRegNo = getX86RegNum(IndexReg.getReg());
470    else // Examples: [ESP+1*<noreg>+4] or [scaled idx]+disp32 (MOD=0,BASE=5)
471      IndexRegNo = 4;
472    emitSIBByte(SS, IndexRegNo, 5);
473  } else {
474    unsigned BaseRegNo = getX86RegNum(BaseReg);
475    unsigned IndexRegNo;
476    if (IndexReg.getReg())
477      IndexRegNo = getX86RegNum(IndexReg.getReg());
478    else
479      IndexRegNo = 4;   // For example [ESP+1*<noreg>+4]
480    emitSIBByte(SS, IndexRegNo, BaseRegNo);
481  }
482
483  // Do we need to output a displacement?
484  if (ForceDisp8) {
485    emitConstant(DispVal, 1);
486  } else if (DispVal != 0 || ForceDisp32) {
487    emitDisplacementField(DispForReloc, DispVal, PCAdj, IsPCRel);
488  }
489}
490
491template<class CodeEmitter>
492void Emitter<CodeEmitter>::emitInstruction(const MachineInstr &MI,
493                                           const TargetInstrDesc *Desc) {
494  DEBUG(dbgs() << MI);
495
496  MCE.processDebugLoc(MI.getDebugLoc(), true);
497
498  unsigned Opcode = Desc->Opcode;
499
500  // Emit the lock opcode prefix as needed.
501  if (Desc->TSFlags & X86II::LOCK)
502    MCE.emitByte(0xF0);
503
504  // Emit segment override opcode prefix as needed.
505  switch (Desc->TSFlags & X86II::SegOvrMask) {
506  case X86II::FS:
507    MCE.emitByte(0x64);
508    break;
509  case X86II::GS:
510    MCE.emitByte(0x65);
511    break;
512  default: llvm_unreachable("Invalid segment!");
513  case 0: break;  // No segment override!
514  }
515
516  // Emit the repeat opcode prefix as needed.
517  if ((Desc->TSFlags & X86II::Op0Mask) == X86II::REP)
518    MCE.emitByte(0xF3);
519
520  // Emit the operand size opcode prefix as needed.
521  if (Desc->TSFlags & X86II::OpSize)
522    MCE.emitByte(0x66);
523
524  // Emit the address size opcode prefix as needed.
525  if (Desc->TSFlags & X86II::AdSize)
526    MCE.emitByte(0x67);
527
528  bool Need0FPrefix = false;
529  switch (Desc->TSFlags & X86II::Op0Mask) {
530  case X86II::TB:  // Two-byte opcode prefix
531  case X86II::T8:  // 0F 38
532  case X86II::TA:  // 0F 3A
533    Need0FPrefix = true;
534    break;
535  case X86II::TF: // F2 0F 38
536    MCE.emitByte(0xF2);
537    Need0FPrefix = true;
538    break;
539  case X86II::REP: break; // already handled.
540  case X86II::XS:   // F3 0F
541    MCE.emitByte(0xF3);
542    Need0FPrefix = true;
543    break;
544  case X86II::XD:   // F2 0F
545    MCE.emitByte(0xF2);
546    Need0FPrefix = true;
547    break;
548  case X86II::D8: case X86II::D9: case X86II::DA: case X86II::DB:
549  case X86II::DC: case X86II::DD: case X86II::DE: case X86II::DF:
550    MCE.emitByte(0xD8+
551                 (((Desc->TSFlags & X86II::Op0Mask)-X86II::D8)
552                                   >> X86II::Op0Shift));
553    break; // Two-byte opcode prefix
554  default: llvm_unreachable("Invalid prefix!");
555  case 0: break;  // No prefix!
556  }
557
558  // Handle REX prefix.
559  if (Is64BitMode) {
560    if (unsigned REX = X86InstrInfo::determineREX(MI))
561      MCE.emitByte(0x40 | REX);
562  }
563
564  // 0x0F escape code must be emitted just before the opcode.
565  if (Need0FPrefix)
566    MCE.emitByte(0x0F);
567
568  switch (Desc->TSFlags & X86II::Op0Mask) {
569  case X86II::TF:    // F2 0F 38
570  case X86II::T8:    // 0F 38
571    MCE.emitByte(0x38);
572    break;
573  case X86II::TA:    // 0F 3A
574    MCE.emitByte(0x3A);
575    break;
576  }
577
578  // If this is a two-address instruction, skip one of the register operands.
579  unsigned NumOps = Desc->getNumOperands();
580  unsigned CurOp = 0;
581  if (NumOps > 1 && Desc->getOperandConstraint(1, TOI::TIED_TO) != -1)
582    ++CurOp;
583  else if (NumOps > 2 && Desc->getOperandConstraint(NumOps-1, TOI::TIED_TO)== 0)
584    // Skip the last source operand that is tied_to the dest reg. e.g. LXADD32
585    --NumOps;
586
587  unsigned char BaseOpcode = X86II::getBaseOpcodeFor(Desc->TSFlags);
588  switch (Desc->TSFlags & X86II::FormMask) {
589  default:
590    llvm_unreachable("Unknown FormMask value in X86 MachineCodeEmitter!");
591  case X86II::Pseudo:
592    // Remember the current PC offset, this is the PIC relocation
593    // base address.
594    switch (Opcode) {
595    default:
596      llvm_unreachable("psuedo instructions should be removed before code"
597                       " emission");
598      break;
599    case TargetOpcode::INLINEASM:
600      // We allow inline assembler nodes with empty bodies - they can
601      // implicitly define registers, which is ok for JIT.
602      if (MI.getOperand(0).getSymbolName()[0])
603        llvm_report_error("JIT does not support inline asm!");
604      break;
605    case TargetOpcode::DBG_LABEL:
606    case TargetOpcode::GC_LABEL:
607    case TargetOpcode::EH_LABEL:
608      MCE.emitLabel(MI.getOperand(0).getMCSymbol());
609      break;
610
611    case TargetOpcode::IMPLICIT_DEF:
612    case TargetOpcode::KILL:
613    case X86::FP_REG_KILL:
614      break;
615    case X86::MOVPC32r: {
616      // This emits the "call" portion of this pseudo instruction.
617      MCE.emitByte(BaseOpcode);
618      emitConstant(0, X86II::getSizeOfImm(Desc->TSFlags));
619      // Remember PIC base.
620      PICBaseOffset = (intptr_t) MCE.getCurrentPCOffset();
621      X86JITInfo *JTI = TM.getJITInfo();
622      JTI->setPICBase(MCE.getCurrentPCValue());
623      break;
624    }
625    }
626    CurOp = NumOps;
627    break;
628  case X86II::RawFrm: {
629    MCE.emitByte(BaseOpcode);
630
631    if (CurOp == NumOps)
632      break;
633
634    const MachineOperand &MO = MI.getOperand(CurOp++);
635
636    DEBUG(dbgs() << "RawFrm CurOp " << CurOp << "\n");
637    DEBUG(dbgs() << "isMBB " << MO.isMBB() << "\n");
638    DEBUG(dbgs() << "isGlobal " << MO.isGlobal() << "\n");
639    DEBUG(dbgs() << "isSymbol " << MO.isSymbol() << "\n");
640    DEBUG(dbgs() << "isImm " << MO.isImm() << "\n");
641
642    if (MO.isMBB()) {
643      emitPCRelativeBlockAddress(MO.getMBB());
644      break;
645    }
646
647    if (MO.isGlobal()) {
648      emitGlobalAddress(MO.getGlobal(), X86::reloc_pcrel_word,
649                        MO.getOffset(), 0);
650      break;
651    }
652
653    if (MO.isSymbol()) {
654      emitExternalSymbolAddress(MO.getSymbolName(), X86::reloc_pcrel_word);
655      break;
656    }
657
658    // FIXME: Only used by hackish MCCodeEmitter, remove when dead.
659    if (MO.isJTI()) {
660      emitJumpTableAddress(MO.getIndex(), X86::reloc_pcrel_word);
661      break;
662    }
663
664    assert(MO.isImm() && "Unknown RawFrm operand!");
665    if (Opcode == X86::CALLpcrel32 || Opcode == X86::CALL64pcrel32) {
666      // Fix up immediate operand for pc relative calls.
667      intptr_t Imm = (intptr_t)MO.getImm();
668      Imm = Imm - MCE.getCurrentPCValue() - 4;
669      emitConstant(Imm, X86II::getSizeOfImm(Desc->TSFlags));
670    } else
671      emitConstant(MO.getImm(), X86II::getSizeOfImm(Desc->TSFlags));
672    break;
673  }
674
675  case X86II::AddRegFrm: {
676    MCE.emitByte(BaseOpcode + getX86RegNum(MI.getOperand(CurOp++).getReg()));
677
678    if (CurOp == NumOps)
679      break;
680
681    const MachineOperand &MO1 = MI.getOperand(CurOp++);
682    unsigned Size = X86II::getSizeOfImm(Desc->TSFlags);
683    if (MO1.isImm()) {
684      emitConstant(MO1.getImm(), Size);
685      break;
686    }
687
688    unsigned rt = Is64BitMode ? X86::reloc_pcrel_word
689      : (IsPIC ? X86::reloc_picrel_word : X86::reloc_absolute_word);
690    if (Opcode == X86::MOV64ri64i32)
691      rt = X86::reloc_absolute_word;  // FIXME: add X86II flag?
692    // This should not occur on Darwin for relocatable objects.
693    if (Opcode == X86::MOV64ri)
694      rt = X86::reloc_absolute_dword;  // FIXME: add X86II flag?
695    if (MO1.isGlobal()) {
696      bool Indirect = gvNeedsNonLazyPtr(MO1, TM);
697      emitGlobalAddress(MO1.getGlobal(), rt, MO1.getOffset(), 0,
698                        Indirect);
699    } else if (MO1.isSymbol())
700      emitExternalSymbolAddress(MO1.getSymbolName(), rt);
701    else if (MO1.isCPI())
702      emitConstPoolAddress(MO1.getIndex(), rt);
703    else if (MO1.isJTI())
704      emitJumpTableAddress(MO1.getIndex(), rt);
705    break;
706  }
707
708  case X86II::MRMDestReg: {
709    MCE.emitByte(BaseOpcode);
710    emitRegModRMByte(MI.getOperand(CurOp).getReg(),
711                     getX86RegNum(MI.getOperand(CurOp+1).getReg()));
712    CurOp += 2;
713    if (CurOp != NumOps)
714      emitConstant(MI.getOperand(CurOp++).getImm(),
715                   X86II::getSizeOfImm(Desc->TSFlags));
716    break;
717  }
718  case X86II::MRMDestMem: {
719    MCE.emitByte(BaseOpcode);
720    emitMemModRMByte(MI, CurOp,
721                     getX86RegNum(MI.getOperand(CurOp + X86AddrNumOperands)
722                                  .getReg()));
723    CurOp +=  X86AddrNumOperands + 1;
724    if (CurOp != NumOps)
725      emitConstant(MI.getOperand(CurOp++).getImm(),
726                   X86II::getSizeOfImm(Desc->TSFlags));
727    break;
728  }
729
730  case X86II::MRMSrcReg:
731    MCE.emitByte(BaseOpcode);
732    emitRegModRMByte(MI.getOperand(CurOp+1).getReg(),
733                     getX86RegNum(MI.getOperand(CurOp).getReg()));
734    CurOp += 2;
735    if (CurOp != NumOps)
736      emitConstant(MI.getOperand(CurOp++).getImm(),
737                   X86II::getSizeOfImm(Desc->TSFlags));
738    break;
739
740  case X86II::MRMSrcMem: {
741    // FIXME: Maybe lea should have its own form?
742    int AddrOperands;
743    if (Opcode == X86::LEA64r || Opcode == X86::LEA64_32r ||
744        Opcode == X86::LEA16r || Opcode == X86::LEA32r)
745      AddrOperands = X86AddrNumOperands - 1; // No segment register
746    else
747      AddrOperands = X86AddrNumOperands;
748
749    intptr_t PCAdj = (CurOp + AddrOperands + 1 != NumOps) ?
750      X86II::getSizeOfImm(Desc->TSFlags) : 0;
751
752    MCE.emitByte(BaseOpcode);
753    emitMemModRMByte(MI, CurOp+1, getX86RegNum(MI.getOperand(CurOp).getReg()),
754                     PCAdj);
755    CurOp += AddrOperands + 1;
756    if (CurOp != NumOps)
757      emitConstant(MI.getOperand(CurOp++).getImm(),
758                   X86II::getSizeOfImm(Desc->TSFlags));
759    break;
760  }
761
762  case X86II::MRM0r: case X86II::MRM1r:
763  case X86II::MRM2r: case X86II::MRM3r:
764  case X86II::MRM4r: case X86II::MRM5r:
765  case X86II::MRM6r: case X86II::MRM7r: {
766    MCE.emitByte(BaseOpcode);
767    emitRegModRMByte(MI.getOperand(CurOp++).getReg(),
768                     (Desc->TSFlags & X86II::FormMask)-X86II::MRM0r);
769
770    if (CurOp == NumOps)
771      break;
772
773    const MachineOperand &MO1 = MI.getOperand(CurOp++);
774    unsigned Size = X86II::getSizeOfImm(Desc->TSFlags);
775    if (MO1.isImm()) {
776      emitConstant(MO1.getImm(), Size);
777      break;
778    }
779
780    unsigned rt = Is64BitMode ? X86::reloc_pcrel_word
781      : (IsPIC ? X86::reloc_picrel_word : X86::reloc_absolute_word);
782    if (Opcode == X86::MOV64ri32)
783      rt = X86::reloc_absolute_word_sext;  // FIXME: add X86II flag?
784    if (MO1.isGlobal()) {
785      bool Indirect = gvNeedsNonLazyPtr(MO1, TM);
786      emitGlobalAddress(MO1.getGlobal(), rt, MO1.getOffset(), 0,
787                        Indirect);
788    } else if (MO1.isSymbol())
789      emitExternalSymbolAddress(MO1.getSymbolName(), rt);
790    else if (MO1.isCPI())
791      emitConstPoolAddress(MO1.getIndex(), rt);
792    else if (MO1.isJTI())
793      emitJumpTableAddress(MO1.getIndex(), rt);
794    break;
795  }
796
797  case X86II::MRM0m: case X86II::MRM1m:
798  case X86II::MRM2m: case X86II::MRM3m:
799  case X86II::MRM4m: case X86II::MRM5m:
800  case X86II::MRM6m: case X86II::MRM7m: {
801    intptr_t PCAdj = (CurOp + X86AddrNumOperands != NumOps) ?
802      (MI.getOperand(CurOp+X86AddrNumOperands).isImm() ?
803          X86II::getSizeOfImm(Desc->TSFlags) : 4) : 0;
804
805    MCE.emitByte(BaseOpcode);
806    emitMemModRMByte(MI, CurOp, (Desc->TSFlags & X86II::FormMask)-X86II::MRM0m,
807                     PCAdj);
808    CurOp += X86AddrNumOperands;
809
810    if (CurOp == NumOps)
811      break;
812
813    const MachineOperand &MO = MI.getOperand(CurOp++);
814    unsigned Size = X86II::getSizeOfImm(Desc->TSFlags);
815    if (MO.isImm()) {
816      emitConstant(MO.getImm(), Size);
817      break;
818    }
819
820    unsigned rt = Is64BitMode ? X86::reloc_pcrel_word
821      : (IsPIC ? X86::reloc_picrel_word : X86::reloc_absolute_word);
822    if (Opcode == X86::MOV64mi32)
823      rt = X86::reloc_absolute_word_sext;  // FIXME: add X86II flag?
824    if (MO.isGlobal()) {
825      bool Indirect = gvNeedsNonLazyPtr(MO, TM);
826      emitGlobalAddress(MO.getGlobal(), rt, MO.getOffset(), 0,
827                        Indirect);
828    } else if (MO.isSymbol())
829      emitExternalSymbolAddress(MO.getSymbolName(), rt);
830    else if (MO.isCPI())
831      emitConstPoolAddress(MO.getIndex(), rt);
832    else if (MO.isJTI())
833      emitJumpTableAddress(MO.getIndex(), rt);
834    break;
835  }
836
837  case X86II::MRMInitReg:
838    MCE.emitByte(BaseOpcode);
839    // Duplicate register, used by things like MOV8r0 (aka xor reg,reg).
840    emitRegModRMByte(MI.getOperand(CurOp).getReg(),
841                     getX86RegNum(MI.getOperand(CurOp).getReg()));
842    ++CurOp;
843    break;
844
845  case X86II::MRM_C1:
846    MCE.emitByte(BaseOpcode);
847    MCE.emitByte(0xC1);
848    break;
849  case X86II::MRM_C8:
850    MCE.emitByte(BaseOpcode);
851    MCE.emitByte(0xC8);
852    break;
853  case X86II::MRM_C9:
854    MCE.emitByte(BaseOpcode);
855    MCE.emitByte(0xC9);
856    break;
857  case X86II::MRM_E8:
858    MCE.emitByte(BaseOpcode);
859    MCE.emitByte(0xE8);
860    break;
861  case X86II::MRM_F0:
862    MCE.emitByte(BaseOpcode);
863    MCE.emitByte(0xF0);
864    break;
865  }
866
867  if (!Desc->isVariadic() && CurOp != NumOps) {
868#ifndef NDEBUG
869    dbgs() << "Cannot encode all operands of: " << MI << "\n";
870#endif
871    llvm_unreachable(0);
872  }
873
874  MCE.processDebugLoc(MI.getDebugLoc(), false);
875}
876