1//===-- X86MCCodeEmitter.cpp - Convert X86 code to machine code -----------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file implements the X86MCCodeEmitter class. 10// 11//===----------------------------------------------------------------------===// 12 13#include "MCTargetDesc/X86BaseInfo.h" 14#include "MCTargetDesc/X86FixupKinds.h" 15#include "MCTargetDesc/X86MCTargetDesc.h" 16#include "llvm/ADT/SmallVector.h" 17#include "llvm/MC/MCCodeEmitter.h" 18#include "llvm/MC/MCContext.h" 19#include "llvm/MC/MCExpr.h" 20#include "llvm/MC/MCFixup.h" 21#include "llvm/MC/MCInst.h" 22#include "llvm/MC/MCInstrDesc.h" 23#include "llvm/MC/MCInstrInfo.h" 24#include "llvm/MC/MCRegisterInfo.h" 25#include "llvm/MC/MCSubtargetInfo.h" 26#include "llvm/MC/MCSymbol.h" 27#include "llvm/Support/ErrorHandling.h" 28#include "llvm/Support/raw_ostream.h" 29#include <cassert> 30#include <cstdint> 31#include <cstdlib> 32 33using namespace llvm; 34 35#define DEBUG_TYPE "mccodeemitter" 36 37namespace { 38 39class X86MCCodeEmitter : public MCCodeEmitter { 40 const MCInstrInfo &MCII; 41 MCContext &Ctx; 42 43public: 44 X86MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx) 45 : MCII(mcii), Ctx(ctx) {} 46 X86MCCodeEmitter(const X86MCCodeEmitter &) = delete; 47 X86MCCodeEmitter &operator=(const X86MCCodeEmitter &) = delete; 48 ~X86MCCodeEmitter() override = default; 49 50 void emitPrefix(const MCInst &MI, raw_ostream &OS, 51 const MCSubtargetInfo &STI) const override; 52 53 void encodeInstruction(const MCInst &MI, raw_ostream &OS, 54 SmallVectorImpl<MCFixup> &Fixups, 55 const MCSubtargetInfo &STI) const override; 56 57private: 58 unsigned getX86RegNum(const MCOperand &MO) const; 59 60 unsigned getX86RegEncoding(const MCInst &MI, unsigned OpNum) const; 61 62 /// \param MI a single low-level machine instruction. 63 /// \param OpNum the operand #. 64 /// \returns true if the OpNumth operand of MI require a bit to be set in 65 /// REX prefix. 66 bool isREXExtendedReg(const MCInst &MI, unsigned OpNum) const; 67 68 void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned ImmSize, 69 MCFixupKind FixupKind, uint64_t StartByte, raw_ostream &OS, 70 SmallVectorImpl<MCFixup> &Fixups, int ImmOffset = 0) const; 71 72 void emitRegModRMByte(const MCOperand &ModRMReg, unsigned RegOpcodeFld, 73 raw_ostream &OS) const; 74 75 void emitSIBByte(unsigned SS, unsigned Index, unsigned Base, 76 raw_ostream &OS) const; 77 78 void emitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField, 79 uint64_t TSFlags, bool HasREX, uint64_t StartByte, 80 raw_ostream &OS, SmallVectorImpl<MCFixup> &Fixups, 81 const MCSubtargetInfo &STI, 82 bool ForceSIB = false) const; 83 84 bool emitPrefixImpl(unsigned &CurOp, const MCInst &MI, 85 const MCSubtargetInfo &STI, raw_ostream &OS) const; 86 87 void emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, 88 raw_ostream &OS) const; 89 90 void emitSegmentOverridePrefix(unsigned SegOperand, const MCInst &MI, 91 raw_ostream &OS) const; 92 93 bool emitOpcodePrefix(int MemOperand, const MCInst &MI, 94 const MCSubtargetInfo &STI, raw_ostream &OS) const; 95 96 bool emitREXPrefix(int MemOperand, const MCInst &MI, raw_ostream &OS) const; 97}; 98 99} // end anonymous namespace 100 101static uint8_t modRMByte(unsigned Mod, unsigned RegOpcode, unsigned RM) { 102 assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!"); 103 return RM | (RegOpcode << 3) | (Mod << 6); 104} 105 106static void emitByte(uint8_t C, raw_ostream &OS) { OS << static_cast<char>(C); } 107 108static void emitConstant(uint64_t Val, unsigned Size, raw_ostream &OS) { 109 // Output the constant in little endian byte order. 110 for (unsigned i = 0; i != Size; ++i) { 111 emitByte(Val & 255, OS); 112 Val >>= 8; 113 } 114} 115 116/// \returns true if this signed displacement fits in a 8-bit sign-extended 117/// field. 118static bool isDisp8(int Value) { return Value == (int8_t)Value; } 119 120/// \returns true if this signed displacement fits in a 8-bit compressed 121/// dispacement field. 122static bool isCDisp8(uint64_t TSFlags, int Value, int &CValue) { 123 assert(((TSFlags & X86II::EncodingMask) == X86II::EVEX) && 124 "Compressed 8-bit displacement is only valid for EVEX inst."); 125 126 unsigned CD8_Scale = 127 (TSFlags & X86II::CD8_Scale_Mask) >> X86II::CD8_Scale_Shift; 128 if (CD8_Scale == 0) { 129 CValue = Value; 130 return isDisp8(Value); 131 } 132 133 unsigned Mask = CD8_Scale - 1; 134 assert((CD8_Scale & Mask) == 0 && "Invalid memory object size."); 135 if (Value & Mask) // Unaligned offset 136 return false; 137 Value /= (int)CD8_Scale; 138 bool Ret = (Value == (int8_t)Value); 139 140 if (Ret) 141 CValue = Value; 142 return Ret; 143} 144 145/// \returns the appropriate fixup kind to use for an immediate in an 146/// instruction with the specified TSFlags. 147static MCFixupKind getImmFixupKind(uint64_t TSFlags) { 148 unsigned Size = X86II::getSizeOfImm(TSFlags); 149 bool isPCRel = X86II::isImmPCRel(TSFlags); 150 151 if (X86II::isImmSigned(TSFlags)) { 152 switch (Size) { 153 default: 154 llvm_unreachable("Unsupported signed fixup size!"); 155 case 4: 156 return MCFixupKind(X86::reloc_signed_4byte); 157 } 158 } 159 return MCFixup::getKindForSize(Size, isPCRel); 160} 161 162/// \param Op operand # of the memory operand. 163/// 164/// \returns true if the specified instruction has a 16-bit memory operand. 165static bool is16BitMemOperand(const MCInst &MI, unsigned Op, 166 const MCSubtargetInfo &STI) { 167 const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg); 168 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg); 169 const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp); 170 171 if (STI.hasFeature(X86::Mode16Bit) && BaseReg.getReg() == 0 && Disp.isImm() && 172 Disp.getImm() < 0x10000) 173 return true; 174 if ((BaseReg.getReg() != 0 && 175 X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg.getReg())) || 176 (IndexReg.getReg() != 0 && 177 X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg.getReg()))) 178 return true; 179 return false; 180} 181 182/// \param Op operand # of the memory operand. 183/// 184/// \returns true if the specified instruction has a 32-bit memory operand. 185static bool is32BitMemOperand(const MCInst &MI, unsigned Op) { 186 const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg); 187 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg); 188 189 if ((BaseReg.getReg() != 0 && 190 X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg.getReg())) || 191 (IndexReg.getReg() != 0 && 192 X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg.getReg()))) 193 return true; 194 if (BaseReg.getReg() == X86::EIP) { 195 assert(IndexReg.getReg() == 0 && "Invalid eip-based address."); 196 return true; 197 } 198 if (IndexReg.getReg() == X86::EIZ) 199 return true; 200 return false; 201} 202 203/// \param Op operand # of the memory operand. 204/// 205/// \returns true if the specified instruction has a 64-bit memory operand. 206#ifndef NDEBUG 207static bool is64BitMemOperand(const MCInst &MI, unsigned Op) { 208 const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg); 209 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg); 210 211 if ((BaseReg.getReg() != 0 && 212 X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg.getReg())) || 213 (IndexReg.getReg() != 0 && 214 X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg.getReg()))) 215 return true; 216 return false; 217} 218#endif 219 220enum GlobalOffsetTableExprKind { GOT_None, GOT_Normal, GOT_SymDiff }; 221 222/// Check if this expression starts with _GLOBAL_OFFSET_TABLE_ and if it is 223/// of the form _GLOBAL_OFFSET_TABLE_-symbol. This is needed to support PIC on 224/// ELF i386 as _GLOBAL_OFFSET_TABLE_ is magical. We check only simple case that 225/// are know to be used: _GLOBAL_OFFSET_TABLE_ by itself or at the start of a 226/// binary expression. 227static GlobalOffsetTableExprKind 228startsWithGlobalOffsetTable(const MCExpr *Expr) { 229 const MCExpr *RHS = nullptr; 230 if (Expr->getKind() == MCExpr::Binary) { 231 const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(Expr); 232 Expr = BE->getLHS(); 233 RHS = BE->getRHS(); 234 } 235 236 if (Expr->getKind() != MCExpr::SymbolRef) 237 return GOT_None; 238 239 const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr); 240 const MCSymbol &S = Ref->getSymbol(); 241 if (S.getName() != "_GLOBAL_OFFSET_TABLE_") 242 return GOT_None; 243 if (RHS && RHS->getKind() == MCExpr::SymbolRef) 244 return GOT_SymDiff; 245 return GOT_Normal; 246} 247 248static bool hasSecRelSymbolRef(const MCExpr *Expr) { 249 if (Expr->getKind() == MCExpr::SymbolRef) { 250 const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr); 251 return Ref->getKind() == MCSymbolRefExpr::VK_SECREL; 252 } 253 return false; 254} 255 256static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) { 257 unsigned Opcode = MI.getOpcode(); 258 const MCInstrDesc &Desc = MCII.get(Opcode); 259 if ((Opcode != X86::CALL64pcrel32 && Opcode != X86::JMP_4 && 260 Opcode != X86::JCC_4) || 261 getImmFixupKind(Desc.TSFlags) != FK_PCRel_4) 262 return false; 263 264 unsigned CurOp = X86II::getOperandBias(Desc); 265 const MCOperand &Op = MI.getOperand(CurOp); 266 if (!Op.isExpr()) 267 return false; 268 269 const MCSymbolRefExpr *Ref = dyn_cast<MCSymbolRefExpr>(Op.getExpr()); 270 return Ref && Ref->getKind() == MCSymbolRefExpr::VK_None; 271} 272 273unsigned X86MCCodeEmitter::getX86RegNum(const MCOperand &MO) const { 274 return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg()) & 0x7; 275} 276 277unsigned X86MCCodeEmitter::getX86RegEncoding(const MCInst &MI, 278 unsigned OpNum) const { 279 return Ctx.getRegisterInfo()->getEncodingValue(MI.getOperand(OpNum).getReg()); 280} 281 282/// \param MI a single low-level machine instruction. 283/// \param OpNum the operand #. 284/// \returns true if the OpNumth operand of MI require a bit to be set in 285/// REX prefix. 286bool X86MCCodeEmitter::isREXExtendedReg(const MCInst &MI, 287 unsigned OpNum) const { 288 return (getX86RegEncoding(MI, OpNum) >> 3) & 1; 289} 290 291void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc, 292 unsigned Size, MCFixupKind FixupKind, 293 uint64_t StartByte, raw_ostream &OS, 294 SmallVectorImpl<MCFixup> &Fixups, 295 int ImmOffset) const { 296 const MCExpr *Expr = nullptr; 297 if (DispOp.isImm()) { 298 // If this is a simple integer displacement that doesn't require a 299 // relocation, emit it now. 300 if (FixupKind != FK_PCRel_1 && FixupKind != FK_PCRel_2 && 301 FixupKind != FK_PCRel_4) { 302 emitConstant(DispOp.getImm() + ImmOffset, Size, OS); 303 return; 304 } 305 Expr = MCConstantExpr::create(DispOp.getImm(), Ctx); 306 } else { 307 Expr = DispOp.getExpr(); 308 } 309 310 // If we have an immoffset, add it to the expression. 311 if ((FixupKind == FK_Data_4 || FixupKind == FK_Data_8 || 312 FixupKind == MCFixupKind(X86::reloc_signed_4byte))) { 313 GlobalOffsetTableExprKind Kind = startsWithGlobalOffsetTable(Expr); 314 if (Kind != GOT_None) { 315 assert(ImmOffset == 0); 316 317 if (Size == 8) { 318 FixupKind = MCFixupKind(X86::reloc_global_offset_table8); 319 } else { 320 assert(Size == 4); 321 FixupKind = MCFixupKind(X86::reloc_global_offset_table); 322 } 323 324 if (Kind == GOT_Normal) 325 ImmOffset = static_cast<int>(OS.tell() - StartByte); 326 } else if (Expr->getKind() == MCExpr::SymbolRef) { 327 if (hasSecRelSymbolRef(Expr)) { 328 FixupKind = MCFixupKind(FK_SecRel_4); 329 } 330 } else if (Expr->getKind() == MCExpr::Binary) { 331 const MCBinaryExpr *Bin = static_cast<const MCBinaryExpr *>(Expr); 332 if (hasSecRelSymbolRef(Bin->getLHS()) || 333 hasSecRelSymbolRef(Bin->getRHS())) { 334 FixupKind = MCFixupKind(FK_SecRel_4); 335 } 336 } 337 } 338 339 // If the fixup is pc-relative, we need to bias the value to be relative to 340 // the start of the field, not the end of the field. 341 if (FixupKind == FK_PCRel_4 || 342 FixupKind == MCFixupKind(X86::reloc_riprel_4byte) || 343 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_movq_load) || 344 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax) || 345 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex) || 346 FixupKind == MCFixupKind(X86::reloc_branch_4byte_pcrel)) { 347 ImmOffset -= 4; 348 // If this is a pc-relative load off _GLOBAL_OFFSET_TABLE_: 349 // leaq _GLOBAL_OFFSET_TABLE_(%rip), %r15 350 // this needs to be a GOTPC32 relocation. 351 if (startsWithGlobalOffsetTable(Expr) != GOT_None) 352 FixupKind = MCFixupKind(X86::reloc_global_offset_table); 353 } 354 if (FixupKind == FK_PCRel_2) 355 ImmOffset -= 2; 356 if (FixupKind == FK_PCRel_1) 357 ImmOffset -= 1; 358 359 if (ImmOffset) 360 Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(ImmOffset, Ctx), 361 Ctx); 362 363 // Emit a symbolic constant as a fixup and 4 zeros. 364 Fixups.push_back(MCFixup::create(static_cast<uint32_t>(OS.tell() - StartByte), 365 Expr, FixupKind, Loc)); 366 emitConstant(0, Size, OS); 367} 368 369void X86MCCodeEmitter::emitRegModRMByte(const MCOperand &ModRMReg, 370 unsigned RegOpcodeFld, 371 raw_ostream &OS) const { 372 emitByte(modRMByte(3, RegOpcodeFld, getX86RegNum(ModRMReg)), OS); 373} 374 375void X86MCCodeEmitter::emitSIBByte(unsigned SS, unsigned Index, unsigned Base, 376 raw_ostream &OS) const { 377 // SIB byte is in the same format as the modRMByte. 378 emitByte(modRMByte(SS, Index, Base), OS); 379} 380 381void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op, 382 unsigned RegOpcodeField, 383 uint64_t TSFlags, bool HasREX, 384 uint64_t StartByte, raw_ostream &OS, 385 SmallVectorImpl<MCFixup> &Fixups, 386 const MCSubtargetInfo &STI, 387 bool ForceSIB) const { 388 const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp); 389 const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg); 390 const MCOperand &Scale = MI.getOperand(Op + X86::AddrScaleAmt); 391 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg); 392 unsigned BaseReg = Base.getReg(); 393 bool HasEVEX = (TSFlags & X86II::EncodingMask) == X86II::EVEX; 394 395 // Handle %rip relative addressing. 396 if (BaseReg == X86::RIP || 397 BaseReg == X86::EIP) { // [disp32+rIP] in X86-64 mode 398 assert(STI.hasFeature(X86::Mode64Bit) && 399 "Rip-relative addressing requires 64-bit mode"); 400 assert(IndexReg.getReg() == 0 && !ForceSIB && 401 "Invalid rip-relative address"); 402 emitByte(modRMByte(0, RegOpcodeField, 5), OS); 403 404 unsigned Opcode = MI.getOpcode(); 405 // movq loads are handled with a special relocation form which allows the 406 // linker to eliminate some loads for GOT references which end up in the 407 // same linkage unit. 408 unsigned FixupKind = [=]() { 409 switch (Opcode) { 410 default: 411 return X86::reloc_riprel_4byte; 412 case X86::MOV64rm: 413 assert(HasREX); 414 return X86::reloc_riprel_4byte_movq_load; 415 case X86::CALL64m: 416 case X86::JMP64m: 417 case X86::TAILJMPm64: 418 case X86::TEST64mr: 419 case X86::ADC64rm: 420 case X86::ADD64rm: 421 case X86::AND64rm: 422 case X86::CMP64rm: 423 case X86::OR64rm: 424 case X86::SBB64rm: 425 case X86::SUB64rm: 426 case X86::XOR64rm: 427 return HasREX ? X86::reloc_riprel_4byte_relax_rex 428 : X86::reloc_riprel_4byte_relax; 429 } 430 }(); 431 432 // rip-relative addressing is actually relative to the *next* instruction. 433 // Since an immediate can follow the mod/rm byte for an instruction, this 434 // means that we need to bias the displacement field of the instruction with 435 // the size of the immediate field. If we have this case, add it into the 436 // expression to emit. 437 // Note: rip-relative addressing using immediate displacement values should 438 // not be adjusted, assuming it was the user's intent. 439 int ImmSize = !Disp.isImm() && X86II::hasImm(TSFlags) 440 ? X86II::getSizeOfImm(TSFlags) 441 : 0; 442 443 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, OS, 444 Fixups, -ImmSize); 445 return; 446 } 447 448 unsigned BaseRegNo = BaseReg ? getX86RegNum(Base) : -1U; 449 450 // 16-bit addressing forms of the ModR/M byte have a different encoding for 451 // the R/M field and are far more limited in which registers can be used. 452 if (is16BitMemOperand(MI, Op, STI)) { 453 if (BaseReg) { 454 // For 32-bit addressing, the row and column values in Table 2-2 are 455 // basically the same. It's AX/CX/DX/BX/SP/BP/SI/DI in that order, with 456 // some special cases. And getX86RegNum reflects that numbering. 457 // For 16-bit addressing it's more fun, as shown in the SDM Vol 2A, 458 // Table 2-1 "16-Bit Addressing Forms with the ModR/M byte". We can only 459 // use SI/DI/BP/BX, which have "row" values 4-7 in no particular order, 460 // while values 0-3 indicate the allowed combinations (base+index) of 461 // those: 0 for BX+SI, 1 for BX+DI, 2 for BP+SI, 3 for BP+DI. 462 // 463 // R16Table[] is a lookup from the normal RegNo, to the row values from 464 // Table 2-1 for 16-bit addressing modes. Where zero means disallowed. 465 static const unsigned R16Table[] = {0, 0, 0, 7, 0, 6, 4, 5}; 466 unsigned RMfield = R16Table[BaseRegNo]; 467 468 assert(RMfield && "invalid 16-bit base register"); 469 470 if (IndexReg.getReg()) { 471 unsigned IndexReg16 = R16Table[getX86RegNum(IndexReg)]; 472 473 assert(IndexReg16 && "invalid 16-bit index register"); 474 // We must have one of SI/DI (4,5), and one of BP/BX (6,7). 475 assert(((IndexReg16 ^ RMfield) & 2) && 476 "invalid 16-bit base/index register combination"); 477 assert(Scale.getImm() == 1 && 478 "invalid scale for 16-bit memory reference"); 479 480 // Allow base/index to appear in either order (although GAS doesn't). 481 if (IndexReg16 & 2) 482 RMfield = (RMfield & 1) | ((7 - IndexReg16) << 1); 483 else 484 RMfield = (IndexReg16 & 1) | ((7 - RMfield) << 1); 485 } 486 487 if (Disp.isImm() && isDisp8(Disp.getImm())) { 488 if (Disp.getImm() == 0 && RMfield != 6) { 489 // There is no displacement; just the register. 490 emitByte(modRMByte(0, RegOpcodeField, RMfield), OS); 491 return; 492 } 493 // Use the [REG]+disp8 form, including for [BP] which cannot be encoded. 494 emitByte(modRMByte(1, RegOpcodeField, RMfield), OS); 495 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups); 496 return; 497 } 498 // This is the [REG]+disp16 case. 499 emitByte(modRMByte(2, RegOpcodeField, RMfield), OS); 500 } else { 501 // There is no BaseReg; this is the plain [disp16] case. 502 emitByte(modRMByte(0, RegOpcodeField, 6), OS); 503 } 504 505 // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases. 506 emitImmediate(Disp, MI.getLoc(), 2, FK_Data_2, StartByte, OS, Fixups); 507 return; 508 } 509 510 // Determine whether a SIB byte is needed. 511 // If no BaseReg, issue a RIP relative instruction only if the MCE can 512 // resolve addresses on-the-fly, otherwise use SIB (Intel Manual 2A, table 513 // 2-7) and absolute references. 514 515 if ( // The SIB byte must be used if there is an index register. 516 !ForceSIB && IndexReg.getReg() == 0 && 517 // The SIB byte must be used if the base is ESP/RSP/R12, all of which 518 // encode to an R/M value of 4, which indicates that a SIB byte is 519 // present. 520 BaseRegNo != N86::ESP && 521 // If there is no base register and we're in 64-bit mode, we need a SIB 522 // byte to emit an addr that is just 'disp32' (the non-RIP relative form). 523 (!STI.hasFeature(X86::Mode64Bit) || BaseReg != 0)) { 524 525 if (BaseReg == 0) { // [disp32] in X86-32 mode 526 emitByte(modRMByte(0, RegOpcodeField, 5), OS); 527 emitImmediate(Disp, MI.getLoc(), 4, FK_Data_4, StartByte, OS, Fixups); 528 return; 529 } 530 531 // If the base is not EBP/ESP and there is no displacement, use simple 532 // indirect register encoding, this handles addresses like [EAX]. The 533 // encoding for [EBP] with no displacement means [disp32] so we handle it 534 // by emitting a displacement of 0 below. 535 if (BaseRegNo != N86::EBP) { 536 if (Disp.isImm() && Disp.getImm() == 0) { 537 emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), OS); 538 return; 539 } 540 541 // If the displacement is @tlscall, treat it as a zero. 542 if (Disp.isExpr()) { 543 auto *Sym = dyn_cast<MCSymbolRefExpr>(Disp.getExpr()); 544 if (Sym && Sym->getKind() == MCSymbolRefExpr::VK_TLSCALL) { 545 // This is exclusively used by call *a@tlscall(base). The relocation 546 // (R_386_TLSCALL or R_X86_64_TLSCALL) applies to the beginning. 547 Fixups.push_back(MCFixup::create(0, Sym, FK_NONE, MI.getLoc())); 548 emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), OS); 549 return; 550 } 551 } 552 } 553 554 // Otherwise, if the displacement fits in a byte, encode as [REG+disp8]. 555 if (Disp.isImm()) { 556 if (!HasEVEX && isDisp8(Disp.getImm())) { 557 emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), OS); 558 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups); 559 return; 560 } 561 // Try EVEX compressed 8-bit displacement first; if failed, fall back to 562 // 32-bit displacement. 563 int CDisp8 = 0; 564 if (HasEVEX && isCDisp8(TSFlags, Disp.getImm(), CDisp8)) { 565 emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), OS); 566 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups, 567 CDisp8 - Disp.getImm()); 568 return; 569 } 570 } 571 572 // Otherwise, emit the most general non-SIB encoding: [REG+disp32] 573 emitByte(modRMByte(2, RegOpcodeField, BaseRegNo), OS); 574 unsigned Opcode = MI.getOpcode(); 575 unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax 576 : X86::reloc_signed_4byte; 577 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, OS, 578 Fixups); 579 return; 580 } 581 582 // We need a SIB byte, so start by outputting the ModR/M byte first 583 assert(IndexReg.getReg() != X86::ESP && IndexReg.getReg() != X86::RSP && 584 "Cannot use ESP as index reg!"); 585 586 bool ForceDisp32 = false; 587 bool ForceDisp8 = false; 588 int CDisp8 = 0; 589 int ImmOffset = 0; 590 if (BaseReg == 0) { 591 // If there is no base register, we emit the special case SIB byte with 592 // MOD=0, BASE=5, to JUST get the index, scale, and displacement. 593 emitByte(modRMByte(0, RegOpcodeField, 4), OS); 594 ForceDisp32 = true; 595 } else if (!Disp.isImm()) { 596 // Emit the normal disp32 encoding. 597 emitByte(modRMByte(2, RegOpcodeField, 4), OS); 598 ForceDisp32 = true; 599 } else if (Disp.getImm() == 0 && 600 // Base reg can't be anything that ends up with '5' as the base 601 // reg, it is the magic [*] nomenclature that indicates no base. 602 BaseRegNo != N86::EBP) { 603 // Emit no displacement ModR/M byte 604 emitByte(modRMByte(0, RegOpcodeField, 4), OS); 605 } else if (!HasEVEX && isDisp8(Disp.getImm())) { 606 // Emit the disp8 encoding. 607 emitByte(modRMByte(1, RegOpcodeField, 4), OS); 608 ForceDisp8 = true; // Make sure to force 8 bit disp if Base=EBP 609 } else if (HasEVEX && isCDisp8(TSFlags, Disp.getImm(), CDisp8)) { 610 // Emit the disp8 encoding. 611 emitByte(modRMByte(1, RegOpcodeField, 4), OS); 612 ForceDisp8 = true; // Make sure to force 8 bit disp if Base=EBP 613 ImmOffset = CDisp8 - Disp.getImm(); 614 } else { 615 // Emit the normal disp32 encoding. 616 emitByte(modRMByte(2, RegOpcodeField, 4), OS); 617 } 618 619 // Calculate what the SS field value should be... 620 static const unsigned SSTable[] = {~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3}; 621 unsigned SS = SSTable[Scale.getImm()]; 622 623 if (BaseReg == 0) { 624 // Handle the SIB byte for the case where there is no base, see Intel 625 // Manual 2A, table 2-7. The displacement has already been output. 626 unsigned IndexRegNo; 627 if (IndexReg.getReg()) 628 IndexRegNo = getX86RegNum(IndexReg); 629 else // Examples: [ESP+1*<noreg>+4] or [scaled idx]+disp32 (MOD=0,BASE=5) 630 IndexRegNo = 4; 631 emitSIBByte(SS, IndexRegNo, 5, OS); 632 } else { 633 unsigned IndexRegNo; 634 if (IndexReg.getReg()) 635 IndexRegNo = getX86RegNum(IndexReg); 636 else 637 IndexRegNo = 4; // For example [ESP+1*<noreg>+4] 638 emitSIBByte(SS, IndexRegNo, getX86RegNum(Base), OS); 639 } 640 641 // Do we need to output a displacement? 642 if (ForceDisp8) 643 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups, 644 ImmOffset); 645 else if (ForceDisp32 || Disp.getImm() != 0) 646 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(X86::reloc_signed_4byte), 647 StartByte, OS, Fixups); 648} 649 650/// Emit all instruction prefixes. 651/// 652/// \returns true if REX prefix is used, otherwise returns false. 653bool X86MCCodeEmitter::emitPrefixImpl(unsigned &CurOp, const MCInst &MI, 654 const MCSubtargetInfo &STI, 655 raw_ostream &OS) const { 656 uint64_t TSFlags = MCII.get(MI.getOpcode()).TSFlags; 657 // Determine where the memory operand starts, if present. 658 int MemoryOperand = X86II::getMemoryOperandNo(TSFlags); 659 // Emit segment override opcode prefix as needed. 660 if (MemoryOperand != -1) { 661 MemoryOperand += CurOp; 662 emitSegmentOverridePrefix(MemoryOperand + X86::AddrSegmentReg, MI, OS); 663 } 664 665 // Emit the repeat opcode prefix as needed. 666 unsigned Flags = MI.getFlags(); 667 if (TSFlags & X86II::REP || Flags & X86::IP_HAS_REPEAT) 668 emitByte(0xF3, OS); 669 if (Flags & X86::IP_HAS_REPEAT_NE) 670 emitByte(0xF2, OS); 671 672 // Emit the address size opcode prefix as needed. 673 bool NeedAddressOverride; 674 uint64_t AdSize = TSFlags & X86II::AdSizeMask; 675 if ((STI.hasFeature(X86::Mode16Bit) && AdSize == X86II::AdSize32) || 676 (STI.hasFeature(X86::Mode32Bit) && AdSize == X86II::AdSize16) || 677 (STI.hasFeature(X86::Mode64Bit) && AdSize == X86II::AdSize32)) { 678 NeedAddressOverride = true; 679 } else if (MemoryOperand < 0) { 680 NeedAddressOverride = false; 681 } else if (STI.hasFeature(X86::Mode64Bit)) { 682 assert(!is16BitMemOperand(MI, MemoryOperand, STI)); 683 NeedAddressOverride = is32BitMemOperand(MI, MemoryOperand); 684 } else if (STI.hasFeature(X86::Mode32Bit)) { 685 assert(!is64BitMemOperand(MI, MemoryOperand)); 686 NeedAddressOverride = is16BitMemOperand(MI, MemoryOperand, STI); 687 } else { 688 assert(STI.hasFeature(X86::Mode16Bit)); 689 assert(!is64BitMemOperand(MI, MemoryOperand)); 690 NeedAddressOverride = !is16BitMemOperand(MI, MemoryOperand, STI); 691 } 692 693 if (NeedAddressOverride) 694 emitByte(0x67, OS); 695 696 // Encoding type for this instruction. 697 uint64_t Encoding = TSFlags & X86II::EncodingMask; 698 bool HasREX = false; 699 if (Encoding) 700 emitVEXOpcodePrefix(MemoryOperand, MI, OS); 701 else 702 HasREX = emitOpcodePrefix(MemoryOperand, MI, STI, OS); 703 704 uint64_t Form = TSFlags & X86II::FormMask; 705 switch (Form) { 706 default: 707 break; 708 case X86II::RawFrmDstSrc: { 709 unsigned siReg = MI.getOperand(1).getReg(); 710 assert(((siReg == X86::SI && MI.getOperand(0).getReg() == X86::DI) || 711 (siReg == X86::ESI && MI.getOperand(0).getReg() == X86::EDI) || 712 (siReg == X86::RSI && MI.getOperand(0).getReg() == X86::RDI)) && 713 "SI and DI register sizes do not match"); 714 // Emit segment override opcode prefix as needed (not for %ds). 715 if (MI.getOperand(2).getReg() != X86::DS) 716 emitSegmentOverridePrefix(2, MI, OS); 717 // Emit AdSize prefix as needed. 718 if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) || 719 (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI)) 720 emitByte(0x67, OS); 721 CurOp += 3; // Consume operands. 722 break; 723 } 724 case X86II::RawFrmSrc: { 725 unsigned siReg = MI.getOperand(0).getReg(); 726 // Emit segment override opcode prefix as needed (not for %ds). 727 if (MI.getOperand(1).getReg() != X86::DS) 728 emitSegmentOverridePrefix(1, MI, OS); 729 // Emit AdSize prefix as needed. 730 if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) || 731 (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI)) 732 emitByte(0x67, OS); 733 CurOp += 2; // Consume operands. 734 break; 735 } 736 case X86II::RawFrmDst: { 737 unsigned siReg = MI.getOperand(0).getReg(); 738 // Emit AdSize prefix as needed. 739 if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::EDI) || 740 (STI.hasFeature(X86::Mode32Bit) && siReg == X86::DI)) 741 emitByte(0x67, OS); 742 ++CurOp; // Consume operand. 743 break; 744 } 745 case X86II::RawFrmMemOffs: { 746 // Emit segment override opcode prefix as needed. 747 emitSegmentOverridePrefix(1, MI, OS); 748 break; 749 } 750 } 751 752 return HasREX; 753} 754 755/// AVX instructions are encoded using a opcode prefix called VEX. 756void X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, 757 raw_ostream &OS) const { 758 const MCInstrDesc &Desc = MCII.get(MI.getOpcode()); 759 uint64_t TSFlags = Desc.TSFlags; 760 761 assert(!(TSFlags & X86II::LOCK) && "Can't have LOCK VEX."); 762 763 uint64_t Encoding = TSFlags & X86II::EncodingMask; 764 bool HasEVEX_K = TSFlags & X86II::EVEX_K; 765 bool HasVEX_4V = TSFlags & X86II::VEX_4V; 766 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC; 767 768 // VEX_R: opcode externsion equivalent to REX.R in 769 // 1's complement (inverted) form 770 // 771 // 1: Same as REX_R=0 (must be 1 in 32-bit mode) 772 // 0: Same as REX_R=1 (64 bit mode only) 773 // 774 uint8_t VEX_R = 0x1; 775 uint8_t EVEX_R2 = 0x1; 776 777 // VEX_X: equivalent to REX.X, only used when a 778 // register is used for index in SIB Byte. 779 // 780 // 1: Same as REX.X=0 (must be 1 in 32-bit mode) 781 // 0: Same as REX.X=1 (64-bit mode only) 782 uint8_t VEX_X = 0x1; 783 784 // VEX_B: 785 // 786 // 1: Same as REX_B=0 (ignored in 32-bit mode) 787 // 0: Same as REX_B=1 (64 bit mode only) 788 // 789 uint8_t VEX_B = 0x1; 790 791 // VEX_W: opcode specific (use like REX.W, or used for 792 // opcode extension, or ignored, depending on the opcode byte) 793 uint8_t VEX_W = (TSFlags & X86II::VEX_W) ? 1 : 0; 794 795 // VEX_5M (VEX m-mmmmm field): 796 // 797 // 0b00000: Reserved for future use 798 // 0b00001: implied 0F leading opcode 799 // 0b00010: implied 0F 38 leading opcode bytes 800 // 0b00011: implied 0F 3A leading opcode bytes 801 // 0b00100-0b11111: Reserved for future use 802 // 0b01000: XOP map select - 08h instructions with imm byte 803 // 0b01001: XOP map select - 09h instructions with no imm byte 804 // 0b01010: XOP map select - 0Ah instructions with imm dword 805 uint8_t VEX_5M; 806 switch (TSFlags & X86II::OpMapMask) { 807 default: 808 llvm_unreachable("Invalid prefix!"); 809 case X86II::TB: 810 VEX_5M = 0x1; 811 break; // 0F 812 case X86II::T8: 813 VEX_5M = 0x2; 814 break; // 0F 38 815 case X86II::TA: 816 VEX_5M = 0x3; 817 break; // 0F 3A 818 case X86II::XOP8: 819 VEX_5M = 0x8; 820 break; 821 case X86II::XOP9: 822 VEX_5M = 0x9; 823 break; 824 case X86II::XOPA: 825 VEX_5M = 0xA; 826 break; 827 } 828 829 // VEX_4V (VEX vvvv field): a register specifier 830 // (in 1's complement form) or 1111 if unused. 831 uint8_t VEX_4V = 0xf; 832 uint8_t EVEX_V2 = 0x1; 833 834 // EVEX_L2/VEX_L (Vector Length): 835 // 836 // L2 L 837 // 0 0: scalar or 128-bit vector 838 // 0 1: 256-bit vector 839 // 1 0: 512-bit vector 840 // 841 uint8_t VEX_L = (TSFlags & X86II::VEX_L) ? 1 : 0; 842 uint8_t EVEX_L2 = (TSFlags & X86II::EVEX_L2) ? 1 : 0; 843 844 // VEX_PP: opcode extension providing equivalent 845 // functionality of a SIMD prefix 846 // 847 // 0b00: None 848 // 0b01: 66 849 // 0b10: F3 850 // 0b11: F2 851 // 852 uint8_t VEX_PP = 0; 853 switch (TSFlags & X86II::OpPrefixMask) { 854 case X86II::PD: 855 VEX_PP = 0x1; 856 break; // 66 857 case X86II::XS: 858 VEX_PP = 0x2; 859 break; // F3 860 case X86II::XD: 861 VEX_PP = 0x3; 862 break; // F2 863 } 864 865 // EVEX_U 866 uint8_t EVEX_U = 1; // Always '1' so far 867 868 // EVEX_z 869 uint8_t EVEX_z = (HasEVEX_K && (TSFlags & X86II::EVEX_Z)) ? 1 : 0; 870 871 // EVEX_b 872 uint8_t EVEX_b = (TSFlags & X86II::EVEX_B) ? 1 : 0; 873 874 // EVEX_rc 875 uint8_t EVEX_rc = 0; 876 877 // EVEX_aaa 878 uint8_t EVEX_aaa = 0; 879 880 bool EncodeRC = false; 881 882 // Classify VEX_B, VEX_4V, VEX_R, VEX_X 883 unsigned NumOps = Desc.getNumOperands(); 884 unsigned CurOp = X86II::getOperandBias(Desc); 885 886 switch (TSFlags & X86II::FormMask) { 887 default: 888 llvm_unreachable("Unexpected form in emitVEXOpcodePrefix!"); 889 case X86II::MRM_C0: 890 case X86II::RawFrm: 891 case X86II::PrefixByte: 892 break; 893 case X86II::MRMDestMemFSIB: 894 case X86II::MRMDestMem: { 895 // MRMDestMem instructions forms: 896 // MemAddr, src1(ModR/M) 897 // MemAddr, src1(VEX_4V), src2(ModR/M) 898 // MemAddr, src1(ModR/M), imm8 899 // 900 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg); 901 VEX_B = ~(BaseRegEnc >> 3) & 1; 902 unsigned IndexRegEnc = 903 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg); 904 VEX_X = ~(IndexRegEnc >> 3) & 1; 905 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV. 906 EVEX_V2 = ~(IndexRegEnc >> 4) & 1; 907 908 CurOp += X86::AddrNumOperands; 909 910 if (HasEVEX_K) 911 EVEX_aaa = getX86RegEncoding(MI, CurOp++); 912 913 if (HasVEX_4V) { 914 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++); 915 VEX_4V = ~VRegEnc & 0xf; 916 EVEX_V2 = ~(VRegEnc >> 4) & 1; 917 } 918 919 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 920 VEX_R = ~(RegEnc >> 3) & 1; 921 EVEX_R2 = ~(RegEnc >> 4) & 1; 922 break; 923 } 924 case X86II::MRMSrcMemFSIB: 925 case X86II::MRMSrcMem: { 926 // MRMSrcMem instructions forms: 927 // src1(ModR/M), MemAddr 928 // src1(ModR/M), src2(VEX_4V), MemAddr 929 // src1(ModR/M), MemAddr, imm8 930 // src1(ModR/M), MemAddr, src2(Imm[7:4]) 931 // 932 // FMA4: 933 // dst(ModR/M.reg), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4]) 934 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 935 VEX_R = ~(RegEnc >> 3) & 1; 936 EVEX_R2 = ~(RegEnc >> 4) & 1; 937 938 if (HasEVEX_K) 939 EVEX_aaa = getX86RegEncoding(MI, CurOp++); 940 941 if (HasVEX_4V) { 942 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++); 943 VEX_4V = ~VRegEnc & 0xf; 944 EVEX_V2 = ~(VRegEnc >> 4) & 1; 945 } 946 947 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg); 948 VEX_B = ~(BaseRegEnc >> 3) & 1; 949 unsigned IndexRegEnc = 950 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg); 951 VEX_X = ~(IndexRegEnc >> 3) & 1; 952 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV. 953 EVEX_V2 = ~(IndexRegEnc >> 4) & 1; 954 955 break; 956 } 957 case X86II::MRMSrcMem4VOp3: { 958 // Instruction format for 4VOp3: 959 // src1(ModR/M), MemAddr, src3(VEX_4V) 960 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 961 VEX_R = ~(RegEnc >> 3) & 1; 962 963 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg); 964 VEX_B = ~(BaseRegEnc >> 3) & 1; 965 unsigned IndexRegEnc = 966 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg); 967 VEX_X = ~(IndexRegEnc >> 3) & 1; 968 969 VEX_4V = ~getX86RegEncoding(MI, CurOp + X86::AddrNumOperands) & 0xf; 970 break; 971 } 972 case X86II::MRMSrcMemOp4: { 973 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M), 974 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 975 VEX_R = ~(RegEnc >> 3) & 1; 976 977 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++); 978 VEX_4V = ~VRegEnc & 0xf; 979 980 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg); 981 VEX_B = ~(BaseRegEnc >> 3) & 1; 982 unsigned IndexRegEnc = 983 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg); 984 VEX_X = ~(IndexRegEnc >> 3) & 1; 985 break; 986 } 987 case X86II::MRM0m: 988 case X86II::MRM1m: 989 case X86II::MRM2m: 990 case X86II::MRM3m: 991 case X86II::MRM4m: 992 case X86II::MRM5m: 993 case X86II::MRM6m: 994 case X86II::MRM7m: { 995 // MRM[0-9]m instructions forms: 996 // MemAddr 997 // src1(VEX_4V), MemAddr 998 if (HasVEX_4V) { 999 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++); 1000 VEX_4V = ~VRegEnc & 0xf; 1001 EVEX_V2 = ~(VRegEnc >> 4) & 1; 1002 } 1003 1004 if (HasEVEX_K) 1005 EVEX_aaa = getX86RegEncoding(MI, CurOp++); 1006 1007 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg); 1008 VEX_B = ~(BaseRegEnc >> 3) & 1; 1009 unsigned IndexRegEnc = 1010 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg); 1011 VEX_X = ~(IndexRegEnc >> 3) & 1; 1012 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV. 1013 EVEX_V2 = ~(IndexRegEnc >> 4) & 1; 1014 1015 break; 1016 } 1017 case X86II::MRMSrcReg: { 1018 // MRMSrcReg instructions forms: 1019 // dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4]) 1020 // dst(ModR/M), src1(ModR/M) 1021 // dst(ModR/M), src1(ModR/M), imm8 1022 // 1023 // FMA4: 1024 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M), 1025 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 1026 VEX_R = ~(RegEnc >> 3) & 1; 1027 EVEX_R2 = ~(RegEnc >> 4) & 1; 1028 1029 if (HasEVEX_K) 1030 EVEX_aaa = getX86RegEncoding(MI, CurOp++); 1031 1032 if (HasVEX_4V) { 1033 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++); 1034 VEX_4V = ~VRegEnc & 0xf; 1035 EVEX_V2 = ~(VRegEnc >> 4) & 1; 1036 } 1037 1038 RegEnc = getX86RegEncoding(MI, CurOp++); 1039 VEX_B = ~(RegEnc >> 3) & 1; 1040 VEX_X = ~(RegEnc >> 4) & 1; 1041 1042 if (EVEX_b) { 1043 if (HasEVEX_RC) { 1044 unsigned RcOperand = NumOps - 1; 1045 assert(RcOperand >= CurOp); 1046 EVEX_rc = MI.getOperand(RcOperand).getImm(); 1047 assert(EVEX_rc <= 3 && "Invalid rounding control!"); 1048 } 1049 EncodeRC = true; 1050 } 1051 break; 1052 } 1053 case X86II::MRMSrcReg4VOp3: { 1054 // Instruction format for 4VOp3: 1055 // src1(ModR/M), src2(ModR/M), src3(VEX_4V) 1056 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 1057 VEX_R = ~(RegEnc >> 3) & 1; 1058 1059 RegEnc = getX86RegEncoding(MI, CurOp++); 1060 VEX_B = ~(RegEnc >> 3) & 1; 1061 1062 VEX_4V = ~getX86RegEncoding(MI, CurOp++) & 0xf; 1063 break; 1064 } 1065 case X86II::MRMSrcRegOp4: { 1066 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M), 1067 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 1068 VEX_R = ~(RegEnc >> 3) & 1; 1069 1070 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++); 1071 VEX_4V = ~VRegEnc & 0xf; 1072 1073 // Skip second register source (encoded in Imm[7:4]) 1074 ++CurOp; 1075 1076 RegEnc = getX86RegEncoding(MI, CurOp++); 1077 VEX_B = ~(RegEnc >> 3) & 1; 1078 VEX_X = ~(RegEnc >> 4) & 1; 1079 break; 1080 } 1081 case X86II::MRMDestReg: { 1082 // MRMDestReg instructions forms: 1083 // dst(ModR/M), src(ModR/M) 1084 // dst(ModR/M), src(ModR/M), imm8 1085 // dst(ModR/M), src1(VEX_4V), src2(ModR/M) 1086 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 1087 VEX_B = ~(RegEnc >> 3) & 1; 1088 VEX_X = ~(RegEnc >> 4) & 1; 1089 1090 if (HasEVEX_K) 1091 EVEX_aaa = getX86RegEncoding(MI, CurOp++); 1092 1093 if (HasVEX_4V) { 1094 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++); 1095 VEX_4V = ~VRegEnc & 0xf; 1096 EVEX_V2 = ~(VRegEnc >> 4) & 1; 1097 } 1098 1099 RegEnc = getX86RegEncoding(MI, CurOp++); 1100 VEX_R = ~(RegEnc >> 3) & 1; 1101 EVEX_R2 = ~(RegEnc >> 4) & 1; 1102 if (EVEX_b) 1103 EncodeRC = true; 1104 break; 1105 } 1106 case X86II::MRMr0: { 1107 // MRMr0 instructions forms: 1108 // 11:rrr:000 1109 // dst(ModR/M) 1110 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 1111 VEX_R = ~(RegEnc >> 3) & 1; 1112 EVEX_R2 = ~(RegEnc >> 4) & 1; 1113 break; 1114 } 1115 case X86II::MRM0r: 1116 case X86II::MRM1r: 1117 case X86II::MRM2r: 1118 case X86II::MRM3r: 1119 case X86II::MRM4r: 1120 case X86II::MRM5r: 1121 case X86II::MRM6r: 1122 case X86II::MRM7r: { 1123 // MRM0r-MRM7r instructions forms: 1124 // dst(VEX_4V), src(ModR/M), imm8 1125 if (HasVEX_4V) { 1126 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++); 1127 VEX_4V = ~VRegEnc & 0xf; 1128 EVEX_V2 = ~(VRegEnc >> 4) & 1; 1129 } 1130 if (HasEVEX_K) 1131 EVEX_aaa = getX86RegEncoding(MI, CurOp++); 1132 1133 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 1134 VEX_B = ~(RegEnc >> 3) & 1; 1135 VEX_X = ~(RegEnc >> 4) & 1; 1136 break; 1137 } 1138 } 1139 1140 if (Encoding == X86II::VEX || Encoding == X86II::XOP) { 1141 // VEX opcode prefix can have 2 or 3 bytes 1142 // 1143 // 3 bytes: 1144 // +-----+ +--------------+ +-------------------+ 1145 // | C4h | | RXB | m-mmmm | | W | vvvv | L | pp | 1146 // +-----+ +--------------+ +-------------------+ 1147 // 2 bytes: 1148 // +-----+ +-------------------+ 1149 // | C5h | | R | vvvv | L | pp | 1150 // +-----+ +-------------------+ 1151 // 1152 // XOP uses a similar prefix: 1153 // +-----+ +--------------+ +-------------------+ 1154 // | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp | 1155 // +-----+ +--------------+ +-------------------+ 1156 uint8_t LastByte = VEX_PP | (VEX_L << 2) | (VEX_4V << 3); 1157 1158 // Can we use the 2 byte VEX prefix? 1159 if (!(MI.getFlags() & X86::IP_USE_VEX3) && Encoding == X86II::VEX && 1160 VEX_B && VEX_X && !VEX_W && (VEX_5M == 1)) { 1161 emitByte(0xC5, OS); 1162 emitByte(LastByte | (VEX_R << 7), OS); 1163 return; 1164 } 1165 1166 // 3 byte VEX prefix 1167 emitByte(Encoding == X86II::XOP ? 0x8F : 0xC4, OS); 1168 emitByte(VEX_R << 7 | VEX_X << 6 | VEX_B << 5 | VEX_5M, OS); 1169 emitByte(LastByte | (VEX_W << 7), OS); 1170 } else { 1171 assert(Encoding == X86II::EVEX && "unknown encoding!"); 1172 // EVEX opcode prefix can have 4 bytes 1173 // 1174 // +-----+ +--------------+ +-------------------+ +------------------------+ 1175 // | 62h | | RXBR' | 00mm | | W | vvvv | U | pp | | z | L'L | b | v' | aaa | 1176 // +-----+ +--------------+ +-------------------+ +------------------------+ 1177 assert((VEX_5M & 0x3) == VEX_5M && 1178 "More than 2 significant bits in VEX.m-mmmm fields for EVEX!"); 1179 1180 emitByte(0x62, OS); 1181 emitByte((VEX_R << 7) | (VEX_X << 6) | (VEX_B << 5) | (EVEX_R2 << 4) | 1182 VEX_5M, 1183 OS); 1184 emitByte((VEX_W << 7) | (VEX_4V << 3) | (EVEX_U << 2) | VEX_PP, OS); 1185 if (EncodeRC) 1186 emitByte((EVEX_z << 7) | (EVEX_rc << 5) | (EVEX_b << 4) | (EVEX_V2 << 3) | 1187 EVEX_aaa, 1188 OS); 1189 else 1190 emitByte((EVEX_z << 7) | (EVEX_L2 << 6) | (VEX_L << 5) | (EVEX_b << 4) | 1191 (EVEX_V2 << 3) | EVEX_aaa, 1192 OS); 1193 } 1194} 1195 1196/// Emit REX prefix which specifies 1197/// 1) 64-bit instructions, 1198/// 2) non-default operand size, and 1199/// 3) use of X86-64 extended registers. 1200/// 1201/// \returns true if REX prefix is used, otherwise returns false. 1202bool X86MCCodeEmitter::emitREXPrefix(int MemOperand, const MCInst &MI, 1203 raw_ostream &OS) const { 1204 uint8_t REX = [&, MemOperand]() { 1205 uint8_t REX = 0; 1206 bool UsesHighByteReg = false; 1207 1208 const MCInstrDesc &Desc = MCII.get(MI.getOpcode()); 1209 uint64_t TSFlags = Desc.TSFlags; 1210 1211 if (TSFlags & X86II::REX_W) 1212 REX |= 1 << 3; // set REX.W 1213 1214 if (MI.getNumOperands() == 0) 1215 return REX; 1216 1217 unsigned NumOps = MI.getNumOperands(); 1218 unsigned CurOp = X86II::getOperandBias(Desc); 1219 1220 // If it accesses SPL, BPL, SIL, or DIL, then it requires a 0x40 REX prefix. 1221 for (unsigned i = CurOp; i != NumOps; ++i) { 1222 const MCOperand &MO = MI.getOperand(i); 1223 if (!MO.isReg()) 1224 continue; 1225 unsigned Reg = MO.getReg(); 1226 if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH) 1227 UsesHighByteReg = true; 1228 if (X86II::isX86_64NonExtLowByteReg(Reg)) 1229 // FIXME: The caller of determineREXPrefix slaps this prefix onto 1230 // anything that returns non-zero. 1231 REX |= 0x40; // REX fixed encoding prefix 1232 } 1233 1234 switch (TSFlags & X86II::FormMask) { 1235 case X86II::AddRegFrm: 1236 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B 1237 break; 1238 case X86II::MRMSrcReg: 1239 case X86II::MRMSrcRegCC: 1240 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R 1241 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B 1242 break; 1243 case X86II::MRMSrcMem: 1244 case X86II::MRMSrcMemCC: 1245 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R 1246 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B 1247 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X 1248 CurOp += X86::AddrNumOperands; 1249 break; 1250 case X86II::MRMDestReg: 1251 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B 1252 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R 1253 break; 1254 case X86II::MRMDestMem: 1255 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B 1256 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X 1257 CurOp += X86::AddrNumOperands; 1258 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R 1259 break; 1260 case X86II::MRMXmCC: 1261 case X86II::MRMXm: 1262 case X86II::MRM0m: 1263 case X86II::MRM1m: 1264 case X86II::MRM2m: 1265 case X86II::MRM3m: 1266 case X86II::MRM4m: 1267 case X86II::MRM5m: 1268 case X86II::MRM6m: 1269 case X86II::MRM7m: 1270 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B 1271 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X 1272 break; 1273 case X86II::MRMXrCC: 1274 case X86II::MRMXr: 1275 case X86II::MRM0r: 1276 case X86II::MRM1r: 1277 case X86II::MRM2r: 1278 case X86II::MRM3r: 1279 case X86II::MRM4r: 1280 case X86II::MRM5r: 1281 case X86II::MRM6r: 1282 case X86II::MRM7r: 1283 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B 1284 break; 1285 case X86II::MRMr0: 1286 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R 1287 break; 1288 case X86II::MRMDestMemFSIB: 1289 llvm_unreachable("FSIB format never need REX prefix!"); 1290 } 1291 if (REX && UsesHighByteReg) 1292 report_fatal_error( 1293 "Cannot encode high byte register in REX-prefixed instruction"); 1294 return REX; 1295 }(); 1296 1297 if (!REX) 1298 return false; 1299 1300 emitByte(0x40 | REX, OS); 1301 return true; 1302} 1303 1304/// Emit segment override opcode prefix as needed. 1305void X86MCCodeEmitter::emitSegmentOverridePrefix(unsigned SegOperand, 1306 const MCInst &MI, 1307 raw_ostream &OS) const { 1308 // Check for explicit segment override on memory operand. 1309 if (unsigned Reg = MI.getOperand(SegOperand).getReg()) 1310 emitByte(X86::getSegmentOverridePrefixForReg(Reg), OS); 1311} 1312 1313/// Emit all instruction prefixes prior to the opcode. 1314/// 1315/// \param MemOperand the operand # of the start of a memory operand if present. 1316/// If not present, it is -1. 1317/// 1318/// \returns true if REX prefix is used, otherwise returns false. 1319bool X86MCCodeEmitter::emitOpcodePrefix(int MemOperand, const MCInst &MI, 1320 const MCSubtargetInfo &STI, 1321 raw_ostream &OS) const { 1322 const MCInstrDesc &Desc = MCII.get(MI.getOpcode()); 1323 uint64_t TSFlags = Desc.TSFlags; 1324 1325 // Emit the operand size opcode prefix as needed. 1326 if ((TSFlags & X86II::OpSizeMask) == 1327 (STI.hasFeature(X86::Mode16Bit) ? X86II::OpSize32 : X86II::OpSize16)) 1328 emitByte(0x66, OS); 1329 1330 // Emit the LOCK opcode prefix. 1331 if (TSFlags & X86II::LOCK || MI.getFlags() & X86::IP_HAS_LOCK) 1332 emitByte(0xF0, OS); 1333 1334 // Emit the NOTRACK opcode prefix. 1335 if (TSFlags & X86II::NOTRACK || MI.getFlags() & X86::IP_HAS_NOTRACK) 1336 emitByte(0x3E, OS); 1337 1338 switch (TSFlags & X86II::OpPrefixMask) { 1339 case X86II::PD: // 66 1340 emitByte(0x66, OS); 1341 break; 1342 case X86II::XS: // F3 1343 emitByte(0xF3, OS); 1344 break; 1345 case X86II::XD: // F2 1346 emitByte(0xF2, OS); 1347 break; 1348 } 1349 1350 // Handle REX prefix. 1351 assert((STI.hasFeature(X86::Mode64Bit) || !(TSFlags & X86II::REX_W)) && 1352 "REX.W requires 64bit mode."); 1353 bool HasREX = STI.hasFeature(X86::Mode64Bit) 1354 ? emitREXPrefix(MemOperand, MI, OS) 1355 : false; 1356 1357 // 0x0F escape code must be emitted just before the opcode. 1358 switch (TSFlags & X86II::OpMapMask) { 1359 case X86II::TB: // Two-byte opcode map 1360 case X86II::T8: // 0F 38 1361 case X86II::TA: // 0F 3A 1362 case X86II::ThreeDNow: // 0F 0F, second 0F emitted by caller. 1363 emitByte(0x0F, OS); 1364 break; 1365 } 1366 1367 switch (TSFlags & X86II::OpMapMask) { 1368 case X86II::T8: // 0F 38 1369 emitByte(0x38, OS); 1370 break; 1371 case X86II::TA: // 0F 3A 1372 emitByte(0x3A, OS); 1373 break; 1374 } 1375 1376 return HasREX; 1377} 1378 1379void X86MCCodeEmitter::emitPrefix(const MCInst &MI, raw_ostream &OS, 1380 const MCSubtargetInfo &STI) const { 1381 unsigned Opcode = MI.getOpcode(); 1382 const MCInstrDesc &Desc = MCII.get(Opcode); 1383 uint64_t TSFlags = Desc.TSFlags; 1384 1385 // Pseudo instructions don't get encoded. 1386 if (X86II::isPseudo(TSFlags)) 1387 return; 1388 1389 unsigned CurOp = X86II::getOperandBias(Desc); 1390 1391 emitPrefixImpl(CurOp, MI, STI, OS); 1392} 1393 1394void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS, 1395 SmallVectorImpl<MCFixup> &Fixups, 1396 const MCSubtargetInfo &STI) const { 1397 unsigned Opcode = MI.getOpcode(); 1398 const MCInstrDesc &Desc = MCII.get(Opcode); 1399 uint64_t TSFlags = Desc.TSFlags; 1400 1401 // Pseudo instructions don't get encoded. 1402 if (X86II::isPseudo(TSFlags)) 1403 return; 1404 1405 unsigned NumOps = Desc.getNumOperands(); 1406 unsigned CurOp = X86II::getOperandBias(Desc); 1407 1408 uint64_t StartByte = OS.tell(); 1409 1410 bool HasREX = emitPrefixImpl(CurOp, MI, STI, OS); 1411 1412 // It uses the VEX.VVVV field? 1413 bool HasVEX_4V = TSFlags & X86II::VEX_4V; 1414 bool HasVEX_I8Reg = (TSFlags & X86II::ImmMask) == X86II::Imm8Reg; 1415 1416 // It uses the EVEX.aaa field? 1417 bool HasEVEX_K = TSFlags & X86II::EVEX_K; 1418 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC; 1419 1420 // Used if a register is encoded in 7:4 of immediate. 1421 unsigned I8RegNum = 0; 1422 1423 uint8_t BaseOpcode = X86II::getBaseOpcodeFor(TSFlags); 1424 1425 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow) 1426 BaseOpcode = 0x0F; // Weird 3DNow! encoding. 1427 1428 unsigned OpcodeOffset = 0; 1429 1430 uint64_t Form = TSFlags & X86II::FormMask; 1431 switch (Form) { 1432 default: 1433 errs() << "FORM: " << Form << "\n"; 1434 llvm_unreachable("Unknown FormMask value in X86MCCodeEmitter!"); 1435 case X86II::Pseudo: 1436 llvm_unreachable("Pseudo instruction shouldn't be emitted"); 1437 case X86II::RawFrmDstSrc: 1438 case X86II::RawFrmSrc: 1439 case X86II::RawFrmDst: 1440 case X86II::PrefixByte: 1441 emitByte(BaseOpcode, OS); 1442 break; 1443 case X86II::AddCCFrm: { 1444 // This will be added to the opcode in the fallthrough. 1445 OpcodeOffset = MI.getOperand(NumOps - 1).getImm(); 1446 assert(OpcodeOffset < 16 && "Unexpected opcode offset!"); 1447 --NumOps; // Drop the operand from the end. 1448 LLVM_FALLTHROUGH; 1449 case X86II::RawFrm: 1450 emitByte(BaseOpcode + OpcodeOffset, OS); 1451 1452 if (!STI.hasFeature(X86::Mode64Bit) || !isPCRel32Branch(MI, MCII)) 1453 break; 1454 1455 const MCOperand &Op = MI.getOperand(CurOp++); 1456 emitImmediate(Op, MI.getLoc(), X86II::getSizeOfImm(TSFlags), 1457 MCFixupKind(X86::reloc_branch_4byte_pcrel), StartByte, OS, 1458 Fixups); 1459 break; 1460 } 1461 case X86II::RawFrmMemOffs: 1462 emitByte(BaseOpcode, OS); 1463 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1464 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags), 1465 StartByte, OS, Fixups); 1466 ++CurOp; // skip segment operand 1467 break; 1468 case X86II::RawFrmImm8: 1469 emitByte(BaseOpcode, OS); 1470 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1471 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags), 1472 StartByte, OS, Fixups); 1473 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1, FK_Data_1, StartByte, 1474 OS, Fixups); 1475 break; 1476 case X86II::RawFrmImm16: 1477 emitByte(BaseOpcode, OS); 1478 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1479 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags), 1480 StartByte, OS, Fixups); 1481 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 2, FK_Data_2, StartByte, 1482 OS, Fixups); 1483 break; 1484 1485 case X86II::AddRegFrm: 1486 emitByte(BaseOpcode + getX86RegNum(MI.getOperand(CurOp++)), OS); 1487 break; 1488 1489 case X86II::MRMDestReg: { 1490 emitByte(BaseOpcode, OS); 1491 unsigned SrcRegNum = CurOp + 1; 1492 1493 if (HasEVEX_K) // Skip writemask 1494 ++SrcRegNum; 1495 1496 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV) 1497 ++SrcRegNum; 1498 1499 emitRegModRMByte(MI.getOperand(CurOp), 1500 getX86RegNum(MI.getOperand(SrcRegNum)), OS); 1501 CurOp = SrcRegNum + 1; 1502 break; 1503 } 1504 case X86II::MRMDestMemFSIB: 1505 case X86II::MRMDestMem: { 1506 emitByte(BaseOpcode, OS); 1507 unsigned SrcRegNum = CurOp + X86::AddrNumOperands; 1508 1509 if (HasEVEX_K) // Skip writemask 1510 ++SrcRegNum; 1511 1512 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV) 1513 ++SrcRegNum; 1514 1515 bool ForceSIB = (Form == X86II::MRMDestMemFSIB); 1516 emitMemModRMByte(MI, CurOp, getX86RegNum(MI.getOperand(SrcRegNum)), TSFlags, 1517 HasREX, StartByte, OS, Fixups, STI, ForceSIB); 1518 CurOp = SrcRegNum + 1; 1519 break; 1520 } 1521 case X86II::MRMSrcReg: { 1522 emitByte(BaseOpcode, OS); 1523 unsigned SrcRegNum = CurOp + 1; 1524 1525 if (HasEVEX_K) // Skip writemask 1526 ++SrcRegNum; 1527 1528 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV) 1529 ++SrcRegNum; 1530 1531 emitRegModRMByte(MI.getOperand(SrcRegNum), 1532 getX86RegNum(MI.getOperand(CurOp)), OS); 1533 CurOp = SrcRegNum + 1; 1534 if (HasVEX_I8Reg) 1535 I8RegNum = getX86RegEncoding(MI, CurOp++); 1536 // do not count the rounding control operand 1537 if (HasEVEX_RC) 1538 --NumOps; 1539 break; 1540 } 1541 case X86II::MRMSrcReg4VOp3: { 1542 emitByte(BaseOpcode, OS); 1543 unsigned SrcRegNum = CurOp + 1; 1544 1545 emitRegModRMByte(MI.getOperand(SrcRegNum), 1546 getX86RegNum(MI.getOperand(CurOp)), OS); 1547 CurOp = SrcRegNum + 1; 1548 ++CurOp; // Encoded in VEX.VVVV 1549 break; 1550 } 1551 case X86II::MRMSrcRegOp4: { 1552 emitByte(BaseOpcode, OS); 1553 unsigned SrcRegNum = CurOp + 1; 1554 1555 // Skip 1st src (which is encoded in VEX_VVVV) 1556 ++SrcRegNum; 1557 1558 // Capture 2nd src (which is encoded in Imm[7:4]) 1559 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg"); 1560 I8RegNum = getX86RegEncoding(MI, SrcRegNum++); 1561 1562 emitRegModRMByte(MI.getOperand(SrcRegNum), 1563 getX86RegNum(MI.getOperand(CurOp)), OS); 1564 CurOp = SrcRegNum + 1; 1565 break; 1566 } 1567 case X86II::MRMSrcRegCC: { 1568 unsigned FirstOp = CurOp++; 1569 unsigned SecondOp = CurOp++; 1570 1571 unsigned CC = MI.getOperand(CurOp++).getImm(); 1572 emitByte(BaseOpcode + CC, OS); 1573 1574 emitRegModRMByte(MI.getOperand(SecondOp), 1575 getX86RegNum(MI.getOperand(FirstOp)), OS); 1576 break; 1577 } 1578 case X86II::MRMSrcMemFSIB: 1579 case X86II::MRMSrcMem: { 1580 unsigned FirstMemOp = CurOp + 1; 1581 1582 if (HasEVEX_K) // Skip writemask 1583 ++FirstMemOp; 1584 1585 if (HasVEX_4V) 1586 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV). 1587 1588 emitByte(BaseOpcode, OS); 1589 1590 bool ForceSIB = (Form == X86II::MRMSrcMemFSIB); 1591 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)), 1592 TSFlags, HasREX, StartByte, OS, Fixups, STI, ForceSIB); 1593 CurOp = FirstMemOp + X86::AddrNumOperands; 1594 if (HasVEX_I8Reg) 1595 I8RegNum = getX86RegEncoding(MI, CurOp++); 1596 break; 1597 } 1598 case X86II::MRMSrcMem4VOp3: { 1599 unsigned FirstMemOp = CurOp + 1; 1600 1601 emitByte(BaseOpcode, OS); 1602 1603 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)), 1604 TSFlags, HasREX, StartByte, OS, Fixups, STI); 1605 CurOp = FirstMemOp + X86::AddrNumOperands; 1606 ++CurOp; // Encoded in VEX.VVVV. 1607 break; 1608 } 1609 case X86II::MRMSrcMemOp4: { 1610 unsigned FirstMemOp = CurOp + 1; 1611 1612 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV). 1613 1614 // Capture second register source (encoded in Imm[7:4]) 1615 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg"); 1616 I8RegNum = getX86RegEncoding(MI, FirstMemOp++); 1617 1618 emitByte(BaseOpcode, OS); 1619 1620 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)), 1621 TSFlags, HasREX, StartByte, OS, Fixups, STI); 1622 CurOp = FirstMemOp + X86::AddrNumOperands; 1623 break; 1624 } 1625 case X86II::MRMSrcMemCC: { 1626 unsigned RegOp = CurOp++; 1627 unsigned FirstMemOp = CurOp; 1628 CurOp = FirstMemOp + X86::AddrNumOperands; 1629 1630 unsigned CC = MI.getOperand(CurOp++).getImm(); 1631 emitByte(BaseOpcode + CC, OS); 1632 1633 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(RegOp)), 1634 TSFlags, HasREX, StartByte, OS, Fixups, STI); 1635 break; 1636 } 1637 1638 case X86II::MRMXrCC: { 1639 unsigned RegOp = CurOp++; 1640 1641 unsigned CC = MI.getOperand(CurOp++).getImm(); 1642 emitByte(BaseOpcode + CC, OS); 1643 emitRegModRMByte(MI.getOperand(RegOp), 0, OS); 1644 break; 1645 } 1646 1647 case X86II::MRMXr: 1648 case X86II::MRM0r: 1649 case X86II::MRM1r: 1650 case X86II::MRM2r: 1651 case X86II::MRM3r: 1652 case X86II::MRM4r: 1653 case X86II::MRM5r: 1654 case X86II::MRM6r: 1655 case X86II::MRM7r: 1656 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV). 1657 ++CurOp; 1658 if (HasEVEX_K) // Skip writemask 1659 ++CurOp; 1660 emitByte(BaseOpcode, OS); 1661 emitRegModRMByte(MI.getOperand(CurOp++), 1662 (Form == X86II::MRMXr) ? 0 : Form - X86II::MRM0r, OS); 1663 break; 1664 case X86II::MRMr0: 1665 emitByte(BaseOpcode, OS); 1666 emitByte(modRMByte(3, getX86RegNum(MI.getOperand(CurOp++)),0), OS); 1667 break; 1668 1669 case X86II::MRMXmCC: { 1670 unsigned FirstMemOp = CurOp; 1671 CurOp = FirstMemOp + X86::AddrNumOperands; 1672 1673 unsigned CC = MI.getOperand(CurOp++).getImm(); 1674 emitByte(BaseOpcode + CC, OS); 1675 1676 emitMemModRMByte(MI, FirstMemOp, 0, TSFlags, HasREX, StartByte, OS, Fixups, 1677 STI); 1678 break; 1679 } 1680 1681 case X86II::MRMXm: 1682 case X86II::MRM0m: 1683 case X86II::MRM1m: 1684 case X86II::MRM2m: 1685 case X86II::MRM3m: 1686 case X86II::MRM4m: 1687 case X86II::MRM5m: 1688 case X86II::MRM6m: 1689 case X86II::MRM7m: 1690 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV). 1691 ++CurOp; 1692 if (HasEVEX_K) // Skip writemask 1693 ++CurOp; 1694 emitByte(BaseOpcode, OS); 1695 emitMemModRMByte(MI, CurOp, 1696 (Form == X86II::MRMXm) ? 0 : Form - X86II::MRM0m, TSFlags, 1697 HasREX, StartByte, OS, Fixups, STI); 1698 CurOp += X86::AddrNumOperands; 1699 break; 1700 1701 case X86II::MRM0X: 1702 case X86II::MRM1X: 1703 case X86II::MRM2X: 1704 case X86II::MRM3X: 1705 case X86II::MRM4X: 1706 case X86II::MRM5X: 1707 case X86II::MRM6X: 1708 case X86II::MRM7X: 1709 emitByte(BaseOpcode, OS); 1710 emitByte(0xC0 + ((Form - X86II::MRM0X) << 3), OS); 1711 break; 1712 1713 case X86II::MRM_C0: 1714 case X86II::MRM_C1: 1715 case X86II::MRM_C2: 1716 case X86II::MRM_C3: 1717 case X86II::MRM_C4: 1718 case X86II::MRM_C5: 1719 case X86II::MRM_C6: 1720 case X86II::MRM_C7: 1721 case X86II::MRM_C8: 1722 case X86II::MRM_C9: 1723 case X86II::MRM_CA: 1724 case X86II::MRM_CB: 1725 case X86II::MRM_CC: 1726 case X86II::MRM_CD: 1727 case X86II::MRM_CE: 1728 case X86II::MRM_CF: 1729 case X86II::MRM_D0: 1730 case X86II::MRM_D1: 1731 case X86II::MRM_D2: 1732 case X86II::MRM_D3: 1733 case X86II::MRM_D4: 1734 case X86II::MRM_D5: 1735 case X86II::MRM_D6: 1736 case X86II::MRM_D7: 1737 case X86II::MRM_D8: 1738 case X86II::MRM_D9: 1739 case X86II::MRM_DA: 1740 case X86II::MRM_DB: 1741 case X86II::MRM_DC: 1742 case X86II::MRM_DD: 1743 case X86II::MRM_DE: 1744 case X86II::MRM_DF: 1745 case X86II::MRM_E0: 1746 case X86II::MRM_E1: 1747 case X86II::MRM_E2: 1748 case X86II::MRM_E3: 1749 case X86II::MRM_E4: 1750 case X86II::MRM_E5: 1751 case X86II::MRM_E6: 1752 case X86II::MRM_E7: 1753 case X86II::MRM_E8: 1754 case X86II::MRM_E9: 1755 case X86II::MRM_EA: 1756 case X86II::MRM_EB: 1757 case X86II::MRM_EC: 1758 case X86II::MRM_ED: 1759 case X86II::MRM_EE: 1760 case X86II::MRM_EF: 1761 case X86II::MRM_F0: 1762 case X86II::MRM_F1: 1763 case X86II::MRM_F2: 1764 case X86II::MRM_F3: 1765 case X86II::MRM_F4: 1766 case X86II::MRM_F5: 1767 case X86II::MRM_F6: 1768 case X86II::MRM_F7: 1769 case X86II::MRM_F8: 1770 case X86II::MRM_F9: 1771 case X86II::MRM_FA: 1772 case X86II::MRM_FB: 1773 case X86II::MRM_FC: 1774 case X86II::MRM_FD: 1775 case X86II::MRM_FE: 1776 case X86II::MRM_FF: 1777 emitByte(BaseOpcode, OS); 1778 emitByte(0xC0 + Form - X86II::MRM_C0, OS); 1779 break; 1780 } 1781 1782 if (HasVEX_I8Reg) { 1783 // The last source register of a 4 operand instruction in AVX is encoded 1784 // in bits[7:4] of a immediate byte. 1785 assert(I8RegNum < 16 && "Register encoding out of range"); 1786 I8RegNum <<= 4; 1787 if (CurOp != NumOps) { 1788 unsigned Val = MI.getOperand(CurOp++).getImm(); 1789 assert(Val < 16 && "Immediate operand value out of range"); 1790 I8RegNum |= Val; 1791 } 1792 emitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), 1, FK_Data_1, 1793 StartByte, OS, Fixups); 1794 } else { 1795 // If there is a remaining operand, it must be a trailing immediate. Emit it 1796 // according to the right size for the instruction. Some instructions 1797 // (SSE4a extrq and insertq) have two trailing immediates. 1798 while (CurOp != NumOps && NumOps - CurOp <= 2) { 1799 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1800 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags), 1801 StartByte, OS, Fixups); 1802 } 1803 } 1804 1805 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow) 1806 emitByte(X86II::getBaseOpcodeFor(TSFlags), OS); 1807 1808 assert(OS.tell() - StartByte <= 15 && 1809 "The size of instruction must be no longer than 15."); 1810#ifndef NDEBUG 1811 // FIXME: Verify. 1812 if (/*!Desc.isVariadic() &&*/ CurOp != NumOps) { 1813 errs() << "Cannot encode all operands of: "; 1814 MI.dump(); 1815 errs() << '\n'; 1816 abort(); 1817 } 1818#endif 1819} 1820 1821MCCodeEmitter *llvm::createX86MCCodeEmitter(const MCInstrInfo &MCII, 1822 const MCRegisterInfo &MRI, 1823 MCContext &Ctx) { 1824 return new X86MCCodeEmitter(MCII, Ctx); 1825} 1826