1326938Sdim//===- utils/TableGen/X86FoldTablesEmitter.cpp - X86 backend-*- C++ -*-===// 2326938Sdim// 3353358Sdim// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4353358Sdim// See https://llvm.org/LICENSE.txt for license information. 5353358Sdim// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6326938Sdim// 7326938Sdim//===----------------------------------------------------------------------===// 8326938Sdim// 9326938Sdim// This tablegen backend is responsible for emitting the memory fold tables of 10326938Sdim// the X86 backend instructions. 11326938Sdim// 12326938Sdim//===----------------------------------------------------------------------===// 13326938Sdim 14326938Sdim#include "CodeGenTarget.h" 15326938Sdim#include "X86RecognizableInstr.h" 16353358Sdim#include "llvm/Support/FormattedStream.h" 17326938Sdim#include "llvm/TableGen/Error.h" 18326938Sdim#include "llvm/TableGen/TableGenBackend.h" 19326938Sdim 20326938Sdimusing namespace llvm; 21326938Sdim 22326938Sdimnamespace { 23326938Sdim 24326938Sdim// 3 possible strategies for the unfolding flag (TB_NO_REVERSE) of the 25326938Sdim// manual added entries. 26326938Sdimenum UnfoldStrategy { 27326938Sdim UNFOLD, // Allow unfolding 28326938Sdim NO_UNFOLD, // Prevent unfolding 29326938Sdim NO_STRATEGY // Make decision according to operands' sizes 30326938Sdim}; 31326938Sdim 32326938Sdim// Represents an entry in the manual mapped instructions set. 33326938Sdimstruct ManualMapEntry { 34326938Sdim const char *RegInstStr; 35326938Sdim const char *MemInstStr; 36326938Sdim UnfoldStrategy Strategy; 37326938Sdim 38326938Sdim ManualMapEntry(const char *RegInstStr, const char *MemInstStr, 39326938Sdim UnfoldStrategy Strategy = NO_STRATEGY) 40326938Sdim : RegInstStr(RegInstStr), MemInstStr(MemInstStr), Strategy(Strategy) {} 41326938Sdim}; 42326938Sdim 43326938Sdimclass IsMatch; 44326938Sdim 45326938Sdim// List of instructions requiring explicitly aligned memory. 46326938Sdimconst char *ExplicitAlign[] = {"MOVDQA", "MOVAPS", "MOVAPD", "MOVNTPS", 47326938Sdim "MOVNTPD", "MOVNTDQ", "MOVNTDQA"}; 48326938Sdim 49326938Sdim// List of instructions NOT requiring explicit memory alignment. 50341825Sdimconst char *ExplicitUnalign[] = {"MOVDQU", "MOVUPS", "MOVUPD", 51341825Sdim "PCMPESTRM", "PCMPESTRI", 52341825Sdim "PCMPISTRM", "PCMPISTRI" }; 53326938Sdim 54326938Sdim// For manually mapping instructions that do not match by their encoding. 55326938Sdimconst ManualMapEntry ManualMapSet[] = { 56326938Sdim { "ADD16ri_DB", "ADD16mi", NO_UNFOLD }, 57326938Sdim { "ADD16ri8_DB", "ADD16mi8", NO_UNFOLD }, 58326938Sdim { "ADD16rr_DB", "ADD16mr", NO_UNFOLD }, 59326938Sdim { "ADD32ri_DB", "ADD32mi", NO_UNFOLD }, 60326938Sdim { "ADD32ri8_DB", "ADD32mi8", NO_UNFOLD }, 61326938Sdim { "ADD32rr_DB", "ADD32mr", NO_UNFOLD }, 62326938Sdim { "ADD64ri32_DB", "ADD64mi32", NO_UNFOLD }, 63326938Sdim { "ADD64ri8_DB", "ADD64mi8", NO_UNFOLD }, 64326938Sdim { "ADD64rr_DB", "ADD64mr", NO_UNFOLD }, 65353358Sdim { "ADD8ri_DB", "ADD8mi", NO_UNFOLD }, 66353358Sdim { "ADD8rr_DB", "ADD8mr", NO_UNFOLD }, 67326938Sdim { "ADD16rr_DB", "ADD16rm", NO_UNFOLD }, 68326938Sdim { "ADD32rr_DB", "ADD32rm", NO_UNFOLD }, 69326938Sdim { "ADD64rr_DB", "ADD64rm", NO_UNFOLD }, 70353358Sdim { "ADD8rr_DB", "ADD8rm", NO_UNFOLD }, 71341825Sdim { "PUSH16r", "PUSH16rmm", UNFOLD }, 72341825Sdim { "PUSH32r", "PUSH32rmm", UNFOLD }, 73341825Sdim { "PUSH64r", "PUSH64rmm", UNFOLD }, 74326938Sdim { "TAILJMPr", "TAILJMPm", UNFOLD }, 75326938Sdim { "TAILJMPr64", "TAILJMPm64", UNFOLD }, 76326938Sdim { "TAILJMPr64_REX", "TAILJMPm64_REX", UNFOLD }, 77326938Sdim}; 78326938Sdim 79326938Sdim 80326938Sdimstatic bool isExplicitAlign(const CodeGenInstruction *Inst) { 81326938Sdim return any_of(ExplicitAlign, [Inst](const char *InstStr) { 82326938Sdim return Inst->TheDef->getName().find(InstStr) != StringRef::npos; 83326938Sdim }); 84326938Sdim} 85326938Sdim 86326938Sdimstatic bool isExplicitUnalign(const CodeGenInstruction *Inst) { 87326938Sdim return any_of(ExplicitUnalign, [Inst](const char *InstStr) { 88326938Sdim return Inst->TheDef->getName().find(InstStr) != StringRef::npos; 89326938Sdim }); 90326938Sdim} 91326938Sdim 92326938Sdimclass X86FoldTablesEmitter { 93326938Sdim RecordKeeper &Records; 94326938Sdim CodeGenTarget Target; 95326938Sdim 96326938Sdim // Represents an entry in the folding table 97326938Sdim class X86FoldTableEntry { 98326938Sdim const CodeGenInstruction *RegInst; 99326938Sdim const CodeGenInstruction *MemInst; 100326938Sdim 101326938Sdim public: 102326938Sdim bool CannotUnfold = false; 103326938Sdim bool IsLoad = false; 104326938Sdim bool IsStore = false; 105326938Sdim bool IsAligned = false; 106326938Sdim unsigned int Alignment = 0; 107326938Sdim 108326938Sdim X86FoldTableEntry(const CodeGenInstruction *RegInst, 109326938Sdim const CodeGenInstruction *MemInst) 110326938Sdim : RegInst(RegInst), MemInst(MemInst) {} 111326938Sdim 112353358Sdim void print(formatted_raw_ostream &OS) const { 113353358Sdim OS.indent(2); 114353358Sdim OS << "{ X86::" << RegInst->TheDef->getName() << ","; 115353358Sdim OS.PadToColumn(40); 116353358Sdim OS << "X86::" << MemInst->TheDef->getName() << ","; 117353358Sdim OS.PadToColumn(75); 118326938Sdim 119353358Sdim if (IsLoad) 120326938Sdim OS << "TB_FOLDED_LOAD | "; 121353358Sdim if (IsStore) 122326938Sdim OS << "TB_FOLDED_STORE | "; 123353358Sdim if (CannotUnfold) 124326938Sdim OS << "TB_NO_REVERSE | "; 125353358Sdim if (IsAligned) 126353358Sdim OS << "TB_ALIGN_" << Alignment << " | "; 127326938Sdim 128326938Sdim OS << "0 },\n"; 129326938Sdim } 130326938Sdim }; 131326938Sdim 132326938Sdim typedef std::vector<X86FoldTableEntry> FoldTable; 133326938Sdim // std::vector for each folding table. 134326938Sdim // Table2Addr - Holds instructions which their memory form performs load+store 135326938Sdim // Table#i - Holds instructions which the their memory form perform a load OR 136326938Sdim // a store, and their #i'th operand is folded. 137326938Sdim FoldTable Table2Addr; 138326938Sdim FoldTable Table0; 139326938Sdim FoldTable Table1; 140326938Sdim FoldTable Table2; 141326938Sdim FoldTable Table3; 142326938Sdim FoldTable Table4; 143326938Sdim 144326938Sdimpublic: 145326938Sdim X86FoldTablesEmitter(RecordKeeper &R) : Records(R), Target(R) {} 146326938Sdim 147326938Sdim // run - Generate the 6 X86 memory fold tables. 148353358Sdim void run(formatted_raw_ostream &OS); 149326938Sdim 150326938Sdimprivate: 151326938Sdim // Decides to which table to add the entry with the given instructions. 152326938Sdim // S sets the strategy of adding the TB_NO_REVERSE flag. 153326938Sdim void updateTables(const CodeGenInstruction *RegInstr, 154326938Sdim const CodeGenInstruction *MemInstr, 155326938Sdim const UnfoldStrategy S = NO_STRATEGY); 156326938Sdim 157326938Sdim // Generates X86FoldTableEntry with the given instructions and fill it with 158326938Sdim // the appropriate flags - then adds it to Table. 159326938Sdim void addEntryWithFlags(FoldTable &Table, const CodeGenInstruction *RegInstr, 160326938Sdim const CodeGenInstruction *MemInstr, 161326938Sdim const UnfoldStrategy S, const unsigned int FoldedInd); 162326938Sdim 163326938Sdim // Print the given table as a static const C++ array of type 164326938Sdim // X86MemoryFoldTableEntry. 165341825Sdim void printTable(const FoldTable &Table, StringRef TableName, 166353358Sdim formatted_raw_ostream &OS) { 167326938Sdim OS << "static const X86MemoryFoldTableEntry MemoryFold" << TableName 168326938Sdim << "[] = {\n"; 169326938Sdim 170326938Sdim for (const X86FoldTableEntry &E : Table) 171353358Sdim E.print(OS); 172326938Sdim 173353358Sdim OS << "};\n\n"; 174326938Sdim } 175326938Sdim}; 176326938Sdim 177326938Sdim// Return true if one of the instruction's operands is a RST register class 178326938Sdimstatic bool hasRSTRegClass(const CodeGenInstruction *Inst) { 179326938Sdim return any_of(Inst->Operands, [](const CGIOperandList::OperandInfo &OpIn) { 180353358Sdim return OpIn.Rec->getName() == "RST" || OpIn.Rec->getName() == "RSTi"; 181326938Sdim }); 182326938Sdim} 183326938Sdim 184326938Sdim// Return true if one of the instruction's operands is a ptr_rc_tailcall 185326938Sdimstatic bool hasPtrTailcallRegClass(const CodeGenInstruction *Inst) { 186326938Sdim return any_of(Inst->Operands, [](const CGIOperandList::OperandInfo &OpIn) { 187326938Sdim return OpIn.Rec->getName() == "ptr_rc_tailcall"; 188326938Sdim }); 189326938Sdim} 190326938Sdim 191326938Sdim// Calculates the integer value representing the BitsInit object 192326938Sdimstatic inline uint64_t getValueFromBitsInit(const BitsInit *B) { 193326938Sdim assert(B->getNumBits() <= sizeof(uint64_t) * 8 && "BitInits' too long!"); 194326938Sdim 195326938Sdim uint64_t Value = 0; 196326938Sdim for (unsigned i = 0, e = B->getNumBits(); i != e; ++i) { 197326938Sdim BitInit *Bit = cast<BitInit>(B->getBit(i)); 198326938Sdim Value |= uint64_t(Bit->getValue()) << i; 199326938Sdim } 200326938Sdim return Value; 201326938Sdim} 202326938Sdim 203326938Sdim// Returns true if the two given BitsInits represent the same integer value 204326938Sdimstatic inline bool equalBitsInits(const BitsInit *B1, const BitsInit *B2) { 205326938Sdim if (B1->getNumBits() != B2->getNumBits()) 206326938Sdim PrintFatalError("Comparing two BitsInits with different sizes!"); 207326938Sdim 208326938Sdim for (unsigned i = 0, e = B1->getNumBits(); i != e; ++i) { 209326938Sdim BitInit *Bit1 = cast<BitInit>(B1->getBit(i)); 210326938Sdim BitInit *Bit2 = cast<BitInit>(B2->getBit(i)); 211326938Sdim if (Bit1->getValue() != Bit2->getValue()) 212326938Sdim return false; 213326938Sdim } 214326938Sdim return true; 215326938Sdim} 216326938Sdim 217326938Sdim// Return the size of the register operand 218326938Sdimstatic inline unsigned int getRegOperandSize(const Record *RegRec) { 219326938Sdim if (RegRec->isSubClassOf("RegisterOperand")) 220326938Sdim RegRec = RegRec->getValueAsDef("RegClass"); 221326938Sdim if (RegRec->isSubClassOf("RegisterClass")) 222326938Sdim return RegRec->getValueAsListOfDefs("RegTypes")[0]->getValueAsInt("Size"); 223326938Sdim 224326938Sdim llvm_unreachable("Register operand's size not known!"); 225326938Sdim} 226326938Sdim 227326938Sdim// Return the size of the memory operand 228326938Sdimstatic inline unsigned int 229326938SdimgetMemOperandSize(const Record *MemRec, const bool IntrinsicSensitive = false) { 230326938Sdim if (MemRec->isSubClassOf("Operand")) { 231326938Sdim // Intrinsic memory instructions use ssmem/sdmem. 232326938Sdim if (IntrinsicSensitive && 233326938Sdim (MemRec->getName() == "sdmem" || MemRec->getName() == "ssmem")) 234326938Sdim return 128; 235326938Sdim 236326938Sdim StringRef Name = 237326938Sdim MemRec->getValueAsDef("ParserMatchClass")->getValueAsString("Name"); 238326938Sdim if (Name == "Mem8") 239326938Sdim return 8; 240326938Sdim if (Name == "Mem16") 241326938Sdim return 16; 242326938Sdim if (Name == "Mem32") 243326938Sdim return 32; 244326938Sdim if (Name == "Mem64") 245326938Sdim return 64; 246326938Sdim if (Name == "Mem80") 247326938Sdim return 80; 248326938Sdim if (Name == "Mem128") 249326938Sdim return 128; 250326938Sdim if (Name == "Mem256") 251326938Sdim return 256; 252326938Sdim if (Name == "Mem512") 253326938Sdim return 512; 254326938Sdim } 255326938Sdim 256326938Sdim llvm_unreachable("Memory operand's size not known!"); 257326938Sdim} 258326938Sdim 259326938Sdim// Return true if the instruction defined as a register flavor. 260326938Sdimstatic inline bool hasRegisterFormat(const Record *Inst) { 261326938Sdim const BitsInit *FormBits = Inst->getValueAsBitsInit("FormBits"); 262326938Sdim uint64_t FormBitsNum = getValueFromBitsInit(FormBits); 263326938Sdim 264326938Sdim // Values from X86Local namespace defined in X86RecognizableInstr.cpp 265326938Sdim return FormBitsNum >= X86Local::MRMDestReg && FormBitsNum <= X86Local::MRM7r; 266326938Sdim} 267326938Sdim 268326938Sdim// Return true if the instruction defined as a memory flavor. 269326938Sdimstatic inline bool hasMemoryFormat(const Record *Inst) { 270326938Sdim const BitsInit *FormBits = Inst->getValueAsBitsInit("FormBits"); 271326938Sdim uint64_t FormBitsNum = getValueFromBitsInit(FormBits); 272326938Sdim 273326938Sdim // Values from X86Local namespace defined in X86RecognizableInstr.cpp 274326938Sdim return FormBitsNum >= X86Local::MRMDestMem && FormBitsNum <= X86Local::MRM7m; 275326938Sdim} 276326938Sdim 277326938Sdimstatic inline bool isNOREXRegClass(const Record *Op) { 278326938Sdim return Op->getName().find("_NOREX") != StringRef::npos; 279326938Sdim} 280326938Sdim 281326938Sdimstatic inline bool isRegisterOperand(const Record *Rec) { 282326938Sdim return Rec->isSubClassOf("RegisterClass") || 283326938Sdim Rec->isSubClassOf("RegisterOperand") || 284326938Sdim Rec->isSubClassOf("PointerLikeRegClass"); 285326938Sdim} 286326938Sdim 287326938Sdimstatic inline bool isMemoryOperand(const Record *Rec) { 288326938Sdim return Rec->isSubClassOf("Operand") && 289326938Sdim Rec->getValueAsString("OperandType") == "OPERAND_MEMORY"; 290326938Sdim} 291326938Sdim 292326938Sdimstatic inline bool isImmediateOperand(const Record *Rec) { 293326938Sdim return Rec->isSubClassOf("Operand") && 294326938Sdim Rec->getValueAsString("OperandType") == "OPERAND_IMMEDIATE"; 295326938Sdim} 296326938Sdim 297326938Sdim// Get the alternative instruction pointed by "FoldGenRegForm" field. 298326938Sdimstatic inline const CodeGenInstruction * 299326938SdimgetAltRegInst(const CodeGenInstruction *I, const RecordKeeper &Records, 300326938Sdim const CodeGenTarget &Target) { 301326938Sdim 302326938Sdim StringRef AltRegInstStr = I->TheDef->getValueAsString("FoldGenRegForm"); 303326938Sdim Record *AltRegInstRec = Records.getDef(AltRegInstStr); 304326938Sdim assert(AltRegInstRec && 305326938Sdim "Alternative register form instruction def not found"); 306326938Sdim CodeGenInstruction &AltRegInst = Target.getInstruction(AltRegInstRec); 307326938Sdim return &AltRegInst; 308326938Sdim} 309326938Sdim 310326938Sdim// Function object - Operator() returns true if the given VEX instruction 311326938Sdim// matches the EVEX instruction of this object. 312326938Sdimclass IsMatch { 313326938Sdim const CodeGenInstruction *MemInst; 314326938Sdim 315326938Sdimpublic: 316326938Sdim IsMatch(const CodeGenInstruction *Inst, const RecordKeeper &Records) 317326938Sdim : MemInst(Inst) {} 318326938Sdim 319326938Sdim bool operator()(const CodeGenInstruction *RegInst) { 320326938Sdim Record *MemRec = MemInst->TheDef; 321326938Sdim Record *RegRec = RegInst->TheDef; 322326938Sdim 323326938Sdim // Return false if one (at least) of the encoding fields of both 324326938Sdim // instructions do not match. 325326938Sdim if (RegRec->getValueAsDef("OpEnc") != MemRec->getValueAsDef("OpEnc") || 326326938Sdim !equalBitsInits(RegRec->getValueAsBitsInit("Opcode"), 327326938Sdim MemRec->getValueAsBitsInit("Opcode")) || 328326938Sdim // VEX/EVEX fields 329326938Sdim RegRec->getValueAsDef("OpPrefix") != 330326938Sdim MemRec->getValueAsDef("OpPrefix") || 331326938Sdim RegRec->getValueAsDef("OpMap") != MemRec->getValueAsDef("OpMap") || 332326938Sdim RegRec->getValueAsDef("OpSize") != MemRec->getValueAsDef("OpSize") || 333341825Sdim RegRec->getValueAsDef("AdSize") != MemRec->getValueAsDef("AdSize") || 334326938Sdim RegRec->getValueAsBit("hasVEX_4V") != 335326938Sdim MemRec->getValueAsBit("hasVEX_4V") || 336326938Sdim RegRec->getValueAsBit("hasEVEX_K") != 337326938Sdim MemRec->getValueAsBit("hasEVEX_K") || 338326938Sdim RegRec->getValueAsBit("hasEVEX_Z") != 339326938Sdim MemRec->getValueAsBit("hasEVEX_Z") || 340341825Sdim // EVEX_B means different things for memory and register forms. 341341825Sdim RegRec->getValueAsBit("hasEVEX_B") != 0 || 342341825Sdim MemRec->getValueAsBit("hasEVEX_B") != 0 || 343326938Sdim RegRec->getValueAsBit("hasEVEX_RC") != 344326938Sdim MemRec->getValueAsBit("hasEVEX_RC") || 345326938Sdim RegRec->getValueAsBit("hasREX_WPrefix") != 346326938Sdim MemRec->getValueAsBit("hasREX_WPrefix") || 347326938Sdim RegRec->getValueAsBit("hasLockPrefix") != 348326938Sdim MemRec->getValueAsBit("hasLockPrefix") || 349341825Sdim RegRec->getValueAsBit("hasNoTrackPrefix") != 350341825Sdim MemRec->getValueAsBit("hasNoTrackPrefix") || 351353358Sdim RegRec->getValueAsBit("hasVEX_L") != 352353358Sdim MemRec->getValueAsBit("hasVEX_L") || 353353358Sdim RegRec->getValueAsBit("hasEVEX_L2") != 354353358Sdim MemRec->getValueAsBit("hasEVEX_L2") || 355353358Sdim RegRec->getValueAsBit("ignoresVEX_L") != 356353358Sdim MemRec->getValueAsBit("ignoresVEX_L") || 357353358Sdim RegRec->getValueAsBit("HasVEX_W") != 358353358Sdim MemRec->getValueAsBit("HasVEX_W") || 359353358Sdim RegRec->getValueAsBit("IgnoresVEX_W") != 360353358Sdim MemRec->getValueAsBit("IgnoresVEX_W") || 361353358Sdim RegRec->getValueAsBit("EVEX_W1_VEX_W0") != 362353358Sdim MemRec->getValueAsBit("EVEX_W1_VEX_W0") || 363326938Sdim // Instruction's format - The register form's "Form" field should be 364326938Sdim // the opposite of the memory form's "Form" field. 365326938Sdim !areOppositeForms(RegRec->getValueAsBitsInit("FormBits"), 366326938Sdim MemRec->getValueAsBitsInit("FormBits")) || 367326938Sdim RegRec->getValueAsBit("isAsmParserOnly") != 368326938Sdim MemRec->getValueAsBit("isAsmParserOnly")) 369326938Sdim return false; 370326938Sdim 371326938Sdim // Make sure the sizes of the operands of both instructions suit each other. 372326938Sdim // This is needed for instructions with intrinsic version (_Int). 373326938Sdim // Where the only difference is the size of the operands. 374326938Sdim // For example: VUCOMISDZrm and Int_VUCOMISDrm 375326938Sdim // Also for instructions that their EVEX version was upgraded to work with 376326938Sdim // k-registers. For example VPCMPEQBrm (xmm output register) and 377326938Sdim // VPCMPEQBZ128rm (k register output register). 378326938Sdim bool ArgFolded = false; 379326938Sdim unsigned MemOutSize = MemRec->getValueAsDag("OutOperandList")->getNumArgs(); 380326938Sdim unsigned RegOutSize = RegRec->getValueAsDag("OutOperandList")->getNumArgs(); 381326938Sdim unsigned MemInSize = MemRec->getValueAsDag("InOperandList")->getNumArgs(); 382326938Sdim unsigned RegInSize = RegRec->getValueAsDag("InOperandList")->getNumArgs(); 383326938Sdim 384326938Sdim // Instructions with one output in their memory form use the memory folded 385326938Sdim // operand as source and destination (Read-Modify-Write). 386326938Sdim unsigned RegStartIdx = 387326938Sdim (MemOutSize + 1 == RegOutSize) && (MemInSize == RegInSize) ? 1 : 0; 388326938Sdim 389326938Sdim for (unsigned i = 0, e = MemInst->Operands.size(); i < e; i++) { 390326938Sdim Record *MemOpRec = MemInst->Operands[i].Rec; 391326938Sdim Record *RegOpRec = RegInst->Operands[i + RegStartIdx].Rec; 392326938Sdim 393326938Sdim if (MemOpRec == RegOpRec) 394326938Sdim continue; 395326938Sdim 396326938Sdim if (isRegisterOperand(MemOpRec) && isRegisterOperand(RegOpRec)) { 397326938Sdim if (getRegOperandSize(MemOpRec) != getRegOperandSize(RegOpRec) || 398326938Sdim isNOREXRegClass(MemOpRec) != isNOREXRegClass(RegOpRec)) 399326938Sdim return false; 400326938Sdim } else if (isMemoryOperand(MemOpRec) && isMemoryOperand(RegOpRec)) { 401326938Sdim if (getMemOperandSize(MemOpRec) != getMemOperandSize(RegOpRec)) 402326938Sdim return false; 403326938Sdim } else if (isImmediateOperand(MemOpRec) && isImmediateOperand(RegOpRec)) { 404326938Sdim if (MemOpRec->getValueAsDef("Type") != RegOpRec->getValueAsDef("Type")) 405326938Sdim return false; 406326938Sdim } else { 407326938Sdim // Only one operand can be folded. 408326938Sdim if (ArgFolded) 409326938Sdim return false; 410326938Sdim 411326938Sdim assert(isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec)); 412326938Sdim ArgFolded = true; 413326938Sdim } 414326938Sdim } 415326938Sdim 416326938Sdim return true; 417326938Sdim } 418326938Sdim 419326938Sdimprivate: 420326938Sdim // Return true of the 2 given forms are the opposite of each other. 421326938Sdim bool areOppositeForms(const BitsInit *RegFormBits, 422326938Sdim const BitsInit *MemFormBits) { 423326938Sdim uint64_t MemFormNum = getValueFromBitsInit(MemFormBits); 424326938Sdim uint64_t RegFormNum = getValueFromBitsInit(RegFormBits); 425326938Sdim 426326938Sdim if ((MemFormNum == X86Local::MRM0m && RegFormNum == X86Local::MRM0r) || 427326938Sdim (MemFormNum == X86Local::MRM1m && RegFormNum == X86Local::MRM1r) || 428326938Sdim (MemFormNum == X86Local::MRM2m && RegFormNum == X86Local::MRM2r) || 429326938Sdim (MemFormNum == X86Local::MRM3m && RegFormNum == X86Local::MRM3r) || 430326938Sdim (MemFormNum == X86Local::MRM4m && RegFormNum == X86Local::MRM4r) || 431326938Sdim (MemFormNum == X86Local::MRM5m && RegFormNum == X86Local::MRM5r) || 432326938Sdim (MemFormNum == X86Local::MRM6m && RegFormNum == X86Local::MRM6r) || 433326938Sdim (MemFormNum == X86Local::MRM7m && RegFormNum == X86Local::MRM7r) || 434326938Sdim (MemFormNum == X86Local::MRMXm && RegFormNum == X86Local::MRMXr) || 435353358Sdim (MemFormNum == X86Local::MRMXmCC && RegFormNum == X86Local::MRMXrCC) || 436326938Sdim (MemFormNum == X86Local::MRMDestMem && 437326938Sdim RegFormNum == X86Local::MRMDestReg) || 438326938Sdim (MemFormNum == X86Local::MRMSrcMem && 439326938Sdim RegFormNum == X86Local::MRMSrcReg) || 440326938Sdim (MemFormNum == X86Local::MRMSrcMem4VOp3 && 441326938Sdim RegFormNum == X86Local::MRMSrcReg4VOp3) || 442326938Sdim (MemFormNum == X86Local::MRMSrcMemOp4 && 443353358Sdim RegFormNum == X86Local::MRMSrcRegOp4) || 444353358Sdim (MemFormNum == X86Local::MRMSrcMemCC && 445353358Sdim RegFormNum == X86Local::MRMSrcRegCC)) 446326938Sdim return true; 447326938Sdim 448326938Sdim return false; 449326938Sdim } 450326938Sdim}; 451326938Sdim 452326938Sdim} // end anonymous namespace 453326938Sdim 454326938Sdimvoid X86FoldTablesEmitter::addEntryWithFlags(FoldTable &Table, 455326938Sdim const CodeGenInstruction *RegInstr, 456326938Sdim const CodeGenInstruction *MemInstr, 457326938Sdim const UnfoldStrategy S, 458326938Sdim const unsigned int FoldedInd) { 459326938Sdim 460326938Sdim X86FoldTableEntry Result = X86FoldTableEntry(RegInstr, MemInstr); 461326938Sdim Record *RegRec = RegInstr->TheDef; 462326938Sdim Record *MemRec = MemInstr->TheDef; 463326938Sdim 464326938Sdim // Only table0 entries should explicitly specify a load or store flag. 465326938Sdim if (&Table == &Table0) { 466326938Sdim unsigned MemInOpsNum = MemRec->getValueAsDag("InOperandList")->getNumArgs(); 467326938Sdim unsigned RegInOpsNum = RegRec->getValueAsDag("InOperandList")->getNumArgs(); 468326938Sdim // If the instruction writes to the folded operand, it will appear as an 469326938Sdim // output in the register form instruction and as an input in the memory 470326938Sdim // form instruction. 471326938Sdim // If the instruction reads from the folded operand, it well appear as in 472326938Sdim // input in both forms. 473326938Sdim if (MemInOpsNum == RegInOpsNum) 474326938Sdim Result.IsLoad = true; 475326938Sdim else 476326938Sdim Result.IsStore = true; 477326938Sdim } 478326938Sdim 479326938Sdim Record *RegOpRec = RegInstr->Operands[FoldedInd].Rec; 480326938Sdim Record *MemOpRec = MemInstr->Operands[FoldedInd].Rec; 481326938Sdim 482326938Sdim // Unfolding code generates a load/store instruction according to the size of 483326938Sdim // the register in the register form instruction. 484326938Sdim // If the register's size is greater than the memory's operand size, do not 485326938Sdim // allow unfolding. 486326938Sdim if (S == UNFOLD) 487326938Sdim Result.CannotUnfold = false; 488326938Sdim else if (S == NO_UNFOLD) 489326938Sdim Result.CannotUnfold = true; 490326938Sdim else if (getRegOperandSize(RegOpRec) > getMemOperandSize(MemOpRec)) 491326938Sdim Result.CannotUnfold = true; // S == NO_STRATEGY 492326938Sdim 493326938Sdim uint64_t Enc = getValueFromBitsInit(RegRec->getValueAsBitsInit("OpEncBits")); 494326938Sdim if (isExplicitAlign(RegInstr)) { 495326938Sdim // The instruction require explicitly aligned memory. 496326938Sdim BitsInit *VectSize = RegRec->getValueAsBitsInit("VectSize"); 497326938Sdim uint64_t Value = getValueFromBitsInit(VectSize); 498326938Sdim Result.IsAligned = true; 499326938Sdim Result.Alignment = Value; 500326938Sdim } else if (Enc != X86Local::XOP && Enc != X86Local::VEX && 501326938Sdim Enc != X86Local::EVEX) { 502326938Sdim // Instructions with VEX encoding do not require alignment. 503326938Sdim if (!isExplicitUnalign(RegInstr) && getMemOperandSize(MemOpRec) > 64) { 504326938Sdim // SSE packed vector instructions require a 16 byte alignment. 505326938Sdim Result.IsAligned = true; 506326938Sdim Result.Alignment = 16; 507326938Sdim } 508326938Sdim } 509326938Sdim 510326938Sdim Table.push_back(Result); 511326938Sdim} 512326938Sdim 513326938Sdimvoid X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInstr, 514326938Sdim const CodeGenInstruction *MemInstr, 515326938Sdim const UnfoldStrategy S) { 516326938Sdim 517326938Sdim Record *RegRec = RegInstr->TheDef; 518326938Sdim Record *MemRec = MemInstr->TheDef; 519326938Sdim unsigned MemOutSize = MemRec->getValueAsDag("OutOperandList")->getNumArgs(); 520326938Sdim unsigned RegOutSize = RegRec->getValueAsDag("OutOperandList")->getNumArgs(); 521326938Sdim unsigned MemInSize = MemRec->getValueAsDag("InOperandList")->getNumArgs(); 522326938Sdim unsigned RegInSize = RegRec->getValueAsDag("InOperandList")->getNumArgs(); 523326938Sdim 524341825Sdim // Instructions which Read-Modify-Write should be added to Table2Addr. 525341825Sdim if (MemOutSize != RegOutSize && MemInSize == RegInSize) { 526326938Sdim addEntryWithFlags(Table2Addr, RegInstr, MemInstr, S, 0); 527326938Sdim return; 528326938Sdim } 529326938Sdim 530326938Sdim if (MemInSize == RegInSize && MemOutSize == RegOutSize) { 531326938Sdim // Load-Folding cases. 532326938Sdim // If the i'th register form operand is a register and the i'th memory form 533326938Sdim // operand is a memory operand, add instructions to Table#i. 534326938Sdim for (unsigned i = RegOutSize, e = RegInstr->Operands.size(); i < e; i++) { 535326938Sdim Record *RegOpRec = RegInstr->Operands[i].Rec; 536326938Sdim Record *MemOpRec = MemInstr->Operands[i].Rec; 537326938Sdim if (isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec)) { 538326938Sdim switch (i) { 539326938Sdim case 0: 540326938Sdim addEntryWithFlags(Table0, RegInstr, MemInstr, S, 0); 541326938Sdim return; 542326938Sdim case 1: 543326938Sdim addEntryWithFlags(Table1, RegInstr, MemInstr, S, 1); 544326938Sdim return; 545326938Sdim case 2: 546326938Sdim addEntryWithFlags(Table2, RegInstr, MemInstr, S, 2); 547326938Sdim return; 548326938Sdim case 3: 549326938Sdim addEntryWithFlags(Table3, RegInstr, MemInstr, S, 3); 550326938Sdim return; 551326938Sdim case 4: 552326938Sdim addEntryWithFlags(Table4, RegInstr, MemInstr, S, 4); 553326938Sdim return; 554326938Sdim } 555326938Sdim } 556326938Sdim } 557326938Sdim } else if (MemInSize == RegInSize + 1 && MemOutSize + 1 == RegOutSize) { 558326938Sdim // Store-Folding cases. 559341825Sdim // If the memory form instruction performs a store, the *output* 560326938Sdim // register of the register form instructions disappear and instead a 561326938Sdim // memory *input* operand appears in the memory form instruction. 562326938Sdim // For example: 563326938Sdim // MOVAPSrr => (outs VR128:$dst), (ins VR128:$src) 564326938Sdim // MOVAPSmr => (outs), (ins f128mem:$dst, VR128:$src) 565326938Sdim Record *RegOpRec = RegInstr->Operands[RegOutSize - 1].Rec; 566326938Sdim Record *MemOpRec = MemInstr->Operands[RegOutSize - 1].Rec; 567341825Sdim if (isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec) && 568341825Sdim getRegOperandSize(RegOpRec) == getMemOperandSize(MemOpRec)) 569326938Sdim addEntryWithFlags(Table0, RegInstr, MemInstr, S, 0); 570326938Sdim } 571326938Sdim 572326938Sdim return; 573326938Sdim} 574326938Sdim 575353358Sdimvoid X86FoldTablesEmitter::run(formatted_raw_ostream &OS) { 576326938Sdim emitSourceFileHeader("X86 fold tables", OS); 577326938Sdim 578326938Sdim // Holds all memory instructions 579326938Sdim std::vector<const CodeGenInstruction *> MemInsts; 580326938Sdim // Holds all register instructions - divided according to opcode. 581326938Sdim std::map<uint8_t, std::vector<const CodeGenInstruction *>> RegInsts; 582326938Sdim 583326938Sdim ArrayRef<const CodeGenInstruction *> NumberedInstructions = 584326938Sdim Target.getInstructionsByEnumValue(); 585326938Sdim 586326938Sdim for (const CodeGenInstruction *Inst : NumberedInstructions) { 587326938Sdim if (!Inst->TheDef->getNameInit() || !Inst->TheDef->isSubClassOf("X86Inst")) 588326938Sdim continue; 589326938Sdim 590326938Sdim const Record *Rec = Inst->TheDef; 591326938Sdim 592326938Sdim // - Do not proceed if the instruction is marked as notMemoryFoldable. 593326938Sdim // - Instructions including RST register class operands are not relevant 594326938Sdim // for memory folding (for further details check the explanation in 595326938Sdim // lib/Target/X86/X86InstrFPStack.td file). 596326938Sdim // - Some instructions (listed in the manual map above) use the register 597326938Sdim // class ptr_rc_tailcall, which can be of a size 32 or 64, to ensure 598326938Sdim // safe mapping of these instruction we manually map them and exclude 599326938Sdim // them from the automation. 600326938Sdim if (Rec->getValueAsBit("isMemoryFoldable") == false || 601326938Sdim hasRSTRegClass(Inst) || hasPtrTailcallRegClass(Inst)) 602326938Sdim continue; 603326938Sdim 604326938Sdim // Add all the memory form instructions to MemInsts, and all the register 605326938Sdim // form instructions to RegInsts[Opc], where Opc in the opcode of each 606326938Sdim // instructions. this helps reducing the runtime of the backend. 607326938Sdim if (hasMemoryFormat(Rec)) 608326938Sdim MemInsts.push_back(Inst); 609326938Sdim else if (hasRegisterFormat(Rec)) { 610326938Sdim uint8_t Opc = getValueFromBitsInit(Rec->getValueAsBitsInit("Opcode")); 611326938Sdim RegInsts[Opc].push_back(Inst); 612326938Sdim } 613326938Sdim } 614326938Sdim 615326938Sdim // For each memory form instruction, try to find its register form 616326938Sdim // instruction. 617326938Sdim for (const CodeGenInstruction *MemInst : MemInsts) { 618326938Sdim uint8_t Opc = 619326938Sdim getValueFromBitsInit(MemInst->TheDef->getValueAsBitsInit("Opcode")); 620326938Sdim 621360784Sdim auto RegInstsIt = RegInsts.find(Opc); 622360784Sdim if (RegInstsIt == RegInsts.end()) 623326938Sdim continue; 624326938Sdim 625326938Sdim // Two forms (memory & register) of the same instruction must have the same 626326938Sdim // opcode. try matching only with register form instructions with the same 627326938Sdim // opcode. 628360784Sdim std::vector<const CodeGenInstruction *> &OpcRegInsts = RegInstsIt->second; 629326938Sdim 630326938Sdim auto Match = find_if(OpcRegInsts, IsMatch(MemInst, Records)); 631326938Sdim if (Match != OpcRegInsts.end()) { 632326938Sdim const CodeGenInstruction *RegInst = *Match; 633326938Sdim // If the matched instruction has it's "FoldGenRegForm" set, map the 634326938Sdim // memory form instruction to the register form instruction pointed by 635326938Sdim // this field 636326938Sdim if (RegInst->TheDef->isValueUnset("FoldGenRegForm")) { 637326938Sdim updateTables(RegInst, MemInst); 638326938Sdim } else { 639326938Sdim const CodeGenInstruction *AltRegInst = 640326938Sdim getAltRegInst(RegInst, Records, Target); 641326938Sdim updateTables(AltRegInst, MemInst); 642326938Sdim } 643326938Sdim OpcRegInsts.erase(Match); 644326938Sdim } 645326938Sdim } 646326938Sdim 647326938Sdim // Add the manually mapped instructions listed above. 648326938Sdim for (const ManualMapEntry &Entry : ManualMapSet) { 649326938Sdim Record *RegInstIter = Records.getDef(Entry.RegInstStr); 650326938Sdim Record *MemInstIter = Records.getDef(Entry.MemInstStr); 651326938Sdim 652326938Sdim updateTables(&(Target.getInstruction(RegInstIter)), 653326938Sdim &(Target.getInstruction(MemInstIter)), Entry.Strategy); 654326938Sdim } 655326938Sdim 656353358Sdim // Print all tables. 657326938Sdim printTable(Table2Addr, "Table2Addr", OS); 658326938Sdim printTable(Table0, "Table0", OS); 659326938Sdim printTable(Table1, "Table1", OS); 660326938Sdim printTable(Table2, "Table2", OS); 661326938Sdim printTable(Table3, "Table3", OS); 662326938Sdim printTable(Table4, "Table4", OS); 663326938Sdim} 664326938Sdim 665326938Sdimnamespace llvm { 666326938Sdim 667353358Sdimvoid EmitX86FoldTables(RecordKeeper &RK, raw_ostream &o) { 668353358Sdim formatted_raw_ostream OS(o); 669326938Sdim X86FoldTablesEmitter(RK).run(OS); 670326938Sdim} 671326938Sdim} // namespace llvm 672