1//===- X86InstructionSelector.cpp -----------------------------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8/// \file 9/// This file implements the targeting of the InstructionSelector class for 10/// X86. 11/// \todo This should be generated by TableGen. 12//===----------------------------------------------------------------------===// 13 14#include "MCTargetDesc/X86BaseInfo.h" 15#include "X86.h" 16#include "X86InstrBuilder.h" 17#include "X86InstrInfo.h" 18#include "X86RegisterBankInfo.h" 19#include "X86RegisterInfo.h" 20#include "X86Subtarget.h" 21#include "X86TargetMachine.h" 22#include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 23#include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h" 24#include "llvm/CodeGen/GlobalISel/RegisterBank.h" 25#include "llvm/CodeGen/GlobalISel/Utils.h" 26#include "llvm/CodeGen/MachineBasicBlock.h" 27#include "llvm/CodeGen/MachineConstantPool.h" 28#include "llvm/CodeGen/MachineFunction.h" 29#include "llvm/CodeGen/MachineInstr.h" 30#include "llvm/CodeGen/MachineInstrBuilder.h" 31#include "llvm/CodeGen/MachineMemOperand.h" 32#include "llvm/CodeGen/MachineOperand.h" 33#include "llvm/CodeGen/MachineRegisterInfo.h" 34#include "llvm/CodeGen/TargetOpcodes.h" 35#include "llvm/CodeGen/TargetRegisterInfo.h" 36#include "llvm/IR/DataLayout.h" 37#include "llvm/IR/InstrTypes.h" 38#include "llvm/IR/IntrinsicsX86.h" 39#include "llvm/Support/AtomicOrdering.h" 40#include "llvm/Support/CodeGen.h" 41#include "llvm/Support/Debug.h" 42#include "llvm/Support/ErrorHandling.h" 43#include "llvm/Support/LowLevelTypeImpl.h" 44#include "llvm/Support/MathExtras.h" 45#include "llvm/Support/raw_ostream.h" 46#include <cassert> 47#include <cstdint> 48#include <tuple> 49 50#define DEBUG_TYPE "X86-isel" 51 52using namespace llvm; 53 54namespace { 55 56#define GET_GLOBALISEL_PREDICATE_BITSET 57#include "X86GenGlobalISel.inc" 58#undef GET_GLOBALISEL_PREDICATE_BITSET 59 60class X86InstructionSelector : public InstructionSelector { 61public: 62 X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI, 63 const X86RegisterBankInfo &RBI); 64 65 bool select(MachineInstr &I) override; 66 static const char *getName() { return DEBUG_TYPE; } 67 68private: 69 /// tblgen-erated 'select' implementation, used as the initial selector for 70 /// the patterns that don't require complex C++. 71 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 72 73 // TODO: remove after supported by Tablegen-erated instruction selection. 74 unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc, 75 Align Alignment) const; 76 77 bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI, 78 MachineFunction &MF) const; 79 bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI, 80 MachineFunction &MF) const; 81 bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI, 82 MachineFunction &MF) const; 83 bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI, 84 MachineFunction &MF) const; 85 bool selectTruncOrPtrToInt(MachineInstr &I, MachineRegisterInfo &MRI, 86 MachineFunction &MF) const; 87 bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI, 88 MachineFunction &MF) const; 89 bool selectAnyext(MachineInstr &I, MachineRegisterInfo &MRI, 90 MachineFunction &MF) const; 91 bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI, 92 MachineFunction &MF) const; 93 bool selectFCmp(MachineInstr &I, MachineRegisterInfo &MRI, 94 MachineFunction &MF) const; 95 bool selectUadde(MachineInstr &I, MachineRegisterInfo &MRI, 96 MachineFunction &MF) const; 97 bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const; 98 bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI, 99 MachineFunction &MF); 100 bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI, 101 MachineFunction &MF); 102 bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI, 103 MachineFunction &MF) const; 104 bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI, 105 MachineFunction &MF) const; 106 bool selectCondBranch(MachineInstr &I, MachineRegisterInfo &MRI, 107 MachineFunction &MF) const; 108 bool selectTurnIntoCOPY(MachineInstr &I, MachineRegisterInfo &MRI, 109 const unsigned DstReg, 110 const TargetRegisterClass *DstRC, 111 const unsigned SrcReg, 112 const TargetRegisterClass *SrcRC) const; 113 bool materializeFP(MachineInstr &I, MachineRegisterInfo &MRI, 114 MachineFunction &MF) const; 115 bool selectImplicitDefOrPHI(MachineInstr &I, MachineRegisterInfo &MRI) const; 116 bool selectDivRem(MachineInstr &I, MachineRegisterInfo &MRI, 117 MachineFunction &MF) const; 118 bool selectIntrinsicWSideEffects(MachineInstr &I, MachineRegisterInfo &MRI, 119 MachineFunction &MF) const; 120 121 // emit insert subreg instruction and insert it before MachineInstr &I 122 bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I, 123 MachineRegisterInfo &MRI, MachineFunction &MF) const; 124 // emit extract subreg instruction and insert it before MachineInstr &I 125 bool emitExtractSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I, 126 MachineRegisterInfo &MRI, MachineFunction &MF) const; 127 128 const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const; 129 const TargetRegisterClass *getRegClass(LLT Ty, unsigned Reg, 130 MachineRegisterInfo &MRI) const; 131 132 const X86TargetMachine &TM; 133 const X86Subtarget &STI; 134 const X86InstrInfo &TII; 135 const X86RegisterInfo &TRI; 136 const X86RegisterBankInfo &RBI; 137 138#define GET_GLOBALISEL_PREDICATES_DECL 139#include "X86GenGlobalISel.inc" 140#undef GET_GLOBALISEL_PREDICATES_DECL 141 142#define GET_GLOBALISEL_TEMPORARIES_DECL 143#include "X86GenGlobalISel.inc" 144#undef GET_GLOBALISEL_TEMPORARIES_DECL 145}; 146 147} // end anonymous namespace 148 149#define GET_GLOBALISEL_IMPL 150#include "X86GenGlobalISel.inc" 151#undef GET_GLOBALISEL_IMPL 152 153X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM, 154 const X86Subtarget &STI, 155 const X86RegisterBankInfo &RBI) 156 : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()), 157 TRI(*STI.getRegisterInfo()), RBI(RBI), 158#define GET_GLOBALISEL_PREDICATES_INIT 159#include "X86GenGlobalISel.inc" 160#undef GET_GLOBALISEL_PREDICATES_INIT 161#define GET_GLOBALISEL_TEMPORARIES_INIT 162#include "X86GenGlobalISel.inc" 163#undef GET_GLOBALISEL_TEMPORARIES_INIT 164{ 165} 166 167// FIXME: This should be target-independent, inferred from the types declared 168// for each class in the bank. 169const TargetRegisterClass * 170X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const { 171 if (RB.getID() == X86::GPRRegBankID) { 172 if (Ty.getSizeInBits() <= 8) 173 return &X86::GR8RegClass; 174 if (Ty.getSizeInBits() == 16) 175 return &X86::GR16RegClass; 176 if (Ty.getSizeInBits() == 32) 177 return &X86::GR32RegClass; 178 if (Ty.getSizeInBits() == 64) 179 return &X86::GR64RegClass; 180 } 181 if (RB.getID() == X86::VECRRegBankID) { 182 if (Ty.getSizeInBits() == 32) 183 return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass; 184 if (Ty.getSizeInBits() == 64) 185 return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass; 186 if (Ty.getSizeInBits() == 128) 187 return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass; 188 if (Ty.getSizeInBits() == 256) 189 return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass; 190 if (Ty.getSizeInBits() == 512) 191 return &X86::VR512RegClass; 192 } 193 194 llvm_unreachable("Unknown RegBank!"); 195} 196 197const TargetRegisterClass * 198X86InstructionSelector::getRegClass(LLT Ty, unsigned Reg, 199 MachineRegisterInfo &MRI) const { 200 const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI); 201 return getRegClass(Ty, RegBank); 202} 203 204static unsigned getSubRegIndex(const TargetRegisterClass *RC) { 205 unsigned SubIdx = X86::NoSubRegister; 206 if (RC == &X86::GR32RegClass) { 207 SubIdx = X86::sub_32bit; 208 } else if (RC == &X86::GR16RegClass) { 209 SubIdx = X86::sub_16bit; 210 } else if (RC == &X86::GR8RegClass) { 211 SubIdx = X86::sub_8bit; 212 } 213 214 return SubIdx; 215} 216 217static const TargetRegisterClass *getRegClassFromGRPhysReg(unsigned Reg) { 218 assert(Register::isPhysicalRegister(Reg)); 219 if (X86::GR64RegClass.contains(Reg)) 220 return &X86::GR64RegClass; 221 if (X86::GR32RegClass.contains(Reg)) 222 return &X86::GR32RegClass; 223 if (X86::GR16RegClass.contains(Reg)) 224 return &X86::GR16RegClass; 225 if (X86::GR8RegClass.contains(Reg)) 226 return &X86::GR8RegClass; 227 228 llvm_unreachable("Unknown RegClass for PhysReg!"); 229} 230 231// Set X86 Opcode and constrain DestReg. 232bool X86InstructionSelector::selectCopy(MachineInstr &I, 233 MachineRegisterInfo &MRI) const { 234 Register DstReg = I.getOperand(0).getReg(); 235 const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI); 236 const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI); 237 238 Register SrcReg = I.getOperand(1).getReg(); 239 const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI); 240 const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI); 241 242 if (Register::isPhysicalRegister(DstReg)) { 243 assert(I.isCopy() && "Generic operators do not allow physical registers"); 244 245 if (DstSize > SrcSize && SrcRegBank.getID() == X86::GPRRegBankID && 246 DstRegBank.getID() == X86::GPRRegBankID) { 247 248 const TargetRegisterClass *SrcRC = 249 getRegClass(MRI.getType(SrcReg), SrcRegBank); 250 const TargetRegisterClass *DstRC = getRegClassFromGRPhysReg(DstReg); 251 252 if (SrcRC != DstRC) { 253 // This case can be generated by ABI lowering, performe anyext 254 Register ExtSrc = MRI.createVirtualRegister(DstRC); 255 BuildMI(*I.getParent(), I, I.getDebugLoc(), 256 TII.get(TargetOpcode::SUBREG_TO_REG)) 257 .addDef(ExtSrc) 258 .addImm(0) 259 .addReg(SrcReg) 260 .addImm(getSubRegIndex(SrcRC)); 261 262 I.getOperand(1).setReg(ExtSrc); 263 } 264 } 265 266 return true; 267 } 268 269 assert((!Register::isPhysicalRegister(SrcReg) || I.isCopy()) && 270 "No phys reg on generic operators"); 271 assert((DstSize == SrcSize || 272 // Copies are a mean to setup initial types, the number of 273 // bits may not exactly match. 274 (Register::isPhysicalRegister(SrcReg) && 275 DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) && 276 "Copy with different width?!"); 277 278 const TargetRegisterClass *DstRC = 279 getRegClass(MRI.getType(DstReg), DstRegBank); 280 281 if (SrcRegBank.getID() == X86::GPRRegBankID && 282 DstRegBank.getID() == X86::GPRRegBankID && SrcSize > DstSize && 283 Register::isPhysicalRegister(SrcReg)) { 284 // Change the physical register to performe truncate. 285 286 const TargetRegisterClass *SrcRC = getRegClassFromGRPhysReg(SrcReg); 287 288 if (DstRC != SrcRC) { 289 I.getOperand(1).setSubReg(getSubRegIndex(DstRC)); 290 I.getOperand(1).substPhysReg(SrcReg, TRI); 291 } 292 } 293 294 // No need to constrain SrcReg. It will get constrained when 295 // we hit another of its use or its defs. 296 // Copies do not have constraints. 297 const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(DstReg); 298 if (!OldRC || !DstRC->hasSubClassEq(OldRC)) { 299 if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 300 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) 301 << " operand\n"); 302 return false; 303 } 304 } 305 I.setDesc(TII.get(X86::COPY)); 306 return true; 307} 308 309bool X86InstructionSelector::select(MachineInstr &I) { 310 assert(I.getParent() && "Instruction should be in a basic block!"); 311 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 312 313 MachineBasicBlock &MBB = *I.getParent(); 314 MachineFunction &MF = *MBB.getParent(); 315 MachineRegisterInfo &MRI = MF.getRegInfo(); 316 317 unsigned Opcode = I.getOpcode(); 318 if (!isPreISelGenericOpcode(Opcode)) { 319 // Certain non-generic instructions also need some special handling. 320 321 if (Opcode == TargetOpcode::LOAD_STACK_GUARD) 322 return false; 323 324 if (I.isCopy()) 325 return selectCopy(I, MRI); 326 327 return true; 328 } 329 330 assert(I.getNumOperands() == I.getNumExplicitOperands() && 331 "Generic instruction has unexpected implicit operands\n"); 332 333 if (selectImpl(I, *CoverageInfo)) 334 return true; 335 336 LLVM_DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs())); 337 338 // TODO: This should be implemented by tblgen. 339 switch (I.getOpcode()) { 340 default: 341 return false; 342 case TargetOpcode::G_STORE: 343 case TargetOpcode::G_LOAD: 344 return selectLoadStoreOp(I, MRI, MF); 345 case TargetOpcode::G_PTR_ADD: 346 case TargetOpcode::G_FRAME_INDEX: 347 return selectFrameIndexOrGep(I, MRI, MF); 348 case TargetOpcode::G_GLOBAL_VALUE: 349 return selectGlobalValue(I, MRI, MF); 350 case TargetOpcode::G_CONSTANT: 351 return selectConstant(I, MRI, MF); 352 case TargetOpcode::G_FCONSTANT: 353 return materializeFP(I, MRI, MF); 354 case TargetOpcode::G_PTRTOINT: 355 case TargetOpcode::G_TRUNC: 356 return selectTruncOrPtrToInt(I, MRI, MF); 357 case TargetOpcode::G_INTTOPTR: 358 return selectCopy(I, MRI); 359 case TargetOpcode::G_ZEXT: 360 return selectZext(I, MRI, MF); 361 case TargetOpcode::G_ANYEXT: 362 return selectAnyext(I, MRI, MF); 363 case TargetOpcode::G_ICMP: 364 return selectCmp(I, MRI, MF); 365 case TargetOpcode::G_FCMP: 366 return selectFCmp(I, MRI, MF); 367 case TargetOpcode::G_UADDE: 368 return selectUadde(I, MRI, MF); 369 case TargetOpcode::G_UNMERGE_VALUES: 370 return selectUnmergeValues(I, MRI, MF); 371 case TargetOpcode::G_MERGE_VALUES: 372 case TargetOpcode::G_CONCAT_VECTORS: 373 return selectMergeValues(I, MRI, MF); 374 case TargetOpcode::G_EXTRACT: 375 return selectExtract(I, MRI, MF); 376 case TargetOpcode::G_INSERT: 377 return selectInsert(I, MRI, MF); 378 case TargetOpcode::G_BRCOND: 379 return selectCondBranch(I, MRI, MF); 380 case TargetOpcode::G_IMPLICIT_DEF: 381 case TargetOpcode::G_PHI: 382 return selectImplicitDefOrPHI(I, MRI); 383 case TargetOpcode::G_SDIV: 384 case TargetOpcode::G_UDIV: 385 case TargetOpcode::G_SREM: 386 case TargetOpcode::G_UREM: 387 return selectDivRem(I, MRI, MF); 388 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 389 return selectIntrinsicWSideEffects(I, MRI, MF); 390 } 391 392 return false; 393} 394 395unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty, 396 const RegisterBank &RB, 397 unsigned Opc, 398 Align Alignment) const { 399 bool Isload = (Opc == TargetOpcode::G_LOAD); 400 bool HasAVX = STI.hasAVX(); 401 bool HasAVX512 = STI.hasAVX512(); 402 bool HasVLX = STI.hasVLX(); 403 404 if (Ty == LLT::scalar(8)) { 405 if (X86::GPRRegBankID == RB.getID()) 406 return Isload ? X86::MOV8rm : X86::MOV8mr; 407 } else if (Ty == LLT::scalar(16)) { 408 if (X86::GPRRegBankID == RB.getID()) 409 return Isload ? X86::MOV16rm : X86::MOV16mr; 410 } else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) { 411 if (X86::GPRRegBankID == RB.getID()) 412 return Isload ? X86::MOV32rm : X86::MOV32mr; 413 if (X86::VECRRegBankID == RB.getID()) 414 return Isload ? (HasAVX512 ? X86::VMOVSSZrm_alt : 415 HasAVX ? X86::VMOVSSrm_alt : 416 X86::MOVSSrm_alt) 417 : (HasAVX512 ? X86::VMOVSSZmr : 418 HasAVX ? X86::VMOVSSmr : 419 X86::MOVSSmr); 420 } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) { 421 if (X86::GPRRegBankID == RB.getID()) 422 return Isload ? X86::MOV64rm : X86::MOV64mr; 423 if (X86::VECRRegBankID == RB.getID()) 424 return Isload ? (HasAVX512 ? X86::VMOVSDZrm_alt : 425 HasAVX ? X86::VMOVSDrm_alt : 426 X86::MOVSDrm_alt) 427 : (HasAVX512 ? X86::VMOVSDZmr : 428 HasAVX ? X86::VMOVSDmr : 429 X86::MOVSDmr); 430 } else if (Ty.isVector() && Ty.getSizeInBits() == 128) { 431 if (Alignment >= Align(16)) 432 return Isload ? (HasVLX ? X86::VMOVAPSZ128rm 433 : HasAVX512 434 ? X86::VMOVAPSZ128rm_NOVLX 435 : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm) 436 : (HasVLX ? X86::VMOVAPSZ128mr 437 : HasAVX512 438 ? X86::VMOVAPSZ128mr_NOVLX 439 : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr); 440 else 441 return Isload ? (HasVLX ? X86::VMOVUPSZ128rm 442 : HasAVX512 443 ? X86::VMOVUPSZ128rm_NOVLX 444 : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm) 445 : (HasVLX ? X86::VMOVUPSZ128mr 446 : HasAVX512 447 ? X86::VMOVUPSZ128mr_NOVLX 448 : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr); 449 } else if (Ty.isVector() && Ty.getSizeInBits() == 256) { 450 if (Alignment >= Align(32)) 451 return Isload ? (HasVLX ? X86::VMOVAPSZ256rm 452 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX 453 : X86::VMOVAPSYrm) 454 : (HasVLX ? X86::VMOVAPSZ256mr 455 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX 456 : X86::VMOVAPSYmr); 457 else 458 return Isload ? (HasVLX ? X86::VMOVUPSZ256rm 459 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX 460 : X86::VMOVUPSYrm) 461 : (HasVLX ? X86::VMOVUPSZ256mr 462 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX 463 : X86::VMOVUPSYmr); 464 } else if (Ty.isVector() && Ty.getSizeInBits() == 512) { 465 if (Alignment >= Align(64)) 466 return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr; 467 else 468 return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr; 469 } 470 return Opc; 471} 472 473// Fill in an address from the given instruction. 474static void X86SelectAddress(const MachineInstr &I, 475 const MachineRegisterInfo &MRI, 476 X86AddressMode &AM) { 477 assert(I.getOperand(0).isReg() && "unsupported opperand."); 478 assert(MRI.getType(I.getOperand(0).getReg()).isPointer() && 479 "unsupported type."); 480 481 if (I.getOpcode() == TargetOpcode::G_PTR_ADD) { 482 if (auto COff = getConstantVRegVal(I.getOperand(2).getReg(), MRI)) { 483 int64_t Imm = *COff; 484 if (isInt<32>(Imm)) { // Check for displacement overflow. 485 AM.Disp = static_cast<int32_t>(Imm); 486 AM.Base.Reg = I.getOperand(1).getReg(); 487 return; 488 } 489 } 490 } else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) { 491 AM.Base.FrameIndex = I.getOperand(1).getIndex(); 492 AM.BaseType = X86AddressMode::FrameIndexBase; 493 return; 494 } 495 496 // Default behavior. 497 AM.Base.Reg = I.getOperand(0).getReg(); 498} 499 500bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I, 501 MachineRegisterInfo &MRI, 502 MachineFunction &MF) const { 503 unsigned Opc = I.getOpcode(); 504 505 assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) && 506 "unexpected instruction"); 507 508 const Register DefReg = I.getOperand(0).getReg(); 509 LLT Ty = MRI.getType(DefReg); 510 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); 511 512 assert(I.hasOneMemOperand()); 513 auto &MemOp = **I.memoperands_begin(); 514 if (MemOp.isAtomic()) { 515 // Note: for unordered operations, we rely on the fact the appropriate MMO 516 // is already on the instruction we're mutating, and thus we don't need to 517 // make any changes. So long as we select an opcode which is capable of 518 // loading or storing the appropriate size atomically, the rest of the 519 // backend is required to respect the MMO state. 520 if (!MemOp.isUnordered()) { 521 LLVM_DEBUG(dbgs() << "Atomic ordering not supported yet\n"); 522 return false; 523 } 524 if (MemOp.getAlign() < Ty.getSizeInBits() / 8) { 525 LLVM_DEBUG(dbgs() << "Unaligned atomics not supported yet\n"); 526 return false; 527 } 528 } 529 530 unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlign()); 531 if (NewOpc == Opc) 532 return false; 533 534 X86AddressMode AM; 535 X86SelectAddress(*MRI.getVRegDef(I.getOperand(1).getReg()), MRI, AM); 536 537 I.setDesc(TII.get(NewOpc)); 538 MachineInstrBuilder MIB(MF, I); 539 if (Opc == TargetOpcode::G_LOAD) { 540 I.RemoveOperand(1); 541 addFullAddress(MIB, AM); 542 } else { 543 // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL) 544 I.RemoveOperand(1); 545 I.RemoveOperand(0); 546 addFullAddress(MIB, AM).addUse(DefReg); 547 } 548 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 549} 550 551static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) { 552 if (Ty == LLT::pointer(0, 64)) 553 return X86::LEA64r; 554 else if (Ty == LLT::pointer(0, 32)) 555 return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r; 556 else 557 llvm_unreachable("Can't get LEA opcode. Unsupported type."); 558} 559 560bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I, 561 MachineRegisterInfo &MRI, 562 MachineFunction &MF) const { 563 unsigned Opc = I.getOpcode(); 564 565 assert((Opc == TargetOpcode::G_FRAME_INDEX || Opc == TargetOpcode::G_PTR_ADD) && 566 "unexpected instruction"); 567 568 const Register DefReg = I.getOperand(0).getReg(); 569 LLT Ty = MRI.getType(DefReg); 570 571 // Use LEA to calculate frame index and GEP 572 unsigned NewOpc = getLeaOP(Ty, STI); 573 I.setDesc(TII.get(NewOpc)); 574 MachineInstrBuilder MIB(MF, I); 575 576 if (Opc == TargetOpcode::G_FRAME_INDEX) { 577 addOffset(MIB, 0); 578 } else { 579 MachineOperand &InxOp = I.getOperand(2); 580 I.addOperand(InxOp); // set IndexReg 581 InxOp.ChangeToImmediate(1); // set Scale 582 MIB.addImm(0).addReg(0); 583 } 584 585 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 586} 587 588bool X86InstructionSelector::selectGlobalValue(MachineInstr &I, 589 MachineRegisterInfo &MRI, 590 MachineFunction &MF) const { 591 assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) && 592 "unexpected instruction"); 593 594 auto GV = I.getOperand(1).getGlobal(); 595 if (GV->isThreadLocal()) { 596 return false; // TODO: we don't support TLS yet. 597 } 598 599 // Can't handle alternate code models yet. 600 if (TM.getCodeModel() != CodeModel::Small) 601 return false; 602 603 X86AddressMode AM; 604 AM.GV = GV; 605 AM.GVOpFlags = STI.classifyGlobalReference(GV); 606 607 // TODO: The ABI requires an extra load. not supported yet. 608 if (isGlobalStubReference(AM.GVOpFlags)) 609 return false; 610 611 // TODO: This reference is relative to the pic base. not supported yet. 612 if (isGlobalRelativeToPICBase(AM.GVOpFlags)) 613 return false; 614 615 if (STI.isPICStyleRIPRel()) { 616 // Use rip-relative addressing. 617 assert(AM.Base.Reg == 0 && AM.IndexReg == 0); 618 AM.Base.Reg = X86::RIP; 619 } 620 621 const Register DefReg = I.getOperand(0).getReg(); 622 LLT Ty = MRI.getType(DefReg); 623 unsigned NewOpc = getLeaOP(Ty, STI); 624 625 I.setDesc(TII.get(NewOpc)); 626 MachineInstrBuilder MIB(MF, I); 627 628 I.RemoveOperand(1); 629 addFullAddress(MIB, AM); 630 631 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 632} 633 634bool X86InstructionSelector::selectConstant(MachineInstr &I, 635 MachineRegisterInfo &MRI, 636 MachineFunction &MF) const { 637 assert((I.getOpcode() == TargetOpcode::G_CONSTANT) && 638 "unexpected instruction"); 639 640 const Register DefReg = I.getOperand(0).getReg(); 641 LLT Ty = MRI.getType(DefReg); 642 643 if (RBI.getRegBank(DefReg, MRI, TRI)->getID() != X86::GPRRegBankID) 644 return false; 645 646 uint64_t Val = 0; 647 if (I.getOperand(1).isCImm()) { 648 Val = I.getOperand(1).getCImm()->getZExtValue(); 649 I.getOperand(1).ChangeToImmediate(Val); 650 } else if (I.getOperand(1).isImm()) { 651 Val = I.getOperand(1).getImm(); 652 } else 653 llvm_unreachable("Unsupported operand type."); 654 655 unsigned NewOpc; 656 switch (Ty.getSizeInBits()) { 657 case 8: 658 NewOpc = X86::MOV8ri; 659 break; 660 case 16: 661 NewOpc = X86::MOV16ri; 662 break; 663 case 32: 664 NewOpc = X86::MOV32ri; 665 break; 666 case 64: 667 // TODO: in case isUInt<32>(Val), X86::MOV32ri can be used 668 if (isInt<32>(Val)) 669 NewOpc = X86::MOV64ri32; 670 else 671 NewOpc = X86::MOV64ri; 672 break; 673 default: 674 llvm_unreachable("Can't select G_CONSTANT, unsupported type."); 675 } 676 677 I.setDesc(TII.get(NewOpc)); 678 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 679} 680 681// Helper function for selectTruncOrPtrToInt and selectAnyext. 682// Returns true if DstRC lives on a floating register class and 683// SrcRC lives on a 128-bit vector class. 684static bool canTurnIntoCOPY(const TargetRegisterClass *DstRC, 685 const TargetRegisterClass *SrcRC) { 686 return (DstRC == &X86::FR32RegClass || DstRC == &X86::FR32XRegClass || 687 DstRC == &X86::FR64RegClass || DstRC == &X86::FR64XRegClass) && 688 (SrcRC == &X86::VR128RegClass || SrcRC == &X86::VR128XRegClass); 689} 690 691bool X86InstructionSelector::selectTurnIntoCOPY( 692 MachineInstr &I, MachineRegisterInfo &MRI, const unsigned DstReg, 693 const TargetRegisterClass *DstRC, const unsigned SrcReg, 694 const TargetRegisterClass *SrcRC) const { 695 696 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 697 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 698 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) 699 << " operand\n"); 700 return false; 701 } 702 I.setDesc(TII.get(X86::COPY)); 703 return true; 704} 705 706bool X86InstructionSelector::selectTruncOrPtrToInt(MachineInstr &I, 707 MachineRegisterInfo &MRI, 708 MachineFunction &MF) const { 709 assert((I.getOpcode() == TargetOpcode::G_TRUNC || 710 I.getOpcode() == TargetOpcode::G_PTRTOINT) && 711 "unexpected instruction"); 712 713 const Register DstReg = I.getOperand(0).getReg(); 714 const Register SrcReg = I.getOperand(1).getReg(); 715 716 const LLT DstTy = MRI.getType(DstReg); 717 const LLT SrcTy = MRI.getType(SrcReg); 718 719 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); 720 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI); 721 722 if (DstRB.getID() != SrcRB.getID()) { 723 LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode()) 724 << " input/output on different banks\n"); 725 return false; 726 } 727 728 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB); 729 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB); 730 731 if (!DstRC || !SrcRC) 732 return false; 733 734 // If that's truncation of the value that lives on the vector class and goes 735 // into the floating class, just replace it with copy, as we are able to 736 // select it as a regular move. 737 if (canTurnIntoCOPY(DstRC, SrcRC)) 738 return selectTurnIntoCOPY(I, MRI, DstReg, DstRC, SrcReg, SrcRC); 739 740 if (DstRB.getID() != X86::GPRRegBankID) 741 return false; 742 743 unsigned SubIdx; 744 if (DstRC == SrcRC) { 745 // Nothing to be done 746 SubIdx = X86::NoSubRegister; 747 } else if (DstRC == &X86::GR32RegClass) { 748 SubIdx = X86::sub_32bit; 749 } else if (DstRC == &X86::GR16RegClass) { 750 SubIdx = X86::sub_16bit; 751 } else if (DstRC == &X86::GR8RegClass) { 752 SubIdx = X86::sub_8bit; 753 } else { 754 return false; 755 } 756 757 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx); 758 759 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 760 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 761 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) 762 << "\n"); 763 return false; 764 } 765 766 I.getOperand(1).setSubReg(SubIdx); 767 768 I.setDesc(TII.get(X86::COPY)); 769 return true; 770} 771 772bool X86InstructionSelector::selectZext(MachineInstr &I, 773 MachineRegisterInfo &MRI, 774 MachineFunction &MF) const { 775 assert((I.getOpcode() == TargetOpcode::G_ZEXT) && "unexpected instruction"); 776 777 const Register DstReg = I.getOperand(0).getReg(); 778 const Register SrcReg = I.getOperand(1).getReg(); 779 780 const LLT DstTy = MRI.getType(DstReg); 781 const LLT SrcTy = MRI.getType(SrcReg); 782 783 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(32)) && 784 "8=>32 Zext is handled by tablegen"); 785 assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(32)) && 786 "16=>32 Zext is handled by tablegen"); 787 788 const static struct ZextEntry { 789 LLT SrcTy; 790 LLT DstTy; 791 unsigned MovOp; 792 bool NeedSubregToReg; 793 } OpTable[] = { 794 {LLT::scalar(8), LLT::scalar(16), X86::MOVZX16rr8, false}, // i8 => i16 795 {LLT::scalar(8), LLT::scalar(64), X86::MOVZX32rr8, true}, // i8 => i64 796 {LLT::scalar(16), LLT::scalar(64), X86::MOVZX32rr16, true}, // i16 => i64 797 {LLT::scalar(32), LLT::scalar(64), 0, true} // i32 => i64 798 }; 799 800 auto ZextEntryIt = 801 std::find_if(std::begin(OpTable), std::end(OpTable), 802 [SrcTy, DstTy](const ZextEntry &El) { 803 return El.DstTy == DstTy && El.SrcTy == SrcTy; 804 }); 805 806 // Here we try to select Zext into a MOVZ and/or SUBREG_TO_REG instruction. 807 if (ZextEntryIt != std::end(OpTable)) { 808 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); 809 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI); 810 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB); 811 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB); 812 813 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 814 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 815 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) 816 << " operand\n"); 817 return false; 818 } 819 820 unsigned TransitRegTo = DstReg; 821 unsigned TransitRegFrom = SrcReg; 822 if (ZextEntryIt->MovOp) { 823 // If we select Zext into MOVZ + SUBREG_TO_REG, we need to have 824 // a transit register in between: create it here. 825 if (ZextEntryIt->NeedSubregToReg) { 826 TransitRegFrom = MRI.createVirtualRegister( 827 getRegClass(LLT::scalar(32), DstReg, MRI)); 828 TransitRegTo = TransitRegFrom; 829 } 830 831 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(ZextEntryIt->MovOp)) 832 .addDef(TransitRegTo) 833 .addReg(SrcReg); 834 } 835 if (ZextEntryIt->NeedSubregToReg) { 836 BuildMI(*I.getParent(), I, I.getDebugLoc(), 837 TII.get(TargetOpcode::SUBREG_TO_REG)) 838 .addDef(DstReg) 839 .addImm(0) 840 .addReg(TransitRegFrom) 841 .addImm(X86::sub_32bit); 842 } 843 I.eraseFromParent(); 844 return true; 845 } 846 847 if (SrcTy != LLT::scalar(1)) 848 return false; 849 850 unsigned AndOpc; 851 if (DstTy == LLT::scalar(8)) 852 AndOpc = X86::AND8ri; 853 else if (DstTy == LLT::scalar(16)) 854 AndOpc = X86::AND16ri8; 855 else if (DstTy == LLT::scalar(32)) 856 AndOpc = X86::AND32ri8; 857 else if (DstTy == LLT::scalar(64)) 858 AndOpc = X86::AND64ri8; 859 else 860 return false; 861 862 unsigned DefReg = SrcReg; 863 if (DstTy != LLT::scalar(8)) { 864 DefReg = MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI)); 865 BuildMI(*I.getParent(), I, I.getDebugLoc(), 866 TII.get(TargetOpcode::SUBREG_TO_REG), DefReg) 867 .addImm(0) 868 .addReg(SrcReg) 869 .addImm(X86::sub_8bit); 870 } 871 872 MachineInstr &AndInst = 873 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AndOpc), DstReg) 874 .addReg(DefReg) 875 .addImm(1); 876 877 constrainSelectedInstRegOperands(AndInst, TII, TRI, RBI); 878 879 I.eraseFromParent(); 880 return true; 881} 882 883bool X86InstructionSelector::selectAnyext(MachineInstr &I, 884 MachineRegisterInfo &MRI, 885 MachineFunction &MF) const { 886 assert((I.getOpcode() == TargetOpcode::G_ANYEXT) && "unexpected instruction"); 887 888 const Register DstReg = I.getOperand(0).getReg(); 889 const Register SrcReg = I.getOperand(1).getReg(); 890 891 const LLT DstTy = MRI.getType(DstReg); 892 const LLT SrcTy = MRI.getType(SrcReg); 893 894 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); 895 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI); 896 897 assert(DstRB.getID() == SrcRB.getID() && 898 "G_ANYEXT input/output on different banks\n"); 899 900 assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() && 901 "G_ANYEXT incorrect operand size"); 902 903 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB); 904 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB); 905 906 // If that's ANY_EXT of the value that lives on the floating class and goes 907 // into the vector class, just replace it with copy, as we are able to select 908 // it as a regular move. 909 if (canTurnIntoCOPY(SrcRC, DstRC)) 910 return selectTurnIntoCOPY(I, MRI, SrcReg, SrcRC, DstReg, DstRC); 911 912 if (DstRB.getID() != X86::GPRRegBankID) 913 return false; 914 915 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 916 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 917 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) 918 << " operand\n"); 919 return false; 920 } 921 922 if (SrcRC == DstRC) { 923 I.setDesc(TII.get(X86::COPY)); 924 return true; 925 } 926 927 BuildMI(*I.getParent(), I, I.getDebugLoc(), 928 TII.get(TargetOpcode::SUBREG_TO_REG)) 929 .addDef(DstReg) 930 .addImm(0) 931 .addReg(SrcReg) 932 .addImm(getSubRegIndex(SrcRC)); 933 934 I.eraseFromParent(); 935 return true; 936} 937 938bool X86InstructionSelector::selectCmp(MachineInstr &I, 939 MachineRegisterInfo &MRI, 940 MachineFunction &MF) const { 941 assert((I.getOpcode() == TargetOpcode::G_ICMP) && "unexpected instruction"); 942 943 X86::CondCode CC; 944 bool SwapArgs; 945 std::tie(CC, SwapArgs) = X86::getX86ConditionCode( 946 (CmpInst::Predicate)I.getOperand(1).getPredicate()); 947 948 Register LHS = I.getOperand(2).getReg(); 949 Register RHS = I.getOperand(3).getReg(); 950 951 if (SwapArgs) 952 std::swap(LHS, RHS); 953 954 unsigned OpCmp; 955 LLT Ty = MRI.getType(LHS); 956 957 switch (Ty.getSizeInBits()) { 958 default: 959 return false; 960 case 8: 961 OpCmp = X86::CMP8rr; 962 break; 963 case 16: 964 OpCmp = X86::CMP16rr; 965 break; 966 case 32: 967 OpCmp = X86::CMP32rr; 968 break; 969 case 64: 970 OpCmp = X86::CMP64rr; 971 break; 972 } 973 974 MachineInstr &CmpInst = 975 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp)) 976 .addReg(LHS) 977 .addReg(RHS); 978 979 MachineInstr &SetInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 980 TII.get(X86::SETCCr), I.getOperand(0).getReg()).addImm(CC); 981 982 constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI); 983 constrainSelectedInstRegOperands(SetInst, TII, TRI, RBI); 984 985 I.eraseFromParent(); 986 return true; 987} 988 989bool X86InstructionSelector::selectFCmp(MachineInstr &I, 990 MachineRegisterInfo &MRI, 991 MachineFunction &MF) const { 992 assert((I.getOpcode() == TargetOpcode::G_FCMP) && "unexpected instruction"); 993 994 Register LhsReg = I.getOperand(2).getReg(); 995 Register RhsReg = I.getOperand(3).getReg(); 996 CmpInst::Predicate Predicate = 997 (CmpInst::Predicate)I.getOperand(1).getPredicate(); 998 999 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction. 1000 static const uint16_t SETFOpcTable[2][3] = { 1001 {X86::COND_E, X86::COND_NP, X86::AND8rr}, 1002 {X86::COND_NE, X86::COND_P, X86::OR8rr}}; 1003 const uint16_t *SETFOpc = nullptr; 1004 switch (Predicate) { 1005 default: 1006 break; 1007 case CmpInst::FCMP_OEQ: 1008 SETFOpc = &SETFOpcTable[0][0]; 1009 break; 1010 case CmpInst::FCMP_UNE: 1011 SETFOpc = &SETFOpcTable[1][0]; 1012 break; 1013 } 1014 1015 // Compute the opcode for the CMP instruction. 1016 unsigned OpCmp; 1017 LLT Ty = MRI.getType(LhsReg); 1018 switch (Ty.getSizeInBits()) { 1019 default: 1020 return false; 1021 case 32: 1022 OpCmp = X86::UCOMISSrr; 1023 break; 1024 case 64: 1025 OpCmp = X86::UCOMISDrr; 1026 break; 1027 } 1028 1029 Register ResultReg = I.getOperand(0).getReg(); 1030 RBI.constrainGenericRegister( 1031 ResultReg, 1032 *getRegClass(LLT::scalar(8), *RBI.getRegBank(ResultReg, MRI, TRI)), MRI); 1033 if (SETFOpc) { 1034 MachineInstr &CmpInst = 1035 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp)) 1036 .addReg(LhsReg) 1037 .addReg(RhsReg); 1038 1039 Register FlagReg1 = MRI.createVirtualRegister(&X86::GR8RegClass); 1040 Register FlagReg2 = MRI.createVirtualRegister(&X86::GR8RegClass); 1041 MachineInstr &Set1 = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 1042 TII.get(X86::SETCCr), FlagReg1).addImm(SETFOpc[0]); 1043 MachineInstr &Set2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 1044 TII.get(X86::SETCCr), FlagReg2).addImm(SETFOpc[1]); 1045 MachineInstr &Set3 = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 1046 TII.get(SETFOpc[2]), ResultReg) 1047 .addReg(FlagReg1) 1048 .addReg(FlagReg2); 1049 constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI); 1050 constrainSelectedInstRegOperands(Set1, TII, TRI, RBI); 1051 constrainSelectedInstRegOperands(Set2, TII, TRI, RBI); 1052 constrainSelectedInstRegOperands(Set3, TII, TRI, RBI); 1053 1054 I.eraseFromParent(); 1055 return true; 1056 } 1057 1058 X86::CondCode CC; 1059 bool SwapArgs; 1060 std::tie(CC, SwapArgs) = X86::getX86ConditionCode(Predicate); 1061 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code."); 1062 1063 if (SwapArgs) 1064 std::swap(LhsReg, RhsReg); 1065 1066 // Emit a compare of LHS/RHS. 1067 MachineInstr &CmpInst = 1068 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp)) 1069 .addReg(LhsReg) 1070 .addReg(RhsReg); 1071 1072 MachineInstr &Set = 1073 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SETCCr), ResultReg).addImm(CC); 1074 constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI); 1075 constrainSelectedInstRegOperands(Set, TII, TRI, RBI); 1076 I.eraseFromParent(); 1077 return true; 1078} 1079 1080bool X86InstructionSelector::selectUadde(MachineInstr &I, 1081 MachineRegisterInfo &MRI, 1082 MachineFunction &MF) const { 1083 assert((I.getOpcode() == TargetOpcode::G_UADDE) && "unexpected instruction"); 1084 1085 const Register DstReg = I.getOperand(0).getReg(); 1086 const Register CarryOutReg = I.getOperand(1).getReg(); 1087 const Register Op0Reg = I.getOperand(2).getReg(); 1088 const Register Op1Reg = I.getOperand(3).getReg(); 1089 Register CarryInReg = I.getOperand(4).getReg(); 1090 1091 const LLT DstTy = MRI.getType(DstReg); 1092 1093 if (DstTy != LLT::scalar(32)) 1094 return false; 1095 1096 // find CarryIn def instruction. 1097 MachineInstr *Def = MRI.getVRegDef(CarryInReg); 1098 while (Def->getOpcode() == TargetOpcode::G_TRUNC) { 1099 CarryInReg = Def->getOperand(1).getReg(); 1100 Def = MRI.getVRegDef(CarryInReg); 1101 } 1102 1103 unsigned Opcode; 1104 if (Def->getOpcode() == TargetOpcode::G_UADDE) { 1105 // carry set by prev ADD. 1106 1107 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), X86::EFLAGS) 1108 .addReg(CarryInReg); 1109 1110 if (!RBI.constrainGenericRegister(CarryInReg, X86::GR32RegClass, MRI)) 1111 return false; 1112 1113 Opcode = X86::ADC32rr; 1114 } else if (auto val = getConstantVRegVal(CarryInReg, MRI)) { 1115 // carry is constant, support only 0. 1116 if (*val != 0) 1117 return false; 1118 1119 Opcode = X86::ADD32rr; 1120 } else 1121 return false; 1122 1123 MachineInstr &AddInst = 1124 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg) 1125 .addReg(Op0Reg) 1126 .addReg(Op1Reg); 1127 1128 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), CarryOutReg) 1129 .addReg(X86::EFLAGS); 1130 1131 if (!constrainSelectedInstRegOperands(AddInst, TII, TRI, RBI) || 1132 !RBI.constrainGenericRegister(CarryOutReg, X86::GR32RegClass, MRI)) 1133 return false; 1134 1135 I.eraseFromParent(); 1136 return true; 1137} 1138 1139bool X86InstructionSelector::selectExtract(MachineInstr &I, 1140 MachineRegisterInfo &MRI, 1141 MachineFunction &MF) const { 1142 assert((I.getOpcode() == TargetOpcode::G_EXTRACT) && 1143 "unexpected instruction"); 1144 1145 const Register DstReg = I.getOperand(0).getReg(); 1146 const Register SrcReg = I.getOperand(1).getReg(); 1147 int64_t Index = I.getOperand(2).getImm(); 1148 1149 const LLT DstTy = MRI.getType(DstReg); 1150 const LLT SrcTy = MRI.getType(SrcReg); 1151 1152 // Meanwile handle vector type only. 1153 if (!DstTy.isVector()) 1154 return false; 1155 1156 if (Index % DstTy.getSizeInBits() != 0) 1157 return false; // Not extract subvector. 1158 1159 if (Index == 0) { 1160 // Replace by extract subreg copy. 1161 if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF)) 1162 return false; 1163 1164 I.eraseFromParent(); 1165 return true; 1166 } 1167 1168 bool HasAVX = STI.hasAVX(); 1169 bool HasAVX512 = STI.hasAVX512(); 1170 bool HasVLX = STI.hasVLX(); 1171 1172 if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) { 1173 if (HasVLX) 1174 I.setDesc(TII.get(X86::VEXTRACTF32x4Z256rr)); 1175 else if (HasAVX) 1176 I.setDesc(TII.get(X86::VEXTRACTF128rr)); 1177 else 1178 return false; 1179 } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) { 1180 if (DstTy.getSizeInBits() == 128) 1181 I.setDesc(TII.get(X86::VEXTRACTF32x4Zrr)); 1182 else if (DstTy.getSizeInBits() == 256) 1183 I.setDesc(TII.get(X86::VEXTRACTF64x4Zrr)); 1184 else 1185 return false; 1186 } else 1187 return false; 1188 1189 // Convert to X86 VEXTRACT immediate. 1190 Index = Index / DstTy.getSizeInBits(); 1191 I.getOperand(2).setImm(Index); 1192 1193 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 1194} 1195 1196bool X86InstructionSelector::emitExtractSubreg(unsigned DstReg, unsigned SrcReg, 1197 MachineInstr &I, 1198 MachineRegisterInfo &MRI, 1199 MachineFunction &MF) const { 1200 const LLT DstTy = MRI.getType(DstReg); 1201 const LLT SrcTy = MRI.getType(SrcReg); 1202 unsigned SubIdx = X86::NoSubRegister; 1203 1204 if (!DstTy.isVector() || !SrcTy.isVector()) 1205 return false; 1206 1207 assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() && 1208 "Incorrect Src/Dst register size"); 1209 1210 if (DstTy.getSizeInBits() == 128) 1211 SubIdx = X86::sub_xmm; 1212 else if (DstTy.getSizeInBits() == 256) 1213 SubIdx = X86::sub_ymm; 1214 else 1215 return false; 1216 1217 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI); 1218 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI); 1219 1220 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx); 1221 1222 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 1223 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 1224 LLVM_DEBUG(dbgs() << "Failed to constrain EXTRACT_SUBREG\n"); 1225 return false; 1226 } 1227 1228 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), DstReg) 1229 .addReg(SrcReg, 0, SubIdx); 1230 1231 return true; 1232} 1233 1234bool X86InstructionSelector::emitInsertSubreg(unsigned DstReg, unsigned SrcReg, 1235 MachineInstr &I, 1236 MachineRegisterInfo &MRI, 1237 MachineFunction &MF) const { 1238 const LLT DstTy = MRI.getType(DstReg); 1239 const LLT SrcTy = MRI.getType(SrcReg); 1240 unsigned SubIdx = X86::NoSubRegister; 1241 1242 // TODO: support scalar types 1243 if (!DstTy.isVector() || !SrcTy.isVector()) 1244 return false; 1245 1246 assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() && 1247 "Incorrect Src/Dst register size"); 1248 1249 if (SrcTy.getSizeInBits() == 128) 1250 SubIdx = X86::sub_xmm; 1251 else if (SrcTy.getSizeInBits() == 256) 1252 SubIdx = X86::sub_ymm; 1253 else 1254 return false; 1255 1256 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI); 1257 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI); 1258 1259 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 1260 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 1261 LLVM_DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n"); 1262 return false; 1263 } 1264 1265 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY)) 1266 .addReg(DstReg, RegState::DefineNoRead, SubIdx) 1267 .addReg(SrcReg); 1268 1269 return true; 1270} 1271 1272bool X86InstructionSelector::selectInsert(MachineInstr &I, 1273 MachineRegisterInfo &MRI, 1274 MachineFunction &MF) const { 1275 assert((I.getOpcode() == TargetOpcode::G_INSERT) && "unexpected instruction"); 1276 1277 const Register DstReg = I.getOperand(0).getReg(); 1278 const Register SrcReg = I.getOperand(1).getReg(); 1279 const Register InsertReg = I.getOperand(2).getReg(); 1280 int64_t Index = I.getOperand(3).getImm(); 1281 1282 const LLT DstTy = MRI.getType(DstReg); 1283 const LLT InsertRegTy = MRI.getType(InsertReg); 1284 1285 // Meanwile handle vector type only. 1286 if (!DstTy.isVector()) 1287 return false; 1288 1289 if (Index % InsertRegTy.getSizeInBits() != 0) 1290 return false; // Not insert subvector. 1291 1292 if (Index == 0 && MRI.getVRegDef(SrcReg)->isImplicitDef()) { 1293 // Replace by subreg copy. 1294 if (!emitInsertSubreg(DstReg, InsertReg, I, MRI, MF)) 1295 return false; 1296 1297 I.eraseFromParent(); 1298 return true; 1299 } 1300 1301 bool HasAVX = STI.hasAVX(); 1302 bool HasAVX512 = STI.hasAVX512(); 1303 bool HasVLX = STI.hasVLX(); 1304 1305 if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) { 1306 if (HasVLX) 1307 I.setDesc(TII.get(X86::VINSERTF32x4Z256rr)); 1308 else if (HasAVX) 1309 I.setDesc(TII.get(X86::VINSERTF128rr)); 1310 else 1311 return false; 1312 } else if (DstTy.getSizeInBits() == 512 && HasAVX512) { 1313 if (InsertRegTy.getSizeInBits() == 128) 1314 I.setDesc(TII.get(X86::VINSERTF32x4Zrr)); 1315 else if (InsertRegTy.getSizeInBits() == 256) 1316 I.setDesc(TII.get(X86::VINSERTF64x4Zrr)); 1317 else 1318 return false; 1319 } else 1320 return false; 1321 1322 // Convert to X86 VINSERT immediate. 1323 Index = Index / InsertRegTy.getSizeInBits(); 1324 1325 I.getOperand(3).setImm(Index); 1326 1327 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 1328} 1329 1330bool X86InstructionSelector::selectUnmergeValues( 1331 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) { 1332 assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) && 1333 "unexpected instruction"); 1334 1335 // Split to extracts. 1336 unsigned NumDefs = I.getNumOperands() - 1; 1337 Register SrcReg = I.getOperand(NumDefs).getReg(); 1338 unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits(); 1339 1340 for (unsigned Idx = 0; Idx < NumDefs; ++Idx) { 1341 MachineInstr &ExtrInst = 1342 *BuildMI(*I.getParent(), I, I.getDebugLoc(), 1343 TII.get(TargetOpcode::G_EXTRACT), I.getOperand(Idx).getReg()) 1344 .addReg(SrcReg) 1345 .addImm(Idx * DefSize); 1346 1347 if (!select(ExtrInst)) 1348 return false; 1349 } 1350 1351 I.eraseFromParent(); 1352 return true; 1353} 1354 1355bool X86InstructionSelector::selectMergeValues( 1356 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) { 1357 assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES || 1358 I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS) && 1359 "unexpected instruction"); 1360 1361 // Split to inserts. 1362 Register DstReg = I.getOperand(0).getReg(); 1363 Register SrcReg0 = I.getOperand(1).getReg(); 1364 1365 const LLT DstTy = MRI.getType(DstReg); 1366 const LLT SrcTy = MRI.getType(SrcReg0); 1367 unsigned SrcSize = SrcTy.getSizeInBits(); 1368 1369 const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI); 1370 1371 // For the first src use insertSubReg. 1372 Register DefReg = MRI.createGenericVirtualRegister(DstTy); 1373 MRI.setRegBank(DefReg, RegBank); 1374 if (!emitInsertSubreg(DefReg, I.getOperand(1).getReg(), I, MRI, MF)) 1375 return false; 1376 1377 for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) { 1378 Register Tmp = MRI.createGenericVirtualRegister(DstTy); 1379 MRI.setRegBank(Tmp, RegBank); 1380 1381 MachineInstr &InsertInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 1382 TII.get(TargetOpcode::G_INSERT), Tmp) 1383 .addReg(DefReg) 1384 .addReg(I.getOperand(Idx).getReg()) 1385 .addImm((Idx - 1) * SrcSize); 1386 1387 DefReg = Tmp; 1388 1389 if (!select(InsertInst)) 1390 return false; 1391 } 1392 1393 MachineInstr &CopyInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 1394 TII.get(TargetOpcode::COPY), DstReg) 1395 .addReg(DefReg); 1396 1397 if (!select(CopyInst)) 1398 return false; 1399 1400 I.eraseFromParent(); 1401 return true; 1402} 1403 1404bool X86InstructionSelector::selectCondBranch(MachineInstr &I, 1405 MachineRegisterInfo &MRI, 1406 MachineFunction &MF) const { 1407 assert((I.getOpcode() == TargetOpcode::G_BRCOND) && "unexpected instruction"); 1408 1409 const Register CondReg = I.getOperand(0).getReg(); 1410 MachineBasicBlock *DestMBB = I.getOperand(1).getMBB(); 1411 1412 MachineInstr &TestInst = 1413 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TEST8ri)) 1414 .addReg(CondReg) 1415 .addImm(1); 1416 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::JCC_1)) 1417 .addMBB(DestMBB).addImm(X86::COND_NE); 1418 1419 constrainSelectedInstRegOperands(TestInst, TII, TRI, RBI); 1420 1421 I.eraseFromParent(); 1422 return true; 1423} 1424 1425bool X86InstructionSelector::materializeFP(MachineInstr &I, 1426 MachineRegisterInfo &MRI, 1427 MachineFunction &MF) const { 1428 assert((I.getOpcode() == TargetOpcode::G_FCONSTANT) && 1429 "unexpected instruction"); 1430 1431 // Can't handle alternate code models yet. 1432 CodeModel::Model CM = TM.getCodeModel(); 1433 if (CM != CodeModel::Small && CM != CodeModel::Large) 1434 return false; 1435 1436 const Register DstReg = I.getOperand(0).getReg(); 1437 const LLT DstTy = MRI.getType(DstReg); 1438 const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI); 1439 Align Alignment = Align(DstTy.getSizeInBytes()); 1440 const DebugLoc &DbgLoc = I.getDebugLoc(); 1441 1442 unsigned Opc = 1443 getLoadStoreOp(DstTy, RegBank, TargetOpcode::G_LOAD, Alignment); 1444 1445 // Create the load from the constant pool. 1446 const ConstantFP *CFP = I.getOperand(1).getFPImm(); 1447 unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(CFP, Alignment); 1448 MachineInstr *LoadInst = nullptr; 1449 unsigned char OpFlag = STI.classifyLocalReference(nullptr); 1450 1451 if (CM == CodeModel::Large && STI.is64Bit()) { 1452 // Under X86-64 non-small code model, GV (and friends) are 64-bits, so 1453 // they cannot be folded into immediate fields. 1454 1455 Register AddrReg = MRI.createVirtualRegister(&X86::GR64RegClass); 1456 BuildMI(*I.getParent(), I, DbgLoc, TII.get(X86::MOV64ri), AddrReg) 1457 .addConstantPoolIndex(CPI, 0, OpFlag); 1458 1459 MachineMemOperand *MMO = MF.getMachineMemOperand( 1460 MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad, 1461 MF.getDataLayout().getPointerSize(), Alignment); 1462 1463 LoadInst = 1464 addDirectMem(BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg), 1465 AddrReg) 1466 .addMemOperand(MMO); 1467 1468 } else if (CM == CodeModel::Small || !STI.is64Bit()) { 1469 // Handle the case when globals fit in our immediate field. 1470 // This is true for X86-32 always and X86-64 when in -mcmodel=small mode. 1471 1472 // x86-32 PIC requires a PIC base register for constant pools. 1473 unsigned PICBase = 0; 1474 if (OpFlag == X86II::MO_PIC_BASE_OFFSET || OpFlag == X86II::MO_GOTOFF) { 1475 // PICBase can be allocated by TII.getGlobalBaseReg(&MF). 1476 // In DAGISEL the code that initialize it generated by the CGBR pass. 1477 return false; // TODO support the mode. 1478 } else if (STI.is64Bit() && TM.getCodeModel() == CodeModel::Small) 1479 PICBase = X86::RIP; 1480 1481 LoadInst = addConstantPoolReference( 1482 BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg), CPI, PICBase, 1483 OpFlag); 1484 } else 1485 return false; 1486 1487 constrainSelectedInstRegOperands(*LoadInst, TII, TRI, RBI); 1488 I.eraseFromParent(); 1489 return true; 1490} 1491 1492bool X86InstructionSelector::selectImplicitDefOrPHI( 1493 MachineInstr &I, MachineRegisterInfo &MRI) const { 1494 assert((I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF || 1495 I.getOpcode() == TargetOpcode::G_PHI) && 1496 "unexpected instruction"); 1497 1498 Register DstReg = I.getOperand(0).getReg(); 1499 1500 if (!MRI.getRegClassOrNull(DstReg)) { 1501 const LLT DstTy = MRI.getType(DstReg); 1502 const TargetRegisterClass *RC = getRegClass(DstTy, DstReg, MRI); 1503 1504 if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) { 1505 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) 1506 << " operand\n"); 1507 return false; 1508 } 1509 } 1510 1511 if (I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 1512 I.setDesc(TII.get(X86::IMPLICIT_DEF)); 1513 else 1514 I.setDesc(TII.get(X86::PHI)); 1515 1516 return true; 1517} 1518 1519bool X86InstructionSelector::selectDivRem(MachineInstr &I, 1520 MachineRegisterInfo &MRI, 1521 MachineFunction &MF) const { 1522 // The implementation of this function is taken from X86FastISel. 1523 assert((I.getOpcode() == TargetOpcode::G_SDIV || 1524 I.getOpcode() == TargetOpcode::G_SREM || 1525 I.getOpcode() == TargetOpcode::G_UDIV || 1526 I.getOpcode() == TargetOpcode::G_UREM) && 1527 "unexpected instruction"); 1528 1529 const Register DstReg = I.getOperand(0).getReg(); 1530 const Register Op1Reg = I.getOperand(1).getReg(); 1531 const Register Op2Reg = I.getOperand(2).getReg(); 1532 1533 const LLT RegTy = MRI.getType(DstReg); 1534 assert(RegTy == MRI.getType(Op1Reg) && RegTy == MRI.getType(Op2Reg) && 1535 "Arguments and return value types must match"); 1536 1537 const RegisterBank *RegRB = RBI.getRegBank(DstReg, MRI, TRI); 1538 if (!RegRB || RegRB->getID() != X86::GPRRegBankID) 1539 return false; 1540 1541 const static unsigned NumTypes = 4; // i8, i16, i32, i64 1542 const static unsigned NumOps = 4; // SDiv, SRem, UDiv, URem 1543 const static bool S = true; // IsSigned 1544 const static bool U = false; // !IsSigned 1545 const static unsigned Copy = TargetOpcode::COPY; 1546 // For the X86 IDIV instruction, in most cases the dividend 1547 // (numerator) must be in a specific register pair highreg:lowreg, 1548 // producing the quotient in lowreg and the remainder in highreg. 1549 // For most data types, to set up the instruction, the dividend is 1550 // copied into lowreg, and lowreg is sign-extended into highreg. The 1551 // exception is i8, where the dividend is defined as a single register rather 1552 // than a register pair, and we therefore directly sign-extend the dividend 1553 // into lowreg, instead of copying, and ignore the highreg. 1554 const static struct DivRemEntry { 1555 // The following portion depends only on the data type. 1556 unsigned SizeInBits; 1557 unsigned LowInReg; // low part of the register pair 1558 unsigned HighInReg; // high part of the register pair 1559 // The following portion depends on both the data type and the operation. 1560 struct DivRemResult { 1561 unsigned OpDivRem; // The specific DIV/IDIV opcode to use. 1562 unsigned OpSignExtend; // Opcode for sign-extending lowreg into 1563 // highreg, or copying a zero into highreg. 1564 unsigned OpCopy; // Opcode for copying dividend into lowreg, or 1565 // zero/sign-extending into lowreg for i8. 1566 unsigned DivRemResultReg; // Register containing the desired result. 1567 bool IsOpSigned; // Whether to use signed or unsigned form. 1568 } ResultTable[NumOps]; 1569 } OpTable[NumTypes] = { 1570 {8, 1571 X86::AX, 1572 0, 1573 { 1574 {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AL, S}, // SDiv 1575 {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AH, S}, // SRem 1576 {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AL, U}, // UDiv 1577 {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AH, U}, // URem 1578 }}, // i8 1579 {16, 1580 X86::AX, 1581 X86::DX, 1582 { 1583 {X86::IDIV16r, X86::CWD, Copy, X86::AX, S}, // SDiv 1584 {X86::IDIV16r, X86::CWD, Copy, X86::DX, S}, // SRem 1585 {X86::DIV16r, X86::MOV32r0, Copy, X86::AX, U}, // UDiv 1586 {X86::DIV16r, X86::MOV32r0, Copy, X86::DX, U}, // URem 1587 }}, // i16 1588 {32, 1589 X86::EAX, 1590 X86::EDX, 1591 { 1592 {X86::IDIV32r, X86::CDQ, Copy, X86::EAX, S}, // SDiv 1593 {X86::IDIV32r, X86::CDQ, Copy, X86::EDX, S}, // SRem 1594 {X86::DIV32r, X86::MOV32r0, Copy, X86::EAX, U}, // UDiv 1595 {X86::DIV32r, X86::MOV32r0, Copy, X86::EDX, U}, // URem 1596 }}, // i32 1597 {64, 1598 X86::RAX, 1599 X86::RDX, 1600 { 1601 {X86::IDIV64r, X86::CQO, Copy, X86::RAX, S}, // SDiv 1602 {X86::IDIV64r, X86::CQO, Copy, X86::RDX, S}, // SRem 1603 {X86::DIV64r, X86::MOV32r0, Copy, X86::RAX, U}, // UDiv 1604 {X86::DIV64r, X86::MOV32r0, Copy, X86::RDX, U}, // URem 1605 }}, // i64 1606 }; 1607 1608 auto OpEntryIt = std::find_if(std::begin(OpTable), std::end(OpTable), 1609 [RegTy](const DivRemEntry &El) { 1610 return El.SizeInBits == RegTy.getSizeInBits(); 1611 }); 1612 if (OpEntryIt == std::end(OpTable)) 1613 return false; 1614 1615 unsigned OpIndex; 1616 switch (I.getOpcode()) { 1617 default: 1618 llvm_unreachable("Unexpected div/rem opcode"); 1619 case TargetOpcode::G_SDIV: 1620 OpIndex = 0; 1621 break; 1622 case TargetOpcode::G_SREM: 1623 OpIndex = 1; 1624 break; 1625 case TargetOpcode::G_UDIV: 1626 OpIndex = 2; 1627 break; 1628 case TargetOpcode::G_UREM: 1629 OpIndex = 3; 1630 break; 1631 } 1632 1633 const DivRemEntry &TypeEntry = *OpEntryIt; 1634 const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex]; 1635 1636 const TargetRegisterClass *RegRC = getRegClass(RegTy, *RegRB); 1637 if (!RBI.constrainGenericRegister(Op1Reg, *RegRC, MRI) || 1638 !RBI.constrainGenericRegister(Op2Reg, *RegRC, MRI) || 1639 !RBI.constrainGenericRegister(DstReg, *RegRC, MRI)) { 1640 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) 1641 << " operand\n"); 1642 return false; 1643 } 1644 1645 // Move op1 into low-order input register. 1646 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpCopy), 1647 TypeEntry.LowInReg) 1648 .addReg(Op1Reg); 1649 // Zero-extend or sign-extend into high-order input register. 1650 if (OpEntry.OpSignExtend) { 1651 if (OpEntry.IsOpSigned) 1652 BuildMI(*I.getParent(), I, I.getDebugLoc(), 1653 TII.get(OpEntry.OpSignExtend)); 1654 else { 1655 Register Zero32 = MRI.createVirtualRegister(&X86::GR32RegClass); 1656 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::MOV32r0), 1657 Zero32); 1658 1659 // Copy the zero into the appropriate sub/super/identical physical 1660 // register. Unfortunately the operations needed are not uniform enough 1661 // to fit neatly into the table above. 1662 if (RegTy.getSizeInBits() == 16) { 1663 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy), 1664 TypeEntry.HighInReg) 1665 .addReg(Zero32, 0, X86::sub_16bit); 1666 } else if (RegTy.getSizeInBits() == 32) { 1667 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy), 1668 TypeEntry.HighInReg) 1669 .addReg(Zero32); 1670 } else if (RegTy.getSizeInBits() == 64) { 1671 BuildMI(*I.getParent(), I, I.getDebugLoc(), 1672 TII.get(TargetOpcode::SUBREG_TO_REG), TypeEntry.HighInReg) 1673 .addImm(0) 1674 .addReg(Zero32) 1675 .addImm(X86::sub_32bit); 1676 } 1677 } 1678 } 1679 // Generate the DIV/IDIV instruction. 1680 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpDivRem)) 1681 .addReg(Op2Reg); 1682 // For i8 remainder, we can't reference ah directly, as we'll end 1683 // up with bogus copies like %r9b = COPY %ah. Reference ax 1684 // instead to prevent ah references in a rex instruction. 1685 // 1686 // The current assumption of the fast register allocator is that isel 1687 // won't generate explicit references to the GR8_NOREX registers. If 1688 // the allocator and/or the backend get enhanced to be more robust in 1689 // that regard, this can be, and should be, removed. 1690 if ((I.getOpcode() == Instruction::SRem || 1691 I.getOpcode() == Instruction::URem) && 1692 OpEntry.DivRemResultReg == X86::AH && STI.is64Bit()) { 1693 Register SourceSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass); 1694 Register ResultSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass); 1695 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy), SourceSuperReg) 1696 .addReg(X86::AX); 1697 1698 // Shift AX right by 8 bits instead of using AH. 1699 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SHR16ri), 1700 ResultSuperReg) 1701 .addReg(SourceSuperReg) 1702 .addImm(8); 1703 1704 // Now reference the 8-bit subreg of the result. 1705 BuildMI(*I.getParent(), I, I.getDebugLoc(), 1706 TII.get(TargetOpcode::SUBREG_TO_REG)) 1707 .addDef(DstReg) 1708 .addImm(0) 1709 .addReg(ResultSuperReg) 1710 .addImm(X86::sub_8bit); 1711 } else { 1712 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), 1713 DstReg) 1714 .addReg(OpEntry.DivRemResultReg); 1715 } 1716 I.eraseFromParent(); 1717 return true; 1718} 1719 1720bool X86InstructionSelector::selectIntrinsicWSideEffects( 1721 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const { 1722 1723 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS && 1724 "unexpected instruction"); 1725 1726 if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap) 1727 return false; 1728 1729 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TRAP)); 1730 1731 I.eraseFromParent(); 1732 return true; 1733} 1734 1735InstructionSelector * 1736llvm::createX86InstructionSelector(const X86TargetMachine &TM, 1737 X86Subtarget &Subtarget, 1738 X86RegisterBankInfo &RBI) { 1739 return new X86InstructionSelector(TM, Subtarget, RBI); 1740} 1741