X86ISelDAGToDAG.cpp revision 296417
154359Sroberto//===- X86ISelDAGToDAG.cpp - A DAG pattern matching inst selector for X86 -===// 2182007Sroberto// 3285612Sdelphij// The LLVM Compiler Infrastructure 4182007Sroberto// 554359Sroberto// This file is distributed under the University of Illinois Open Source 654359Sroberto// License. See LICENSE.TXT for details. 754359Sroberto// 854359Sroberto//===----------------------------------------------------------------------===// 9182007Sroberto// 10285612Sdelphij// This file defines a DAG pattern matching instruction selector for X86, 1154359Sroberto// converting from a legalized dag to a X86 dag. 12182007Sroberto// 13182007Sroberto//===----------------------------------------------------------------------===// 14182007Sroberto 15182007Sroberto#include "X86.h" 16182007Sroberto#include "X86InstrBuilder.h" 17182007Sroberto#include "X86MachineFunctionInfo.h" 18182007Sroberto#include "X86RegisterInfo.h" 19182007Sroberto#include "X86Subtarget.h" 20182007Sroberto#include "X86TargetMachine.h" 21182007Sroberto#include "llvm/ADT/Statistic.h" 22182007Sroberto#include "llvm/CodeGen/MachineFrameInfo.h" 23182007Sroberto#include "llvm/CodeGen/MachineFunction.h" 24182007Sroberto#include "llvm/CodeGen/MachineInstrBuilder.h" 25182007Sroberto#include "llvm/CodeGen/MachineRegisterInfo.h" 26182007Sroberto#include "llvm/CodeGen/SelectionDAGISel.h" 27182007Sroberto#include "llvm/IR/Function.h" 28182007Sroberto#include "llvm/IR/Instructions.h" 29182007Sroberto#include "llvm/IR/Intrinsics.h" 30182007Sroberto#include "llvm/IR/Type.h" 31182007Sroberto#include "llvm/Support/Debug.h" 32182007Sroberto#include "llvm/Support/ErrorHandling.h" 33182007Sroberto#include "llvm/Support/MathExtras.h" 34182007Sroberto#include "llvm/Support/raw_ostream.h" 35182007Sroberto#include "llvm/Target/TargetMachine.h" 3654359Sroberto#include "llvm/Target/TargetOptions.h" 3754359Sroberto#include <stdint.h> 3854359Srobertousing namespace llvm; 3954359Sroberto 4054359Sroberto#define DEBUG_TYPE "x86-isel" 4154359Sroberto 4254359SrobertoSTATISTIC(NumLoadMoved, "Number of loads moved below TokenFactor"); 4354359Sroberto 4454359Sroberto//===----------------------------------------------------------------------===// 4554359Sroberto// Pattern Matcher Implementation 46182007Sroberto//===----------------------------------------------------------------------===// 4754359Sroberto 4854359Srobertonamespace { 4954359Sroberto /// This corresponds to X86AddressMode, but uses SDValue's instead of register 5054359Sroberto /// numbers for the leaves of the matched tree. 5154359Sroberto struct X86ISelAddressMode { 5254359Sroberto enum { 5354359Sroberto RegBase, 5454359Sroberto FrameIndexBase 5554359Sroberto } BaseType; 5654359Sroberto 5754359Sroberto // This is really a union, discriminated by BaseType! 5854359Sroberto SDValue Base_Reg; 5954359Sroberto int Base_FrameIndex; 6054359Sroberto 6154359Sroberto unsigned Scale; 6254359Sroberto SDValue IndexReg; 6354359Sroberto int32_t Disp; 6454359Sroberto SDValue Segment; 6554359Sroberto const GlobalValue *GV; 6654359Sroberto const Constant *CP; 6754359Sroberto const BlockAddress *BlockAddr; 6854359Sroberto const char *ES; 6954359Sroberto MCSymbol *MCSym; 7054359Sroberto int JT; 7154359Sroberto unsigned Align; // CP alignment. 7254359Sroberto unsigned char SymbolFlags; // X86II::MO_* 7354359Sroberto 7454359Sroberto X86ISelAddressMode() 7554359Sroberto : BaseType(RegBase), Base_FrameIndex(0), Scale(1), IndexReg(), Disp(0), 7654359Sroberto Segment(), GV(nullptr), CP(nullptr), BlockAddr(nullptr), ES(nullptr), 7754359Sroberto MCSym(nullptr), JT(-1), Align(0), SymbolFlags(X86II::MO_NO_FLAG) {} 7854359Sroberto 7954359Sroberto bool hasSymbolicDisplacement() const { 80285612Sdelphij return GV != nullptr || CP != nullptr || ES != nullptr || 81285612Sdelphij MCSym != nullptr || JT != -1 || BlockAddr != nullptr; 82285612Sdelphij } 83285612Sdelphij 84285612Sdelphij bool hasBaseOrIndexReg() const { 85285612Sdelphij return BaseType == FrameIndexBase || 86285612Sdelphij IndexReg.getNode() != nullptr || Base_Reg.getNode() != nullptr; 87285612Sdelphij } 88285612Sdelphij 8954359Sroberto /// Return true if this addressing mode is already RIP-relative. 9054359Sroberto bool isRIPRelative() const { 9154359Sroberto if (BaseType != RegBase) return false; 9254359Sroberto if (RegisterSDNode *RegNode = 9354359Sroberto dyn_cast_or_null<RegisterSDNode>(Base_Reg.getNode())) 9454359Sroberto return RegNode->getReg() == X86::RIP; 9554359Sroberto return false; 9654359Sroberto } 9754359Sroberto 9854359Sroberto void setBaseReg(SDValue Reg) { 9954359Sroberto BaseType = RegBase; 10054359Sroberto Base_Reg = Reg; 10154359Sroberto } 10254359Sroberto 10354359Sroberto#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 10454359Sroberto void dump() { 10554359Sroberto dbgs() << "X86ISelAddressMode " << this << '\n'; 10654359Sroberto dbgs() << "Base_Reg "; 10754359Sroberto if (Base_Reg.getNode()) 10854359Sroberto Base_Reg.getNode()->dump(); 10954359Sroberto else 11054359Sroberto dbgs() << "nul"; 11154359Sroberto dbgs() << " Base.FrameIndex " << Base_FrameIndex << '\n' 11254359Sroberto << " Scale" << Scale << '\n' 113285612Sdelphij << "IndexReg "; 11454359Sroberto if (IndexReg.getNode()) 11554359Sroberto IndexReg.getNode()->dump(); 11654359Sroberto else 11754359Sroberto dbgs() << "nul"; 11854359Sroberto dbgs() << " Disp " << Disp << '\n' 11954359Sroberto << "GV "; 12054359Sroberto if (GV) 12154359Sroberto GV->dump(); 12254359Sroberto else 12354359Sroberto dbgs() << "nul"; 12454359Sroberto dbgs() << " CP "; 12554359Sroberto if (CP) 12654359Sroberto CP->dump(); 12754359Sroberto else 12854359Sroberto dbgs() << "nul"; 12954359Sroberto dbgs() << '\n' 13054359Sroberto << "ES "; 13154359Sroberto if (ES) 13254359Sroberto dbgs() << ES; 13354359Sroberto else 13454359Sroberto dbgs() << "nul"; 13554359Sroberto dbgs() << " MCSym "; 13654359Sroberto if (MCSym) 13754359Sroberto dbgs() << MCSym; 13854359Sroberto else 13954359Sroberto dbgs() << "nul"; 14054359Sroberto dbgs() << " JT" << JT << " Align" << Align << '\n'; 14154359Sroberto } 14254359Sroberto#endif 14354359Sroberto }; 14454359Sroberto} 14554359Sroberto 14654359Srobertonamespace { 14754359Sroberto //===--------------------------------------------------------------------===// 14854359Sroberto /// ISel - X86-specific code to select X86 machine instructions for 14954359Sroberto /// SelectionDAG operations. 15054359Sroberto /// 15154359Sroberto class X86DAGToDAGISel final : public SelectionDAGISel { 15254359Sroberto /// Keep a pointer to the X86Subtarget around so that we can 15354359Sroberto /// make the right decision when generating code for different targets. 15454359Sroberto const X86Subtarget *Subtarget; 15554359Sroberto 15654359Sroberto /// If true, selector should try to optimize for code size instead of 15754359Sroberto /// performance. 15854359Sroberto bool OptForSize; 15954359Sroberto 16054359Sroberto public: 16154359Sroberto explicit X86DAGToDAGISel(X86TargetMachine &tm, CodeGenOpt::Level OptLevel) 16254359Sroberto : SelectionDAGISel(tm, OptLevel), OptForSize(false) {} 16354359Sroberto 16454359Sroberto const char *getPassName() const override { 16554359Sroberto return "X86 DAG->DAG Instruction Selection"; 16654359Sroberto } 16754359Sroberto 16854359Sroberto bool runOnMachineFunction(MachineFunction &MF) override { 169285612Sdelphij // Reset the subtarget each time through. 17054359Sroberto Subtarget = &MF.getSubtarget<X86Subtarget>(); 17154359Sroberto SelectionDAGISel::runOnMachineFunction(MF); 17254359Sroberto return true; 17354359Sroberto } 17454359Sroberto 17554359Sroberto void EmitFunctionEntryCode() override; 17654359Sroberto 17754359Sroberto bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override; 17854359Sroberto 17954359Sroberto void PreprocessISelDAG() override; 18054359Sroberto 18154359Sroberto inline bool immSext8(SDNode *N) const { 18254359Sroberto return isInt<8>(cast<ConstantSDNode>(N)->getSExtValue()); 18354359Sroberto } 18454359Sroberto 18554359Sroberto // True if the 64-bit immediate fits in a 32-bit sign-extended field. 18654359Sroberto inline bool i64immSExt32(SDNode *N) const { 18754359Sroberto uint64_t v = cast<ConstantSDNode>(N)->getZExtValue(); 18854359Sroberto return (int64_t)v == (int32_t)v; 189285612Sdelphij } 19054359Sroberto 19154359Sroberto// Include the pieces autogenerated from the target description. 192285612Sdelphij#include "X86GenDAGISel.inc" 19354359Sroberto 19454359Sroberto private: 19554359Sroberto SDNode *Select(SDNode *N) override; 19654359Sroberto SDNode *selectGather(SDNode *N, unsigned Opc); 19754359Sroberto SDNode *selectAtomicLoadArith(SDNode *Node, MVT NVT); 19854359Sroberto 19954359Sroberto bool foldOffsetIntoAddress(uint64_t Offset, X86ISelAddressMode &AM); 20054359Sroberto bool matchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM); 20154359Sroberto bool matchWrapper(SDValue N, X86ISelAddressMode &AM); 20254359Sroberto bool matchAddress(SDValue N, X86ISelAddressMode &AM); 20354359Sroberto bool matchAdd(SDValue N, X86ISelAddressMode &AM, unsigned Depth); 20454359Sroberto bool matchAddressRecursively(SDValue N, X86ISelAddressMode &AM, 20554359Sroberto unsigned Depth); 20654359Sroberto bool matchAddressBase(SDValue N, X86ISelAddressMode &AM); 20754359Sroberto bool selectAddr(SDNode *Parent, SDValue N, SDValue &Base, 20854359Sroberto SDValue &Scale, SDValue &Index, SDValue &Disp, 20954359Sroberto SDValue &Segment); 21054359Sroberto bool selectVectorAddr(SDNode *Parent, SDValue N, SDValue &Base, 21154359Sroberto SDValue &Scale, SDValue &Index, SDValue &Disp, 21254359Sroberto SDValue &Segment); 21354359Sroberto bool selectMOV64Imm32(SDValue N, SDValue &Imm); 21454359Sroberto bool selectLEAAddr(SDValue N, SDValue &Base, 21554359Sroberto SDValue &Scale, SDValue &Index, SDValue &Disp, 21654359Sroberto SDValue &Segment); 21754359Sroberto bool selectLEA64_32Addr(SDValue N, SDValue &Base, 21854359Sroberto SDValue &Scale, SDValue &Index, SDValue &Disp, 21954359Sroberto SDValue &Segment); 220285612Sdelphij bool selectTLSADDRAddr(SDValue N, SDValue &Base, 22154359Sroberto SDValue &Scale, SDValue &Index, SDValue &Disp, 22254359Sroberto SDValue &Segment); 22354359Sroberto bool selectScalarSSELoad(SDNode *Root, SDValue N, 22454359Sroberto SDValue &Base, SDValue &Scale, 225285612Sdelphij SDValue &Index, SDValue &Disp, 22654359Sroberto SDValue &Segment, 22754359Sroberto SDValue &NodeWithChain); 22854359Sroberto 22954359Sroberto bool tryFoldLoad(SDNode *P, SDValue N, 23054359Sroberto SDValue &Base, SDValue &Scale, 23154359Sroberto SDValue &Index, SDValue &Disp, 23254359Sroberto SDValue &Segment); 23354359Sroberto 23454359Sroberto /// Implement addressing mode selection for inline asm expressions. 23554359Sroberto bool SelectInlineAsmMemoryOperand(const SDValue &Op, 236285612Sdelphij unsigned ConstraintID, 23754359Sroberto std::vector<SDValue> &OutOps) override; 23854359Sroberto 23954359Sroberto void emitSpecialCodeForMain(); 24054359Sroberto 24154359Sroberto inline void getAddressOperands(X86ISelAddressMode &AM, SDLoc DL, 242285612Sdelphij SDValue &Base, SDValue &Scale, 24354359Sroberto SDValue &Index, SDValue &Disp, 24454359Sroberto SDValue &Segment) { 24554359Sroberto Base = (AM.BaseType == X86ISelAddressMode::FrameIndexBase) 24654359Sroberto ? CurDAG->getTargetFrameIndex( 24754359Sroberto AM.Base_FrameIndex, 24854359Sroberto TLI->getPointerTy(CurDAG->getDataLayout())) 24954359Sroberto : AM.Base_Reg; 25054359Sroberto Scale = getI8Imm(AM.Scale, DL); 25154359Sroberto Index = AM.IndexReg; 25254359Sroberto // These are 32-bit even in 64-bit mode since RIP-relative offset 25354359Sroberto // is 32-bit. 25454359Sroberto if (AM.GV) 25554359Sroberto Disp = CurDAG->getTargetGlobalAddress(AM.GV, SDLoc(), 25654359Sroberto MVT::i32, AM.Disp, 25754359Sroberto AM.SymbolFlags); 25854359Sroberto else if (AM.CP) 25954359Sroberto Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, 26054359Sroberto AM.Align, AM.Disp, AM.SymbolFlags); 26154359Sroberto else if (AM.ES) { 26256746Sroberto assert(!AM.Disp && "Non-zero displacement is ignored with ES."); 26354359Sroberto Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags); 264285612Sdelphij } else if (AM.MCSym) { 26554359Sroberto assert(!AM.Disp && "Non-zero displacement is ignored with MCSym."); 26654359Sroberto assert(AM.SymbolFlags == 0 && "oo"); 26754359Sroberto Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32); 26854359Sroberto } else if (AM.JT != -1) { 26954359Sroberto assert(!AM.Disp && "Non-zero displacement is ignored with JT."); 27054359Sroberto Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags); 27154359Sroberto } else if (AM.BlockAddr) 27254359Sroberto Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp, 27354359Sroberto AM.SymbolFlags); 27454359Sroberto else 27554359Sroberto Disp = CurDAG->getTargetConstant(AM.Disp, DL, MVT::i32); 27654359Sroberto 27754359Sroberto if (AM.Segment.getNode()) 27854359Sroberto Segment = AM.Segment; 27954359Sroberto else 28054359Sroberto Segment = CurDAG->getRegister(0, MVT::i32); 28154359Sroberto } 28254359Sroberto 28354359Sroberto // Utility function to determine whether we should avoid selecting 28454359Sroberto // immediate forms of instructions for better code size or not. 28554359Sroberto // At a high level, we'd like to avoid such instructions when 28654359Sroberto // we have similar constants used within the same basic block 28754359Sroberto // that can be kept in a register. 288285612Sdelphij // 289285612Sdelphij bool shouldAvoidImmediateInstFormsForSize(SDNode *N) const { 29054359Sroberto uint32_t UseCount = 0; 29154359Sroberto 29254359Sroberto // Do not want to hoist if we're not optimizing for size. 29354359Sroberto // TODO: We'd like to remove this restriction. 29454359Sroberto // See the comment in X86InstrInfo.td for more info. 295285612Sdelphij if (!OptForSize) 29654359Sroberto return false; 29754359Sroberto 29854359Sroberto // Walk all the users of the immediate. 29954359Sroberto for (SDNode::use_iterator UI = N->use_begin(), 30054359Sroberto UE = N->use_end(); (UI != UE) && (UseCount < 2); ++UI) { 301285612Sdelphij 30254359Sroberto SDNode *User = *UI; 30354359Sroberto 30454359Sroberto // This user is already selected. Count it as a legitimate use and 30554359Sroberto // move on. 30654359Sroberto if (User->isMachineOpcode()) { 30754359Sroberto UseCount++; 30854359Sroberto continue; 30954359Sroberto } 310285612Sdelphij 31154359Sroberto // We want to count stores of immediates as real uses. 312285612Sdelphij if (User->getOpcode() == ISD::STORE && 31354359Sroberto User->getOperand(1).getNode() == N) { 31454359Sroberto UseCount++; 31554359Sroberto continue; 31654359Sroberto } 31754359Sroberto 31854359Sroberto // We don't currently match users that have > 2 operands (except 31954359Sroberto // for stores, which are handled above) 32054359Sroberto // Those instruction won't match in ISEL, for now, and would 32154359Sroberto // be counted incorrectly. 32254359Sroberto // This may change in the future as we add additional instruction 32354359Sroberto // types. 32454359Sroberto if (User->getNumOperands() != 2) 32554359Sroberto continue; 32654359Sroberto 32754359Sroberto // Immediates that are used for offsets as part of stack 32854359Sroberto // manipulation should be left alone. These are typically 32954359Sroberto // used to indicate SP offsets for argument passing and 33054359Sroberto // will get pulled into stores/pushes (implicitly). 331285612Sdelphij if (User->getOpcode() == X86ISD::ADD || 33254359Sroberto User->getOpcode() == ISD::ADD || 33354359Sroberto User->getOpcode() == X86ISD::SUB || 33454359Sroberto User->getOpcode() == ISD::SUB) { 33554359Sroberto 33654359Sroberto // Find the other operand of the add/sub. 33754359Sroberto SDValue OtherOp = User->getOperand(0); 33854359Sroberto if (OtherOp.getNode() == N) 33954359Sroberto OtherOp = User->getOperand(1); 34054359Sroberto 34154359Sroberto // Don't count if the other operand is SP. 34254359Sroberto RegisterSDNode *RegNode; 34354359Sroberto if (OtherOp->getOpcode() == ISD::CopyFromReg && 34454359Sroberto (RegNode = dyn_cast_or_null<RegisterSDNode>( 345285612Sdelphij OtherOp->getOperand(1).getNode()))) 34654359Sroberto if ((RegNode->getReg() == X86::ESP) || 34754359Sroberto (RegNode->getReg() == X86::RSP)) 34854359Sroberto continue; 34954359Sroberto } 35054359Sroberto 351285612Sdelphij // ... otherwise, count this and move on. 35254359Sroberto UseCount++; 35354359Sroberto } 35454359Sroberto 35554359Sroberto // If we have more than 1 use, then recommend for hoisting. 35654359Sroberto return (UseCount > 1); 35754359Sroberto } 35854359Sroberto 35954359Sroberto /// Return a target constant with the specified value of type i8. 360285612Sdelphij inline SDValue getI8Imm(unsigned Imm, SDLoc DL) { 361285612Sdelphij return CurDAG->getTargetConstant(Imm, DL, MVT::i8); 362285612Sdelphij } 363285612Sdelphij 364285612Sdelphij /// Return a target constant with the specified value, of type i32. 36554359Sroberto inline SDValue getI32Imm(unsigned Imm, SDLoc DL) { 36654359Sroberto return CurDAG->getTargetConstant(Imm, DL, MVT::i32); 36754359Sroberto } 36854359Sroberto 36954359Sroberto /// Return an SDNode that returns the value of the global base register. 37054359Sroberto /// Output instructions required to initialize the global base register, 37154359Sroberto /// if necessary. 37254359Sroberto SDNode *getGlobalBaseReg(); 37354359Sroberto 37454359Sroberto /// Return a reference to the TargetMachine, casted to the target-specific 37554359Sroberto /// type. 37654359Sroberto const X86TargetMachine &getTargetMachine() const { 37754359Sroberto return static_cast<const X86TargetMachine &>(TM); 37854359Sroberto } 37954359Sroberto 38054359Sroberto /// Return a reference to the TargetInstrInfo, casted to the target-specific 38154359Sroberto /// type. 38254359Sroberto const X86InstrInfo *getInstrInfo() const { 38354359Sroberto return Subtarget->getInstrInfo(); 38454359Sroberto } 38554359Sroberto 38654359Sroberto /// \brief Address-mode matching performs shift-of-and to and-of-shift 38754359Sroberto /// reassociation in order to expose more scaled addressing 38854359Sroberto /// opportunities. 38954359Sroberto bool ComplexPatternFuncMutatesDAG() const override { 39054359Sroberto return true; 39154359Sroberto } 39254359Sroberto }; 39354359Sroberto} 39454359Sroberto 39554359Sroberto 39654359Srobertobool 39754359SrobertoX86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const { 39854359Sroberto if (OptLevel == CodeGenOpt::None) return false; 39954359Sroberto 40054359Sroberto if (!N.hasOneUse()) 40154359Sroberto return false; 40254359Sroberto 40354359Sroberto if (N.getOpcode() != ISD::LOAD) 40454359Sroberto return true; 40554359Sroberto 40654359Sroberto // If N is a load, do additional profitability checks. 40754359Sroberto if (U == Root) { 40854359Sroberto switch (U->getOpcode()) { 40954359Sroberto default: break; 41054359Sroberto case X86ISD::ADD: 41154359Sroberto case X86ISD::SUB: 41254359Sroberto case X86ISD::AND: 41354359Sroberto case X86ISD::XOR: 41454359Sroberto case X86ISD::OR: 41554359Sroberto case ISD::ADD: 41654359Sroberto case ISD::ADDC: 41754359Sroberto case ISD::ADDE: 41854359Sroberto case ISD::AND: 419285612Sdelphij case ISD::OR: 42054359Sroberto case ISD::XOR: { 42154359Sroberto SDValue Op1 = U->getOperand(1); 42254359Sroberto 42354359Sroberto // If the other operand is a 8-bit immediate we should fold the immediate 42454359Sroberto // instead. This reduces code size. 42554359Sroberto // e.g. 42654359Sroberto // movl 4(%esp), %eax 42754359Sroberto // addl $4, %eax 42854359Sroberto // vs. 42954359Sroberto // movl $4, %eax 43054359Sroberto // addl 4(%esp), %eax 43154359Sroberto // The former is 2 bytes shorter. In case where the increment is 1, then 432285612Sdelphij // the saving can be 4 bytes (by using incl %eax). 433285612Sdelphij if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Op1)) 43454359Sroberto if (Imm->getAPIntValue().isSignedIntN(8)) 43554359Sroberto return false; 43654359Sroberto 43754359Sroberto // If the other operand is a TLS address, we should fold it instead. 43854359Sroberto // This produces 43954359Sroberto // movl %gs:0, %eax 44054359Sroberto // leal i@NTPOFF(%eax), %eax 44154359Sroberto // instead of 44254359Sroberto // movl $i@NTPOFF, %eax 44354359Sroberto // addl %gs:0, %eax 44454359Sroberto // if the block also has an access to a second TLS address this will save 44554359Sroberto // a load. 44654359Sroberto // FIXME: This is probably also true for non-TLS addresses. 44754359Sroberto if (Op1.getOpcode() == X86ISD::Wrapper) { 44854359Sroberto SDValue Val = Op1.getOperand(0); 44954359Sroberto if (Val.getOpcode() == ISD::TargetGlobalTLSAddress) 45054359Sroberto return false; 45154359Sroberto } 45254359Sroberto } 45354359Sroberto } 45454359Sroberto } 45554359Sroberto 45654359Sroberto return true; 45754359Sroberto} 45854359Sroberto 45954359Sroberto/// Replace the original chain operand of the call with 46054359Sroberto/// load's chain operand and move load below the call's chain operand. 46154359Srobertostatic void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load, 46254359Sroberto SDValue Call, SDValue OrigChain) { 463285612Sdelphij SmallVector<SDValue, 8> Ops; 46454359Sroberto SDValue Chain = OrigChain.getOperand(0); 46554359Sroberto if (Chain.getNode() == Load.getNode()) 46654359Sroberto Ops.push_back(Load.getOperand(0)); 46754359Sroberto else { 46854359Sroberto assert(Chain.getOpcode() == ISD::TokenFactor && 46954359Sroberto "Unexpected chain operand"); 47054359Sroberto for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) 47154359Sroberto if (Chain.getOperand(i).getNode() == Load.getNode()) 47254359Sroberto Ops.push_back(Load.getOperand(0)); 47354359Sroberto else 47454359Sroberto Ops.push_back(Chain.getOperand(i)); 47554359Sroberto SDValue NewChain = 47654359Sroberto CurDAG->getNode(ISD::TokenFactor, SDLoc(Load), MVT::Other, Ops); 47754359Sroberto Ops.clear(); 47854359Sroberto Ops.push_back(NewChain); 479285612Sdelphij } 480285612Sdelphij Ops.append(OrigChain->op_begin() + 1, OrigChain->op_end()); 481285612Sdelphij CurDAG->UpdateNodeOperands(OrigChain.getNode(), Ops); 48254359Sroberto CurDAG->UpdateNodeOperands(Load.getNode(), Call.getOperand(0), 48354359Sroberto Load.getOperand(1), Load.getOperand(2)); 484285612Sdelphij 48554359Sroberto Ops.clear(); 48654359Sroberto Ops.push_back(SDValue(Load.getNode(), 1)); 48754359Sroberto Ops.append(Call->op_begin() + 1, Call->op_end()); 48854359Sroberto CurDAG->UpdateNodeOperands(Call.getNode(), Ops); 48954359Sroberto} 49054359Sroberto 491285612Sdelphij/// Return true if call address is a load and it can be 49254359Sroberto/// moved below CALLSEQ_START and the chains leading up to the call. 49354359Sroberto/// Return the CALLSEQ_START by reference as a second output. 49454359Sroberto/// In the case of a tail call, there isn't a callseq node between the call 49554359Sroberto/// chain and the load. 49654359Srobertostatic bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq) { 49754359Sroberto // The transformation is somewhat dangerous if the call's chain was glued to 498285612Sdelphij // the call. After MoveBelowOrigChain the load is moved between the call and 49954359Sroberto // the chain, this can create a cycle if the load is not folded. So it is 50054359Sroberto // *really* important that we are sure the load will be folded. 50154359Sroberto if (Callee.getNode() == Chain.getNode() || !Callee.hasOneUse()) 502285612Sdelphij return false; 50354359Sroberto LoadSDNode *LD = dyn_cast<LoadSDNode>(Callee.getNode()); 50454359Sroberto if (!LD || 50554359Sroberto LD->isVolatile() || 50654359Sroberto LD->getAddressingMode() != ISD::UNINDEXED || 50754359Sroberto LD->getExtensionType() != ISD::NON_EXTLOAD) 50854359Sroberto return false; 509285612Sdelphij 51054359Sroberto // Now let's find the callseq_start. 51154359Sroberto while (HasCallSeq && Chain.getOpcode() != ISD::CALLSEQ_START) { 51254359Sroberto if (!Chain.hasOneUse()) 51354359Sroberto return false; 51454359Sroberto Chain = Chain.getOperand(0); 51554359Sroberto } 51654359Sroberto 51754359Sroberto if (!Chain.getNumOperands()) 51854359Sroberto return false; 51954359Sroberto // Since we are not checking for AA here, conservatively abort if the chain 52054359Sroberto // writes to memory. It's not safe to move the callee (a load) across a store. 52154359Sroberto if (isa<MemSDNode>(Chain.getNode()) && 52254359Sroberto cast<MemSDNode>(Chain.getNode())->writeMem()) 52354359Sroberto return false; 52454359Sroberto if (Chain.getOperand(0).getNode() == Callee.getNode()) 52554359Sroberto return true; 52654359Sroberto if (Chain.getOperand(0).getOpcode() == ISD::TokenFactor && 52754359Sroberto Callee.getValue(1).isOperandOf(Chain.getOperand(0).getNode()) && 52854359Sroberto Callee.getValue(1).hasOneUse()) 52954359Sroberto return true; 53054359Sroberto return false; 53154359Sroberto} 53254359Sroberto 53354359Srobertovoid X86DAGToDAGISel::PreprocessISelDAG() { 534285612Sdelphij // OptForSize is used in pattern predicates that isel is matching. 53554359Sroberto OptForSize = MF->getFunction()->optForSize(); 53654359Sroberto 53754359Sroberto for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 53854359Sroberto E = CurDAG->allnodes_end(); I != E; ) { 53954359Sroberto SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues. 54054359Sroberto 54154359Sroberto if (OptLevel != CodeGenOpt::None && 54254359Sroberto // Only does this when target favors doesn't favor register indirect 543182007Sroberto // call. 54454359Sroberto ((N->getOpcode() == X86ISD::CALL && !Subtarget->callRegIndirect()) || 545182007Sroberto (N->getOpcode() == X86ISD::TC_RETURN && 54654359Sroberto // Only does this if load can be folded into TC_RETURN. 54754359Sroberto (Subtarget->is64Bit() || 54854359Sroberto getTargetMachine().getRelocationModel() != Reloc::PIC_)))) { 54954359Sroberto /// Also try moving call address load from outside callseq_start to just 55054359Sroberto /// before the call to allow it to be folded. 55154359Sroberto /// 55254359Sroberto /// [Load chain] 55354359Sroberto /// ^ 55454359Sroberto /// | 55554359Sroberto /// [Load] 55654359Sroberto /// ^ ^ 55754359Sroberto /// | | 55854359Sroberto /// / \-- 55954359Sroberto /// / | 56054359Sroberto ///[CALLSEQ_START] | 56154359Sroberto /// ^ | 56254359Sroberto /// | | 56354359Sroberto /// [LOAD/C2Reg] | 56454359Sroberto /// | | 56554359Sroberto /// \ / 56654359Sroberto /// \ / 56754359Sroberto /// [CALL] 56854359Sroberto bool HasCallSeq = N->getOpcode() == X86ISD::CALL; 56954359Sroberto SDValue Chain = N->getOperand(0); 57054359Sroberto SDValue Load = N->getOperand(1); 57154359Sroberto if (!isCalleeLoad(Load, Chain, HasCallSeq)) 572285612Sdelphij continue; 57354359Sroberto moveBelowOrigChain(CurDAG, Load, SDValue(N, 0), Chain); 574285612Sdelphij ++NumLoadMoved; 57554359Sroberto continue; 576285612Sdelphij } 57754359Sroberto 57854359Sroberto // Lower fpround and fpextend nodes that target the FP stack to be store and 57954359Sroberto // load to the stack. This is a gross hack. We would like to simply mark 58054359Sroberto // these as being illegal, but when we do that, legalize produces these when 58154359Sroberto // it expands calls, then expands these in the same legalize pass. We would 582285612Sdelphij // like dag combine to be able to hack on these between the call expansion 58354359Sroberto // and the node legalization. As such this pass basically does "really 58454359Sroberto // late" legalization of these inline with the X86 isel pass. 58554359Sroberto // FIXME: This should only happen when not compiled with -O0. 58654359Sroberto if (N->getOpcode() != ISD::FP_ROUND && N->getOpcode() != ISD::FP_EXTEND) 58754359Sroberto continue; 58854359Sroberto 58954359Sroberto MVT SrcVT = N->getOperand(0).getSimpleValueType(); 59054359Sroberto MVT DstVT = N->getSimpleValueType(0); 59154359Sroberto 59254359Sroberto // If any of the sources are vectors, no fp stack involved. 59354359Sroberto if (SrcVT.isVector() || DstVT.isVector()) 59454359Sroberto continue; 59554359Sroberto 59654359Sroberto // If the source and destination are SSE registers, then this is a legal 59754359Sroberto // conversion that should not be lowered. 59854359Sroberto const X86TargetLowering *X86Lowering = 59954359Sroberto static_cast<const X86TargetLowering *>(TLI); 60054359Sroberto bool SrcIsSSE = X86Lowering->isScalarFPTypeInSSEReg(SrcVT); 60154359Sroberto bool DstIsSSE = X86Lowering->isScalarFPTypeInSSEReg(DstVT); 60254359Sroberto if (SrcIsSSE && DstIsSSE) 60354359Sroberto continue; 60454359Sroberto 605285612Sdelphij if (!SrcIsSSE && !DstIsSSE) { 60654359Sroberto // If this is an FPStack extension, it is a noop. 60754359Sroberto if (N->getOpcode() == ISD::FP_EXTEND) 60854359Sroberto continue; 60954359Sroberto // If this is a value-preserving FPStack truncation, it is a noop. 61054359Sroberto if (N->getConstantOperandVal(1)) 61154359Sroberto continue; 61254359Sroberto } 61354359Sroberto 61454359Sroberto // Here we could have an FP stack truncation or an FPStack <-> SSE convert. 61554359Sroberto // FPStack has extload and truncstore. SSE can fold direct loads into other 61654359Sroberto // operations. Based on this, decide what we want to do. 61754359Sroberto MVT MemVT; 61854359Sroberto if (N->getOpcode() == ISD::FP_ROUND) 61954359Sroberto MemVT = DstVT; // FP_ROUND must use DstVT, we can't do a 'trunc load'. 62054359Sroberto else 62154359Sroberto MemVT = SrcIsSSE ? SrcVT : DstVT; 62254359Sroberto 62354359Sroberto SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT); 62454359Sroberto SDLoc dl(N); 62554359Sroberto 62654359Sroberto // FIXME: optimize the case where the src/dest is a load or store? 62754359Sroberto SDValue Store = CurDAG->getTruncStore(CurDAG->getEntryNode(), dl, 62854359Sroberto N->getOperand(0), 62954359Sroberto MemTmp, MachinePointerInfo(), MemVT, 63054359Sroberto false, false, 0); 63154359Sroberto SDValue Result = CurDAG->getExtLoad(ISD::EXTLOAD, dl, DstVT, Store, MemTmp, 63254359Sroberto MachinePointerInfo(), 63354359Sroberto MemVT, false, false, false, 0); 63454359Sroberto 63554359Sroberto // We're about to replace all uses of the FP_ROUND/FP_EXTEND with the 63654359Sroberto // extload we created. This will cause general havok on the dag because 63754359Sroberto // anything below the conversion could be folded into other existing nodes. 638285612Sdelphij // To avoid invalidating 'I', back it up to the convert node. 63954359Sroberto --I; 640285612Sdelphij CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 64154359Sroberto 64254359Sroberto // Now that we did that, the node is dead. Increment the iterator to the 64354359Sroberto // next node to process, then delete N. 64454359Sroberto ++I; 64554359Sroberto CurDAG->DeleteNode(N); 646285612Sdelphij } 64754359Sroberto} 64854359Sroberto 64954359Sroberto 65054359Sroberto/// Emit any code that needs to be executed only in the main function. 65154359Srobertovoid X86DAGToDAGISel::emitSpecialCodeForMain() { 65254359Sroberto if (Subtarget->isTargetCygMing()) { 65354359Sroberto TargetLowering::ArgListTy Args; 65454359Sroberto auto &DL = CurDAG->getDataLayout(); 65554359Sroberto 65654359Sroberto TargetLowering::CallLoweringInfo CLI(*CurDAG); 65754359Sroberto CLI.setChain(CurDAG->getRoot()) 65854359Sroberto .setCallee(CallingConv::C, Type::getVoidTy(*CurDAG->getContext()), 65954359Sroberto CurDAG->getExternalSymbol("__main", TLI->getPointerTy(DL)), 66054359Sroberto std::move(Args), 0); 66154359Sroberto const TargetLowering &TLI = CurDAG->getTargetLoweringInfo(); 66254359Sroberto std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI); 66354359Sroberto CurDAG->setRoot(Result.second); 66454359Sroberto } 66554359Sroberto} 66654359Sroberto 66754359Srobertovoid X86DAGToDAGISel::EmitFunctionEntryCode() { 66854359Sroberto // If this is main, emit special code for main. 66954359Sroberto if (const Function *Fn = MF->getFunction()) 67054359Sroberto if (Fn->hasExternalLinkage() && Fn->getName() == "main") 67154359Sroberto emitSpecialCodeForMain(); 67254359Sroberto} 67354359Sroberto 67454359Srobertostatic bool isDispSafeForFrameIndex(int64_t Val) { 67554359Sroberto // On 64-bit platforms, we can run into an issue where a frame index 67654359Sroberto // includes a displacement that, when added to the explicit displacement, 67754359Sroberto // will overflow the displacement field. Assuming that the frame index 67854359Sroberto // displacement fits into a 31-bit integer (which is only slightly more 679285612Sdelphij // aggressive than the current fundamental assumption that it fits into 68054359Sroberto // a 32-bit integer), a 31-bit disp should always be safe. 68154359Sroberto return isInt<31>(Val); 68254359Sroberto} 68354359Sroberto 68454359Srobertobool X86DAGToDAGISel::foldOffsetIntoAddress(uint64_t Offset, 68554359Sroberto X86ISelAddressMode &AM) { 68654359Sroberto // Cannot combine ExternalSymbol displacements with integer offsets. 68754359Sroberto if (Offset != 0 && (AM.ES || AM.MCSym)) 68854359Sroberto return true; 68954359Sroberto int64_t Val = AM.Disp + Offset; 69054359Sroberto CodeModel::Model M = TM.getCodeModel(); 69154359Sroberto if (Subtarget->is64Bit()) { 69254359Sroberto if (!X86::isOffsetSuitableForCodeModel(Val, M, 69354359Sroberto AM.hasSymbolicDisplacement())) 69454359Sroberto return true; 69554359Sroberto // In addition to the checks required for a register base, check that 69654359Sroberto // we do not try to use an unsafe Disp with a frame index. 697285612Sdelphij if (AM.BaseType == X86ISelAddressMode::FrameIndexBase && 69854359Sroberto !isDispSafeForFrameIndex(Val)) 69954359Sroberto return true; 70054359Sroberto } 70154359Sroberto AM.Disp = Val; 70254359Sroberto return false; 70354359Sroberto 70454359Sroberto} 70554359Sroberto 70654359Srobertobool X86DAGToDAGISel::matchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM){ 70754359Sroberto SDValue Address = N->getOperand(1); 70854359Sroberto 70954359Sroberto // load gs:0 -> GS segment register. 71054359Sroberto // load fs:0 -> FS segment register. 71154359Sroberto // 71254359Sroberto // This optimization is valid because the GNU TLS model defines that 713285612Sdelphij // gs:0 (or fs:0 on X86-64) contains its own address. 71454359Sroberto // For more information see http://people.redhat.com/drepper/tls.pdf 71554359Sroberto if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Address)) 71654359Sroberto if (C->getSExtValue() == 0 && AM.Segment.getNode() == nullptr && 71754359Sroberto Subtarget->isTargetLinux()) 71854359Sroberto switch (N->getPointerInfo().getAddrSpace()) { 71954359Sroberto case 256: 72054359Sroberto AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); 72154359Sroberto return false; 72254359Sroberto case 257: 72354359Sroberto AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); 72454359Sroberto return false; 72554359Sroberto } 72654359Sroberto 72754359Sroberto return true; 72854359Sroberto} 72954359Sroberto 73054359Sroberto/// Try to match X86ISD::Wrapper and X86ISD::WrapperRIP nodes into an addressing 73154359Sroberto/// mode. These wrap things that will resolve down into a symbol reference. 73254359Sroberto/// If no match is possible, this returns true, otherwise it returns false. 73354359Srobertobool X86DAGToDAGISel::matchWrapper(SDValue N, X86ISelAddressMode &AM) { 73454359Sroberto // If the addressing mode already has a symbol as the displacement, we can 73554359Sroberto // never match another symbol. 73654359Sroberto if (AM.hasSymbolicDisplacement()) 73754359Sroberto return true; 73854359Sroberto 73954359Sroberto SDValue N0 = N.getOperand(0); 74054359Sroberto CodeModel::Model M = TM.getCodeModel(); 74154359Sroberto 74254359Sroberto // Handle X86-64 rip-relative addresses. We check this before checking direct 74354359Sroberto // folding because RIP is preferable to non-RIP accesses. 74454359Sroberto if (Subtarget->is64Bit() && N.getOpcode() == X86ISD::WrapperRIP && 74554359Sroberto // Under X86-64 non-small code model, GV (and friends) are 64-bits, so 74654359Sroberto // they cannot be folded into immediate fields. 74754359Sroberto // FIXME: This can be improved for kernel and other models? 74854359Sroberto (M == CodeModel::Small || M == CodeModel::Kernel)) { 74954359Sroberto // Base and index reg must be 0 in order to use %rip as base. 75054359Sroberto if (AM.hasBaseOrIndexReg()) 75154359Sroberto return true; 75254359Sroberto if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N0)) { 75354359Sroberto X86ISelAddressMode Backup = AM; 75454359Sroberto AM.GV = G->getGlobal(); 75554359Sroberto AM.SymbolFlags = G->getTargetFlags(); 75654359Sroberto if (foldOffsetIntoAddress(G->getOffset(), AM)) { 75754359Sroberto AM = Backup; 75854359Sroberto return true; 75954359Sroberto } 760285612Sdelphij } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) { 76154359Sroberto X86ISelAddressMode Backup = AM; 76254359Sroberto AM.CP = CP->getConstVal(); 76354359Sroberto AM.Align = CP->getAlignment(); 76454359Sroberto AM.SymbolFlags = CP->getTargetFlags(); 76554359Sroberto if (foldOffsetIntoAddress(CP->getOffset(), AM)) { 76654359Sroberto AM = Backup; 76754359Sroberto return true; 76854359Sroberto } 76954359Sroberto } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) { 77054359Sroberto AM.ES = S->getSymbol(); 77154359Sroberto AM.SymbolFlags = S->getTargetFlags(); 77254359Sroberto } else if (auto *S = dyn_cast<MCSymbolSDNode>(N0)) { 77354359Sroberto AM.MCSym = S->getMCSymbol(); 77454359Sroberto } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) { 77554359Sroberto AM.JT = J->getIndex(); 77654359Sroberto AM.SymbolFlags = J->getTargetFlags(); 77754359Sroberto } else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(N0)) { 778285612Sdelphij X86ISelAddressMode Backup = AM; 77954359Sroberto AM.BlockAddr = BA->getBlockAddress(); 78054359Sroberto AM.SymbolFlags = BA->getTargetFlags(); 78154359Sroberto if (foldOffsetIntoAddress(BA->getOffset(), AM)) { 78254359Sroberto AM = Backup; 78354359Sroberto return true; 78454359Sroberto } 78554359Sroberto } else 78654359Sroberto llvm_unreachable("Unhandled symbol reference node."); 78754359Sroberto 78854359Sroberto if (N.getOpcode() == X86ISD::WrapperRIP) 78954359Sroberto AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64)); 79054359Sroberto return false; 79154359Sroberto } 79254359Sroberto 79354359Sroberto // Handle the case when globals fit in our immediate field: This is true for 794285612Sdelphij // X86-32 always and X86-64 when in -mcmodel=small mode. In 64-bit 79554359Sroberto // mode, this only applies to a non-RIP-relative computation. 79654359Sroberto if (!Subtarget->is64Bit() || 79754359Sroberto M == CodeModel::Small || M == CodeModel::Kernel) { 79854359Sroberto assert(N.getOpcode() != X86ISD::WrapperRIP && 79954359Sroberto "RIP-relative addressing already handled"); 80054359Sroberto if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N0)) { 80154359Sroberto AM.GV = G->getGlobal(); 80254359Sroberto AM.Disp += G->getOffset(); 80354359Sroberto AM.SymbolFlags = G->getTargetFlags(); 80454359Sroberto } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) { 80554359Sroberto AM.CP = CP->getConstVal(); 80654359Sroberto AM.Align = CP->getAlignment(); 80754359Sroberto AM.Disp += CP->getOffset(); 80854359Sroberto AM.SymbolFlags = CP->getTargetFlags(); 80954359Sroberto } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) { 81054359Sroberto AM.ES = S->getSymbol(); 81154359Sroberto AM.SymbolFlags = S->getTargetFlags(); 81254359Sroberto } else if (auto *S = dyn_cast<MCSymbolSDNode>(N0)) { 81354359Sroberto AM.MCSym = S->getMCSymbol(); 81454359Sroberto } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) { 81554359Sroberto AM.JT = J->getIndex(); 81654359Sroberto AM.SymbolFlags = J->getTargetFlags(); 81754359Sroberto } else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(N0)) { 81854359Sroberto AM.BlockAddr = BA->getBlockAddress(); 81954359Sroberto AM.Disp += BA->getOffset(); 82054359Sroberto AM.SymbolFlags = BA->getTargetFlags(); 82154359Sroberto } else 82254359Sroberto llvm_unreachable("Unhandled symbol reference node."); 82354359Sroberto return false; 82454359Sroberto } 82554359Sroberto 82654359Sroberto return true; 82754359Sroberto} 82854359Sroberto 82954359Sroberto/// Add the specified node to the specified addressing mode, returning true if 83054359Sroberto/// it cannot be done. This just pattern matches for the addressing mode. 83154359Srobertobool X86DAGToDAGISel::matchAddress(SDValue N, X86ISelAddressMode &AM) { 83254359Sroberto if (matchAddressRecursively(N, AM, 0)) 83354359Sroberto return true; 83454359Sroberto 83554359Sroberto // Post-processing: Convert lea(,%reg,2) to lea(%reg,%reg), which has 83654359Sroberto // a smaller encoding and avoids a scaled-index. 83754359Sroberto if (AM.Scale == 2 && 83854359Sroberto AM.BaseType == X86ISelAddressMode::RegBase && 83954359Sroberto AM.Base_Reg.getNode() == nullptr) { 84054359Sroberto AM.Base_Reg = AM.IndexReg; 84154359Sroberto AM.Scale = 1; 84254359Sroberto } 84354359Sroberto 84454359Sroberto // Post-processing: Convert foo to foo(%rip), even in non-PIC mode, 84554359Sroberto // because it has a smaller encoding. 84654359Sroberto // TODO: Which other code models can use this? 847285612Sdelphij if (TM.getCodeModel() == CodeModel::Small && 84854359Sroberto Subtarget->is64Bit() && 84954359Sroberto AM.Scale == 1 && 85054359Sroberto AM.BaseType == X86ISelAddressMode::RegBase && 85154359Sroberto AM.Base_Reg.getNode() == nullptr && 85254359Sroberto AM.IndexReg.getNode() == nullptr && 85354359Sroberto AM.SymbolFlags == X86II::MO_NO_FLAG && 85454359Sroberto AM.hasSymbolicDisplacement()) 85554359Sroberto AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64); 85654359Sroberto 85754359Sroberto return false; 85854359Sroberto} 85954359Sroberto 86054359Srobertobool X86DAGToDAGISel::matchAdd(SDValue N, X86ISelAddressMode &AM, 86154359Sroberto unsigned Depth) { 86254359Sroberto // Add an artificial use to this node so that we can keep track of 86354359Sroberto // it if it gets CSE'd with a different node. 86454359Sroberto HandleSDNode Handle(N); 86554359Sroberto 86654359Sroberto X86ISelAddressMode Backup = AM; 86754359Sroberto if (!matchAddressRecursively(N.getOperand(0), AM, Depth+1) && 86854359Sroberto !matchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1)) 86954359Sroberto return false; 87054359Sroberto AM = Backup; 87154359Sroberto 87254359Sroberto // Try again after commuting the operands. 87354359Sroberto if (!matchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1) && 87454359Sroberto !matchAddressRecursively(Handle.getValue().getOperand(0), AM, Depth+1)) 87554359Sroberto return false; 87654359Sroberto AM = Backup; 87754359Sroberto 87854359Sroberto // If we couldn't fold both operands into the address at the same time, 87954359Sroberto // see if we can just put each operand into a register and fold at least 88054359Sroberto // the add. 88154359Sroberto if (AM.BaseType == X86ISelAddressMode::RegBase && 88254359Sroberto !AM.Base_Reg.getNode() && 88354359Sroberto !AM.IndexReg.getNode()) { 88454359Sroberto N = Handle.getValue(); 88554359Sroberto AM.Base_Reg = N.getOperand(0); 886285612Sdelphij AM.IndexReg = N.getOperand(1); 88754359Sroberto AM.Scale = 1; 88854359Sroberto return false; 88954359Sroberto } 89054359Sroberto N = Handle.getValue(); 89154359Sroberto return true; 89254359Sroberto} 89354359Sroberto 89454359Sroberto// Insert a node into the DAG at least before the Pos node's position. This 89554359Sroberto// will reposition the node as needed, and will assign it a node ID that is <= 89654359Sroberto// the Pos node's ID. Note that this does *not* preserve the uniqueness of node 89754359Sroberto// IDs! The selection DAG must no longer depend on their uniqueness when this 89854359Sroberto// is used. 89954359Srobertostatic void insertDAGNode(SelectionDAG &DAG, SDValue Pos, SDValue N) { 90054359Sroberto if (N.getNode()->getNodeId() == -1 || 90154359Sroberto N.getNode()->getNodeId() > Pos.getNode()->getNodeId()) { 90254359Sroberto DAG.RepositionNode(Pos.getNode()->getIterator(), N.getNode()); 90354359Sroberto N.getNode()->setNodeId(Pos.getNode()->getNodeId()); 90454359Sroberto } 90554359Sroberto} 90654359Sroberto 90754359Sroberto// Transform "(X >> (8-C1)) & (0xff << C1)" to "((X >> 8) & 0xff) << C1" if 90854359Sroberto// safe. This allows us to convert the shift and and into an h-register 90954359Sroberto// extract and a scaled index. Returns false if the simplification is 91054359Sroberto// performed. 911285612Sdelphijstatic bool foldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N, 91254359Sroberto uint64_t Mask, 91354359Sroberto SDValue Shift, SDValue X, 91454359Sroberto X86ISelAddressMode &AM) { 91554359Sroberto if (Shift.getOpcode() != ISD::SRL || 91654359Sroberto !isa<ConstantSDNode>(Shift.getOperand(1)) || 91754359Sroberto !Shift.hasOneUse()) 91854359Sroberto return true; 91954359Sroberto 92054359Sroberto int ScaleLog = 8 - Shift.getConstantOperandVal(1); 92154359Sroberto if (ScaleLog <= 0 || ScaleLog >= 4 || 922285612Sdelphij Mask != (0xffu << ScaleLog)) 923285612Sdelphij return true; 92454359Sroberto 92554359Sroberto MVT VT = N.getSimpleValueType(); 92654359Sroberto SDLoc DL(N); 92754359Sroberto SDValue Eight = DAG.getConstant(8, DL, MVT::i8); 92854359Sroberto SDValue NewMask = DAG.getConstant(0xff, DL, VT); 92954359Sroberto SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, X, Eight); 93054359Sroberto SDValue And = DAG.getNode(ISD::AND, DL, VT, Srl, NewMask); 93154359Sroberto SDValue ShlCount = DAG.getConstant(ScaleLog, DL, MVT::i8); 93254359Sroberto SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And, ShlCount); 933285612Sdelphij 93454359Sroberto // Insert the new nodes into the topological ordering. We must do this in 93554359Sroberto // a valid topological ordering as nothing is going to go back and re-sort 93654359Sroberto // these nodes. We continually insert before 'N' in sequence as this is 93754359Sroberto // essentially a pre-flattened and pre-sorted sequence of nodes. There is no 93854359Sroberto // hierarchy left to express. 93954359Sroberto insertDAGNode(DAG, N, Eight); 94054359Sroberto insertDAGNode(DAG, N, Srl); 94154359Sroberto insertDAGNode(DAG, N, NewMask); 94254359Sroberto insertDAGNode(DAG, N, And); 94354359Sroberto insertDAGNode(DAG, N, ShlCount); 94454359Sroberto insertDAGNode(DAG, N, Shl); 94554359Sroberto DAG.ReplaceAllUsesWith(N, Shl); 94654359Sroberto AM.IndexReg = And; 94754359Sroberto AM.Scale = (1 << ScaleLog); 94854359Sroberto return false; 94954359Sroberto} 95054359Sroberto 95154359Sroberto// Transforms "(X << C1) & C2" to "(X & (C2>>C1)) << C1" if safe and if this 95254359Sroberto// allows us to fold the shift into this addressing mode. Returns false if the 95354359Sroberto// transform succeeded. 95454359Srobertostatic bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N, 95554359Sroberto uint64_t Mask, 95654359Sroberto SDValue Shift, SDValue X, 95754359Sroberto X86ISelAddressMode &AM) { 95854359Sroberto if (Shift.getOpcode() != ISD::SHL || 95954359Sroberto !isa<ConstantSDNode>(Shift.getOperand(1))) 96054359Sroberto return true; 96154359Sroberto 96254359Sroberto // Not likely to be profitable if either the AND or SHIFT node has more 96354359Sroberto // than one use (unless all uses are for address computation). Besides, 96454359Sroberto // isel mechanism requires their node ids to be reused. 96554359Sroberto if (!N.hasOneUse() || !Shift.hasOneUse()) 96654359Sroberto return true; 96754359Sroberto 96854359Sroberto // Verify that the shift amount is something we can fold. 96954359Sroberto unsigned ShiftAmt = Shift.getConstantOperandVal(1); 97054359Sroberto if (ShiftAmt != 1 && ShiftAmt != 2 && ShiftAmt != 3) 97154359Sroberto return true; 97254359Sroberto 97354359Sroberto MVT VT = N.getSimpleValueType(); 97454359Sroberto SDLoc DL(N); 97554359Sroberto SDValue NewMask = DAG.getConstant(Mask >> ShiftAmt, DL, VT); 97654359Sroberto SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, NewMask); 97754359Sroberto SDValue NewShift = DAG.getNode(ISD::SHL, DL, VT, NewAnd, Shift.getOperand(1)); 97854359Sroberto 97954359Sroberto // Insert the new nodes into the topological ordering. We must do this in 98054359Sroberto // a valid topological ordering as nothing is going to go back and re-sort 98154359Sroberto // these nodes. We continually insert before 'N' in sequence as this is 98254359Sroberto // essentially a pre-flattened and pre-sorted sequence of nodes. There is no 98354359Sroberto // hierarchy left to express. 98454359Sroberto insertDAGNode(DAG, N, NewMask); 98554359Sroberto insertDAGNode(DAG, N, NewAnd); 98654359Sroberto insertDAGNode(DAG, N, NewShift); 98754359Sroberto DAG.ReplaceAllUsesWith(N, NewShift); 98854359Sroberto 98954359Sroberto AM.Scale = 1 << ShiftAmt; 99054359Sroberto AM.IndexReg = NewAnd; 99154359Sroberto return false; 99254359Sroberto} 99354359Sroberto 99454359Sroberto// Implement some heroics to detect shifts of masked values where the mask can 99554359Sroberto// be replaced by extending the shift and undoing that in the addressing mode 99654359Sroberto// scale. Patterns such as (shl (srl x, c1), c2) are canonicalized into (and 99754359Sroberto// (srl x, SHIFT), MASK) by DAGCombines that don't know the shl can be done in 99854359Sroberto// the addressing mode. This results in code such as: 99954359Sroberto// 100054359Sroberto// int f(short *y, int *lookup_table) { 100154359Sroberto// ... 100254359Sroberto// return *y + lookup_table[*y >> 11]; 100354359Sroberto// } 100454359Sroberto// 100554359Sroberto// Turning into: 100654359Sroberto// movzwl (%rdi), %eax 100754359Sroberto// movl %eax, %ecx 100854359Sroberto// shrl $11, %ecx 100954359Sroberto// addl (%rsi,%rcx,4), %eax 101054359Sroberto// 101154359Sroberto// Instead of: 101254359Sroberto// movzwl (%rdi), %eax 1013285612Sdelphij// movl %eax, %ecx 1014285612Sdelphij// shrl $9, %ecx 101554359Sroberto// andl $124, %rcx 101654359Sroberto// addl (%rsi,%rcx), %eax 101754359Sroberto// 101854359Sroberto// Note that this function assumes the mask is provided as a mask *after* the 101954359Sroberto// value is shifted. The input chain may or may not match that, but computing 102054359Sroberto// such a mask is trivial. 102154359Srobertostatic bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N, 102254359Sroberto uint64_t Mask, 102354359Sroberto SDValue Shift, SDValue X, 102454359Sroberto X86ISelAddressMode &AM) { 102554359Sroberto if (Shift.getOpcode() != ISD::SRL || !Shift.hasOneUse() || 102654359Sroberto !isa<ConstantSDNode>(Shift.getOperand(1))) 102754359Sroberto return true; 102854359Sroberto 102954359Sroberto unsigned ShiftAmt = Shift.getConstantOperandVal(1); 103054359Sroberto unsigned MaskLZ = countLeadingZeros(Mask); 103154359Sroberto unsigned MaskTZ = countTrailingZeros(Mask); 103254359Sroberto 103354359Sroberto // The amount of shift we're trying to fit into the addressing mode is taken 103454359Sroberto // from the trailing zeros of the mask. 103554359Sroberto unsigned AMShiftAmt = MaskTZ; 103654359Sroberto 103754359Sroberto // There is nothing we can do here unless the mask is removing some bits. 103854359Sroberto // Also, the addressing mode can only represent shifts of 1, 2, or 3 bits. 103954359Sroberto if (AMShiftAmt <= 0 || AMShiftAmt > 3) return true; 104054359Sroberto 1041285612Sdelphij // We also need to ensure that mask is a continuous run of bits. 104254359Sroberto if (countTrailingOnes(Mask >> MaskTZ) + MaskTZ + MaskLZ != 64) return true; 104354359Sroberto 104454359Sroberto // Scale the leading zero count down based on the actual size of the value. 104554359Sroberto // Also scale it down based on the size of the shift. 104654359Sroberto MaskLZ -= (64 - X.getSimpleValueType().getSizeInBits()) + ShiftAmt; 104754359Sroberto 104854359Sroberto // The final check is to ensure that any masked out high bits of X are 104954359Sroberto // already known to be zero. Otherwise, the mask has a semantic impact 105054359Sroberto // other than masking out a couple of low bits. Unfortunately, because of 105154359Sroberto // the mask, zero extensions will be removed from operands in some cases. 105254359Sroberto // This code works extra hard to look through extensions because we can 105354359Sroberto // replace them with zero extensions cheaply if necessary. 105454359Sroberto bool ReplacingAnyExtend = false; 105554359Sroberto if (X.getOpcode() == ISD::ANY_EXTEND) { 105654359Sroberto unsigned ExtendBits = X.getSimpleValueType().getSizeInBits() - 105754359Sroberto X.getOperand(0).getSimpleValueType().getSizeInBits(); 105854359Sroberto // Assume that we'll replace the any-extend with a zero-extend, and 105954359Sroberto // narrow the search to the extended value. 106054359Sroberto X = X.getOperand(0); 106154359Sroberto MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits; 106254359Sroberto ReplacingAnyExtend = true; 106354359Sroberto } 106454359Sroberto APInt MaskedHighBits = 106554359Sroberto APInt::getHighBitsSet(X.getSimpleValueType().getSizeInBits(), MaskLZ); 106654359Sroberto APInt KnownZero, KnownOne; 1067285612Sdelphij DAG.computeKnownBits(X, KnownZero, KnownOne); 106854359Sroberto if (MaskedHighBits != KnownZero) return true; 1069285612Sdelphij 107054359Sroberto // We've identified a pattern that can be transformed into a single shift 107154359Sroberto // and an addressing mode. Make it so. 107254359Sroberto MVT VT = N.getSimpleValueType(); 107354359Sroberto if (ReplacingAnyExtend) { 107454359Sroberto assert(X.getValueType() != VT); 107554359Sroberto // We looked through an ANY_EXTEND node, insert a ZERO_EXTEND. 107654359Sroberto SDValue NewX = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(X), VT, X); 107754359Sroberto insertDAGNode(DAG, N, NewX); 107854359Sroberto X = NewX; 107954359Sroberto } 108054359Sroberto SDLoc DL(N); 108154359Sroberto SDValue NewSRLAmt = DAG.getConstant(ShiftAmt + AMShiftAmt, DL, MVT::i8); 108254359Sroberto SDValue NewSRL = DAG.getNode(ISD::SRL, DL, VT, X, NewSRLAmt); 108354359Sroberto SDValue NewSHLAmt = DAG.getConstant(AMShiftAmt, DL, MVT::i8); 108454359Sroberto SDValue NewSHL = DAG.getNode(ISD::SHL, DL, VT, NewSRL, NewSHLAmt); 108554359Sroberto 108654359Sroberto // Insert the new nodes into the topological ordering. We must do this in 108754359Sroberto // a valid topological ordering as nothing is going to go back and re-sort 108854359Sroberto // these nodes. We continually insert before 'N' in sequence as this is 108954359Sroberto // essentially a pre-flattened and pre-sorted sequence of nodes. There is no 109054359Sroberto // hierarchy left to express. 109154359Sroberto insertDAGNode(DAG, N, NewSRLAmt); 109254359Sroberto insertDAGNode(DAG, N, NewSRL); 109354359Sroberto insertDAGNode(DAG, N, NewSHLAmt); 109454359Sroberto insertDAGNode(DAG, N, NewSHL); 109554359Sroberto DAG.ReplaceAllUsesWith(N, NewSHL); 109654359Sroberto 109754359Sroberto AM.Scale = 1 << AMShiftAmt; 109854359Sroberto AM.IndexReg = NewSRL; 109954359Sroberto return false; 110054359Sroberto} 110154359Sroberto 1102285612Sdelphijbool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM, 110354359Sroberto unsigned Depth) { 110454359Sroberto SDLoc dl(N); 110554359Sroberto DEBUG({ 1106285612Sdelphij dbgs() << "MatchAddress: "; 110754359Sroberto AM.dump(); 110854359Sroberto }); 110954359Sroberto // Limit recursion. 111054359Sroberto if (Depth > 5) 111154359Sroberto return matchAddressBase(N, AM); 111254359Sroberto 111354359Sroberto // If this is already a %rip relative address, we can only merge immediates 111454359Sroberto // into it. Instead of handling this in every case, we handle it here. 1115285612Sdelphij // RIP relative addressing: %rip + 32-bit displacement! 111654359Sroberto if (AM.isRIPRelative()) { 111754359Sroberto // FIXME: JumpTable and ExternalSymbol address currently don't like 111854359Sroberto // displacements. It isn't very important, but this should be fixed for 111954359Sroberto // consistency. 112054359Sroberto if (!(AM.ES || AM.MCSym) && AM.JT != -1) 112154359Sroberto return true; 112254359Sroberto 112354359Sroberto if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N)) 112454359Sroberto if (!foldOffsetIntoAddress(Cst->getSExtValue(), AM)) 112554359Sroberto return false; 112654359Sroberto return true; 112754359Sroberto } 112854359Sroberto 112954359Sroberto switch (N.getOpcode()) { 113054359Sroberto default: break; 113154359Sroberto case ISD::LOCAL_RECOVER: { 113254359Sroberto if (!AM.hasSymbolicDisplacement() && AM.Disp == 0) 113354359Sroberto if (const auto *ESNode = dyn_cast<MCSymbolSDNode>(N.getOperand(0))) { 113454359Sroberto // Use the symbol and don't prefix it. 113554359Sroberto AM.MCSym = ESNode->getMCSymbol(); 113654359Sroberto return false; 113754359Sroberto } 113854359Sroberto break; 113954359Sroberto } 114054359Sroberto case ISD::Constant: { 114154359Sroberto uint64_t Val = cast<ConstantSDNode>(N)->getSExtValue(); 114254359Sroberto if (!foldOffsetIntoAddress(Val, AM)) 114354359Sroberto return false; 114454359Sroberto break; 114554359Sroberto } 114654359Sroberto 114754359Sroberto case X86ISD::Wrapper: 114854359Sroberto case X86ISD::WrapperRIP: 114954359Sroberto if (!matchWrapper(N, AM)) 115054359Sroberto return false; 1151285612Sdelphij break; 115254359Sroberto 115354359Sroberto case ISD::LOAD: 115454359Sroberto if (!matchLoadInAddress(cast<LoadSDNode>(N), AM)) 115554359Sroberto return false; 115654359Sroberto break; 115754359Sroberto 115854359Sroberto case ISD::FrameIndex: 115954359Sroberto if (AM.BaseType == X86ISelAddressMode::RegBase && 116054359Sroberto AM.Base_Reg.getNode() == nullptr && 116154359Sroberto (!Subtarget->is64Bit() || isDispSafeForFrameIndex(AM.Disp))) { 116254359Sroberto AM.BaseType = X86ISelAddressMode::FrameIndexBase; 116354359Sroberto AM.Base_FrameIndex = cast<FrameIndexSDNode>(N)->getIndex(); 116454359Sroberto return false; 116554359Sroberto } 1166285612Sdelphij break; 116754359Sroberto 116854359Sroberto case ISD::SHL: 116954359Sroberto if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) 117054359Sroberto break; 117154359Sroberto 117254359Sroberto if (ConstantSDNode 117354359Sroberto *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1))) { 117454359Sroberto unsigned Val = CN->getZExtValue(); 117554359Sroberto // Note that we handle x<<1 as (,x,2) rather than (x,x) here so 117654359Sroberto // that the base operand remains free for further matching. If 117754359Sroberto // the base doesn't end up getting used, a post-processing step 117854359Sroberto // in MatchAddress turns (,x,2) into (x,x), which is cheaper. 117954359Sroberto if (Val == 1 || Val == 2 || Val == 3) { 118054359Sroberto AM.Scale = 1 << Val; 118154359Sroberto SDValue ShVal = N.getNode()->getOperand(0); 118254359Sroberto 118354359Sroberto // Okay, we know that we have a scale by now. However, if the scaled 118454359Sroberto // value is an add of something and a constant, we can fold the 118554359Sroberto // constant into the disp field here. 118654359Sroberto if (CurDAG->isBaseWithConstantOffset(ShVal)) { 118754359Sroberto AM.IndexReg = ShVal.getNode()->getOperand(0); 118854359Sroberto ConstantSDNode *AddVal = 118954359Sroberto cast<ConstantSDNode>(ShVal.getNode()->getOperand(1)); 119054359Sroberto uint64_t Disp = (uint64_t)AddVal->getSExtValue() << Val; 119154359Sroberto if (!foldOffsetIntoAddress(Disp, AM)) 119254359Sroberto return false; 1193285612Sdelphij } 119454359Sroberto 119554359Sroberto AM.IndexReg = ShVal; 119654359Sroberto return false; 119754359Sroberto } 119854359Sroberto } 119954359Sroberto break; 120054359Sroberto 120154359Sroberto case ISD::SRL: { 120254359Sroberto // Scale must not be used already. 120354359Sroberto if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) break; 120454359Sroberto 120554359Sroberto SDValue And = N.getOperand(0); 120654359Sroberto if (And.getOpcode() != ISD::AND) break; 120754359Sroberto SDValue X = And.getOperand(0); 120854359Sroberto 120954359Sroberto // We only handle up to 64-bit values here as those are what matter for 121054359Sroberto // addressing mode optimizations. 121154359Sroberto if (X.getSimpleValueType().getSizeInBits() > 64) break; 121254359Sroberto 121354359Sroberto // The mask used for the transform is expected to be post-shift, but we 121454359Sroberto // found the shift first so just apply the shift to the mask before passing 121554359Sroberto // it down. 121654359Sroberto if (!isa<ConstantSDNode>(N.getOperand(1)) || 121754359Sroberto !isa<ConstantSDNode>(And.getOperand(1))) 121854359Sroberto break; 121954359Sroberto uint64_t Mask = And.getConstantOperandVal(1) >> N.getConstantOperandVal(1); 122054359Sroberto 1221285612Sdelphij // Try to fold the mask and shift into the scale, and return false if we 122254359Sroberto // succeed. 122354359Sroberto if (!foldMaskAndShiftToScale(*CurDAG, N, Mask, N, X, AM)) 122454359Sroberto return false; 122554359Sroberto break; 122654359Sroberto } 122754359Sroberto 122854359Sroberto case ISD::SMUL_LOHI: 122954359Sroberto case ISD::UMUL_LOHI: 123054359Sroberto // A mul_lohi where we need the low part can be folded as a plain multiply. 123154359Sroberto if (N.getResNo() != 0) break; 123254359Sroberto // FALL THROUGH 123354359Sroberto case ISD::MUL: 123454359Sroberto case X86ISD::MUL_IMM: 123554359Sroberto // X*[3,5,9] -> X+X*[2,4,8] 123654359Sroberto if (AM.BaseType == X86ISelAddressMode::RegBase && 123754359Sroberto AM.Base_Reg.getNode() == nullptr && 123854359Sroberto AM.IndexReg.getNode() == nullptr) { 123954359Sroberto if (ConstantSDNode 124054359Sroberto *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1))) 124154359Sroberto if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 || 124254359Sroberto CN->getZExtValue() == 9) { 124354359Sroberto AM.Scale = unsigned(CN->getZExtValue())-1; 124454359Sroberto 124554359Sroberto SDValue MulVal = N.getNode()->getOperand(0); 124654359Sroberto SDValue Reg; 124754359Sroberto 124854359Sroberto // Okay, we know that we have a scale by now. However, if the scaled 124954359Sroberto // value is an add of something and a constant, we can fold the 125054359Sroberto // constant into the disp field here. 125154359Sroberto if (MulVal.getNode()->getOpcode() == ISD::ADD && MulVal.hasOneUse() && 125254359Sroberto isa<ConstantSDNode>(MulVal.getNode()->getOperand(1))) { 125354359Sroberto Reg = MulVal.getNode()->getOperand(0); 1254285612Sdelphij ConstantSDNode *AddVal = 125554359Sroberto cast<ConstantSDNode>(MulVal.getNode()->getOperand(1)); 125654359Sroberto uint64_t Disp = AddVal->getSExtValue() * CN->getZExtValue(); 125754359Sroberto if (foldOffsetIntoAddress(Disp, AM)) 125854359Sroberto Reg = N.getNode()->getOperand(0); 125954359Sroberto } else { 1260285612Sdelphij Reg = N.getNode()->getOperand(0); 126154359Sroberto } 126254359Sroberto 126354359Sroberto AM.IndexReg = AM.Base_Reg = Reg; 126454359Sroberto return false; 126554359Sroberto } 126654359Sroberto } 126754359Sroberto break; 126854359Sroberto 126954359Sroberto case ISD::SUB: { 127054359Sroberto // Given A-B, if A can be completely folded into the address and 127154359Sroberto // the index field with the index field unused, use -B as the index. 127254359Sroberto // This is a win if a has multiple parts that can be folded into 127354359Sroberto // the address. Also, this saves a mov if the base register has 1274285612Sdelphij // other uses, since it avoids a two-address sub instruction, however 127554359Sroberto // it costs an additional mov if the index register has other uses. 127654359Sroberto 127754359Sroberto // Add an artificial use to this node so that we can keep track of 127854359Sroberto // it if it gets CSE'd with a different node. 127954359Sroberto HandleSDNode Handle(N); 128054359Sroberto 128154359Sroberto // Test if the LHS of the sub can be folded. 128254359Sroberto X86ISelAddressMode Backup = AM; 128354359Sroberto if (matchAddressRecursively(N.getNode()->getOperand(0), AM, Depth+1)) { 128454359Sroberto AM = Backup; 1285285612Sdelphij break; 1286285612Sdelphij } 128754359Sroberto // Test if the index field is free for use. 128854359Sroberto if (AM.IndexReg.getNode() || AM.isRIPRelative()) { 128954359Sroberto AM = Backup; 129054359Sroberto break; 129154359Sroberto } 129254359Sroberto 129354359Sroberto int Cost = 0; 129454359Sroberto SDValue RHS = Handle.getValue().getNode()->getOperand(1); 1295285612Sdelphij // If the RHS involves a register with multiple uses, this 129654359Sroberto // transformation incurs an extra mov, due to the neg instruction 129754359Sroberto // clobbering its operand. 129854359Sroberto if (!RHS.getNode()->hasOneUse() || 129954359Sroberto RHS.getNode()->getOpcode() == ISD::CopyFromReg || 130054359Sroberto RHS.getNode()->getOpcode() == ISD::TRUNCATE || 130154359Sroberto RHS.getNode()->getOpcode() == ISD::ANY_EXTEND || 130254359Sroberto (RHS.getNode()->getOpcode() == ISD::ZERO_EXTEND && 130354359Sroberto RHS.getNode()->getOperand(0).getValueType() == MVT::i32)) 130454359Sroberto ++Cost; 130554359Sroberto // If the base is a register with multiple uses, this 130654359Sroberto // transformation may save a mov. 130754359Sroberto if ((AM.BaseType == X86ISelAddressMode::RegBase && 130854359Sroberto AM.Base_Reg.getNode() && 130954359Sroberto !AM.Base_Reg.getNode()->hasOneUse()) || 131054359Sroberto AM.BaseType == X86ISelAddressMode::FrameIndexBase) 131154359Sroberto --Cost; 131254359Sroberto // If the folded LHS was interesting, this transformation saves 131354359Sroberto // address arithmetic. 131454359Sroberto if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) + 131554359Sroberto ((AM.Disp != 0) && (Backup.Disp == 0)) + 131654359Sroberto (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2) 1317285612Sdelphij --Cost; 131854359Sroberto // If it doesn't look like it may be an overall win, don't do it. 131954359Sroberto if (Cost >= 0) { 132054359Sroberto AM = Backup; 132154359Sroberto break; 132254359Sroberto } 132354359Sroberto 132454359Sroberto // Ok, the transformation is legal and appears profitable. Go for it. 132554359Sroberto SDValue Zero = CurDAG->getConstant(0, dl, N.getValueType()); 132654359Sroberto SDValue Neg = CurDAG->getNode(ISD::SUB, dl, N.getValueType(), Zero, RHS); 132754359Sroberto AM.IndexReg = Neg; 132854359Sroberto AM.Scale = 1; 132954359Sroberto 1330182007Sroberto // Insert the new nodes into the topological ordering. 1331182007Sroberto insertDAGNode(*CurDAG, N, Zero); 1332182007Sroberto insertDAGNode(*CurDAG, N, Neg); 1333182007Sroberto return false; 1334182007Sroberto } 1335182007Sroberto 1336182007Sroberto case ISD::ADD: 1337182007Sroberto if (!matchAdd(N, AM, Depth)) 1338182007Sroberto return false; 133956746Sroberto break; 134056746Sroberto 134156746Sroberto case ISD::OR: 134254359Sroberto // We want to look through a transform in InstCombine and DAGCombiner that 134354359Sroberto // turns 'add' into 'or', so we can treat this 'or' exactly like an 'add'. 134454359Sroberto // Example: (or (and x, 1), (shl y, 3)) --> (add (and x, 1), (shl y, 3)) 134554359Sroberto // An 'lea' can then be used to match the shift (multiply) and add: 134654359Sroberto // and $1, %esi 134754359Sroberto // lea (%rsi, %rdi, 8), %rax 134854359Sroberto if (CurDAG->haveNoCommonBitsSet(N.getOperand(0), N.getOperand(1)) && 134954359Sroberto !matchAdd(N, AM, Depth)) 135054359Sroberto return false; 135154359Sroberto break; 135254359Sroberto 135354359Sroberto case ISD::AND: { 135454359Sroberto // Perform some heroic transforms on an and of a constant-count shift 135554359Sroberto // with a constant to enable use of the scaled offset field. 135654359Sroberto 135754359Sroberto // Scale must not be used already. 135854359Sroberto if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) break; 135954359Sroberto 136054359Sroberto SDValue Shift = N.getOperand(0); 136154359Sroberto if (Shift.getOpcode() != ISD::SRL && Shift.getOpcode() != ISD::SHL) break; 136254359Sroberto SDValue X = Shift.getOperand(0); 136354359Sroberto 136454359Sroberto // We only handle up to 64-bit values here as those are what matter for 136554359Sroberto // addressing mode optimizations. 136654359Sroberto if (X.getSimpleValueType().getSizeInBits() > 64) break; 136754359Sroberto 1368 if (!isa<ConstantSDNode>(N.getOperand(1))) 1369 break; 1370 uint64_t Mask = N.getConstantOperandVal(1); 1371 1372 // Try to fold the mask and shift into an extract and scale. 1373 if (!foldMaskAndShiftToExtract(*CurDAG, N, Mask, Shift, X, AM)) 1374 return false; 1375 1376 // Try to fold the mask and shift directly into the scale. 1377 if (!foldMaskAndShiftToScale(*CurDAG, N, Mask, Shift, X, AM)) 1378 return false; 1379 1380 // Try to swap the mask and shift to place shifts which can be done as 1381 // a scale on the outside of the mask. 1382 if (!foldMaskedShiftToScaledMask(*CurDAG, N, Mask, Shift, X, AM)) 1383 return false; 1384 break; 1385 } 1386 } 1387 1388 return matchAddressBase(N, AM); 1389} 1390 1391/// Helper for MatchAddress. Add the specified node to the 1392/// specified addressing mode without any further recursion. 1393bool X86DAGToDAGISel::matchAddressBase(SDValue N, X86ISelAddressMode &AM) { 1394 // Is the base register already occupied? 1395 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) { 1396 // If so, check to see if the scale index register is set. 1397 if (!AM.IndexReg.getNode()) { 1398 AM.IndexReg = N; 1399 AM.Scale = 1; 1400 return false; 1401 } 1402 1403 // Otherwise, we cannot select it. 1404 return true; 1405 } 1406 1407 // Default, generate it as a register. 1408 AM.BaseType = X86ISelAddressMode::RegBase; 1409 AM.Base_Reg = N; 1410 return false; 1411} 1412 1413bool X86DAGToDAGISel::selectVectorAddr(SDNode *Parent, SDValue N, SDValue &Base, 1414 SDValue &Scale, SDValue &Index, 1415 SDValue &Disp, SDValue &Segment) { 1416 1417 MaskedGatherScatterSDNode *Mgs = dyn_cast<MaskedGatherScatterSDNode>(Parent); 1418 if (!Mgs) 1419 return false; 1420 X86ISelAddressMode AM; 1421 unsigned AddrSpace = Mgs->getPointerInfo().getAddrSpace(); 1422 // AddrSpace 256 -> GS, 257 -> FS. 1423 if (AddrSpace == 256) 1424 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); 1425 if (AddrSpace == 257) 1426 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); 1427 1428 SDLoc DL(N); 1429 Base = Mgs->getBasePtr(); 1430 Index = Mgs->getIndex(); 1431 unsigned ScalarSize = Mgs->getValue().getValueType().getScalarSizeInBits(); 1432 Scale = getI8Imm(ScalarSize/8, DL); 1433 1434 // If Base is 0, the whole address is in index and the Scale is 1 1435 if (isa<ConstantSDNode>(Base)) { 1436 assert(cast<ConstantSDNode>(Base)->isNullValue() && 1437 "Unexpected base in gather/scatter"); 1438 Scale = getI8Imm(1, DL); 1439 Base = CurDAG->getRegister(0, MVT::i32); 1440 } 1441 if (AM.Segment.getNode()) 1442 Segment = AM.Segment; 1443 else 1444 Segment = CurDAG->getRegister(0, MVT::i32); 1445 Disp = CurDAG->getTargetConstant(0, DL, MVT::i32); 1446 return true; 1447} 1448 1449/// Returns true if it is able to pattern match an addressing mode. 1450/// It returns the operands which make up the maximal addressing mode it can 1451/// match by reference. 1452/// 1453/// Parent is the parent node of the addr operand that is being matched. It 1454/// is always a load, store, atomic node, or null. It is only null when 1455/// checking memory operands for inline asm nodes. 1456bool X86DAGToDAGISel::selectAddr(SDNode *Parent, SDValue N, SDValue &Base, 1457 SDValue &Scale, SDValue &Index, 1458 SDValue &Disp, SDValue &Segment) { 1459 X86ISelAddressMode AM; 1460 1461 if (Parent && 1462 // This list of opcodes are all the nodes that have an "addr:$ptr" operand 1463 // that are not a MemSDNode, and thus don't have proper addrspace info. 1464 Parent->getOpcode() != ISD::INTRINSIC_W_CHAIN && // unaligned loads, fixme 1465 Parent->getOpcode() != ISD::INTRINSIC_VOID && // nontemporal stores 1466 Parent->getOpcode() != X86ISD::TLSCALL && // Fixme 1467 Parent->getOpcode() != X86ISD::EH_SJLJ_SETJMP && // setjmp 1468 Parent->getOpcode() != X86ISD::EH_SJLJ_LONGJMP) { // longjmp 1469 unsigned AddrSpace = 1470 cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace(); 1471 // AddrSpace 256 -> GS, 257 -> FS. 1472 if (AddrSpace == 256) 1473 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); 1474 if (AddrSpace == 257) 1475 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); 1476 } 1477 1478 if (matchAddress(N, AM)) 1479 return false; 1480 1481 MVT VT = N.getSimpleValueType(); 1482 if (AM.BaseType == X86ISelAddressMode::RegBase) { 1483 if (!AM.Base_Reg.getNode()) 1484 AM.Base_Reg = CurDAG->getRegister(0, VT); 1485 } 1486 1487 if (!AM.IndexReg.getNode()) 1488 AM.IndexReg = CurDAG->getRegister(0, VT); 1489 1490 getAddressOperands(AM, SDLoc(N), Base, Scale, Index, Disp, Segment); 1491 return true; 1492} 1493 1494/// Match a scalar SSE load. In particular, we want to match a load whose top 1495/// elements are either undef or zeros. The load flavor is derived from the 1496/// type of N, which is either v4f32 or v2f64. 1497/// 1498/// We also return: 1499/// PatternChainNode: this is the matched node that has a chain input and 1500/// output. 1501bool X86DAGToDAGISel::selectScalarSSELoad(SDNode *Root, 1502 SDValue N, SDValue &Base, 1503 SDValue &Scale, SDValue &Index, 1504 SDValue &Disp, SDValue &Segment, 1505 SDValue &PatternNodeWithChain) { 1506 if (N.getOpcode() == ISD::SCALAR_TO_VECTOR) { 1507 PatternNodeWithChain = N.getOperand(0); 1508 if (ISD::isNON_EXTLoad(PatternNodeWithChain.getNode()) && 1509 PatternNodeWithChain.hasOneUse() && 1510 IsProfitableToFold(N.getOperand(0), N.getNode(), Root) && 1511 IsLegalToFold(N.getOperand(0), N.getNode(), Root, OptLevel)) { 1512 LoadSDNode *LD = cast<LoadSDNode>(PatternNodeWithChain); 1513 if (!selectAddr(LD, LD->getBasePtr(), Base, Scale, Index, Disp, Segment)) 1514 return false; 1515 return true; 1516 } 1517 } 1518 1519 // Also handle the case where we explicitly require zeros in the top 1520 // elements. This is a vector shuffle from the zero vector. 1521 if (N.getOpcode() == X86ISD::VZEXT_MOVL && N.getNode()->hasOneUse() && 1522 // Check to see if the top elements are all zeros (or bitcast of zeros). 1523 N.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR && 1524 N.getOperand(0).getNode()->hasOneUse() && 1525 ISD::isNON_EXTLoad(N.getOperand(0).getOperand(0).getNode()) && 1526 N.getOperand(0).getOperand(0).hasOneUse() && 1527 IsProfitableToFold(N.getOperand(0), N.getNode(), Root) && 1528 IsLegalToFold(N.getOperand(0), N.getNode(), Root, OptLevel)) { 1529 // Okay, this is a zero extending load. Fold it. 1530 LoadSDNode *LD = cast<LoadSDNode>(N.getOperand(0).getOperand(0)); 1531 if (!selectAddr(LD, LD->getBasePtr(), Base, Scale, Index, Disp, Segment)) 1532 return false; 1533 PatternNodeWithChain = SDValue(LD, 0); 1534 return true; 1535 } 1536 return false; 1537} 1538 1539 1540bool X86DAGToDAGISel::selectMOV64Imm32(SDValue N, SDValue &Imm) { 1541 if (const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) { 1542 uint64_t ImmVal = CN->getZExtValue(); 1543 if ((uint32_t)ImmVal != (uint64_t)ImmVal) 1544 return false; 1545 1546 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), MVT::i64); 1547 return true; 1548 } 1549 1550 // In static codegen with small code model, we can get the address of a label 1551 // into a register with 'movl'. TableGen has already made sure we're looking 1552 // at a label of some kind. 1553 assert(N->getOpcode() == X86ISD::Wrapper && 1554 "Unexpected node type for MOV32ri64"); 1555 N = N.getOperand(0); 1556 1557 if (N->getOpcode() != ISD::TargetConstantPool && 1558 N->getOpcode() != ISD::TargetJumpTable && 1559 N->getOpcode() != ISD::TargetGlobalAddress && 1560 N->getOpcode() != ISD::TargetExternalSymbol && 1561 N->getOpcode() != ISD::MCSymbol && 1562 N->getOpcode() != ISD::TargetBlockAddress) 1563 return false; 1564 1565 Imm = N; 1566 return TM.getCodeModel() == CodeModel::Small; 1567} 1568 1569bool X86DAGToDAGISel::selectLEA64_32Addr(SDValue N, SDValue &Base, 1570 SDValue &Scale, SDValue &Index, 1571 SDValue &Disp, SDValue &Segment) { 1572 if (!selectLEAAddr(N, Base, Scale, Index, Disp, Segment)) 1573 return false; 1574 1575 SDLoc DL(N); 1576 RegisterSDNode *RN = dyn_cast<RegisterSDNode>(Base); 1577 if (RN && RN->getReg() == 0) 1578 Base = CurDAG->getRegister(0, MVT::i64); 1579 else if (Base.getValueType() == MVT::i32 && !dyn_cast<FrameIndexSDNode>(Base)) { 1580 // Base could already be %rip, particularly in the x32 ABI. 1581 Base = SDValue(CurDAG->getMachineNode( 1582 TargetOpcode::SUBREG_TO_REG, DL, MVT::i64, 1583 CurDAG->getTargetConstant(0, DL, MVT::i64), 1584 Base, 1585 CurDAG->getTargetConstant(X86::sub_32bit, DL, MVT::i32)), 1586 0); 1587 } 1588 1589 RN = dyn_cast<RegisterSDNode>(Index); 1590 if (RN && RN->getReg() == 0) 1591 Index = CurDAG->getRegister(0, MVT::i64); 1592 else { 1593 assert(Index.getValueType() == MVT::i32 && 1594 "Expect to be extending 32-bit registers for use in LEA"); 1595 Index = SDValue(CurDAG->getMachineNode( 1596 TargetOpcode::SUBREG_TO_REG, DL, MVT::i64, 1597 CurDAG->getTargetConstant(0, DL, MVT::i64), 1598 Index, 1599 CurDAG->getTargetConstant(X86::sub_32bit, DL, 1600 MVT::i32)), 1601 0); 1602 } 1603 1604 return true; 1605} 1606 1607/// Calls SelectAddr and determines if the maximal addressing 1608/// mode it matches can be cost effectively emitted as an LEA instruction. 1609bool X86DAGToDAGISel::selectLEAAddr(SDValue N, 1610 SDValue &Base, SDValue &Scale, 1611 SDValue &Index, SDValue &Disp, 1612 SDValue &Segment) { 1613 X86ISelAddressMode AM; 1614 1615 // Set AM.Segment to prevent MatchAddress from using one. LEA doesn't support 1616 // segments. 1617 SDValue Copy = AM.Segment; 1618 SDValue T = CurDAG->getRegister(0, MVT::i32); 1619 AM.Segment = T; 1620 if (matchAddress(N, AM)) 1621 return false; 1622 assert (T == AM.Segment); 1623 AM.Segment = Copy; 1624 1625 MVT VT = N.getSimpleValueType(); 1626 unsigned Complexity = 0; 1627 if (AM.BaseType == X86ISelAddressMode::RegBase) 1628 if (AM.Base_Reg.getNode()) 1629 Complexity = 1; 1630 else 1631 AM.Base_Reg = CurDAG->getRegister(0, VT); 1632 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase) 1633 Complexity = 4; 1634 1635 if (AM.IndexReg.getNode()) 1636 Complexity++; 1637 else 1638 AM.IndexReg = CurDAG->getRegister(0, VT); 1639 1640 // Don't match just leal(,%reg,2). It's cheaper to do addl %reg, %reg, or with 1641 // a simple shift. 1642 if (AM.Scale > 1) 1643 Complexity++; 1644 1645 // FIXME: We are artificially lowering the criteria to turn ADD %reg, $GA 1646 // to a LEA. This is determined with some experimentation but is by no means 1647 // optimal (especially for code size consideration). LEA is nice because of 1648 // its three-address nature. Tweak the cost function again when we can run 1649 // convertToThreeAddress() at register allocation time. 1650 if (AM.hasSymbolicDisplacement()) { 1651 // For X86-64, always use LEA to materialize RIP-relative addresses. 1652 if (Subtarget->is64Bit()) 1653 Complexity = 4; 1654 else 1655 Complexity += 2; 1656 } 1657 1658 if (AM.Disp && (AM.Base_Reg.getNode() || AM.IndexReg.getNode())) 1659 Complexity++; 1660 1661 // If it isn't worth using an LEA, reject it. 1662 if (Complexity <= 2) 1663 return false; 1664 1665 getAddressOperands(AM, SDLoc(N), Base, Scale, Index, Disp, Segment); 1666 return true; 1667} 1668 1669/// This is only run on TargetGlobalTLSAddress nodes. 1670bool X86DAGToDAGISel::selectTLSADDRAddr(SDValue N, SDValue &Base, 1671 SDValue &Scale, SDValue &Index, 1672 SDValue &Disp, SDValue &Segment) { 1673 assert(N.getOpcode() == ISD::TargetGlobalTLSAddress); 1674 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 1675 1676 X86ISelAddressMode AM; 1677 AM.GV = GA->getGlobal(); 1678 AM.Disp += GA->getOffset(); 1679 AM.Base_Reg = CurDAG->getRegister(0, N.getValueType()); 1680 AM.SymbolFlags = GA->getTargetFlags(); 1681 1682 if (N.getValueType() == MVT::i32) { 1683 AM.Scale = 1; 1684 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32); 1685 } else { 1686 AM.IndexReg = CurDAG->getRegister(0, MVT::i64); 1687 } 1688 1689 getAddressOperands(AM, SDLoc(N), Base, Scale, Index, Disp, Segment); 1690 return true; 1691} 1692 1693 1694bool X86DAGToDAGISel::tryFoldLoad(SDNode *P, SDValue N, 1695 SDValue &Base, SDValue &Scale, 1696 SDValue &Index, SDValue &Disp, 1697 SDValue &Segment) { 1698 if (!ISD::isNON_EXTLoad(N.getNode()) || 1699 !IsProfitableToFold(N, P, P) || 1700 !IsLegalToFold(N, P, P, OptLevel)) 1701 return false; 1702 1703 return selectAddr(N.getNode(), 1704 N.getOperand(1), Base, Scale, Index, Disp, Segment); 1705} 1706 1707/// Return an SDNode that returns the value of the global base register. 1708/// Output instructions required to initialize the global base register, 1709/// if necessary. 1710SDNode *X86DAGToDAGISel::getGlobalBaseReg() { 1711 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF); 1712 auto &DL = MF->getDataLayout(); 1713 return CurDAG->getRegister(GlobalBaseReg, TLI->getPointerTy(DL)).getNode(); 1714} 1715 1716/// Atomic opcode table 1717/// 1718enum AtomicOpc { 1719 ADD, 1720 SUB, 1721 INC, 1722 DEC, 1723 OR, 1724 AND, 1725 XOR, 1726 AtomicOpcEnd 1727}; 1728 1729enum AtomicSz { 1730 ConstantI8, 1731 I8, 1732 SextConstantI16, 1733 ConstantI16, 1734 I16, 1735 SextConstantI32, 1736 ConstantI32, 1737 I32, 1738 SextConstantI64, 1739 ConstantI64, 1740 I64, 1741 AtomicSzEnd 1742}; 1743 1744static const uint16_t AtomicOpcTbl[AtomicOpcEnd][AtomicSzEnd] = { 1745 { 1746 X86::LOCK_ADD8mi, 1747 X86::LOCK_ADD8mr, 1748 X86::LOCK_ADD16mi8, 1749 X86::LOCK_ADD16mi, 1750 X86::LOCK_ADD16mr, 1751 X86::LOCK_ADD32mi8, 1752 X86::LOCK_ADD32mi, 1753 X86::LOCK_ADD32mr, 1754 X86::LOCK_ADD64mi8, 1755 X86::LOCK_ADD64mi32, 1756 X86::LOCK_ADD64mr, 1757 }, 1758 { 1759 X86::LOCK_SUB8mi, 1760 X86::LOCK_SUB8mr, 1761 X86::LOCK_SUB16mi8, 1762 X86::LOCK_SUB16mi, 1763 X86::LOCK_SUB16mr, 1764 X86::LOCK_SUB32mi8, 1765 X86::LOCK_SUB32mi, 1766 X86::LOCK_SUB32mr, 1767 X86::LOCK_SUB64mi8, 1768 X86::LOCK_SUB64mi32, 1769 X86::LOCK_SUB64mr, 1770 }, 1771 { 1772 0, 1773 X86::LOCK_INC8m, 1774 0, 1775 0, 1776 X86::LOCK_INC16m, 1777 0, 1778 0, 1779 X86::LOCK_INC32m, 1780 0, 1781 0, 1782 X86::LOCK_INC64m, 1783 }, 1784 { 1785 0, 1786 X86::LOCK_DEC8m, 1787 0, 1788 0, 1789 X86::LOCK_DEC16m, 1790 0, 1791 0, 1792 X86::LOCK_DEC32m, 1793 0, 1794 0, 1795 X86::LOCK_DEC64m, 1796 }, 1797 { 1798 X86::LOCK_OR8mi, 1799 X86::LOCK_OR8mr, 1800 X86::LOCK_OR16mi8, 1801 X86::LOCK_OR16mi, 1802 X86::LOCK_OR16mr, 1803 X86::LOCK_OR32mi8, 1804 X86::LOCK_OR32mi, 1805 X86::LOCK_OR32mr, 1806 X86::LOCK_OR64mi8, 1807 X86::LOCK_OR64mi32, 1808 X86::LOCK_OR64mr, 1809 }, 1810 { 1811 X86::LOCK_AND8mi, 1812 X86::LOCK_AND8mr, 1813 X86::LOCK_AND16mi8, 1814 X86::LOCK_AND16mi, 1815 X86::LOCK_AND16mr, 1816 X86::LOCK_AND32mi8, 1817 X86::LOCK_AND32mi, 1818 X86::LOCK_AND32mr, 1819 X86::LOCK_AND64mi8, 1820 X86::LOCK_AND64mi32, 1821 X86::LOCK_AND64mr, 1822 }, 1823 { 1824 X86::LOCK_XOR8mi, 1825 X86::LOCK_XOR8mr, 1826 X86::LOCK_XOR16mi8, 1827 X86::LOCK_XOR16mi, 1828 X86::LOCK_XOR16mr, 1829 X86::LOCK_XOR32mi8, 1830 X86::LOCK_XOR32mi, 1831 X86::LOCK_XOR32mr, 1832 X86::LOCK_XOR64mi8, 1833 X86::LOCK_XOR64mi32, 1834 X86::LOCK_XOR64mr, 1835 } 1836}; 1837 1838// Return the target constant operand for atomic-load-op and do simple 1839// translations, such as from atomic-load-add to lock-sub. The return value is 1840// one of the following 3 cases: 1841// + target-constant, the operand could be supported as a target constant. 1842// + empty, the operand is not needed any more with the new op selected. 1843// + non-empty, otherwise. 1844static SDValue getAtomicLoadArithTargetConstant(SelectionDAG *CurDAG, 1845 SDLoc dl, 1846 enum AtomicOpc &Op, MVT NVT, 1847 SDValue Val, 1848 const X86Subtarget *Subtarget) { 1849 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val)) { 1850 int64_t CNVal = CN->getSExtValue(); 1851 // Quit if not 32-bit imm. 1852 if ((int32_t)CNVal != CNVal) 1853 return Val; 1854 // Quit if INT32_MIN: it would be negated as it is negative and overflow, 1855 // producing an immediate that does not fit in the 32 bits available for 1856 // an immediate operand to sub. However, it still fits in 32 bits for the 1857 // add (since it is not negated) so we can return target-constant. 1858 if (CNVal == INT32_MIN) 1859 return CurDAG->getTargetConstant(CNVal, dl, NVT); 1860 // For atomic-load-add, we could do some optimizations. 1861 if (Op == ADD) { 1862 // Translate to INC/DEC if ADD by 1 or -1. 1863 if (((CNVal == 1) || (CNVal == -1)) && !Subtarget->slowIncDec()) { 1864 Op = (CNVal == 1) ? INC : DEC; 1865 // No more constant operand after being translated into INC/DEC. 1866 return SDValue(); 1867 } 1868 // Translate to SUB if ADD by negative value. 1869 if (CNVal < 0) { 1870 Op = SUB; 1871 CNVal = -CNVal; 1872 } 1873 } 1874 return CurDAG->getTargetConstant(CNVal, dl, NVT); 1875 } 1876 1877 // If the value operand is single-used, try to optimize it. 1878 if (Op == ADD && Val.hasOneUse()) { 1879 // Translate (atomic-load-add ptr (sub 0 x)) back to (lock-sub x). 1880 if (Val.getOpcode() == ISD::SUB && X86::isZeroNode(Val.getOperand(0))) { 1881 Op = SUB; 1882 return Val.getOperand(1); 1883 } 1884 // A special case for i16, which needs truncating as, in most cases, it's 1885 // promoted to i32. We will translate 1886 // (atomic-load-add (truncate (sub 0 x))) to (lock-sub (EXTRACT_SUBREG x)) 1887 if (Val.getOpcode() == ISD::TRUNCATE && NVT == MVT::i16 && 1888 Val.getOperand(0).getOpcode() == ISD::SUB && 1889 X86::isZeroNode(Val.getOperand(0).getOperand(0))) { 1890 Op = SUB; 1891 Val = Val.getOperand(0); 1892 return CurDAG->getTargetExtractSubreg(X86::sub_16bit, dl, NVT, 1893 Val.getOperand(1)); 1894 } 1895 } 1896 1897 return Val; 1898} 1899 1900SDNode *X86DAGToDAGISel::selectAtomicLoadArith(SDNode *Node, MVT NVT) { 1901 if (Node->hasAnyUseOfValue(0)) 1902 return nullptr; 1903 1904 SDLoc dl(Node); 1905 1906 // Optimize common patterns for __sync_or_and_fetch and similar arith 1907 // operations where the result is not used. This allows us to use the "lock" 1908 // version of the arithmetic instruction. 1909 SDValue Chain = Node->getOperand(0); 1910 SDValue Ptr = Node->getOperand(1); 1911 SDValue Val = Node->getOperand(2); 1912 SDValue Base, Scale, Index, Disp, Segment; 1913 if (!selectAddr(Node, Ptr, Base, Scale, Index, Disp, Segment)) 1914 return nullptr; 1915 1916 // Which index into the table. 1917 enum AtomicOpc Op; 1918 switch (Node->getOpcode()) { 1919 default: 1920 return nullptr; 1921 case ISD::ATOMIC_LOAD_OR: 1922 Op = OR; 1923 break; 1924 case ISD::ATOMIC_LOAD_AND: 1925 Op = AND; 1926 break; 1927 case ISD::ATOMIC_LOAD_XOR: 1928 Op = XOR; 1929 break; 1930 case ISD::ATOMIC_LOAD_ADD: 1931 Op = ADD; 1932 break; 1933 } 1934 1935 Val = getAtomicLoadArithTargetConstant(CurDAG, dl, Op, NVT, Val, Subtarget); 1936 bool isUnOp = !Val.getNode(); 1937 bool isCN = Val.getNode() && (Val.getOpcode() == ISD::TargetConstant); 1938 1939 unsigned Opc = 0; 1940 switch (NVT.SimpleTy) { 1941 default: return nullptr; 1942 case MVT::i8: 1943 if (isCN) 1944 Opc = AtomicOpcTbl[Op][ConstantI8]; 1945 else 1946 Opc = AtomicOpcTbl[Op][I8]; 1947 break; 1948 case MVT::i16: 1949 if (isCN) { 1950 if (immSext8(Val.getNode())) 1951 Opc = AtomicOpcTbl[Op][SextConstantI16]; 1952 else 1953 Opc = AtomicOpcTbl[Op][ConstantI16]; 1954 } else 1955 Opc = AtomicOpcTbl[Op][I16]; 1956 break; 1957 case MVT::i32: 1958 if (isCN) { 1959 if (immSext8(Val.getNode())) 1960 Opc = AtomicOpcTbl[Op][SextConstantI32]; 1961 else 1962 Opc = AtomicOpcTbl[Op][ConstantI32]; 1963 } else 1964 Opc = AtomicOpcTbl[Op][I32]; 1965 break; 1966 case MVT::i64: 1967 if (isCN) { 1968 if (immSext8(Val.getNode())) 1969 Opc = AtomicOpcTbl[Op][SextConstantI64]; 1970 else if (i64immSExt32(Val.getNode())) 1971 Opc = AtomicOpcTbl[Op][ConstantI64]; 1972 else 1973 llvm_unreachable("True 64 bits constant in SelectAtomicLoadArith"); 1974 } else 1975 Opc = AtomicOpcTbl[Op][I64]; 1976 break; 1977 } 1978 1979 assert(Opc != 0 && "Invalid arith lock transform!"); 1980 1981 // Building the new node. 1982 SDValue Ret; 1983 if (isUnOp) { 1984 SDValue Ops[] = { Base, Scale, Index, Disp, Segment, Chain }; 1985 Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops), 0); 1986 } else { 1987 SDValue Ops[] = { Base, Scale, Index, Disp, Segment, Val, Chain }; 1988 Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops), 0); 1989 } 1990 1991 // Copying the MachineMemOperand. 1992 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1993 MemOp[0] = cast<MemSDNode>(Node)->getMemOperand(); 1994 cast<MachineSDNode>(Ret)->setMemRefs(MemOp, MemOp + 1); 1995 1996 // We need to have two outputs as that is what the original instruction had. 1997 // So we add a dummy, undefined output. This is safe as we checked first 1998 // that no-one uses our output anyway. 1999 SDValue Undef = SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, 2000 dl, NVT), 0); 2001 SDValue RetVals[] = { Undef, Ret }; 2002 return CurDAG->getMergeValues(RetVals, dl).getNode(); 2003} 2004 2005/// Test whether the given X86ISD::CMP node has any uses which require the SF 2006/// or OF bits to be accurate. 2007static bool hasNoSignedComparisonUses(SDNode *N) { 2008 // Examine each user of the node. 2009 for (SDNode::use_iterator UI = N->use_begin(), 2010 UE = N->use_end(); UI != UE; ++UI) { 2011 // Only examine CopyToReg uses. 2012 if (UI->getOpcode() != ISD::CopyToReg) 2013 return false; 2014 // Only examine CopyToReg uses that copy to EFLAGS. 2015 if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != 2016 X86::EFLAGS) 2017 return false; 2018 // Examine each user of the CopyToReg use. 2019 for (SDNode::use_iterator FlagUI = UI->use_begin(), 2020 FlagUE = UI->use_end(); FlagUI != FlagUE; ++FlagUI) { 2021 // Only examine the Flag result. 2022 if (FlagUI.getUse().getResNo() != 1) continue; 2023 // Anything unusual: assume conservatively. 2024 if (!FlagUI->isMachineOpcode()) return false; 2025 // Examine the opcode of the user. 2026 switch (FlagUI->getMachineOpcode()) { 2027 // These comparisons don't treat the most significant bit specially. 2028 case X86::SETAr: case X86::SETAEr: case X86::SETBr: case X86::SETBEr: 2029 case X86::SETEr: case X86::SETNEr: case X86::SETPr: case X86::SETNPr: 2030 case X86::SETAm: case X86::SETAEm: case X86::SETBm: case X86::SETBEm: 2031 case X86::SETEm: case X86::SETNEm: case X86::SETPm: case X86::SETNPm: 2032 case X86::JA_1: case X86::JAE_1: case X86::JB_1: case X86::JBE_1: 2033 case X86::JE_1: case X86::JNE_1: case X86::JP_1: case X86::JNP_1: 2034 case X86::CMOVA16rr: case X86::CMOVA16rm: 2035 case X86::CMOVA32rr: case X86::CMOVA32rm: 2036 case X86::CMOVA64rr: case X86::CMOVA64rm: 2037 case X86::CMOVAE16rr: case X86::CMOVAE16rm: 2038 case X86::CMOVAE32rr: case X86::CMOVAE32rm: 2039 case X86::CMOVAE64rr: case X86::CMOVAE64rm: 2040 case X86::CMOVB16rr: case X86::CMOVB16rm: 2041 case X86::CMOVB32rr: case X86::CMOVB32rm: 2042 case X86::CMOVB64rr: case X86::CMOVB64rm: 2043 case X86::CMOVBE16rr: case X86::CMOVBE16rm: 2044 case X86::CMOVBE32rr: case X86::CMOVBE32rm: 2045 case X86::CMOVBE64rr: case X86::CMOVBE64rm: 2046 case X86::CMOVE16rr: case X86::CMOVE16rm: 2047 case X86::CMOVE32rr: case X86::CMOVE32rm: 2048 case X86::CMOVE64rr: case X86::CMOVE64rm: 2049 case X86::CMOVNE16rr: case X86::CMOVNE16rm: 2050 case X86::CMOVNE32rr: case X86::CMOVNE32rm: 2051 case X86::CMOVNE64rr: case X86::CMOVNE64rm: 2052 case X86::CMOVNP16rr: case X86::CMOVNP16rm: 2053 case X86::CMOVNP32rr: case X86::CMOVNP32rm: 2054 case X86::CMOVNP64rr: case X86::CMOVNP64rm: 2055 case X86::CMOVP16rr: case X86::CMOVP16rm: 2056 case X86::CMOVP32rr: case X86::CMOVP32rm: 2057 case X86::CMOVP64rr: case X86::CMOVP64rm: 2058 continue; 2059 // Anything else: assume conservatively. 2060 default: return false; 2061 } 2062 } 2063 } 2064 return true; 2065} 2066 2067/// Check whether or not the chain ending in StoreNode is suitable for doing 2068/// the {load; increment or decrement; store} to modify transformation. 2069static bool isLoadIncOrDecStore(StoreSDNode *StoreNode, unsigned Opc, 2070 SDValue StoredVal, SelectionDAG *CurDAG, 2071 LoadSDNode* &LoadNode, SDValue &InputChain) { 2072 2073 // is the value stored the result of a DEC or INC? 2074 if (!(Opc == X86ISD::DEC || Opc == X86ISD::INC)) return false; 2075 2076 // is the stored value result 0 of the load? 2077 if (StoredVal.getResNo() != 0) return false; 2078 2079 // are there other uses of the loaded value than the inc or dec? 2080 if (!StoredVal.getNode()->hasNUsesOfValue(1, 0)) return false; 2081 2082 // is the store non-extending and non-indexed? 2083 if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal()) 2084 return false; 2085 2086 SDValue Load = StoredVal->getOperand(0); 2087 // Is the stored value a non-extending and non-indexed load? 2088 if (!ISD::isNormalLoad(Load.getNode())) return false; 2089 2090 // Return LoadNode by reference. 2091 LoadNode = cast<LoadSDNode>(Load); 2092 // is the size of the value one that we can handle? (i.e. 64, 32, 16, or 8) 2093 EVT LdVT = LoadNode->getMemoryVT(); 2094 if (LdVT != MVT::i64 && LdVT != MVT::i32 && LdVT != MVT::i16 && 2095 LdVT != MVT::i8) 2096 return false; 2097 2098 // Is store the only read of the loaded value? 2099 if (!Load.hasOneUse()) 2100 return false; 2101 2102 // Is the address of the store the same as the load? 2103 if (LoadNode->getBasePtr() != StoreNode->getBasePtr() || 2104 LoadNode->getOffset() != StoreNode->getOffset()) 2105 return false; 2106 2107 // Check if the chain is produced by the load or is a TokenFactor with 2108 // the load output chain as an operand. Return InputChain by reference. 2109 SDValue Chain = StoreNode->getChain(); 2110 2111 bool ChainCheck = false; 2112 if (Chain == Load.getValue(1)) { 2113 ChainCheck = true; 2114 InputChain = LoadNode->getChain(); 2115 } else if (Chain.getOpcode() == ISD::TokenFactor) { 2116 SmallVector<SDValue, 4> ChainOps; 2117 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) { 2118 SDValue Op = Chain.getOperand(i); 2119 if (Op == Load.getValue(1)) { 2120 ChainCheck = true; 2121 continue; 2122 } 2123 2124 // Make sure using Op as part of the chain would not cause a cycle here. 2125 // In theory, we could check whether the chain node is a predecessor of 2126 // the load. But that can be very expensive. Instead visit the uses and 2127 // make sure they all have smaller node id than the load. 2128 int LoadId = LoadNode->getNodeId(); 2129 for (SDNode::use_iterator UI = Op.getNode()->use_begin(), 2130 UE = UI->use_end(); UI != UE; ++UI) { 2131 if (UI.getUse().getResNo() != 0) 2132 continue; 2133 if (UI->getNodeId() > LoadId) 2134 return false; 2135 } 2136 2137 ChainOps.push_back(Op); 2138 } 2139 2140 if (ChainCheck) 2141 // Make a new TokenFactor with all the other input chains except 2142 // for the load. 2143 InputChain = CurDAG->getNode(ISD::TokenFactor, SDLoc(Chain), 2144 MVT::Other, ChainOps); 2145 } 2146 if (!ChainCheck) 2147 return false; 2148 2149 return true; 2150} 2151 2152/// Get the appropriate X86 opcode for an in-memory increment or decrement. 2153/// Opc should be X86ISD::DEC or X86ISD::INC. 2154static unsigned getFusedLdStOpcode(EVT &LdVT, unsigned Opc) { 2155 if (Opc == X86ISD::DEC) { 2156 if (LdVT == MVT::i64) return X86::DEC64m; 2157 if (LdVT == MVT::i32) return X86::DEC32m; 2158 if (LdVT == MVT::i16) return X86::DEC16m; 2159 if (LdVT == MVT::i8) return X86::DEC8m; 2160 } else { 2161 assert(Opc == X86ISD::INC && "unrecognized opcode"); 2162 if (LdVT == MVT::i64) return X86::INC64m; 2163 if (LdVT == MVT::i32) return X86::INC32m; 2164 if (LdVT == MVT::i16) return X86::INC16m; 2165 if (LdVT == MVT::i8) return X86::INC8m; 2166 } 2167 llvm_unreachable("unrecognized size for LdVT"); 2168} 2169 2170/// Customized ISel for GATHER operations. 2171SDNode *X86DAGToDAGISel::selectGather(SDNode *Node, unsigned Opc) { 2172 // Operands of Gather: VSrc, Base, VIdx, VMask, Scale 2173 SDValue Chain = Node->getOperand(0); 2174 SDValue VSrc = Node->getOperand(2); 2175 SDValue Base = Node->getOperand(3); 2176 SDValue VIdx = Node->getOperand(4); 2177 SDValue VMask = Node->getOperand(5); 2178 ConstantSDNode *Scale = dyn_cast<ConstantSDNode>(Node->getOperand(6)); 2179 if (!Scale) 2180 return nullptr; 2181 2182 SDVTList VTs = CurDAG->getVTList(VSrc.getValueType(), VSrc.getValueType(), 2183 MVT::Other); 2184 2185 SDLoc DL(Node); 2186 2187 // Memory Operands: Base, Scale, Index, Disp, Segment 2188 SDValue Disp = CurDAG->getTargetConstant(0, DL, MVT::i32); 2189 SDValue Segment = CurDAG->getRegister(0, MVT::i32); 2190 const SDValue Ops[] = { VSrc, Base, getI8Imm(Scale->getSExtValue(), DL), VIdx, 2191 Disp, Segment, VMask, Chain}; 2192 SDNode *ResNode = CurDAG->getMachineNode(Opc, DL, VTs, Ops); 2193 // Node has 2 outputs: VDst and MVT::Other. 2194 // ResNode has 3 outputs: VDst, VMask_wb, and MVT::Other. 2195 // We replace VDst of Node with VDst of ResNode, and Other of Node with Other 2196 // of ResNode. 2197 ReplaceUses(SDValue(Node, 0), SDValue(ResNode, 0)); 2198 ReplaceUses(SDValue(Node, 1), SDValue(ResNode, 2)); 2199 return ResNode; 2200} 2201 2202SDNode *X86DAGToDAGISel::Select(SDNode *Node) { 2203 MVT NVT = Node->getSimpleValueType(0); 2204 unsigned Opc, MOpc; 2205 unsigned Opcode = Node->getOpcode(); 2206 SDLoc dl(Node); 2207 2208 DEBUG(dbgs() << "Selecting: "; Node->dump(CurDAG); dbgs() << '\n'); 2209 2210 if (Node->isMachineOpcode()) { 2211 DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << '\n'); 2212 Node->setNodeId(-1); 2213 return nullptr; // Already selected. 2214 } 2215 2216 switch (Opcode) { 2217 default: break; 2218 case ISD::BRIND: { 2219 if (Subtarget->isTargetNaCl()) 2220 // NaCl has its own pass where jmp %r32 are converted to jmp %r64. We 2221 // leave the instruction alone. 2222 break; 2223 if (Subtarget->isTarget64BitILP32()) { 2224 // Converts a 32-bit register to a 64-bit, zero-extended version of 2225 // it. This is needed because x86-64 can do many things, but jmp %r32 2226 // ain't one of them. 2227 const SDValue &Target = Node->getOperand(1); 2228 assert(Target.getSimpleValueType() == llvm::MVT::i32); 2229 SDValue ZextTarget = CurDAG->getZExtOrTrunc(Target, dl, EVT(MVT::i64)); 2230 SDValue Brind = CurDAG->getNode(ISD::BRIND, dl, MVT::Other, 2231 Node->getOperand(0), ZextTarget); 2232 ReplaceUses(SDValue(Node, 0), Brind); 2233 SelectCode(ZextTarget.getNode()); 2234 SelectCode(Brind.getNode()); 2235 return nullptr; 2236 } 2237 break; 2238 } 2239 case ISD::INTRINSIC_W_CHAIN: { 2240 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue(); 2241 switch (IntNo) { 2242 default: break; 2243 case Intrinsic::x86_avx2_gather_d_pd: 2244 case Intrinsic::x86_avx2_gather_d_pd_256: 2245 case Intrinsic::x86_avx2_gather_q_pd: 2246 case Intrinsic::x86_avx2_gather_q_pd_256: 2247 case Intrinsic::x86_avx2_gather_d_ps: 2248 case Intrinsic::x86_avx2_gather_d_ps_256: 2249 case Intrinsic::x86_avx2_gather_q_ps: 2250 case Intrinsic::x86_avx2_gather_q_ps_256: 2251 case Intrinsic::x86_avx2_gather_d_q: 2252 case Intrinsic::x86_avx2_gather_d_q_256: 2253 case Intrinsic::x86_avx2_gather_q_q: 2254 case Intrinsic::x86_avx2_gather_q_q_256: 2255 case Intrinsic::x86_avx2_gather_d_d: 2256 case Intrinsic::x86_avx2_gather_d_d_256: 2257 case Intrinsic::x86_avx2_gather_q_d: 2258 case Intrinsic::x86_avx2_gather_q_d_256: { 2259 if (!Subtarget->hasAVX2()) 2260 break; 2261 unsigned Opc; 2262 switch (IntNo) { 2263 default: llvm_unreachable("Impossible intrinsic"); 2264 case Intrinsic::x86_avx2_gather_d_pd: Opc = X86::VGATHERDPDrm; break; 2265 case Intrinsic::x86_avx2_gather_d_pd_256: Opc = X86::VGATHERDPDYrm; break; 2266 case Intrinsic::x86_avx2_gather_q_pd: Opc = X86::VGATHERQPDrm; break; 2267 case Intrinsic::x86_avx2_gather_q_pd_256: Opc = X86::VGATHERQPDYrm; break; 2268 case Intrinsic::x86_avx2_gather_d_ps: Opc = X86::VGATHERDPSrm; break; 2269 case Intrinsic::x86_avx2_gather_d_ps_256: Opc = X86::VGATHERDPSYrm; break; 2270 case Intrinsic::x86_avx2_gather_q_ps: Opc = X86::VGATHERQPSrm; break; 2271 case Intrinsic::x86_avx2_gather_q_ps_256: Opc = X86::VGATHERQPSYrm; break; 2272 case Intrinsic::x86_avx2_gather_d_q: Opc = X86::VPGATHERDQrm; break; 2273 case Intrinsic::x86_avx2_gather_d_q_256: Opc = X86::VPGATHERDQYrm; break; 2274 case Intrinsic::x86_avx2_gather_q_q: Opc = X86::VPGATHERQQrm; break; 2275 case Intrinsic::x86_avx2_gather_q_q_256: Opc = X86::VPGATHERQQYrm; break; 2276 case Intrinsic::x86_avx2_gather_d_d: Opc = X86::VPGATHERDDrm; break; 2277 case Intrinsic::x86_avx2_gather_d_d_256: Opc = X86::VPGATHERDDYrm; break; 2278 case Intrinsic::x86_avx2_gather_q_d: Opc = X86::VPGATHERQDrm; break; 2279 case Intrinsic::x86_avx2_gather_q_d_256: Opc = X86::VPGATHERQDYrm; break; 2280 } 2281 SDNode *RetVal = selectGather(Node, Opc); 2282 if (RetVal) 2283 // We already called ReplaceUses inside SelectGather. 2284 return nullptr; 2285 break; 2286 } 2287 } 2288 break; 2289 } 2290 case X86ISD::GlobalBaseReg: 2291 return getGlobalBaseReg(); 2292 2293 case X86ISD::SHRUNKBLEND: { 2294 // SHRUNKBLEND selects like a regular VSELECT. 2295 SDValue VSelect = CurDAG->getNode( 2296 ISD::VSELECT, SDLoc(Node), Node->getValueType(0), Node->getOperand(0), 2297 Node->getOperand(1), Node->getOperand(2)); 2298 ReplaceUses(SDValue(Node, 0), VSelect); 2299 SelectCode(VSelect.getNode()); 2300 // We already called ReplaceUses. 2301 return nullptr; 2302 } 2303 2304 case ISD::ATOMIC_LOAD_XOR: 2305 case ISD::ATOMIC_LOAD_AND: 2306 case ISD::ATOMIC_LOAD_OR: 2307 case ISD::ATOMIC_LOAD_ADD: { 2308 SDNode *RetVal = selectAtomicLoadArith(Node, NVT); 2309 if (RetVal) 2310 return RetVal; 2311 break; 2312 } 2313 case ISD::AND: 2314 case ISD::OR: 2315 case ISD::XOR: { 2316 // For operations of the form (x << C1) op C2, check if we can use a smaller 2317 // encoding for C2 by transforming it into (x op (C2>>C1)) << C1. 2318 SDValue N0 = Node->getOperand(0); 2319 SDValue N1 = Node->getOperand(1); 2320 2321 if (N0->getOpcode() != ISD::SHL || !N0->hasOneUse()) 2322 break; 2323 2324 // i8 is unshrinkable, i16 should be promoted to i32. 2325 if (NVT != MVT::i32 && NVT != MVT::i64) 2326 break; 2327 2328 ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N1); 2329 ConstantSDNode *ShlCst = dyn_cast<ConstantSDNode>(N0->getOperand(1)); 2330 if (!Cst || !ShlCst) 2331 break; 2332 2333 int64_t Val = Cst->getSExtValue(); 2334 uint64_t ShlVal = ShlCst->getZExtValue(); 2335 2336 // Make sure that we don't change the operation by removing bits. 2337 // This only matters for OR and XOR, AND is unaffected. 2338 uint64_t RemovedBitsMask = (1ULL << ShlVal) - 1; 2339 if (Opcode != ISD::AND && (Val & RemovedBitsMask) != 0) 2340 break; 2341 2342 unsigned ShlOp, AddOp, Op; 2343 MVT CstVT = NVT; 2344 2345 // Check the minimum bitwidth for the new constant. 2346 // TODO: AND32ri is the same as AND64ri32 with zext imm. 2347 // TODO: MOV32ri+OR64r is cheaper than MOV64ri64+OR64rr 2348 // TODO: Using 16 and 8 bit operations is also possible for or32 & xor32. 2349 if (!isInt<8>(Val) && isInt<8>(Val >> ShlVal)) 2350 CstVT = MVT::i8; 2351 else if (!isInt<32>(Val) && isInt<32>(Val >> ShlVal)) 2352 CstVT = MVT::i32; 2353 2354 // Bail if there is no smaller encoding. 2355 if (NVT == CstVT) 2356 break; 2357 2358 switch (NVT.SimpleTy) { 2359 default: llvm_unreachable("Unsupported VT!"); 2360 case MVT::i32: 2361 assert(CstVT == MVT::i8); 2362 ShlOp = X86::SHL32ri; 2363 AddOp = X86::ADD32rr; 2364 2365 switch (Opcode) { 2366 default: llvm_unreachable("Impossible opcode"); 2367 case ISD::AND: Op = X86::AND32ri8; break; 2368 case ISD::OR: Op = X86::OR32ri8; break; 2369 case ISD::XOR: Op = X86::XOR32ri8; break; 2370 } 2371 break; 2372 case MVT::i64: 2373 assert(CstVT == MVT::i8 || CstVT == MVT::i32); 2374 ShlOp = X86::SHL64ri; 2375 AddOp = X86::ADD64rr; 2376 2377 switch (Opcode) { 2378 default: llvm_unreachable("Impossible opcode"); 2379 case ISD::AND: Op = CstVT==MVT::i8? X86::AND64ri8 : X86::AND64ri32; break; 2380 case ISD::OR: Op = CstVT==MVT::i8? X86::OR64ri8 : X86::OR64ri32; break; 2381 case ISD::XOR: Op = CstVT==MVT::i8? X86::XOR64ri8 : X86::XOR64ri32; break; 2382 } 2383 break; 2384 } 2385 2386 // Emit the smaller op and the shift. 2387 SDValue NewCst = CurDAG->getTargetConstant(Val >> ShlVal, dl, CstVT); 2388 SDNode *New = CurDAG->getMachineNode(Op, dl, NVT, N0->getOperand(0),NewCst); 2389 if (ShlVal == 1) 2390 return CurDAG->SelectNodeTo(Node, AddOp, NVT, SDValue(New, 0), 2391 SDValue(New, 0)); 2392 return CurDAG->SelectNodeTo(Node, ShlOp, NVT, SDValue(New, 0), 2393 getI8Imm(ShlVal, dl)); 2394 } 2395 case X86ISD::UMUL8: 2396 case X86ISD::SMUL8: { 2397 SDValue N0 = Node->getOperand(0); 2398 SDValue N1 = Node->getOperand(1); 2399 2400 Opc = (Opcode == X86ISD::SMUL8 ? X86::IMUL8r : X86::MUL8r); 2401 2402 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::AL, 2403 N0, SDValue()).getValue(1); 2404 2405 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32); 2406 SDValue Ops[] = {N1, InFlag}; 2407 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); 2408 2409 ReplaceUses(SDValue(Node, 0), SDValue(CNode, 0)); 2410 ReplaceUses(SDValue(Node, 1), SDValue(CNode, 1)); 2411 return nullptr; 2412 } 2413 2414 case X86ISD::UMUL: { 2415 SDValue N0 = Node->getOperand(0); 2416 SDValue N1 = Node->getOperand(1); 2417 2418 unsigned LoReg; 2419 switch (NVT.SimpleTy) { 2420 default: llvm_unreachable("Unsupported VT!"); 2421 case MVT::i8: LoReg = X86::AL; Opc = X86::MUL8r; break; 2422 case MVT::i16: LoReg = X86::AX; Opc = X86::MUL16r; break; 2423 case MVT::i32: LoReg = X86::EAX; Opc = X86::MUL32r; break; 2424 case MVT::i64: LoReg = X86::RAX; Opc = X86::MUL64r; break; 2425 } 2426 2427 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg, 2428 N0, SDValue()).getValue(1); 2429 2430 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::i32); 2431 SDValue Ops[] = {N1, InFlag}; 2432 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); 2433 2434 ReplaceUses(SDValue(Node, 0), SDValue(CNode, 0)); 2435 ReplaceUses(SDValue(Node, 1), SDValue(CNode, 1)); 2436 ReplaceUses(SDValue(Node, 2), SDValue(CNode, 2)); 2437 return nullptr; 2438 } 2439 2440 case ISD::SMUL_LOHI: 2441 case ISD::UMUL_LOHI: { 2442 SDValue N0 = Node->getOperand(0); 2443 SDValue N1 = Node->getOperand(1); 2444 2445 bool isSigned = Opcode == ISD::SMUL_LOHI; 2446 bool hasBMI2 = Subtarget->hasBMI2(); 2447 if (!isSigned) { 2448 switch (NVT.SimpleTy) { 2449 default: llvm_unreachable("Unsupported VT!"); 2450 case MVT::i8: Opc = X86::MUL8r; MOpc = X86::MUL8m; break; 2451 case MVT::i16: Opc = X86::MUL16r; MOpc = X86::MUL16m; break; 2452 case MVT::i32: Opc = hasBMI2 ? X86::MULX32rr : X86::MUL32r; 2453 MOpc = hasBMI2 ? X86::MULX32rm : X86::MUL32m; break; 2454 case MVT::i64: Opc = hasBMI2 ? X86::MULX64rr : X86::MUL64r; 2455 MOpc = hasBMI2 ? X86::MULX64rm : X86::MUL64m; break; 2456 } 2457 } else { 2458 switch (NVT.SimpleTy) { 2459 default: llvm_unreachable("Unsupported VT!"); 2460 case MVT::i8: Opc = X86::IMUL8r; MOpc = X86::IMUL8m; break; 2461 case MVT::i16: Opc = X86::IMUL16r; MOpc = X86::IMUL16m; break; 2462 case MVT::i32: Opc = X86::IMUL32r; MOpc = X86::IMUL32m; break; 2463 case MVT::i64: Opc = X86::IMUL64r; MOpc = X86::IMUL64m; break; 2464 } 2465 } 2466 2467 unsigned SrcReg, LoReg, HiReg; 2468 switch (Opc) { 2469 default: llvm_unreachable("Unknown MUL opcode!"); 2470 case X86::IMUL8r: 2471 case X86::MUL8r: 2472 SrcReg = LoReg = X86::AL; HiReg = X86::AH; 2473 break; 2474 case X86::IMUL16r: 2475 case X86::MUL16r: 2476 SrcReg = LoReg = X86::AX; HiReg = X86::DX; 2477 break; 2478 case X86::IMUL32r: 2479 case X86::MUL32r: 2480 SrcReg = LoReg = X86::EAX; HiReg = X86::EDX; 2481 break; 2482 case X86::IMUL64r: 2483 case X86::MUL64r: 2484 SrcReg = LoReg = X86::RAX; HiReg = X86::RDX; 2485 break; 2486 case X86::MULX32rr: 2487 SrcReg = X86::EDX; LoReg = HiReg = 0; 2488 break; 2489 case X86::MULX64rr: 2490 SrcReg = X86::RDX; LoReg = HiReg = 0; 2491 break; 2492 } 2493 2494 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 2495 bool foldedLoad = tryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 2496 // Multiply is commmutative. 2497 if (!foldedLoad) { 2498 foldedLoad = tryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 2499 if (foldedLoad) 2500 std::swap(N0, N1); 2501 } 2502 2503 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, SrcReg, 2504 N0, SDValue()).getValue(1); 2505 SDValue ResHi, ResLo; 2506 2507 if (foldedLoad) { 2508 SDValue Chain; 2509 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), 2510 InFlag }; 2511 if (MOpc == X86::MULX32rm || MOpc == X86::MULX64rm) { 2512 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::Other, MVT::Glue); 2513 SDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); 2514 ResHi = SDValue(CNode, 0); 2515 ResLo = SDValue(CNode, 1); 2516 Chain = SDValue(CNode, 2); 2517 InFlag = SDValue(CNode, 3); 2518 } else { 2519 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue); 2520 SDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); 2521 Chain = SDValue(CNode, 0); 2522 InFlag = SDValue(CNode, 1); 2523 } 2524 2525 // Update the chain. 2526 ReplaceUses(N1.getValue(1), Chain); 2527 } else { 2528 SDValue Ops[] = { N1, InFlag }; 2529 if (Opc == X86::MULX32rr || Opc == X86::MULX64rr) { 2530 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::Glue); 2531 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); 2532 ResHi = SDValue(CNode, 0); 2533 ResLo = SDValue(CNode, 1); 2534 InFlag = SDValue(CNode, 2); 2535 } else { 2536 SDVTList VTs = CurDAG->getVTList(MVT::Glue); 2537 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); 2538 InFlag = SDValue(CNode, 0); 2539 } 2540 } 2541 2542 // Prevent use of AH in a REX instruction by referencing AX instead. 2543 if (HiReg == X86::AH && Subtarget->is64Bit() && 2544 !SDValue(Node, 1).use_empty()) { 2545 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 2546 X86::AX, MVT::i16, InFlag); 2547 InFlag = Result.getValue(2); 2548 // Get the low part if needed. Don't use getCopyFromReg for aliasing 2549 // registers. 2550 if (!SDValue(Node, 0).use_empty()) 2551 ReplaceUses(SDValue(Node, 1), 2552 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); 2553 2554 // Shift AX down 8 bits. 2555 Result = SDValue(CurDAG->getMachineNode(X86::SHR16ri, dl, MVT::i16, 2556 Result, 2557 CurDAG->getTargetConstant(8, dl, MVT::i8)), 2558 0); 2559 // Then truncate it down to i8. 2560 ReplaceUses(SDValue(Node, 1), 2561 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); 2562 } 2563 // Copy the low half of the result, if it is needed. 2564 if (!SDValue(Node, 0).use_empty()) { 2565 if (!ResLo.getNode()) { 2566 assert(LoReg && "Register for low half is not defined!"); 2567 ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg, NVT, 2568 InFlag); 2569 InFlag = ResLo.getValue(2); 2570 } 2571 ReplaceUses(SDValue(Node, 0), ResLo); 2572 DEBUG(dbgs() << "=> "; ResLo.getNode()->dump(CurDAG); dbgs() << '\n'); 2573 } 2574 // Copy the high half of the result, if it is needed. 2575 if (!SDValue(Node, 1).use_empty()) { 2576 if (!ResHi.getNode()) { 2577 assert(HiReg && "Register for high half is not defined!"); 2578 ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg, NVT, 2579 InFlag); 2580 InFlag = ResHi.getValue(2); 2581 } 2582 ReplaceUses(SDValue(Node, 1), ResHi); 2583 DEBUG(dbgs() << "=> "; ResHi.getNode()->dump(CurDAG); dbgs() << '\n'); 2584 } 2585 2586 return nullptr; 2587 } 2588 2589 case ISD::SDIVREM: 2590 case ISD::UDIVREM: 2591 case X86ISD::SDIVREM8_SEXT_HREG: 2592 case X86ISD::UDIVREM8_ZEXT_HREG: { 2593 SDValue N0 = Node->getOperand(0); 2594 SDValue N1 = Node->getOperand(1); 2595 2596 bool isSigned = (Opcode == ISD::SDIVREM || 2597 Opcode == X86ISD::SDIVREM8_SEXT_HREG); 2598 if (!isSigned) { 2599 switch (NVT.SimpleTy) { 2600 default: llvm_unreachable("Unsupported VT!"); 2601 case MVT::i8: Opc = X86::DIV8r; MOpc = X86::DIV8m; break; 2602 case MVT::i16: Opc = X86::DIV16r; MOpc = X86::DIV16m; break; 2603 case MVT::i32: Opc = X86::DIV32r; MOpc = X86::DIV32m; break; 2604 case MVT::i64: Opc = X86::DIV64r; MOpc = X86::DIV64m; break; 2605 } 2606 } else { 2607 switch (NVT.SimpleTy) { 2608 default: llvm_unreachable("Unsupported VT!"); 2609 case MVT::i8: Opc = X86::IDIV8r; MOpc = X86::IDIV8m; break; 2610 case MVT::i16: Opc = X86::IDIV16r; MOpc = X86::IDIV16m; break; 2611 case MVT::i32: Opc = X86::IDIV32r; MOpc = X86::IDIV32m; break; 2612 case MVT::i64: Opc = X86::IDIV64r; MOpc = X86::IDIV64m; break; 2613 } 2614 } 2615 2616 unsigned LoReg, HiReg, ClrReg; 2617 unsigned SExtOpcode; 2618 switch (NVT.SimpleTy) { 2619 default: llvm_unreachable("Unsupported VT!"); 2620 case MVT::i8: 2621 LoReg = X86::AL; ClrReg = HiReg = X86::AH; 2622 SExtOpcode = X86::CBW; 2623 break; 2624 case MVT::i16: 2625 LoReg = X86::AX; HiReg = X86::DX; 2626 ClrReg = X86::DX; 2627 SExtOpcode = X86::CWD; 2628 break; 2629 case MVT::i32: 2630 LoReg = X86::EAX; ClrReg = HiReg = X86::EDX; 2631 SExtOpcode = X86::CDQ; 2632 break; 2633 case MVT::i64: 2634 LoReg = X86::RAX; ClrReg = HiReg = X86::RDX; 2635 SExtOpcode = X86::CQO; 2636 break; 2637 } 2638 2639 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 2640 bool foldedLoad = tryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 2641 bool signBitIsZero = CurDAG->SignBitIsZero(N0); 2642 2643 SDValue InFlag; 2644 if (NVT == MVT::i8 && (!isSigned || signBitIsZero)) { 2645 // Special case for div8, just use a move with zero extension to AX to 2646 // clear the upper 8 bits (AH). 2647 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Move, Chain; 2648 if (tryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) { 2649 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0) }; 2650 Move = 2651 SDValue(CurDAG->getMachineNode(X86::MOVZX32rm8, dl, MVT::i32, 2652 MVT::Other, Ops), 0); 2653 Chain = Move.getValue(1); 2654 ReplaceUses(N0.getValue(1), Chain); 2655 } else { 2656 Move = 2657 SDValue(CurDAG->getMachineNode(X86::MOVZX32rr8, dl, MVT::i32, N0),0); 2658 Chain = CurDAG->getEntryNode(); 2659 } 2660 Chain = CurDAG->getCopyToReg(Chain, dl, X86::EAX, Move, SDValue()); 2661 InFlag = Chain.getValue(1); 2662 } else { 2663 InFlag = 2664 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, 2665 LoReg, N0, SDValue()).getValue(1); 2666 if (isSigned && !signBitIsZero) { 2667 // Sign extend the low part into the high part. 2668 InFlag = 2669 SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InFlag),0); 2670 } else { 2671 // Zero out the high part, effectively zero extending the input. 2672 SDValue ClrNode = SDValue(CurDAG->getMachineNode(X86::MOV32r0, dl, NVT), 0); 2673 switch (NVT.SimpleTy) { 2674 case MVT::i16: 2675 ClrNode = 2676 SDValue(CurDAG->getMachineNode( 2677 TargetOpcode::EXTRACT_SUBREG, dl, MVT::i16, ClrNode, 2678 CurDAG->getTargetConstant(X86::sub_16bit, dl, 2679 MVT::i32)), 2680 0); 2681 break; 2682 case MVT::i32: 2683 break; 2684 case MVT::i64: 2685 ClrNode = 2686 SDValue(CurDAG->getMachineNode( 2687 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64, 2688 CurDAG->getTargetConstant(0, dl, MVT::i64), ClrNode, 2689 CurDAG->getTargetConstant(X86::sub_32bit, dl, 2690 MVT::i32)), 2691 0); 2692 break; 2693 default: 2694 llvm_unreachable("Unexpected division source"); 2695 } 2696 2697 InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg, 2698 ClrNode, InFlag).getValue(1); 2699 } 2700 } 2701 2702 if (foldedLoad) { 2703 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), 2704 InFlag }; 2705 SDNode *CNode = 2706 CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops); 2707 InFlag = SDValue(CNode, 1); 2708 // Update the chain. 2709 ReplaceUses(N1.getValue(1), SDValue(CNode, 0)); 2710 } else { 2711 InFlag = 2712 SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, N1, InFlag), 0); 2713 } 2714 2715 // Prevent use of AH in a REX instruction by explicitly copying it to 2716 // an ABCD_L register. 2717 // 2718 // The current assumption of the register allocator is that isel 2719 // won't generate explicit references to the GR8_ABCD_H registers. If 2720 // the allocator and/or the backend get enhanced to be more robust in 2721 // that regard, this can be, and should be, removed. 2722 if (HiReg == X86::AH && !SDValue(Node, 1).use_empty()) { 2723 SDValue AHCopy = CurDAG->getRegister(X86::AH, MVT::i8); 2724 unsigned AHExtOpcode = 2725 isSigned ? X86::MOVSX32_NOREXrr8 : X86::MOVZX32_NOREXrr8; 2726 2727 SDNode *RNode = CurDAG->getMachineNode(AHExtOpcode, dl, MVT::i32, 2728 MVT::Glue, AHCopy, InFlag); 2729 SDValue Result(RNode, 0); 2730 InFlag = SDValue(RNode, 1); 2731 2732 if (Opcode == X86ISD::UDIVREM8_ZEXT_HREG || 2733 Opcode == X86ISD::SDIVREM8_SEXT_HREG) { 2734 if (Node->getValueType(1) == MVT::i64) { 2735 // It's not possible to directly movsx AH to a 64bit register, because 2736 // the latter needs the REX prefix, but the former can't have it. 2737 assert(Opcode != X86ISD::SDIVREM8_SEXT_HREG && 2738 "Unexpected i64 sext of h-register"); 2739 Result = 2740 SDValue(CurDAG->getMachineNode( 2741 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64, 2742 CurDAG->getTargetConstant(0, dl, MVT::i64), Result, 2743 CurDAG->getTargetConstant(X86::sub_32bit, dl, 2744 MVT::i32)), 2745 0); 2746 } 2747 } else { 2748 Result = 2749 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result); 2750 } 2751 ReplaceUses(SDValue(Node, 1), Result); 2752 DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 2753 } 2754 // Copy the division (low) result, if it is needed. 2755 if (!SDValue(Node, 0).use_empty()) { 2756 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 2757 LoReg, NVT, InFlag); 2758 InFlag = Result.getValue(2); 2759 ReplaceUses(SDValue(Node, 0), Result); 2760 DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 2761 } 2762 // Copy the remainder (high) result, if it is needed. 2763 if (!SDValue(Node, 1).use_empty()) { 2764 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 2765 HiReg, NVT, InFlag); 2766 InFlag = Result.getValue(2); 2767 ReplaceUses(SDValue(Node, 1), Result); 2768 DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 2769 } 2770 return nullptr; 2771 } 2772 2773 case X86ISD::CMP: 2774 case X86ISD::SUB: { 2775 // Sometimes a SUB is used to perform comparison. 2776 if (Opcode == X86ISD::SUB && Node->hasAnyUseOfValue(0)) 2777 // This node is not a CMP. 2778 break; 2779 SDValue N0 = Node->getOperand(0); 2780 SDValue N1 = Node->getOperand(1); 2781 2782 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && 2783 hasNoSignedComparisonUses(Node)) 2784 N0 = N0.getOperand(0); 2785 2786 // Look for (X86cmp (and $op, $imm), 0) and see if we can convert it to 2787 // use a smaller encoding. 2788 // Look past the truncate if CMP is the only use of it. 2789 if ((N0.getNode()->getOpcode() == ISD::AND || 2790 (N0.getResNo() == 0 && N0.getNode()->getOpcode() == X86ISD::AND)) && 2791 N0.getNode()->hasOneUse() && 2792 N0.getValueType() != MVT::i8 && 2793 X86::isZeroNode(N1)) { 2794 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getNode()->getOperand(1)); 2795 if (!C) break; 2796 2797 // For example, convert "testl %eax, $8" to "testb %al, $8" 2798 if ((C->getZExtValue() & ~UINT64_C(0xff)) == 0 && 2799 (!(C->getZExtValue() & 0x80) || 2800 hasNoSignedComparisonUses(Node))) { 2801 SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), dl, MVT::i8); 2802 SDValue Reg = N0.getNode()->getOperand(0); 2803 2804 // On x86-32, only the ABCD registers have 8-bit subregisters. 2805 if (!Subtarget->is64Bit()) { 2806 const TargetRegisterClass *TRC; 2807 switch (N0.getSimpleValueType().SimpleTy) { 2808 case MVT::i32: TRC = &X86::GR32_ABCDRegClass; break; 2809 case MVT::i16: TRC = &X86::GR16_ABCDRegClass; break; 2810 default: llvm_unreachable("Unsupported TEST operand type!"); 2811 } 2812 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32); 2813 Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl, 2814 Reg.getValueType(), Reg, RC), 0); 2815 } 2816 2817 // Extract the l-register. 2818 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, 2819 MVT::i8, Reg); 2820 2821 // Emit a testb. 2822 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST8ri, dl, MVT::i32, 2823 Subreg, Imm); 2824 // Replace SUB|CMP with TEST, since SUB has two outputs while TEST has 2825 // one, do not call ReplaceAllUsesWith. 2826 ReplaceUses(SDValue(Node, (Opcode == X86ISD::SUB ? 1 : 0)), 2827 SDValue(NewNode, 0)); 2828 return nullptr; 2829 } 2830 2831 // For example, "testl %eax, $2048" to "testb %ah, $8". 2832 if ((C->getZExtValue() & ~UINT64_C(0xff00)) == 0 && 2833 (!(C->getZExtValue() & 0x8000) || 2834 hasNoSignedComparisonUses(Node))) { 2835 // Shift the immediate right by 8 bits. 2836 SDValue ShiftedImm = CurDAG->getTargetConstant(C->getZExtValue() >> 8, 2837 dl, MVT::i8); 2838 SDValue Reg = N0.getNode()->getOperand(0); 2839 2840 // Put the value in an ABCD register. 2841 const TargetRegisterClass *TRC; 2842 switch (N0.getSimpleValueType().SimpleTy) { 2843 case MVT::i64: TRC = &X86::GR64_ABCDRegClass; break; 2844 case MVT::i32: TRC = &X86::GR32_ABCDRegClass; break; 2845 case MVT::i16: TRC = &X86::GR16_ABCDRegClass; break; 2846 default: llvm_unreachable("Unsupported TEST operand type!"); 2847 } 2848 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32); 2849 Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl, 2850 Reg.getValueType(), Reg, RC), 0); 2851 2852 // Extract the h-register. 2853 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl, 2854 MVT::i8, Reg); 2855 2856 // Emit a testb. The EXTRACT_SUBREG becomes a COPY that can only 2857 // target GR8_NOREX registers, so make sure the register class is 2858 // forced. 2859 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST8ri_NOREX, dl, 2860 MVT::i32, Subreg, ShiftedImm); 2861 // Replace SUB|CMP with TEST, since SUB has two outputs while TEST has 2862 // one, do not call ReplaceAllUsesWith. 2863 ReplaceUses(SDValue(Node, (Opcode == X86ISD::SUB ? 1 : 0)), 2864 SDValue(NewNode, 0)); 2865 return nullptr; 2866 } 2867 2868 // For example, "testl %eax, $32776" to "testw %ax, $32776". 2869 if ((C->getZExtValue() & ~UINT64_C(0xffff)) == 0 && 2870 N0.getValueType() != MVT::i16 && 2871 (!(C->getZExtValue() & 0x8000) || 2872 hasNoSignedComparisonUses(Node))) { 2873 SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), dl, 2874 MVT::i16); 2875 SDValue Reg = N0.getNode()->getOperand(0); 2876 2877 // Extract the 16-bit subregister. 2878 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_16bit, dl, 2879 MVT::i16, Reg); 2880 2881 // Emit a testw. 2882 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST16ri, dl, MVT::i32, 2883 Subreg, Imm); 2884 // Replace SUB|CMP with TEST, since SUB has two outputs while TEST has 2885 // one, do not call ReplaceAllUsesWith. 2886 ReplaceUses(SDValue(Node, (Opcode == X86ISD::SUB ? 1 : 0)), 2887 SDValue(NewNode, 0)); 2888 return nullptr; 2889 } 2890 2891 // For example, "testq %rax, $268468232" to "testl %eax, $268468232". 2892 if ((C->getZExtValue() & ~UINT64_C(0xffffffff)) == 0 && 2893 N0.getValueType() == MVT::i64 && 2894 (!(C->getZExtValue() & 0x80000000) || 2895 hasNoSignedComparisonUses(Node))) { 2896 SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), dl, 2897 MVT::i32); 2898 SDValue Reg = N0.getNode()->getOperand(0); 2899 2900 // Extract the 32-bit subregister. 2901 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_32bit, dl, 2902 MVT::i32, Reg); 2903 2904 // Emit a testl. 2905 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST32ri, dl, MVT::i32, 2906 Subreg, Imm); 2907 // Replace SUB|CMP with TEST, since SUB has two outputs while TEST has 2908 // one, do not call ReplaceAllUsesWith. 2909 ReplaceUses(SDValue(Node, (Opcode == X86ISD::SUB ? 1 : 0)), 2910 SDValue(NewNode, 0)); 2911 return nullptr; 2912 } 2913 } 2914 break; 2915 } 2916 case ISD::STORE: { 2917 // Change a chain of {load; incr or dec; store} of the same value into 2918 // a simple increment or decrement through memory of that value, if the 2919 // uses of the modified value and its address are suitable. 2920 // The DEC64m tablegen pattern is currently not able to match the case where 2921 // the EFLAGS on the original DEC are used. (This also applies to 2922 // {INC,DEC}X{64,32,16,8}.) 2923 // We'll need to improve tablegen to allow flags to be transferred from a 2924 // node in the pattern to the result node. probably with a new keyword 2925 // for example, we have this 2926 // def DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst), "dec{q}\t$dst", 2927 // [(store (add (loadi64 addr:$dst), -1), addr:$dst), 2928 // (implicit EFLAGS)]>; 2929 // but maybe need something like this 2930 // def DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst), "dec{q}\t$dst", 2931 // [(store (add (loadi64 addr:$dst), -1), addr:$dst), 2932 // (transferrable EFLAGS)]>; 2933 2934 StoreSDNode *StoreNode = cast<StoreSDNode>(Node); 2935 SDValue StoredVal = StoreNode->getOperand(1); 2936 unsigned Opc = StoredVal->getOpcode(); 2937 2938 LoadSDNode *LoadNode = nullptr; 2939 SDValue InputChain; 2940 if (!isLoadIncOrDecStore(StoreNode, Opc, StoredVal, CurDAG, 2941 LoadNode, InputChain)) 2942 break; 2943 2944 SDValue Base, Scale, Index, Disp, Segment; 2945 if (!selectAddr(LoadNode, LoadNode->getBasePtr(), 2946 Base, Scale, Index, Disp, Segment)) 2947 break; 2948 2949 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(2); 2950 MemOp[0] = StoreNode->getMemOperand(); 2951 MemOp[1] = LoadNode->getMemOperand(); 2952 const SDValue Ops[] = { Base, Scale, Index, Disp, Segment, InputChain }; 2953 EVT LdVT = LoadNode->getMemoryVT(); 2954 unsigned newOpc = getFusedLdStOpcode(LdVT, Opc); 2955 MachineSDNode *Result = CurDAG->getMachineNode(newOpc, 2956 SDLoc(Node), 2957 MVT::i32, MVT::Other, Ops); 2958 Result->setMemRefs(MemOp, MemOp + 2); 2959 2960 ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1)); 2961 ReplaceUses(SDValue(StoredVal.getNode(), 1), SDValue(Result, 0)); 2962 2963 return Result; 2964 } 2965 } 2966 2967 SDNode *ResNode = SelectCode(Node); 2968 2969 DEBUG(dbgs() << "=> "; 2970 if (ResNode == nullptr || ResNode == Node) 2971 Node->dump(CurDAG); 2972 else 2973 ResNode->dump(CurDAG); 2974 dbgs() << '\n'); 2975 2976 return ResNode; 2977} 2978 2979bool X86DAGToDAGISel:: 2980SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, 2981 std::vector<SDValue> &OutOps) { 2982 SDValue Op0, Op1, Op2, Op3, Op4; 2983 switch (ConstraintID) { 2984 default: 2985 llvm_unreachable("Unexpected asm memory constraint"); 2986 case InlineAsm::Constraint_i: 2987 // FIXME: It seems strange that 'i' is needed here since it's supposed to 2988 // be an immediate and not a memory constraint. 2989 // Fallthrough. 2990 case InlineAsm::Constraint_o: // offsetable ?? 2991 case InlineAsm::Constraint_v: // not offsetable ?? 2992 case InlineAsm::Constraint_m: // memory 2993 case InlineAsm::Constraint_X: 2994 if (!selectAddr(nullptr, Op, Op0, Op1, Op2, Op3, Op4)) 2995 return true; 2996 break; 2997 } 2998 2999 OutOps.push_back(Op0); 3000 OutOps.push_back(Op1); 3001 OutOps.push_back(Op2); 3002 OutOps.push_back(Op3); 3003 OutOps.push_back(Op4); 3004 return false; 3005} 3006 3007/// This pass converts a legalized DAG into a X86-specific DAG, 3008/// ready for instruction scheduling. 3009FunctionPass *llvm::createX86ISelDag(X86TargetMachine &TM, 3010 CodeGenOpt::Level OptLevel) { 3011 return new X86DAGToDAGISel(TM, OptLevel); 3012} 3013