X86ISelDAGToDAG.cpp revision 221345
1193323Sed//===- X86ISelDAGToDAG.cpp - A DAG pattern matching inst selector for X86 -===// 2193323Sed// 3193323Sed// The LLVM Compiler Infrastructure 4193323Sed// 5193323Sed// This file is distributed under the University of Illinois Open Source 6193323Sed// License. See LICENSE.TXT for details. 7193323Sed// 8193323Sed//===----------------------------------------------------------------------===// 9193323Sed// 10193323Sed// This file defines a DAG pattern matching instruction selector for X86, 11193323Sed// converting from a legalized dag to a X86 dag. 12193323Sed// 13193323Sed//===----------------------------------------------------------------------===// 14193323Sed 15193323Sed#define DEBUG_TYPE "x86-isel" 16193323Sed#include "X86.h" 17193323Sed#include "X86InstrBuilder.h" 18193323Sed#include "X86MachineFunctionInfo.h" 19193323Sed#include "X86RegisterInfo.h" 20193323Sed#include "X86Subtarget.h" 21193323Sed#include "X86TargetMachine.h" 22193323Sed#include "llvm/Instructions.h" 23193323Sed#include "llvm/Intrinsics.h" 24193323Sed#include "llvm/Support/CFG.h" 25193323Sed#include "llvm/Type.h" 26193323Sed#include "llvm/CodeGen/MachineConstantPool.h" 27193323Sed#include "llvm/CodeGen/MachineFunction.h" 28193323Sed#include "llvm/CodeGen/MachineFrameInfo.h" 29193323Sed#include "llvm/CodeGen/MachineInstrBuilder.h" 30193323Sed#include "llvm/CodeGen/MachineRegisterInfo.h" 31193323Sed#include "llvm/CodeGen/SelectionDAGISel.h" 32193323Sed#include "llvm/Target/TargetMachine.h" 33193323Sed#include "llvm/Target/TargetOptions.h" 34193323Sed#include "llvm/Support/Debug.h" 35198090Srdivacky#include "llvm/Support/ErrorHandling.h" 36193323Sed#include "llvm/Support/MathExtras.h" 37198090Srdivacky#include "llvm/Support/raw_ostream.h" 38193323Sed#include "llvm/ADT/SmallPtrSet.h" 39193323Sed#include "llvm/ADT/Statistic.h" 40193323Sedusing namespace llvm; 41193323Sed 42193323SedSTATISTIC(NumLoadMoved, "Number of loads moved below TokenFactor"); 43193323Sed 44193323Sed//===----------------------------------------------------------------------===// 45193323Sed// Pattern Matcher Implementation 46193323Sed//===----------------------------------------------------------------------===// 47193323Sed 48193323Sednamespace { 49193323Sed /// X86ISelAddressMode - This corresponds to X86AddressMode, but uses 50193323Sed /// SDValue's instead of register numbers for the leaves of the matched 51193323Sed /// tree. 52193323Sed struct X86ISelAddressMode { 53193323Sed enum { 54193323Sed RegBase, 55193323Sed FrameIndexBase 56193323Sed } BaseType; 57193323Sed 58207618Srdivacky // This is really a union, discriminated by BaseType! 59207618Srdivacky SDValue Base_Reg; 60207618Srdivacky int Base_FrameIndex; 61193323Sed 62193323Sed unsigned Scale; 63193323Sed SDValue IndexReg; 64193323Sed int32_t Disp; 65193323Sed SDValue Segment; 66207618Srdivacky const GlobalValue *GV; 67207618Srdivacky const Constant *CP; 68207618Srdivacky const BlockAddress *BlockAddr; 69193323Sed const char *ES; 70193323Sed int JT; 71193323Sed unsigned Align; // CP alignment. 72195098Sed unsigned char SymbolFlags; // X86II::MO_* 73193323Sed 74193323Sed X86ISelAddressMode() 75207618Srdivacky : BaseType(RegBase), Base_FrameIndex(0), Scale(1), IndexReg(), Disp(0), 76198892Srdivacky Segment(), GV(0), CP(0), BlockAddr(0), ES(0), JT(-1), Align(0), 77198090Srdivacky SymbolFlags(X86II::MO_NO_FLAG) { 78193323Sed } 79193323Sed 80193323Sed bool hasSymbolicDisplacement() const { 81198892Srdivacky return GV != 0 || CP != 0 || ES != 0 || JT != -1 || BlockAddr != 0; 82193323Sed } 83195098Sed 84195098Sed bool hasBaseOrIndexReg() const { 85207618Srdivacky return IndexReg.getNode() != 0 || Base_Reg.getNode() != 0; 86195098Sed } 87195098Sed 88195098Sed /// isRIPRelative - Return true if this addressing mode is already RIP 89195098Sed /// relative. 90195098Sed bool isRIPRelative() const { 91195098Sed if (BaseType != RegBase) return false; 92195098Sed if (RegisterSDNode *RegNode = 93207618Srdivacky dyn_cast_or_null<RegisterSDNode>(Base_Reg.getNode())) 94195098Sed return RegNode->getReg() == X86::RIP; 95195098Sed return false; 96195098Sed } 97195098Sed 98195098Sed void setBaseReg(SDValue Reg) { 99195098Sed BaseType = RegBase; 100207618Srdivacky Base_Reg = Reg; 101195098Sed } 102193323Sed 103193323Sed void dump() { 104202375Srdivacky dbgs() << "X86ISelAddressMode " << this << '\n'; 105207618Srdivacky dbgs() << "Base_Reg "; 106207618Srdivacky if (Base_Reg.getNode() != 0) 107207618Srdivacky Base_Reg.getNode()->dump(); 108198090Srdivacky else 109202375Srdivacky dbgs() << "nul"; 110207618Srdivacky dbgs() << " Base.FrameIndex " << Base_FrameIndex << '\n' 111198090Srdivacky << " Scale" << Scale << '\n' 112198090Srdivacky << "IndexReg "; 113198090Srdivacky if (IndexReg.getNode() != 0) 114198090Srdivacky IndexReg.getNode()->dump(); 115198090Srdivacky else 116202375Srdivacky dbgs() << "nul"; 117202375Srdivacky dbgs() << " Disp " << Disp << '\n' 118198090Srdivacky << "GV "; 119198090Srdivacky if (GV) 120198090Srdivacky GV->dump(); 121198090Srdivacky else 122202375Srdivacky dbgs() << "nul"; 123202375Srdivacky dbgs() << " CP "; 124198090Srdivacky if (CP) 125198090Srdivacky CP->dump(); 126198090Srdivacky else 127202375Srdivacky dbgs() << "nul"; 128202375Srdivacky dbgs() << '\n' 129198090Srdivacky << "ES "; 130198090Srdivacky if (ES) 131202375Srdivacky dbgs() << ES; 132198090Srdivacky else 133202375Srdivacky dbgs() << "nul"; 134202375Srdivacky dbgs() << " JT" << JT << " Align" << Align << '\n'; 135193323Sed } 136193323Sed }; 137193323Sed} 138193323Sed 139193323Sednamespace { 140193323Sed //===--------------------------------------------------------------------===// 141193323Sed /// ISel - X86 specific code to select X86 machine instructions for 142193323Sed /// SelectionDAG operations. 143193323Sed /// 144198892Srdivacky class X86DAGToDAGISel : public SelectionDAGISel { 145193323Sed /// X86Lowering - This object fully describes how to lower LLVM code to an 146193323Sed /// X86-specific SelectionDAG. 147207618Srdivacky const X86TargetLowering &X86Lowering; 148193323Sed 149193323Sed /// Subtarget - Keep a pointer to the X86Subtarget around so that we can 150193323Sed /// make the right decision when generating code for different targets. 151193323Sed const X86Subtarget *Subtarget; 152193323Sed 153193323Sed /// OptForSize - If true, selector should try to optimize for code size 154193323Sed /// instead of performance. 155193323Sed bool OptForSize; 156193323Sed 157193323Sed public: 158193323Sed explicit X86DAGToDAGISel(X86TargetMachine &tm, CodeGenOpt::Level OptLevel) 159193323Sed : SelectionDAGISel(tm, OptLevel), 160193399Sed X86Lowering(*tm.getTargetLowering()), 161193399Sed Subtarget(&tm.getSubtarget<X86Subtarget>()), 162193323Sed OptForSize(false) {} 163193323Sed 164193323Sed virtual const char *getPassName() const { 165193323Sed return "X86 DAG->DAG Instruction Selection"; 166193323Sed } 167193323Sed 168207618Srdivacky virtual void EmitFunctionEntryCode(); 169193323Sed 170203954Srdivacky virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const; 171193323Sed 172204642Srdivacky virtual void PreprocessISelDAG(); 173203954Srdivacky 174212904Sdim inline bool immSext8(SDNode *N) const { 175212904Sdim return isInt<8>(cast<ConstantSDNode>(N)->getSExtValue()); 176212904Sdim } 177212904Sdim 178212904Sdim // i64immSExt32 predicate - True if the 64-bit immediate fits in a 32-bit 179212904Sdim // sign extended field. 180212904Sdim inline bool i64immSExt32(SDNode *N) const { 181212904Sdim uint64_t v = cast<ConstantSDNode>(N)->getZExtValue(); 182212904Sdim return (int64_t)v == (int32_t)v; 183212904Sdim } 184212904Sdim 185193323Sed// Include the pieces autogenerated from the target description. 186193323Sed#include "X86GenDAGISel.inc" 187193323Sed 188193323Sed private: 189202375Srdivacky SDNode *Select(SDNode *N); 190193323Sed SDNode *SelectAtomic64(SDNode *Node, unsigned Opc); 191198090Srdivacky SDNode *SelectAtomicLoadAdd(SDNode *Node, EVT NVT); 192193323Sed 193218893Sdim bool MatchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM); 194193323Sed bool MatchWrapper(SDValue N, X86ISelAddressMode &AM); 195198090Srdivacky bool MatchAddress(SDValue N, X86ISelAddressMode &AM); 196198090Srdivacky bool MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM, 197198090Srdivacky unsigned Depth); 198193323Sed bool MatchAddressBase(SDValue N, X86ISelAddressMode &AM); 199218893Sdim bool SelectAddr(SDNode *Parent, SDValue N, SDValue &Base, 200193323Sed SDValue &Scale, SDValue &Index, SDValue &Disp, 201193323Sed SDValue &Segment); 202218893Sdim bool SelectLEAAddr(SDValue N, SDValue &Base, 203210299Sed SDValue &Scale, SDValue &Index, SDValue &Disp, 204210299Sed SDValue &Segment); 205218893Sdim bool SelectTLSADDRAddr(SDValue N, SDValue &Base, 206210299Sed SDValue &Scale, SDValue &Index, SDValue &Disp, 207210299Sed SDValue &Segment); 208204642Srdivacky bool SelectScalarSSELoad(SDNode *Root, SDValue N, 209204642Srdivacky SDValue &Base, SDValue &Scale, 210193323Sed SDValue &Index, SDValue &Disp, 211193323Sed SDValue &Segment, 212204642Srdivacky SDValue &NodeWithChain); 213204642Srdivacky 214202375Srdivacky bool TryFoldLoad(SDNode *P, SDValue N, 215193323Sed SDValue &Base, SDValue &Scale, 216193323Sed SDValue &Index, SDValue &Disp, 217193323Sed SDValue &Segment); 218204642Srdivacky 219193323Sed /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 220193323Sed /// inline asm expressions. 221193323Sed virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, 222193323Sed char ConstraintCode, 223193323Sed std::vector<SDValue> &OutOps); 224193323Sed 225193323Sed void EmitSpecialCodeForMain(MachineBasicBlock *BB, MachineFrameInfo *MFI); 226193323Sed 227193323Sed inline void getAddressOperands(X86ISelAddressMode &AM, SDValue &Base, 228193323Sed SDValue &Scale, SDValue &Index, 229193323Sed SDValue &Disp, SDValue &Segment) { 230193323Sed Base = (AM.BaseType == X86ISelAddressMode::FrameIndexBase) ? 231207618Srdivacky CurDAG->getTargetFrameIndex(AM.Base_FrameIndex, TLI.getPointerTy()) : 232207618Srdivacky AM.Base_Reg; 233193323Sed Scale = getI8Imm(AM.Scale); 234193323Sed Index = AM.IndexReg; 235193323Sed // These are 32-bit even in 64-bit mode since RIP relative offset 236193323Sed // is 32-bit. 237193323Sed if (AM.GV) 238210299Sed Disp = CurDAG->getTargetGlobalAddress(AM.GV, DebugLoc(), 239210299Sed MVT::i32, AM.Disp, 240195098Sed AM.SymbolFlags); 241193323Sed else if (AM.CP) 242193323Sed Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, 243195098Sed AM.Align, AM.Disp, AM.SymbolFlags); 244193323Sed else if (AM.ES) 245195098Sed Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags); 246193323Sed else if (AM.JT != -1) 247195098Sed Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags); 248198892Srdivacky else if (AM.BlockAddr) 249199989Srdivacky Disp = CurDAG->getBlockAddress(AM.BlockAddr, MVT::i32, 250199989Srdivacky true, AM.SymbolFlags); 251193323Sed else 252193323Sed Disp = CurDAG->getTargetConstant(AM.Disp, MVT::i32); 253193323Sed 254193323Sed if (AM.Segment.getNode()) 255193323Sed Segment = AM.Segment; 256193323Sed else 257193323Sed Segment = CurDAG->getRegister(0, MVT::i32); 258193323Sed } 259193323Sed 260193323Sed /// getI8Imm - Return a target constant with the specified value, of type 261193323Sed /// i8. 262193323Sed inline SDValue getI8Imm(unsigned Imm) { 263193323Sed return CurDAG->getTargetConstant(Imm, MVT::i8); 264193323Sed } 265193323Sed 266193323Sed /// getI32Imm - Return a target constant with the specified value, of type 267193323Sed /// i32. 268193323Sed inline SDValue getI32Imm(unsigned Imm) { 269193323Sed return CurDAG->getTargetConstant(Imm, MVT::i32); 270193323Sed } 271193323Sed 272193323Sed /// getGlobalBaseReg - Return an SDNode that returns the value of 273193323Sed /// the global base register. Output instructions required to 274193323Sed /// initialize the global base register, if necessary. 275193323Sed /// 276193323Sed SDNode *getGlobalBaseReg(); 277193323Sed 278193399Sed /// getTargetMachine - Return a reference to the TargetMachine, casted 279193399Sed /// to the target-specific type. 280193399Sed const X86TargetMachine &getTargetMachine() { 281193399Sed return static_cast<const X86TargetMachine &>(TM); 282193399Sed } 283193399Sed 284193399Sed /// getInstrInfo - Return a reference to the TargetInstrInfo, casted 285193399Sed /// to the target-specific type. 286193399Sed const X86InstrInfo *getInstrInfo() { 287193399Sed return getTargetMachine().getInstrInfo(); 288193399Sed } 289193323Sed }; 290193323Sed} 291193323Sed 292193323Sed 293203954Srdivackybool 294203954SrdivackyX86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const { 295193323Sed if (OptLevel == CodeGenOpt::None) return false; 296193323Sed 297203954Srdivacky if (!N.hasOneUse()) 298203954Srdivacky return false; 299203954Srdivacky 300203954Srdivacky if (N.getOpcode() != ISD::LOAD) 301203954Srdivacky return true; 302203954Srdivacky 303203954Srdivacky // If N is a load, do additional profitability checks. 304203954Srdivacky if (U == Root) { 305193323Sed switch (U->getOpcode()) { 306193323Sed default: break; 307202375Srdivacky case X86ISD::ADD: 308202375Srdivacky case X86ISD::SUB: 309202375Srdivacky case X86ISD::AND: 310202375Srdivacky case X86ISD::XOR: 311202375Srdivacky case X86ISD::OR: 312193323Sed case ISD::ADD: 313193323Sed case ISD::ADDC: 314193323Sed case ISD::ADDE: 315193323Sed case ISD::AND: 316193323Sed case ISD::OR: 317193323Sed case ISD::XOR: { 318193323Sed SDValue Op1 = U->getOperand(1); 319193323Sed 320193323Sed // If the other operand is a 8-bit immediate we should fold the immediate 321193323Sed // instead. This reduces code size. 322193323Sed // e.g. 323193323Sed // movl 4(%esp), %eax 324193323Sed // addl $4, %eax 325193323Sed // vs. 326193323Sed // movl $4, %eax 327193323Sed // addl 4(%esp), %eax 328193323Sed // The former is 2 bytes shorter. In case where the increment is 1, then 329193323Sed // the saving can be 4 bytes (by using incl %eax). 330193323Sed if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Op1)) 331193323Sed if (Imm->getAPIntValue().isSignedIntN(8)) 332193323Sed return false; 333193323Sed 334193323Sed // If the other operand is a TLS address, we should fold it instead. 335193323Sed // This produces 336193323Sed // movl %gs:0, %eax 337193323Sed // leal i@NTPOFF(%eax), %eax 338193323Sed // instead of 339193323Sed // movl $i@NTPOFF, %eax 340193323Sed // addl %gs:0, %eax 341193323Sed // if the block also has an access to a second TLS address this will save 342193323Sed // a load. 343193323Sed // FIXME: This is probably also true for non TLS addresses. 344193323Sed if (Op1.getOpcode() == X86ISD::Wrapper) { 345193323Sed SDValue Val = Op1.getOperand(0); 346193323Sed if (Val.getOpcode() == ISD::TargetGlobalTLSAddress) 347193323Sed return false; 348193323Sed } 349193323Sed } 350193323Sed } 351203954Srdivacky } 352193323Sed 353203954Srdivacky return true; 354203954Srdivacky} 355203954Srdivacky 356205218Srdivacky/// MoveBelowCallOrigChain - Replace the original chain operand of the call with 357205218Srdivacky/// load's chain operand and move load below the call's chain operand. 358205218Srdivackystatic void MoveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load, 359205218Srdivacky SDValue Call, SDValue OrigChain) { 360193323Sed SmallVector<SDValue, 8> Ops; 361205218Srdivacky SDValue Chain = OrigChain.getOperand(0); 362193323Sed if (Chain.getNode() == Load.getNode()) 363193323Sed Ops.push_back(Load.getOperand(0)); 364193323Sed else { 365193323Sed assert(Chain.getOpcode() == ISD::TokenFactor && 366205218Srdivacky "Unexpected chain operand"); 367193323Sed for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) 368193323Sed if (Chain.getOperand(i).getNode() == Load.getNode()) 369193323Sed Ops.push_back(Load.getOperand(0)); 370193323Sed else 371193323Sed Ops.push_back(Chain.getOperand(i)); 372193323Sed SDValue NewChain = 373193323Sed CurDAG->getNode(ISD::TokenFactor, Load.getDebugLoc(), 374193323Sed MVT::Other, &Ops[0], Ops.size()); 375193323Sed Ops.clear(); 376193323Sed Ops.push_back(NewChain); 377193323Sed } 378205218Srdivacky for (unsigned i = 1, e = OrigChain.getNumOperands(); i != e; ++i) 379205218Srdivacky Ops.push_back(OrigChain.getOperand(i)); 380210299Sed CurDAG->UpdateNodeOperands(OrigChain.getNode(), &Ops[0], Ops.size()); 381210299Sed CurDAG->UpdateNodeOperands(Load.getNode(), Call.getOperand(0), 382193323Sed Load.getOperand(1), Load.getOperand(2)); 383193323Sed Ops.clear(); 384193323Sed Ops.push_back(SDValue(Load.getNode(), 1)); 385193323Sed for (unsigned i = 1, e = Call.getNode()->getNumOperands(); i != e; ++i) 386193323Sed Ops.push_back(Call.getOperand(i)); 387210299Sed CurDAG->UpdateNodeOperands(Call.getNode(), &Ops[0], Ops.size()); 388193323Sed} 389193323Sed 390193323Sed/// isCalleeLoad - Return true if call address is a load and it can be 391193323Sed/// moved below CALLSEQ_START and the chains leading up to the call. 392193323Sed/// Return the CALLSEQ_START by reference as a second output. 393205218Srdivacky/// In the case of a tail call, there isn't a callseq node between the call 394205218Srdivacky/// chain and the load. 395205218Srdivackystatic bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq) { 396193323Sed if (Callee.getNode() == Chain.getNode() || !Callee.hasOneUse()) 397193323Sed return false; 398193323Sed LoadSDNode *LD = dyn_cast<LoadSDNode>(Callee.getNode()); 399193323Sed if (!LD || 400193323Sed LD->isVolatile() || 401193323Sed LD->getAddressingMode() != ISD::UNINDEXED || 402193323Sed LD->getExtensionType() != ISD::NON_EXTLOAD) 403193323Sed return false; 404193323Sed 405193323Sed // Now let's find the callseq_start. 406205218Srdivacky while (HasCallSeq && Chain.getOpcode() != ISD::CALLSEQ_START) { 407193323Sed if (!Chain.hasOneUse()) 408193323Sed return false; 409193323Sed Chain = Chain.getOperand(0); 410193323Sed } 411205218Srdivacky 412205218Srdivacky if (!Chain.getNumOperands()) 413205218Srdivacky return false; 414193323Sed if (Chain.getOperand(0).getNode() == Callee.getNode()) 415193323Sed return true; 416193323Sed if (Chain.getOperand(0).getOpcode() == ISD::TokenFactor && 417198090Srdivacky Callee.getValue(1).isOperandOf(Chain.getOperand(0).getNode()) && 418198090Srdivacky Callee.getValue(1).hasOneUse()) 419193323Sed return true; 420193323Sed return false; 421193323Sed} 422193323Sed 423204642Srdivackyvoid X86DAGToDAGISel::PreprocessISelDAG() { 424204792Srdivacky // OptForSize is used in pattern predicates that isel is matching. 425204642Srdivacky OptForSize = MF->getFunction()->hasFnAttr(Attribute::OptimizeForSize); 426204642Srdivacky 427204642Srdivacky for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 428204642Srdivacky E = CurDAG->allnodes_end(); I != E; ) { 429204642Srdivacky SDNode *N = I++; // Preincrement iterator to avoid invalidation issues. 430193323Sed 431205218Srdivacky if (OptLevel != CodeGenOpt::None && 432205218Srdivacky (N->getOpcode() == X86ISD::CALL || 433205218Srdivacky N->getOpcode() == X86ISD::TC_RETURN)) { 434193323Sed /// Also try moving call address load from outside callseq_start to just 435193323Sed /// before the call to allow it to be folded. 436193323Sed /// 437193323Sed /// [Load chain] 438193323Sed /// ^ 439193323Sed /// | 440193323Sed /// [Load] 441193323Sed /// ^ ^ 442193323Sed /// | | 443193323Sed /// / \-- 444193323Sed /// / | 445193323Sed ///[CALLSEQ_START] | 446193323Sed /// ^ | 447193323Sed /// | | 448193323Sed /// [LOAD/C2Reg] | 449193323Sed /// | | 450193323Sed /// \ / 451193323Sed /// \ / 452193323Sed /// [CALL] 453205218Srdivacky bool HasCallSeq = N->getOpcode() == X86ISD::CALL; 454204642Srdivacky SDValue Chain = N->getOperand(0); 455204642Srdivacky SDValue Load = N->getOperand(1); 456205218Srdivacky if (!isCalleeLoad(Load, Chain, HasCallSeq)) 457193323Sed continue; 458205218Srdivacky MoveBelowOrigChain(CurDAG, Load, SDValue(N, 0), Chain); 459193323Sed ++NumLoadMoved; 460193323Sed continue; 461193323Sed } 462204642Srdivacky 463204642Srdivacky // Lower fpround and fpextend nodes that target the FP stack to be store and 464204642Srdivacky // load to the stack. This is a gross hack. We would like to simply mark 465204642Srdivacky // these as being illegal, but when we do that, legalize produces these when 466204642Srdivacky // it expands calls, then expands these in the same legalize pass. We would 467204642Srdivacky // like dag combine to be able to hack on these between the call expansion 468204642Srdivacky // and the node legalization. As such this pass basically does "really 469204642Srdivacky // late" legalization of these inline with the X86 isel pass. 470204642Srdivacky // FIXME: This should only happen when not compiled with -O0. 471193323Sed if (N->getOpcode() != ISD::FP_ROUND && N->getOpcode() != ISD::FP_EXTEND) 472193323Sed continue; 473193323Sed 474193323Sed // If the source and destination are SSE registers, then this is a legal 475193323Sed // conversion that should not be lowered. 476198090Srdivacky EVT SrcVT = N->getOperand(0).getValueType(); 477198090Srdivacky EVT DstVT = N->getValueType(0); 478193323Sed bool SrcIsSSE = X86Lowering.isScalarFPTypeInSSEReg(SrcVT); 479193323Sed bool DstIsSSE = X86Lowering.isScalarFPTypeInSSEReg(DstVT); 480193323Sed if (SrcIsSSE && DstIsSSE) 481193323Sed continue; 482193323Sed 483193323Sed if (!SrcIsSSE && !DstIsSSE) { 484193323Sed // If this is an FPStack extension, it is a noop. 485193323Sed if (N->getOpcode() == ISD::FP_EXTEND) 486193323Sed continue; 487193323Sed // If this is a value-preserving FPStack truncation, it is a noop. 488193323Sed if (N->getConstantOperandVal(1)) 489193323Sed continue; 490193323Sed } 491193323Sed 492193323Sed // Here we could have an FP stack truncation or an FPStack <-> SSE convert. 493193323Sed // FPStack has extload and truncstore. SSE can fold direct loads into other 494193323Sed // operations. Based on this, decide what we want to do. 495198090Srdivacky EVT MemVT; 496193323Sed if (N->getOpcode() == ISD::FP_ROUND) 497193323Sed MemVT = DstVT; // FP_ROUND must use DstVT, we can't do a 'trunc load'. 498193323Sed else 499193323Sed MemVT = SrcIsSSE ? SrcVT : DstVT; 500193323Sed 501193323Sed SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT); 502193323Sed DebugLoc dl = N->getDebugLoc(); 503193323Sed 504193323Sed // FIXME: optimize the case where the src/dest is a load or store? 505193323Sed SDValue Store = CurDAG->getTruncStore(CurDAG->getEntryNode(), dl, 506193323Sed N->getOperand(0), 507218893Sdim MemTmp, MachinePointerInfo(), MemVT, 508203954Srdivacky false, false, 0); 509218893Sdim SDValue Result = CurDAG->getExtLoad(ISD::EXTLOAD, dl, DstVT, Store, MemTmp, 510218893Sdim MachinePointerInfo(), 511218893Sdim MemVT, false, false, 0); 512193323Sed 513193323Sed // We're about to replace all uses of the FP_ROUND/FP_EXTEND with the 514193323Sed // extload we created. This will cause general havok on the dag because 515193323Sed // anything below the conversion could be folded into other existing nodes. 516193323Sed // To avoid invalidating 'I', back it up to the convert node. 517193323Sed --I; 518193323Sed CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 519193323Sed 520193323Sed // Now that we did that, the node is dead. Increment the iterator to the 521193323Sed // next node to process, then delete N. 522193323Sed ++I; 523193323Sed CurDAG->DeleteNode(N); 524193323Sed } 525193323Sed} 526193323Sed 527193323Sed 528193323Sed/// EmitSpecialCodeForMain - Emit any code that needs to be executed only in 529193323Sed/// the main function. 530193323Sedvoid X86DAGToDAGISel::EmitSpecialCodeForMain(MachineBasicBlock *BB, 531193323Sed MachineFrameInfo *MFI) { 532193323Sed const TargetInstrInfo *TII = TM.getInstrInfo(); 533218893Sdim if (Subtarget->isTargetCygMing()) { 534218893Sdim unsigned CallOp = 535218893Sdim Subtarget->is64Bit() ? X86::WINCALL64pcrel32 : X86::CALLpcrel32; 536206124Srdivacky BuildMI(BB, DebugLoc(), 537218893Sdim TII->get(CallOp)).addExternalSymbol("__main"); 538218893Sdim } 539193323Sed} 540193323Sed 541207618Srdivackyvoid X86DAGToDAGISel::EmitFunctionEntryCode() { 542193323Sed // If this is main, emit special code for main. 543207618Srdivacky if (const Function *Fn = MF->getFunction()) 544207618Srdivacky if (Fn->hasExternalLinkage() && Fn->getName() == "main") 545207618Srdivacky EmitSpecialCodeForMain(MF->begin(), MF->getFrameInfo()); 546193323Sed} 547193323Sed 548193323Sed 549218893Sdimbool X86DAGToDAGISel::MatchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM){ 550218893Sdim SDValue Address = N->getOperand(1); 551218893Sdim 552218893Sdim // load gs:0 -> GS segment register. 553218893Sdim // load fs:0 -> FS segment register. 554218893Sdim // 555193323Sed // This optimization is valid because the GNU TLS model defines that 556193323Sed // gs:0 (or fs:0 on X86-64) contains its own address. 557193323Sed // For more information see http://people.redhat.com/drepper/tls.pdf 558218893Sdim if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Address)) 559218893Sdim if (C->getSExtValue() == 0 && AM.Segment.getNode() == 0 && 560218893Sdim Subtarget->isTargetELF()) 561218893Sdim switch (N->getPointerInfo().getAddrSpace()) { 562218893Sdim case 256: 563218893Sdim AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); 564218893Sdim return false; 565218893Sdim case 257: 566218893Sdim AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); 567218893Sdim return false; 568218893Sdim } 569218893Sdim 570193323Sed return true; 571193323Sed} 572193323Sed 573195098Sed/// MatchWrapper - Try to match X86ISD::Wrapper and X86ISD::WrapperRIP nodes 574195098Sed/// into an addressing mode. These wrap things that will resolve down into a 575195098Sed/// symbol reference. If no match is possible, this returns true, otherwise it 576198090Srdivacky/// returns false. 577193323Sedbool X86DAGToDAGISel::MatchWrapper(SDValue N, X86ISelAddressMode &AM) { 578195098Sed // If the addressing mode already has a symbol as the displacement, we can 579195098Sed // never match another symbol. 580193323Sed if (AM.hasSymbolicDisplacement()) 581193323Sed return true; 582193323Sed 583193323Sed SDValue N0 = N.getOperand(0); 584198090Srdivacky CodeModel::Model M = TM.getCodeModel(); 585198090Srdivacky 586195098Sed // Handle X86-64 rip-relative addresses. We check this before checking direct 587195098Sed // folding because RIP is preferable to non-RIP accesses. 588195098Sed if (Subtarget->is64Bit() && 589195098Sed // Under X86-64 non-small code model, GV (and friends) are 64-bits, so 590195098Sed // they cannot be folded into immediate fields. 591195098Sed // FIXME: This can be improved for kernel and other models? 592198090Srdivacky (M == CodeModel::Small || M == CodeModel::Kernel) && 593195098Sed // Base and index reg must be 0 in order to use %rip as base and lowering 594195098Sed // must allow RIP. 595195098Sed !AM.hasBaseOrIndexReg() && N.getOpcode() == X86ISD::WrapperRIP) { 596195098Sed if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N0)) { 597195098Sed int64_t Offset = AM.Disp + G->getOffset(); 598198090Srdivacky if (!X86::isOffsetSuitableForCodeModel(Offset, M)) return true; 599195098Sed AM.GV = G->getGlobal(); 600195098Sed AM.Disp = Offset; 601195098Sed AM.SymbolFlags = G->getTargetFlags(); 602195098Sed } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) { 603195098Sed int64_t Offset = AM.Disp + CP->getOffset(); 604198090Srdivacky if (!X86::isOffsetSuitableForCodeModel(Offset, M)) return true; 605195098Sed AM.CP = CP->getConstVal(); 606195098Sed AM.Align = CP->getAlignment(); 607195098Sed AM.Disp = Offset; 608195098Sed AM.SymbolFlags = CP->getTargetFlags(); 609195098Sed } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) { 610195098Sed AM.ES = S->getSymbol(); 611195098Sed AM.SymbolFlags = S->getTargetFlags(); 612198892Srdivacky } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) { 613195098Sed AM.JT = J->getIndex(); 614195098Sed AM.SymbolFlags = J->getTargetFlags(); 615198892Srdivacky } else { 616198892Srdivacky AM.BlockAddr = cast<BlockAddressSDNode>(N0)->getBlockAddress(); 617199989Srdivacky AM.SymbolFlags = cast<BlockAddressSDNode>(N0)->getTargetFlags(); 618193323Sed } 619198090Srdivacky 620195098Sed if (N.getOpcode() == X86ISD::WrapperRIP) 621195098Sed AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64)); 622195098Sed return false; 623195098Sed } 624195098Sed 625195098Sed // Handle the case when globals fit in our immediate field: This is true for 626195098Sed // X86-32 always and X86-64 when in -static -mcmodel=small mode. In 64-bit 627195098Sed // mode, this results in a non-RIP-relative computation. 628195098Sed if (!Subtarget->is64Bit() || 629198090Srdivacky ((M == CodeModel::Small || M == CodeModel::Kernel) && 630195098Sed TM.getRelocationModel() == Reloc::Static)) { 631195098Sed if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N0)) { 632195098Sed AM.GV = G->getGlobal(); 633195098Sed AM.Disp += G->getOffset(); 634195098Sed AM.SymbolFlags = G->getTargetFlags(); 635195098Sed } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) { 636193323Sed AM.CP = CP->getConstVal(); 637193323Sed AM.Align = CP->getAlignment(); 638195098Sed AM.Disp += CP->getOffset(); 639195098Sed AM.SymbolFlags = CP->getTargetFlags(); 640195098Sed } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) { 641195098Sed AM.ES = S->getSymbol(); 642195098Sed AM.SymbolFlags = S->getTargetFlags(); 643198892Srdivacky } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) { 644195098Sed AM.JT = J->getIndex(); 645195098Sed AM.SymbolFlags = J->getTargetFlags(); 646198892Srdivacky } else { 647198892Srdivacky AM.BlockAddr = cast<BlockAddressSDNode>(N0)->getBlockAddress(); 648199989Srdivacky AM.SymbolFlags = cast<BlockAddressSDNode>(N0)->getTargetFlags(); 649193323Sed } 650193323Sed return false; 651193323Sed } 652193323Sed 653193323Sed return true; 654193323Sed} 655193323Sed 656193323Sed/// MatchAddress - Add the specified node to the specified addressing mode, 657193323Sed/// returning true if it cannot be done. This just pattern matches for the 658193323Sed/// addressing mode. 659198090Srdivackybool X86DAGToDAGISel::MatchAddress(SDValue N, X86ISelAddressMode &AM) { 660210299Sed if (MatchAddressRecursively(N, AM, 0)) 661198090Srdivacky return true; 662198090Srdivacky 663198090Srdivacky // Post-processing: Convert lea(,%reg,2) to lea(%reg,%reg), which has 664198090Srdivacky // a smaller encoding and avoids a scaled-index. 665198090Srdivacky if (AM.Scale == 2 && 666198090Srdivacky AM.BaseType == X86ISelAddressMode::RegBase && 667207618Srdivacky AM.Base_Reg.getNode() == 0) { 668207618Srdivacky AM.Base_Reg = AM.IndexReg; 669198090Srdivacky AM.Scale = 1; 670198090Srdivacky } 671198090Srdivacky 672198090Srdivacky // Post-processing: Convert foo to foo(%rip), even in non-PIC mode, 673198090Srdivacky // because it has a smaller encoding. 674198090Srdivacky // TODO: Which other code models can use this? 675198090Srdivacky if (TM.getCodeModel() == CodeModel::Small && 676198090Srdivacky Subtarget->is64Bit() && 677198090Srdivacky AM.Scale == 1 && 678198090Srdivacky AM.BaseType == X86ISelAddressMode::RegBase && 679207618Srdivacky AM.Base_Reg.getNode() == 0 && 680198090Srdivacky AM.IndexReg.getNode() == 0 && 681198090Srdivacky AM.SymbolFlags == X86II::MO_NO_FLAG && 682198090Srdivacky AM.hasSymbolicDisplacement()) 683207618Srdivacky AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64); 684198090Srdivacky 685198090Srdivacky return false; 686198090Srdivacky} 687198090Srdivacky 688198090Srdivackybool X86DAGToDAGISel::MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM, 689198090Srdivacky unsigned Depth) { 690193323Sed bool is64Bit = Subtarget->is64Bit(); 691193323Sed DebugLoc dl = N.getDebugLoc(); 692198090Srdivacky DEBUG({ 693202375Srdivacky dbgs() << "MatchAddress: "; 694198090Srdivacky AM.dump(); 695198090Srdivacky }); 696193323Sed // Limit recursion. 697193323Sed if (Depth > 5) 698193323Sed return MatchAddressBase(N, AM); 699198090Srdivacky 700198090Srdivacky CodeModel::Model M = TM.getCodeModel(); 701198090Srdivacky 702195098Sed // If this is already a %rip relative address, we can only merge immediates 703195098Sed // into it. Instead of handling this in every case, we handle it here. 704193323Sed // RIP relative addressing: %rip + 32-bit displacement! 705195098Sed if (AM.isRIPRelative()) { 706195098Sed // FIXME: JumpTable and ExternalSymbol address currently don't like 707195098Sed // displacements. It isn't very important, but this should be fixed for 708195098Sed // consistency. 709195098Sed if (!AM.ES && AM.JT != -1) return true; 710198090Srdivacky 711195098Sed if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N)) { 712195098Sed int64_t Val = AM.Disp + Cst->getSExtValue(); 713198090Srdivacky if (X86::isOffsetSuitableForCodeModel(Val, M, 714198090Srdivacky AM.hasSymbolicDisplacement())) { 715195098Sed AM.Disp = Val; 716193323Sed return false; 717193323Sed } 718193323Sed } 719193323Sed return true; 720193323Sed } 721193323Sed 722193323Sed switch (N.getOpcode()) { 723193323Sed default: break; 724193323Sed case ISD::Constant: { 725193323Sed uint64_t Val = cast<ConstantSDNode>(N)->getSExtValue(); 726198090Srdivacky if (!is64Bit || 727198090Srdivacky X86::isOffsetSuitableForCodeModel(AM.Disp + Val, M, 728198090Srdivacky AM.hasSymbolicDisplacement())) { 729193323Sed AM.Disp += Val; 730193323Sed return false; 731193323Sed } 732193323Sed break; 733193323Sed } 734193323Sed 735193323Sed case X86ISD::Wrapper: 736195098Sed case X86ISD::WrapperRIP: 737193323Sed if (!MatchWrapper(N, AM)) 738193323Sed return false; 739193323Sed break; 740193323Sed 741193323Sed case ISD::LOAD: 742218893Sdim if (!MatchLoadInAddress(cast<LoadSDNode>(N), AM)) 743193323Sed return false; 744193323Sed break; 745193323Sed 746193323Sed case ISD::FrameIndex: 747193323Sed if (AM.BaseType == X86ISelAddressMode::RegBase 748207618Srdivacky && AM.Base_Reg.getNode() == 0) { 749193323Sed AM.BaseType = X86ISelAddressMode::FrameIndexBase; 750207618Srdivacky AM.Base_FrameIndex = cast<FrameIndexSDNode>(N)->getIndex(); 751193323Sed return false; 752193323Sed } 753193323Sed break; 754193323Sed 755193323Sed case ISD::SHL: 756195098Sed if (AM.IndexReg.getNode() != 0 || AM.Scale != 1) 757193323Sed break; 758193323Sed 759193323Sed if (ConstantSDNode 760193323Sed *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1))) { 761193323Sed unsigned Val = CN->getZExtValue(); 762198090Srdivacky // Note that we handle x<<1 as (,x,2) rather than (x,x) here so 763198090Srdivacky // that the base operand remains free for further matching. If 764198090Srdivacky // the base doesn't end up getting used, a post-processing step 765198090Srdivacky // in MatchAddress turns (,x,2) into (x,x), which is cheaper. 766193323Sed if (Val == 1 || Val == 2 || Val == 3) { 767193323Sed AM.Scale = 1 << Val; 768193323Sed SDValue ShVal = N.getNode()->getOperand(0); 769193323Sed 770193323Sed // Okay, we know that we have a scale by now. However, if the scaled 771193323Sed // value is an add of something and a constant, we can fold the 772193323Sed // constant into the disp field here. 773218893Sdim if (CurDAG->isBaseWithConstantOffset(ShVal)) { 774193323Sed AM.IndexReg = ShVal.getNode()->getOperand(0); 775193323Sed ConstantSDNode *AddVal = 776193323Sed cast<ConstantSDNode>(ShVal.getNode()->getOperand(1)); 777193323Sed uint64_t Disp = AM.Disp + (AddVal->getSExtValue() << Val); 778198090Srdivacky if (!is64Bit || 779198090Srdivacky X86::isOffsetSuitableForCodeModel(Disp, M, 780198090Srdivacky AM.hasSymbolicDisplacement())) 781193323Sed AM.Disp = Disp; 782193323Sed else 783193323Sed AM.IndexReg = ShVal; 784193323Sed } else { 785193323Sed AM.IndexReg = ShVal; 786193323Sed } 787193323Sed return false; 788193323Sed } 789193323Sed break; 790193323Sed } 791193323Sed 792193323Sed case ISD::SMUL_LOHI: 793193323Sed case ISD::UMUL_LOHI: 794193323Sed // A mul_lohi where we need the low part can be folded as a plain multiply. 795193323Sed if (N.getResNo() != 0) break; 796193323Sed // FALL THROUGH 797193323Sed case ISD::MUL: 798193323Sed case X86ISD::MUL_IMM: 799193323Sed // X*[3,5,9] -> X+X*[2,4,8] 800193323Sed if (AM.BaseType == X86ISelAddressMode::RegBase && 801207618Srdivacky AM.Base_Reg.getNode() == 0 && 802195098Sed AM.IndexReg.getNode() == 0) { 803193323Sed if (ConstantSDNode 804193323Sed *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1))) 805193323Sed if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 || 806193323Sed CN->getZExtValue() == 9) { 807193323Sed AM.Scale = unsigned(CN->getZExtValue())-1; 808193323Sed 809193323Sed SDValue MulVal = N.getNode()->getOperand(0); 810193323Sed SDValue Reg; 811193323Sed 812193323Sed // Okay, we know that we have a scale by now. However, if the scaled 813193323Sed // value is an add of something and a constant, we can fold the 814193323Sed // constant into the disp field here. 815193323Sed if (MulVal.getNode()->getOpcode() == ISD::ADD && MulVal.hasOneUse() && 816193323Sed isa<ConstantSDNode>(MulVal.getNode()->getOperand(1))) { 817193323Sed Reg = MulVal.getNode()->getOperand(0); 818193323Sed ConstantSDNode *AddVal = 819193323Sed cast<ConstantSDNode>(MulVal.getNode()->getOperand(1)); 820193323Sed uint64_t Disp = AM.Disp + AddVal->getSExtValue() * 821193323Sed CN->getZExtValue(); 822198090Srdivacky if (!is64Bit || 823198090Srdivacky X86::isOffsetSuitableForCodeModel(Disp, M, 824198090Srdivacky AM.hasSymbolicDisplacement())) 825193323Sed AM.Disp = Disp; 826193323Sed else 827193323Sed Reg = N.getNode()->getOperand(0); 828193323Sed } else { 829193323Sed Reg = N.getNode()->getOperand(0); 830193323Sed } 831193323Sed 832207618Srdivacky AM.IndexReg = AM.Base_Reg = Reg; 833193323Sed return false; 834193323Sed } 835193323Sed } 836193323Sed break; 837193323Sed 838193323Sed case ISD::SUB: { 839193323Sed // Given A-B, if A can be completely folded into the address and 840193323Sed // the index field with the index field unused, use -B as the index. 841193323Sed // This is a win if a has multiple parts that can be folded into 842193323Sed // the address. Also, this saves a mov if the base register has 843193323Sed // other uses, since it avoids a two-address sub instruction, however 844193323Sed // it costs an additional mov if the index register has other uses. 845193323Sed 846210299Sed // Add an artificial use to this node so that we can keep track of 847210299Sed // it if it gets CSE'd with a different node. 848210299Sed HandleSDNode Handle(N); 849210299Sed 850193323Sed // Test if the LHS of the sub can be folded. 851193323Sed X86ISelAddressMode Backup = AM; 852210299Sed if (MatchAddressRecursively(N.getNode()->getOperand(0), AM, Depth+1)) { 853193323Sed AM = Backup; 854193323Sed break; 855193323Sed } 856193323Sed // Test if the index field is free for use. 857195098Sed if (AM.IndexReg.getNode() || AM.isRIPRelative()) { 858193323Sed AM = Backup; 859193323Sed break; 860193323Sed } 861205407Srdivacky 862193323Sed int Cost = 0; 863210299Sed SDValue RHS = Handle.getValue().getNode()->getOperand(1); 864193323Sed // If the RHS involves a register with multiple uses, this 865193323Sed // transformation incurs an extra mov, due to the neg instruction 866193323Sed // clobbering its operand. 867193323Sed if (!RHS.getNode()->hasOneUse() || 868193323Sed RHS.getNode()->getOpcode() == ISD::CopyFromReg || 869193323Sed RHS.getNode()->getOpcode() == ISD::TRUNCATE || 870193323Sed RHS.getNode()->getOpcode() == ISD::ANY_EXTEND || 871193323Sed (RHS.getNode()->getOpcode() == ISD::ZERO_EXTEND && 872193323Sed RHS.getNode()->getOperand(0).getValueType() == MVT::i32)) 873193323Sed ++Cost; 874193323Sed // If the base is a register with multiple uses, this 875193323Sed // transformation may save a mov. 876193323Sed if ((AM.BaseType == X86ISelAddressMode::RegBase && 877207618Srdivacky AM.Base_Reg.getNode() && 878207618Srdivacky !AM.Base_Reg.getNode()->hasOneUse()) || 879193323Sed AM.BaseType == X86ISelAddressMode::FrameIndexBase) 880193323Sed --Cost; 881193323Sed // If the folded LHS was interesting, this transformation saves 882193323Sed // address arithmetic. 883193323Sed if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) + 884193323Sed ((AM.Disp != 0) && (Backup.Disp == 0)) + 885193323Sed (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2) 886193323Sed --Cost; 887193323Sed // If it doesn't look like it may be an overall win, don't do it. 888193323Sed if (Cost >= 0) { 889193323Sed AM = Backup; 890193323Sed break; 891193323Sed } 892193323Sed 893193323Sed // Ok, the transformation is legal and appears profitable. Go for it. 894193323Sed SDValue Zero = CurDAG->getConstant(0, N.getValueType()); 895193323Sed SDValue Neg = CurDAG->getNode(ISD::SUB, dl, N.getValueType(), Zero, RHS); 896193323Sed AM.IndexReg = Neg; 897193323Sed AM.Scale = 1; 898193323Sed 899193323Sed // Insert the new nodes into the topological ordering. 900193323Sed if (Zero.getNode()->getNodeId() == -1 || 901193323Sed Zero.getNode()->getNodeId() > N.getNode()->getNodeId()) { 902193323Sed CurDAG->RepositionNode(N.getNode(), Zero.getNode()); 903193323Sed Zero.getNode()->setNodeId(N.getNode()->getNodeId()); 904193323Sed } 905193323Sed if (Neg.getNode()->getNodeId() == -1 || 906193323Sed Neg.getNode()->getNodeId() > N.getNode()->getNodeId()) { 907193323Sed CurDAG->RepositionNode(N.getNode(), Neg.getNode()); 908193323Sed Neg.getNode()->setNodeId(N.getNode()->getNodeId()); 909193323Sed } 910193323Sed return false; 911193323Sed } 912193323Sed 913193323Sed case ISD::ADD: { 914210299Sed // Add an artificial use to this node so that we can keep track of 915210299Sed // it if it gets CSE'd with a different node. 916210299Sed HandleSDNode Handle(N); 917210299Sed 918193323Sed X86ISelAddressMode Backup = AM; 919218893Sdim if (!MatchAddressRecursively(N.getOperand(0), AM, Depth+1) && 920218893Sdim !MatchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1)) 921210299Sed return false; 922210299Sed AM = Backup; 923218893Sdim 924205407Srdivacky // Try again after commuting the operands. 925218893Sdim if (!MatchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1)&& 926218893Sdim !MatchAddressRecursively(Handle.getValue().getOperand(0), AM, Depth+1)) 927210299Sed return false; 928193323Sed AM = Backup; 929193323Sed 930193323Sed // If we couldn't fold both operands into the address at the same time, 931193323Sed // see if we can just put each operand into a register and fold at least 932193323Sed // the add. 933193323Sed if (AM.BaseType == X86ISelAddressMode::RegBase && 934207618Srdivacky !AM.Base_Reg.getNode() && 935195098Sed !AM.IndexReg.getNode()) { 936218893Sdim N = Handle.getValue(); 937218893Sdim AM.Base_Reg = N.getOperand(0); 938218893Sdim AM.IndexReg = N.getOperand(1); 939193323Sed AM.Scale = 1; 940193323Sed return false; 941193323Sed } 942218893Sdim N = Handle.getValue(); 943193323Sed break; 944193323Sed } 945193323Sed 946193323Sed case ISD::OR: 947193323Sed // Handle "X | C" as "X + C" iff X is known to have C bits clear. 948218893Sdim if (CurDAG->isBaseWithConstantOffset(N)) { 949193323Sed X86ISelAddressMode Backup = AM; 950207618Srdivacky ConstantSDNode *CN = cast<ConstantSDNode>(N.getOperand(1)); 951193323Sed uint64_t Offset = CN->getSExtValue(); 952205407Srdivacky 953193323Sed // Start with the LHS as an addr mode. 954210299Sed if (!MatchAddressRecursively(N.getOperand(0), AM, Depth+1) && 955193323Sed // Address could not have picked a GV address for the displacement. 956193323Sed AM.GV == NULL && 957193323Sed // On x86-64, the resultant disp must fit in 32-bits. 958198090Srdivacky (!is64Bit || 959198090Srdivacky X86::isOffsetSuitableForCodeModel(AM.Disp + Offset, M, 960205407Srdivacky AM.hasSymbolicDisplacement()))) { 961193323Sed AM.Disp += Offset; 962193323Sed return false; 963193323Sed } 964193323Sed AM = Backup; 965193323Sed } 966193323Sed break; 967193323Sed 968193323Sed case ISD::AND: { 969193323Sed // Perform some heroic transforms on an and of a constant-count shift 970193323Sed // with a constant to enable use of the scaled offset field. 971193323Sed 972193323Sed SDValue Shift = N.getOperand(0); 973193323Sed if (Shift.getNumOperands() != 2) break; 974193323Sed 975193323Sed // Scale must not be used already. 976193323Sed if (AM.IndexReg.getNode() != 0 || AM.Scale != 1) break; 977193323Sed 978193323Sed SDValue X = Shift.getOperand(0); 979193323Sed ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N.getOperand(1)); 980193323Sed ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(Shift.getOperand(1)); 981193323Sed if (!C1 || !C2) break; 982193323Sed 983193323Sed // Handle "(X >> (8-C1)) & C2" as "(X >> 8) & 0xff)" if safe. This 984193323Sed // allows us to convert the shift and and into an h-register extract and 985193323Sed // a scaled index. 986193323Sed if (Shift.getOpcode() == ISD::SRL && Shift.hasOneUse()) { 987193323Sed unsigned ScaleLog = 8 - C1->getZExtValue(); 988193323Sed if (ScaleLog > 0 && ScaleLog < 4 && 989193323Sed C2->getZExtValue() == (UINT64_C(0xff) << ScaleLog)) { 990193323Sed SDValue Eight = CurDAG->getConstant(8, MVT::i8); 991193323Sed SDValue Mask = CurDAG->getConstant(0xff, N.getValueType()); 992193323Sed SDValue Srl = CurDAG->getNode(ISD::SRL, dl, N.getValueType(), 993193323Sed X, Eight); 994193323Sed SDValue And = CurDAG->getNode(ISD::AND, dl, N.getValueType(), 995193323Sed Srl, Mask); 996193323Sed SDValue ShlCount = CurDAG->getConstant(ScaleLog, MVT::i8); 997193323Sed SDValue Shl = CurDAG->getNode(ISD::SHL, dl, N.getValueType(), 998193323Sed And, ShlCount); 999193323Sed 1000193323Sed // Insert the new nodes into the topological ordering. 1001193323Sed if (Eight.getNode()->getNodeId() == -1 || 1002193323Sed Eight.getNode()->getNodeId() > X.getNode()->getNodeId()) { 1003193323Sed CurDAG->RepositionNode(X.getNode(), Eight.getNode()); 1004193323Sed Eight.getNode()->setNodeId(X.getNode()->getNodeId()); 1005193323Sed } 1006193323Sed if (Mask.getNode()->getNodeId() == -1 || 1007193323Sed Mask.getNode()->getNodeId() > X.getNode()->getNodeId()) { 1008193323Sed CurDAG->RepositionNode(X.getNode(), Mask.getNode()); 1009193323Sed Mask.getNode()->setNodeId(X.getNode()->getNodeId()); 1010193323Sed } 1011193323Sed if (Srl.getNode()->getNodeId() == -1 || 1012193323Sed Srl.getNode()->getNodeId() > Shift.getNode()->getNodeId()) { 1013193323Sed CurDAG->RepositionNode(Shift.getNode(), Srl.getNode()); 1014193323Sed Srl.getNode()->setNodeId(Shift.getNode()->getNodeId()); 1015193323Sed } 1016193323Sed if (And.getNode()->getNodeId() == -1 || 1017193323Sed And.getNode()->getNodeId() > N.getNode()->getNodeId()) { 1018193323Sed CurDAG->RepositionNode(N.getNode(), And.getNode()); 1019193323Sed And.getNode()->setNodeId(N.getNode()->getNodeId()); 1020193323Sed } 1021193323Sed if (ShlCount.getNode()->getNodeId() == -1 || 1022193323Sed ShlCount.getNode()->getNodeId() > X.getNode()->getNodeId()) { 1023193323Sed CurDAG->RepositionNode(X.getNode(), ShlCount.getNode()); 1024193323Sed ShlCount.getNode()->setNodeId(N.getNode()->getNodeId()); 1025193323Sed } 1026193323Sed if (Shl.getNode()->getNodeId() == -1 || 1027193323Sed Shl.getNode()->getNodeId() > N.getNode()->getNodeId()) { 1028193323Sed CurDAG->RepositionNode(N.getNode(), Shl.getNode()); 1029193323Sed Shl.getNode()->setNodeId(N.getNode()->getNodeId()); 1030193323Sed } 1031210299Sed CurDAG->ReplaceAllUsesWith(N, Shl); 1032193323Sed AM.IndexReg = And; 1033193323Sed AM.Scale = (1 << ScaleLog); 1034193323Sed return false; 1035193323Sed } 1036193323Sed } 1037193323Sed 1038193323Sed // Handle "(X << C1) & C2" as "(X & (C2>>C1)) << C1" if safe and if this 1039193323Sed // allows us to fold the shift into this addressing mode. 1040193323Sed if (Shift.getOpcode() != ISD::SHL) break; 1041193323Sed 1042193323Sed // Not likely to be profitable if either the AND or SHIFT node has more 1043193323Sed // than one use (unless all uses are for address computation). Besides, 1044193323Sed // isel mechanism requires their node ids to be reused. 1045193323Sed if (!N.hasOneUse() || !Shift.hasOneUse()) 1046193323Sed break; 1047193323Sed 1048193323Sed // Verify that the shift amount is something we can fold. 1049193323Sed unsigned ShiftCst = C1->getZExtValue(); 1050193323Sed if (ShiftCst != 1 && ShiftCst != 2 && ShiftCst != 3) 1051193323Sed break; 1052193323Sed 1053193323Sed // Get the new AND mask, this folds to a constant. 1054193323Sed SDValue NewANDMask = CurDAG->getNode(ISD::SRL, dl, N.getValueType(), 1055193323Sed SDValue(C2, 0), SDValue(C1, 0)); 1056193323Sed SDValue NewAND = CurDAG->getNode(ISD::AND, dl, N.getValueType(), X, 1057193323Sed NewANDMask); 1058193323Sed SDValue NewSHIFT = CurDAG->getNode(ISD::SHL, dl, N.getValueType(), 1059193323Sed NewAND, SDValue(C1, 0)); 1060193323Sed 1061193323Sed // Insert the new nodes into the topological ordering. 1062193323Sed if (C1->getNodeId() > X.getNode()->getNodeId()) { 1063193323Sed CurDAG->RepositionNode(X.getNode(), C1); 1064193323Sed C1->setNodeId(X.getNode()->getNodeId()); 1065193323Sed } 1066193323Sed if (NewANDMask.getNode()->getNodeId() == -1 || 1067193323Sed NewANDMask.getNode()->getNodeId() > X.getNode()->getNodeId()) { 1068193323Sed CurDAG->RepositionNode(X.getNode(), NewANDMask.getNode()); 1069193323Sed NewANDMask.getNode()->setNodeId(X.getNode()->getNodeId()); 1070193323Sed } 1071193323Sed if (NewAND.getNode()->getNodeId() == -1 || 1072193323Sed NewAND.getNode()->getNodeId() > Shift.getNode()->getNodeId()) { 1073193323Sed CurDAG->RepositionNode(Shift.getNode(), NewAND.getNode()); 1074193323Sed NewAND.getNode()->setNodeId(Shift.getNode()->getNodeId()); 1075193323Sed } 1076193323Sed if (NewSHIFT.getNode()->getNodeId() == -1 || 1077193323Sed NewSHIFT.getNode()->getNodeId() > N.getNode()->getNodeId()) { 1078193323Sed CurDAG->RepositionNode(N.getNode(), NewSHIFT.getNode()); 1079193323Sed NewSHIFT.getNode()->setNodeId(N.getNode()->getNodeId()); 1080193323Sed } 1081193323Sed 1082210299Sed CurDAG->ReplaceAllUsesWith(N, NewSHIFT); 1083193323Sed 1084193323Sed AM.Scale = 1 << ShiftCst; 1085193323Sed AM.IndexReg = NewAND; 1086193323Sed return false; 1087193323Sed } 1088193323Sed } 1089193323Sed 1090193323Sed return MatchAddressBase(N, AM); 1091193323Sed} 1092193323Sed 1093193323Sed/// MatchAddressBase - Helper for MatchAddress. Add the specified node to the 1094193323Sed/// specified addressing mode without any further recursion. 1095193323Sedbool X86DAGToDAGISel::MatchAddressBase(SDValue N, X86ISelAddressMode &AM) { 1096193323Sed // Is the base register already occupied? 1097207618Srdivacky if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) { 1098193323Sed // If so, check to see if the scale index register is set. 1099195098Sed if (AM.IndexReg.getNode() == 0) { 1100193323Sed AM.IndexReg = N; 1101193323Sed AM.Scale = 1; 1102193323Sed return false; 1103193323Sed } 1104193323Sed 1105193323Sed // Otherwise, we cannot select it. 1106193323Sed return true; 1107193323Sed } 1108193323Sed 1109193323Sed // Default, generate it as a register. 1110193323Sed AM.BaseType = X86ISelAddressMode::RegBase; 1111207618Srdivacky AM.Base_Reg = N; 1112193323Sed return false; 1113193323Sed} 1114193323Sed 1115193323Sed/// SelectAddr - returns true if it is able pattern match an addressing mode. 1116193323Sed/// It returns the operands which make up the maximal addressing mode it can 1117193323Sed/// match by reference. 1118218893Sdim/// 1119218893Sdim/// Parent is the parent node of the addr operand that is being matched. It 1120218893Sdim/// is always a load, store, atomic node, or null. It is only null when 1121218893Sdim/// checking memory operands for inline asm nodes. 1122218893Sdimbool X86DAGToDAGISel::SelectAddr(SDNode *Parent, SDValue N, SDValue &Base, 1123193323Sed SDValue &Scale, SDValue &Index, 1124193323Sed SDValue &Disp, SDValue &Segment) { 1125193323Sed X86ISelAddressMode AM; 1126218893Sdim 1127218893Sdim if (Parent && 1128218893Sdim // This list of opcodes are all the nodes that have an "addr:$ptr" operand 1129218893Sdim // that are not a MemSDNode, and thus don't have proper addrspace info. 1130218893Sdim Parent->getOpcode() != ISD::INTRINSIC_W_CHAIN && // unaligned loads, fixme 1131218893Sdim Parent->getOpcode() != ISD::INTRINSIC_VOID && // nontemporal stores 1132218893Sdim Parent->getOpcode() != X86ISD::TLSCALL) { // Fixme 1133218893Sdim unsigned AddrSpace = 1134218893Sdim cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace(); 1135218893Sdim // AddrSpace 256 -> GS, 257 -> FS. 1136218893Sdim if (AddrSpace == 256) 1137218893Sdim AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); 1138218893Sdim if (AddrSpace == 257) 1139218893Sdim AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); 1140218893Sdim } 1141218893Sdim 1142201360Srdivacky if (MatchAddress(N, AM)) 1143193323Sed return false; 1144193323Sed 1145198090Srdivacky EVT VT = N.getValueType(); 1146193323Sed if (AM.BaseType == X86ISelAddressMode::RegBase) { 1147207618Srdivacky if (!AM.Base_Reg.getNode()) 1148207618Srdivacky AM.Base_Reg = CurDAG->getRegister(0, VT); 1149193323Sed } 1150193323Sed 1151193323Sed if (!AM.IndexReg.getNode()) 1152193323Sed AM.IndexReg = CurDAG->getRegister(0, VT); 1153193323Sed 1154193323Sed getAddressOperands(AM, Base, Scale, Index, Disp, Segment); 1155193323Sed return true; 1156193323Sed} 1157193323Sed 1158193323Sed/// SelectScalarSSELoad - Match a scalar SSE load. In particular, we want to 1159193323Sed/// match a load whose top elements are either undef or zeros. The load flavor 1160193323Sed/// is derived from the type of N, which is either v4f32 or v2f64. 1161204642Srdivacky/// 1162204642Srdivacky/// We also return: 1163204642Srdivacky/// PatternChainNode: this is the matched node that has a chain input and 1164204642Srdivacky/// output. 1165204642Srdivackybool X86DAGToDAGISel::SelectScalarSSELoad(SDNode *Root, 1166193323Sed SDValue N, SDValue &Base, 1167193323Sed SDValue &Scale, SDValue &Index, 1168193323Sed SDValue &Disp, SDValue &Segment, 1169204642Srdivacky SDValue &PatternNodeWithChain) { 1170193323Sed if (N.getOpcode() == ISD::SCALAR_TO_VECTOR) { 1171204642Srdivacky PatternNodeWithChain = N.getOperand(0); 1172204642Srdivacky if (ISD::isNON_EXTLoad(PatternNodeWithChain.getNode()) && 1173204642Srdivacky PatternNodeWithChain.hasOneUse() && 1174204642Srdivacky IsProfitableToFold(N.getOperand(0), N.getNode(), Root) && 1175207618Srdivacky IsLegalToFold(N.getOperand(0), N.getNode(), Root, OptLevel)) { 1176204642Srdivacky LoadSDNode *LD = cast<LoadSDNode>(PatternNodeWithChain); 1177218893Sdim if (!SelectAddr(LD, LD->getBasePtr(), Base, Scale, Index, Disp, Segment)) 1178193323Sed return false; 1179193323Sed return true; 1180193323Sed } 1181193323Sed } 1182193323Sed 1183193323Sed // Also handle the case where we explicitly require zeros in the top 1184193323Sed // elements. This is a vector shuffle from the zero vector. 1185193323Sed if (N.getOpcode() == X86ISD::VZEXT_MOVL && N.getNode()->hasOneUse() && 1186193323Sed // Check to see if the top elements are all zeros (or bitcast of zeros). 1187193323Sed N.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR && 1188193323Sed N.getOperand(0).getNode()->hasOneUse() && 1189193323Sed ISD::isNON_EXTLoad(N.getOperand(0).getOperand(0).getNode()) && 1190204642Srdivacky N.getOperand(0).getOperand(0).hasOneUse() && 1191204642Srdivacky IsProfitableToFold(N.getOperand(0), N.getNode(), Root) && 1192207618Srdivacky IsLegalToFold(N.getOperand(0), N.getNode(), Root, OptLevel)) { 1193193323Sed // Okay, this is a zero extending load. Fold it. 1194193323Sed LoadSDNode *LD = cast<LoadSDNode>(N.getOperand(0).getOperand(0)); 1195218893Sdim if (!SelectAddr(LD, LD->getBasePtr(), Base, Scale, Index, Disp, Segment)) 1196193323Sed return false; 1197204642Srdivacky PatternNodeWithChain = SDValue(LD, 0); 1198193323Sed return true; 1199193323Sed } 1200193323Sed return false; 1201193323Sed} 1202193323Sed 1203193323Sed 1204193323Sed/// SelectLEAAddr - it calls SelectAddr and determines if the maximal addressing 1205193323Sed/// mode it matches can be cost effectively emitted as an LEA instruction. 1206218893Sdimbool X86DAGToDAGISel::SelectLEAAddr(SDValue N, 1207193323Sed SDValue &Base, SDValue &Scale, 1208210299Sed SDValue &Index, SDValue &Disp, 1209210299Sed SDValue &Segment) { 1210193323Sed X86ISelAddressMode AM; 1211193323Sed 1212193323Sed // Set AM.Segment to prevent MatchAddress from using one. LEA doesn't support 1213193323Sed // segments. 1214193323Sed SDValue Copy = AM.Segment; 1215193323Sed SDValue T = CurDAG->getRegister(0, MVT::i32); 1216193323Sed AM.Segment = T; 1217193323Sed if (MatchAddress(N, AM)) 1218193323Sed return false; 1219193323Sed assert (T == AM.Segment); 1220193323Sed AM.Segment = Copy; 1221193323Sed 1222198090Srdivacky EVT VT = N.getValueType(); 1223193323Sed unsigned Complexity = 0; 1224193323Sed if (AM.BaseType == X86ISelAddressMode::RegBase) 1225207618Srdivacky if (AM.Base_Reg.getNode()) 1226193323Sed Complexity = 1; 1227193323Sed else 1228207618Srdivacky AM.Base_Reg = CurDAG->getRegister(0, VT); 1229193323Sed else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase) 1230193323Sed Complexity = 4; 1231193323Sed 1232193323Sed if (AM.IndexReg.getNode()) 1233193323Sed Complexity++; 1234193323Sed else 1235193323Sed AM.IndexReg = CurDAG->getRegister(0, VT); 1236193323Sed 1237193323Sed // Don't match just leal(,%reg,2). It's cheaper to do addl %reg, %reg, or with 1238193323Sed // a simple shift. 1239193323Sed if (AM.Scale > 1) 1240193323Sed Complexity++; 1241193323Sed 1242193323Sed // FIXME: We are artificially lowering the criteria to turn ADD %reg, $GA 1243193323Sed // to a LEA. This is determined with some expermentation but is by no means 1244193323Sed // optimal (especially for code size consideration). LEA is nice because of 1245193323Sed // its three-address nature. Tweak the cost function again when we can run 1246193323Sed // convertToThreeAddress() at register allocation time. 1247193323Sed if (AM.hasSymbolicDisplacement()) { 1248193323Sed // For X86-64, we should always use lea to materialize RIP relative 1249193323Sed // addresses. 1250193323Sed if (Subtarget->is64Bit()) 1251193323Sed Complexity = 4; 1252193323Sed else 1253193323Sed Complexity += 2; 1254193323Sed } 1255193323Sed 1256207618Srdivacky if (AM.Disp && (AM.Base_Reg.getNode() || AM.IndexReg.getNode())) 1257193323Sed Complexity++; 1258193323Sed 1259198090Srdivacky // If it isn't worth using an LEA, reject it. 1260198090Srdivacky if (Complexity <= 2) 1261198090Srdivacky return false; 1262198090Srdivacky 1263198090Srdivacky getAddressOperands(AM, Base, Scale, Index, Disp, Segment); 1264198090Srdivacky return true; 1265193323Sed} 1266193323Sed 1267194612Sed/// SelectTLSADDRAddr - This is only run on TargetGlobalTLSAddress nodes. 1268218893Sdimbool X86DAGToDAGISel::SelectTLSADDRAddr(SDValue N, SDValue &Base, 1269194612Sed SDValue &Scale, SDValue &Index, 1270210299Sed SDValue &Disp, SDValue &Segment) { 1271194612Sed assert(N.getOpcode() == ISD::TargetGlobalTLSAddress); 1272194612Sed const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 1273210299Sed 1274194612Sed X86ISelAddressMode AM; 1275194612Sed AM.GV = GA->getGlobal(); 1276194612Sed AM.Disp += GA->getOffset(); 1277207618Srdivacky AM.Base_Reg = CurDAG->getRegister(0, N.getValueType()); 1278195098Sed AM.SymbolFlags = GA->getTargetFlags(); 1279195098Sed 1280194612Sed if (N.getValueType() == MVT::i32) { 1281194612Sed AM.Scale = 1; 1282194612Sed AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32); 1283194612Sed } else { 1284194612Sed AM.IndexReg = CurDAG->getRegister(0, MVT::i64); 1285194612Sed } 1286194612Sed 1287194612Sed getAddressOperands(AM, Base, Scale, Index, Disp, Segment); 1288194612Sed return true; 1289194612Sed} 1290194612Sed 1291194612Sed 1292202375Srdivackybool X86DAGToDAGISel::TryFoldLoad(SDNode *P, SDValue N, 1293193323Sed SDValue &Base, SDValue &Scale, 1294193323Sed SDValue &Index, SDValue &Disp, 1295193323Sed SDValue &Segment) { 1296204642Srdivacky if (!ISD::isNON_EXTLoad(N.getNode()) || 1297204642Srdivacky !IsProfitableToFold(N, P, P) || 1298207618Srdivacky !IsLegalToFold(N, P, P, OptLevel)) 1299204642Srdivacky return false; 1300204642Srdivacky 1301218893Sdim return SelectAddr(N.getNode(), 1302218893Sdim N.getOperand(1), Base, Scale, Index, Disp, Segment); 1303193323Sed} 1304193323Sed 1305193323Sed/// getGlobalBaseReg - Return an SDNode that returns the value of 1306193323Sed/// the global base register. Output instructions required to 1307193323Sed/// initialize the global base register, if necessary. 1308193323Sed/// 1309193323SedSDNode *X86DAGToDAGISel::getGlobalBaseReg() { 1310193399Sed unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF); 1311193323Sed return CurDAG->getRegister(GlobalBaseReg, TLI.getPointerTy()).getNode(); 1312193323Sed} 1313193323Sed 1314193323SedSDNode *X86DAGToDAGISel::SelectAtomic64(SDNode *Node, unsigned Opc) { 1315193323Sed SDValue Chain = Node->getOperand(0); 1316193323Sed SDValue In1 = Node->getOperand(1); 1317193323Sed SDValue In2L = Node->getOperand(2); 1318193323Sed SDValue In2H = Node->getOperand(3); 1319193323Sed SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 1320218893Sdim if (!SelectAddr(Node, In1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) 1321193323Sed return NULL; 1322198090Srdivacky MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1323198090Srdivacky MemOp[0] = cast<MemSDNode>(Node)->getMemOperand(); 1324198090Srdivacky const SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, In2L, In2H, Chain}; 1325198090Srdivacky SDNode *ResNode = CurDAG->getMachineNode(Opc, Node->getDebugLoc(), 1326198090Srdivacky MVT::i32, MVT::i32, MVT::Other, Ops, 1327198090Srdivacky array_lengthof(Ops)); 1328198090Srdivacky cast<MachineSDNode>(ResNode)->setMemRefs(MemOp, MemOp + 1); 1329198090Srdivacky return ResNode; 1330193323Sed} 1331193323Sed 1332198090SrdivackySDNode *X86DAGToDAGISel::SelectAtomicLoadAdd(SDNode *Node, EVT NVT) { 1333198090Srdivacky if (Node->hasAnyUseOfValue(0)) 1334198090Srdivacky return 0; 1335198090Srdivacky 1336198090Srdivacky // Optimize common patterns for __sync_add_and_fetch and 1337198090Srdivacky // __sync_sub_and_fetch where the result is not used. This allows us 1338198090Srdivacky // to use "lock" version of add, sub, inc, dec instructions. 1339198090Srdivacky // FIXME: Do not use special instructions but instead add the "lock" 1340198090Srdivacky // prefix to the target node somehow. The extra information will then be 1341198090Srdivacky // transferred to machine instruction and it denotes the prefix. 1342198090Srdivacky SDValue Chain = Node->getOperand(0); 1343198090Srdivacky SDValue Ptr = Node->getOperand(1); 1344198090Srdivacky SDValue Val = Node->getOperand(2); 1345198090Srdivacky SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 1346218893Sdim if (!SelectAddr(Node, Ptr, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) 1347198090Srdivacky return 0; 1348198090Srdivacky 1349198090Srdivacky bool isInc = false, isDec = false, isSub = false, isCN = false; 1350198090Srdivacky ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val); 1351198090Srdivacky if (CN) { 1352198090Srdivacky isCN = true; 1353198090Srdivacky int64_t CNVal = CN->getSExtValue(); 1354198090Srdivacky if (CNVal == 1) 1355198090Srdivacky isInc = true; 1356198090Srdivacky else if (CNVal == -1) 1357198090Srdivacky isDec = true; 1358198090Srdivacky else if (CNVal >= 0) 1359198090Srdivacky Val = CurDAG->getTargetConstant(CNVal, NVT); 1360198090Srdivacky else { 1361198090Srdivacky isSub = true; 1362198090Srdivacky Val = CurDAG->getTargetConstant(-CNVal, NVT); 1363198090Srdivacky } 1364198090Srdivacky } else if (Val.hasOneUse() && 1365198090Srdivacky Val.getOpcode() == ISD::SUB && 1366198090Srdivacky X86::isZeroNode(Val.getOperand(0))) { 1367198090Srdivacky isSub = true; 1368198090Srdivacky Val = Val.getOperand(1); 1369198090Srdivacky } 1370198090Srdivacky 1371198090Srdivacky unsigned Opc = 0; 1372198090Srdivacky switch (NVT.getSimpleVT().SimpleTy) { 1373198090Srdivacky default: return 0; 1374198090Srdivacky case MVT::i8: 1375198090Srdivacky if (isInc) 1376198090Srdivacky Opc = X86::LOCK_INC8m; 1377198090Srdivacky else if (isDec) 1378198090Srdivacky Opc = X86::LOCK_DEC8m; 1379198090Srdivacky else if (isSub) { 1380198090Srdivacky if (isCN) 1381198090Srdivacky Opc = X86::LOCK_SUB8mi; 1382198090Srdivacky else 1383198090Srdivacky Opc = X86::LOCK_SUB8mr; 1384198090Srdivacky } else { 1385198090Srdivacky if (isCN) 1386198090Srdivacky Opc = X86::LOCK_ADD8mi; 1387198090Srdivacky else 1388198090Srdivacky Opc = X86::LOCK_ADD8mr; 1389198090Srdivacky } 1390198090Srdivacky break; 1391198090Srdivacky case MVT::i16: 1392198090Srdivacky if (isInc) 1393198090Srdivacky Opc = X86::LOCK_INC16m; 1394198090Srdivacky else if (isDec) 1395198090Srdivacky Opc = X86::LOCK_DEC16m; 1396198090Srdivacky else if (isSub) { 1397198090Srdivacky if (isCN) { 1398212904Sdim if (immSext8(Val.getNode())) 1399198090Srdivacky Opc = X86::LOCK_SUB16mi8; 1400198090Srdivacky else 1401198090Srdivacky Opc = X86::LOCK_SUB16mi; 1402198090Srdivacky } else 1403198090Srdivacky Opc = X86::LOCK_SUB16mr; 1404198090Srdivacky } else { 1405198090Srdivacky if (isCN) { 1406212904Sdim if (immSext8(Val.getNode())) 1407198090Srdivacky Opc = X86::LOCK_ADD16mi8; 1408198090Srdivacky else 1409198090Srdivacky Opc = X86::LOCK_ADD16mi; 1410198090Srdivacky } else 1411198090Srdivacky Opc = X86::LOCK_ADD16mr; 1412198090Srdivacky } 1413198090Srdivacky break; 1414198090Srdivacky case MVT::i32: 1415198090Srdivacky if (isInc) 1416198090Srdivacky Opc = X86::LOCK_INC32m; 1417198090Srdivacky else if (isDec) 1418198090Srdivacky Opc = X86::LOCK_DEC32m; 1419198090Srdivacky else if (isSub) { 1420198090Srdivacky if (isCN) { 1421212904Sdim if (immSext8(Val.getNode())) 1422198090Srdivacky Opc = X86::LOCK_SUB32mi8; 1423198090Srdivacky else 1424198090Srdivacky Opc = X86::LOCK_SUB32mi; 1425198090Srdivacky } else 1426198090Srdivacky Opc = X86::LOCK_SUB32mr; 1427198090Srdivacky } else { 1428198090Srdivacky if (isCN) { 1429212904Sdim if (immSext8(Val.getNode())) 1430198090Srdivacky Opc = X86::LOCK_ADD32mi8; 1431198090Srdivacky else 1432198090Srdivacky Opc = X86::LOCK_ADD32mi; 1433198090Srdivacky } else 1434198090Srdivacky Opc = X86::LOCK_ADD32mr; 1435198090Srdivacky } 1436198090Srdivacky break; 1437198090Srdivacky case MVT::i64: 1438198090Srdivacky if (isInc) 1439198090Srdivacky Opc = X86::LOCK_INC64m; 1440198090Srdivacky else if (isDec) 1441198090Srdivacky Opc = X86::LOCK_DEC64m; 1442198090Srdivacky else if (isSub) { 1443198090Srdivacky Opc = X86::LOCK_SUB64mr; 1444198090Srdivacky if (isCN) { 1445212904Sdim if (immSext8(Val.getNode())) 1446198090Srdivacky Opc = X86::LOCK_SUB64mi8; 1447212904Sdim else if (i64immSExt32(Val.getNode())) 1448198090Srdivacky Opc = X86::LOCK_SUB64mi32; 1449198090Srdivacky } 1450198090Srdivacky } else { 1451198090Srdivacky Opc = X86::LOCK_ADD64mr; 1452198090Srdivacky if (isCN) { 1453212904Sdim if (immSext8(Val.getNode())) 1454198090Srdivacky Opc = X86::LOCK_ADD64mi8; 1455212904Sdim else if (i64immSExt32(Val.getNode())) 1456198090Srdivacky Opc = X86::LOCK_ADD64mi32; 1457198090Srdivacky } 1458198090Srdivacky } 1459198090Srdivacky break; 1460198090Srdivacky } 1461198090Srdivacky 1462198090Srdivacky DebugLoc dl = Node->getDebugLoc(); 1463203954Srdivacky SDValue Undef = SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, 1464198090Srdivacky dl, NVT), 0); 1465198090Srdivacky MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1466198090Srdivacky MemOp[0] = cast<MemSDNode>(Node)->getMemOperand(); 1467198090Srdivacky if (isInc || isDec) { 1468198090Srdivacky SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain }; 1469198090Srdivacky SDValue Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 6), 0); 1470198090Srdivacky cast<MachineSDNode>(Ret)->setMemRefs(MemOp, MemOp + 1); 1471198090Srdivacky SDValue RetVals[] = { Undef, Ret }; 1472198090Srdivacky return CurDAG->getMergeValues(RetVals, 2, dl).getNode(); 1473198090Srdivacky } else { 1474198090Srdivacky SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Val, Chain }; 1475198090Srdivacky SDValue Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 7), 0); 1476198090Srdivacky cast<MachineSDNode>(Ret)->setMemRefs(MemOp, MemOp + 1); 1477198090Srdivacky SDValue RetVals[] = { Undef, Ret }; 1478198090Srdivacky return CurDAG->getMergeValues(RetVals, 2, dl).getNode(); 1479198090Srdivacky } 1480198090Srdivacky} 1481198090Srdivacky 1482198090Srdivacky/// HasNoSignedComparisonUses - Test whether the given X86ISD::CMP node has 1483198090Srdivacky/// any uses which require the SF or OF bits to be accurate. 1484198090Srdivackystatic bool HasNoSignedComparisonUses(SDNode *N) { 1485198090Srdivacky // Examine each user of the node. 1486198090Srdivacky for (SDNode::use_iterator UI = N->use_begin(), 1487198090Srdivacky UE = N->use_end(); UI != UE; ++UI) { 1488198090Srdivacky // Only examine CopyToReg uses. 1489198090Srdivacky if (UI->getOpcode() != ISD::CopyToReg) 1490198090Srdivacky return false; 1491198090Srdivacky // Only examine CopyToReg uses that copy to EFLAGS. 1492198090Srdivacky if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != 1493198090Srdivacky X86::EFLAGS) 1494198090Srdivacky return false; 1495198090Srdivacky // Examine each user of the CopyToReg use. 1496198090Srdivacky for (SDNode::use_iterator FlagUI = UI->use_begin(), 1497198090Srdivacky FlagUE = UI->use_end(); FlagUI != FlagUE; ++FlagUI) { 1498198090Srdivacky // Only examine the Flag result. 1499198090Srdivacky if (FlagUI.getUse().getResNo() != 1) continue; 1500198090Srdivacky // Anything unusual: assume conservatively. 1501198090Srdivacky if (!FlagUI->isMachineOpcode()) return false; 1502198090Srdivacky // Examine the opcode of the user. 1503198090Srdivacky switch (FlagUI->getMachineOpcode()) { 1504198090Srdivacky // These comparisons don't treat the most significant bit specially. 1505198090Srdivacky case X86::SETAr: case X86::SETAEr: case X86::SETBr: case X86::SETBEr: 1506198090Srdivacky case X86::SETEr: case X86::SETNEr: case X86::SETPr: case X86::SETNPr: 1507198090Srdivacky case X86::SETAm: case X86::SETAEm: case X86::SETBm: case X86::SETBEm: 1508198090Srdivacky case X86::SETEm: case X86::SETNEm: case X86::SETPm: case X86::SETNPm: 1509203954Srdivacky case X86::JA_4: case X86::JAE_4: case X86::JB_4: case X86::JBE_4: 1510203954Srdivacky case X86::JE_4: case X86::JNE_4: case X86::JP_4: case X86::JNP_4: 1511198090Srdivacky case X86::CMOVA16rr: case X86::CMOVA16rm: 1512198090Srdivacky case X86::CMOVA32rr: case X86::CMOVA32rm: 1513198090Srdivacky case X86::CMOVA64rr: case X86::CMOVA64rm: 1514198090Srdivacky case X86::CMOVAE16rr: case X86::CMOVAE16rm: 1515198090Srdivacky case X86::CMOVAE32rr: case X86::CMOVAE32rm: 1516198090Srdivacky case X86::CMOVAE64rr: case X86::CMOVAE64rm: 1517198090Srdivacky case X86::CMOVB16rr: case X86::CMOVB16rm: 1518198090Srdivacky case X86::CMOVB32rr: case X86::CMOVB32rm: 1519198090Srdivacky case X86::CMOVB64rr: case X86::CMOVB64rm: 1520198090Srdivacky case X86::CMOVBE16rr: case X86::CMOVBE16rm: 1521198090Srdivacky case X86::CMOVBE32rr: case X86::CMOVBE32rm: 1522198090Srdivacky case X86::CMOVBE64rr: case X86::CMOVBE64rm: 1523198090Srdivacky case X86::CMOVE16rr: case X86::CMOVE16rm: 1524198090Srdivacky case X86::CMOVE32rr: case X86::CMOVE32rm: 1525198090Srdivacky case X86::CMOVE64rr: case X86::CMOVE64rm: 1526198090Srdivacky case X86::CMOVNE16rr: case X86::CMOVNE16rm: 1527198090Srdivacky case X86::CMOVNE32rr: case X86::CMOVNE32rm: 1528198090Srdivacky case X86::CMOVNE64rr: case X86::CMOVNE64rm: 1529198090Srdivacky case X86::CMOVNP16rr: case X86::CMOVNP16rm: 1530198090Srdivacky case X86::CMOVNP32rr: case X86::CMOVNP32rm: 1531198090Srdivacky case X86::CMOVNP64rr: case X86::CMOVNP64rm: 1532198090Srdivacky case X86::CMOVP16rr: case X86::CMOVP16rm: 1533198090Srdivacky case X86::CMOVP32rr: case X86::CMOVP32rm: 1534198090Srdivacky case X86::CMOVP64rr: case X86::CMOVP64rm: 1535198090Srdivacky continue; 1536198090Srdivacky // Anything else: assume conservatively. 1537198090Srdivacky default: return false; 1538198090Srdivacky } 1539198090Srdivacky } 1540198090Srdivacky } 1541198090Srdivacky return true; 1542198090Srdivacky} 1543198090Srdivacky 1544202375SrdivackySDNode *X86DAGToDAGISel::Select(SDNode *Node) { 1545198090Srdivacky EVT NVT = Node->getValueType(0); 1546193323Sed unsigned Opc, MOpc; 1547193323Sed unsigned Opcode = Node->getOpcode(); 1548193323Sed DebugLoc dl = Node->getDebugLoc(); 1549193323Sed 1550204642Srdivacky DEBUG(dbgs() << "Selecting: "; Node->dump(CurDAG); dbgs() << '\n'); 1551193323Sed 1552193323Sed if (Node->isMachineOpcode()) { 1553204642Srdivacky DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << '\n'); 1554193323Sed return NULL; // Already selected. 1555193323Sed } 1556193323Sed 1557193323Sed switch (Opcode) { 1558198090Srdivacky default: break; 1559198090Srdivacky case X86ISD::GlobalBaseReg: 1560198090Srdivacky return getGlobalBaseReg(); 1561193323Sed 1562198090Srdivacky case X86ISD::ATOMOR64_DAG: 1563198090Srdivacky return SelectAtomic64(Node, X86::ATOMOR6432); 1564198090Srdivacky case X86ISD::ATOMXOR64_DAG: 1565198090Srdivacky return SelectAtomic64(Node, X86::ATOMXOR6432); 1566198090Srdivacky case X86ISD::ATOMADD64_DAG: 1567198090Srdivacky return SelectAtomic64(Node, X86::ATOMADD6432); 1568198090Srdivacky case X86ISD::ATOMSUB64_DAG: 1569198090Srdivacky return SelectAtomic64(Node, X86::ATOMSUB6432); 1570198090Srdivacky case X86ISD::ATOMNAND64_DAG: 1571198090Srdivacky return SelectAtomic64(Node, X86::ATOMNAND6432); 1572198090Srdivacky case X86ISD::ATOMAND64_DAG: 1573198090Srdivacky return SelectAtomic64(Node, X86::ATOMAND6432); 1574198090Srdivacky case X86ISD::ATOMSWAP64_DAG: 1575198090Srdivacky return SelectAtomic64(Node, X86::ATOMSWAP6432); 1576193323Sed 1577198090Srdivacky case ISD::ATOMIC_LOAD_ADD: { 1578198090Srdivacky SDNode *RetVal = SelectAtomicLoadAdd(Node, NVT); 1579198090Srdivacky if (RetVal) 1580198090Srdivacky return RetVal; 1581198090Srdivacky break; 1582198090Srdivacky } 1583221345Sdim case ISD::AND: 1584221345Sdim case ISD::OR: 1585221345Sdim case ISD::XOR: { 1586221345Sdim // For operations of the form (x << C1) op C2, check if we can use a smaller 1587221345Sdim // encoding for C2 by transforming it into (x op (C2>>C1)) << C1. 1588221345Sdim SDValue N0 = Node->getOperand(0); 1589221345Sdim SDValue N1 = Node->getOperand(1); 1590221345Sdim 1591221345Sdim if (N0->getOpcode() != ISD::SHL || !N0->hasOneUse()) 1592221345Sdim break; 1593221345Sdim 1594221345Sdim // i8 is unshrinkable, i16 should be promoted to i32. 1595221345Sdim if (NVT != MVT::i32 && NVT != MVT::i64) 1596221345Sdim break; 1597221345Sdim 1598221345Sdim ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N1); 1599221345Sdim ConstantSDNode *ShlCst = dyn_cast<ConstantSDNode>(N0->getOperand(1)); 1600221345Sdim if (!Cst || !ShlCst) 1601221345Sdim break; 1602221345Sdim 1603221345Sdim int64_t Val = Cst->getSExtValue(); 1604221345Sdim uint64_t ShlVal = ShlCst->getZExtValue(); 1605221345Sdim 1606221345Sdim // Make sure that we don't change the operation by removing bits. 1607221345Sdim // This only matters for OR and XOR, AND is unaffected. 1608221345Sdim if (Opcode != ISD::AND && ((Val >> ShlVal) << ShlVal) != Val) 1609221345Sdim break; 1610221345Sdim 1611221345Sdim unsigned ShlOp, Op = 0; 1612221345Sdim EVT CstVT = NVT; 1613221345Sdim 1614221345Sdim // Check the minimum bitwidth for the new constant. 1615221345Sdim // TODO: AND32ri is the same as AND64ri32 with zext imm. 1616221345Sdim // TODO: MOV32ri+OR64r is cheaper than MOV64ri64+OR64rr 1617221345Sdim // TODO: Using 16 and 8 bit operations is also possible for or32 & xor32. 1618221345Sdim if (!isInt<8>(Val) && isInt<8>(Val >> ShlVal)) 1619221345Sdim CstVT = MVT::i8; 1620221345Sdim else if (!isInt<32>(Val) && isInt<32>(Val >> ShlVal)) 1621221345Sdim CstVT = MVT::i32; 1622221345Sdim 1623221345Sdim // Bail if there is no smaller encoding. 1624221345Sdim if (NVT == CstVT) 1625221345Sdim break; 1626221345Sdim 1627221345Sdim switch (NVT.getSimpleVT().SimpleTy) { 1628221345Sdim default: llvm_unreachable("Unsupported VT!"); 1629221345Sdim case MVT::i32: 1630221345Sdim assert(CstVT == MVT::i8); 1631221345Sdim ShlOp = X86::SHL32ri; 1632221345Sdim 1633221345Sdim switch (Opcode) { 1634221345Sdim case ISD::AND: Op = X86::AND32ri8; break; 1635221345Sdim case ISD::OR: Op = X86::OR32ri8; break; 1636221345Sdim case ISD::XOR: Op = X86::XOR32ri8; break; 1637221345Sdim } 1638221345Sdim break; 1639221345Sdim case MVT::i64: 1640221345Sdim assert(CstVT == MVT::i8 || CstVT == MVT::i32); 1641221345Sdim ShlOp = X86::SHL64ri; 1642221345Sdim 1643221345Sdim switch (Opcode) { 1644221345Sdim case ISD::AND: Op = CstVT==MVT::i8? X86::AND64ri8 : X86::AND64ri32; break; 1645221345Sdim case ISD::OR: Op = CstVT==MVT::i8? X86::OR64ri8 : X86::OR64ri32; break; 1646221345Sdim case ISD::XOR: Op = CstVT==MVT::i8? X86::XOR64ri8 : X86::XOR64ri32; break; 1647221345Sdim } 1648221345Sdim break; 1649221345Sdim } 1650221345Sdim 1651221345Sdim // Emit the smaller op and the shift. 1652221345Sdim SDValue NewCst = CurDAG->getTargetConstant(Val >> ShlVal, CstVT); 1653221345Sdim SDNode *New = CurDAG->getMachineNode(Op, dl, NVT, N0->getOperand(0),NewCst); 1654221345Sdim return CurDAG->SelectNodeTo(Node, ShlOp, NVT, SDValue(New, 0), 1655221345Sdim getI8Imm(ShlVal)); 1656221345Sdim break; 1657221345Sdim } 1658218893Sdim case X86ISD::UMUL: { 1659218893Sdim SDValue N0 = Node->getOperand(0); 1660218893Sdim SDValue N1 = Node->getOperand(1); 1661218893Sdim 1662218893Sdim unsigned LoReg; 1663218893Sdim switch (NVT.getSimpleVT().SimpleTy) { 1664218893Sdim default: llvm_unreachable("Unsupported VT!"); 1665218893Sdim case MVT::i8: LoReg = X86::AL; Opc = X86::MUL8r; break; 1666218893Sdim case MVT::i16: LoReg = X86::AX; Opc = X86::MUL16r; break; 1667218893Sdim case MVT::i32: LoReg = X86::EAX; Opc = X86::MUL32r; break; 1668218893Sdim case MVT::i64: LoReg = X86::RAX; Opc = X86::MUL64r; break; 1669218893Sdim } 1670218893Sdim 1671218893Sdim SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg, 1672218893Sdim N0, SDValue()).getValue(1); 1673218893Sdim 1674218893Sdim SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::i32); 1675218893Sdim SDValue Ops[] = {N1, InFlag}; 1676218893Sdim SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops, 2); 1677218893Sdim 1678218893Sdim ReplaceUses(SDValue(Node, 0), SDValue(CNode, 0)); 1679218893Sdim ReplaceUses(SDValue(Node, 1), SDValue(CNode, 1)); 1680218893Sdim ReplaceUses(SDValue(Node, 2), SDValue(CNode, 2)); 1681218893Sdim return NULL; 1682218893Sdim } 1683218893Sdim 1684198090Srdivacky case ISD::SMUL_LOHI: 1685198090Srdivacky case ISD::UMUL_LOHI: { 1686198090Srdivacky SDValue N0 = Node->getOperand(0); 1687198090Srdivacky SDValue N1 = Node->getOperand(1); 1688193323Sed 1689198090Srdivacky bool isSigned = Opcode == ISD::SMUL_LOHI; 1690198090Srdivacky if (!isSigned) { 1691198090Srdivacky switch (NVT.getSimpleVT().SimpleTy) { 1692198090Srdivacky default: llvm_unreachable("Unsupported VT!"); 1693198090Srdivacky case MVT::i8: Opc = X86::MUL8r; MOpc = X86::MUL8m; break; 1694198090Srdivacky case MVT::i16: Opc = X86::MUL16r; MOpc = X86::MUL16m; break; 1695198090Srdivacky case MVT::i32: Opc = X86::MUL32r; MOpc = X86::MUL32m; break; 1696198090Srdivacky case MVT::i64: Opc = X86::MUL64r; MOpc = X86::MUL64m; break; 1697193323Sed } 1698198090Srdivacky } else { 1699198090Srdivacky switch (NVT.getSimpleVT().SimpleTy) { 1700198090Srdivacky default: llvm_unreachable("Unsupported VT!"); 1701198090Srdivacky case MVT::i8: Opc = X86::IMUL8r; MOpc = X86::IMUL8m; break; 1702198090Srdivacky case MVT::i16: Opc = X86::IMUL16r; MOpc = X86::IMUL16m; break; 1703198090Srdivacky case MVT::i32: Opc = X86::IMUL32r; MOpc = X86::IMUL32m; break; 1704198090Srdivacky case MVT::i64: Opc = X86::IMUL64r; MOpc = X86::IMUL64m; break; 1705193323Sed } 1706198090Srdivacky } 1707193323Sed 1708198090Srdivacky unsigned LoReg, HiReg; 1709198090Srdivacky switch (NVT.getSimpleVT().SimpleTy) { 1710198090Srdivacky default: llvm_unreachable("Unsupported VT!"); 1711198090Srdivacky case MVT::i8: LoReg = X86::AL; HiReg = X86::AH; break; 1712198090Srdivacky case MVT::i16: LoReg = X86::AX; HiReg = X86::DX; break; 1713198090Srdivacky case MVT::i32: LoReg = X86::EAX; HiReg = X86::EDX; break; 1714198090Srdivacky case MVT::i64: LoReg = X86::RAX; HiReg = X86::RDX; break; 1715198090Srdivacky } 1716193323Sed 1717198090Srdivacky SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 1718202375Srdivacky bool foldedLoad = TryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 1719198090Srdivacky // Multiply is commmutative. 1720198090Srdivacky if (!foldedLoad) { 1721202375Srdivacky foldedLoad = TryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 1722198090Srdivacky if (foldedLoad) 1723198090Srdivacky std::swap(N0, N1); 1724198090Srdivacky } 1725193323Sed 1726198090Srdivacky SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg, 1727198090Srdivacky N0, SDValue()).getValue(1); 1728198090Srdivacky 1729198090Srdivacky if (foldedLoad) { 1730198090Srdivacky SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), 1731198090Srdivacky InFlag }; 1732198090Srdivacky SDNode *CNode = 1733218893Sdim CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops, 1734198090Srdivacky array_lengthof(Ops)); 1735198090Srdivacky InFlag = SDValue(CNode, 1); 1736218893Sdim 1737198090Srdivacky // Update the chain. 1738198090Srdivacky ReplaceUses(N1.getValue(1), SDValue(CNode, 0)); 1739198090Srdivacky } else { 1740218893Sdim SDNode *CNode = CurDAG->getMachineNode(Opc, dl, MVT::Glue, N1, InFlag); 1741218893Sdim InFlag = SDValue(CNode, 0); 1742198090Srdivacky } 1743198090Srdivacky 1744210299Sed // Prevent use of AH in a REX instruction by referencing AX instead. 1745210299Sed if (HiReg == X86::AH && Subtarget->is64Bit() && 1746210299Sed !SDValue(Node, 1).use_empty()) { 1747210299Sed SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1748210299Sed X86::AX, MVT::i16, InFlag); 1749210299Sed InFlag = Result.getValue(2); 1750210299Sed // Get the low part if needed. Don't use getCopyFromReg for aliasing 1751210299Sed // registers. 1752210299Sed if (!SDValue(Node, 0).use_empty()) 1753210299Sed ReplaceUses(SDValue(Node, 1), 1754210299Sed CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); 1755210299Sed 1756210299Sed // Shift AX down 8 bits. 1757210299Sed Result = SDValue(CurDAG->getMachineNode(X86::SHR16ri, dl, MVT::i16, 1758210299Sed Result, 1759210299Sed CurDAG->getTargetConstant(8, MVT::i8)), 0); 1760210299Sed // Then truncate it down to i8. 1761210299Sed ReplaceUses(SDValue(Node, 1), 1762210299Sed CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); 1763210299Sed } 1764198090Srdivacky // Copy the low half of the result, if it is needed. 1765202375Srdivacky if (!SDValue(Node, 0).use_empty()) { 1766198090Srdivacky SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1767198090Srdivacky LoReg, NVT, InFlag); 1768198090Srdivacky InFlag = Result.getValue(2); 1769202375Srdivacky ReplaceUses(SDValue(Node, 0), Result); 1770204642Srdivacky DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 1771198090Srdivacky } 1772198090Srdivacky // Copy the high half of the result, if it is needed. 1773202375Srdivacky if (!SDValue(Node, 1).use_empty()) { 1774210299Sed SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1775210299Sed HiReg, NVT, InFlag); 1776210299Sed InFlag = Result.getValue(2); 1777202375Srdivacky ReplaceUses(SDValue(Node, 1), Result); 1778204642Srdivacky DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 1779198090Srdivacky } 1780218893Sdim 1781198090Srdivacky return NULL; 1782198090Srdivacky } 1783193323Sed 1784198090Srdivacky case ISD::SDIVREM: 1785198090Srdivacky case ISD::UDIVREM: { 1786198090Srdivacky SDValue N0 = Node->getOperand(0); 1787198090Srdivacky SDValue N1 = Node->getOperand(1); 1788193323Sed 1789198090Srdivacky bool isSigned = Opcode == ISD::SDIVREM; 1790198090Srdivacky if (!isSigned) { 1791198090Srdivacky switch (NVT.getSimpleVT().SimpleTy) { 1792198090Srdivacky default: llvm_unreachable("Unsupported VT!"); 1793198090Srdivacky case MVT::i8: Opc = X86::DIV8r; MOpc = X86::DIV8m; break; 1794198090Srdivacky case MVT::i16: Opc = X86::DIV16r; MOpc = X86::DIV16m; break; 1795198090Srdivacky case MVT::i32: Opc = X86::DIV32r; MOpc = X86::DIV32m; break; 1796198090Srdivacky case MVT::i64: Opc = X86::DIV64r; MOpc = X86::DIV64m; break; 1797193323Sed } 1798198090Srdivacky } else { 1799198090Srdivacky switch (NVT.getSimpleVT().SimpleTy) { 1800198090Srdivacky default: llvm_unreachable("Unsupported VT!"); 1801198090Srdivacky case MVT::i8: Opc = X86::IDIV8r; MOpc = X86::IDIV8m; break; 1802198090Srdivacky case MVT::i16: Opc = X86::IDIV16r; MOpc = X86::IDIV16m; break; 1803198090Srdivacky case MVT::i32: Opc = X86::IDIV32r; MOpc = X86::IDIV32m; break; 1804198090Srdivacky case MVT::i64: Opc = X86::IDIV64r; MOpc = X86::IDIV64m; break; 1805198090Srdivacky } 1806198090Srdivacky } 1807193323Sed 1808201360Srdivacky unsigned LoReg, HiReg, ClrReg; 1809198090Srdivacky unsigned ClrOpcode, SExtOpcode; 1810198090Srdivacky switch (NVT.getSimpleVT().SimpleTy) { 1811198090Srdivacky default: llvm_unreachable("Unsupported VT!"); 1812198090Srdivacky case MVT::i8: 1813201360Srdivacky LoReg = X86::AL; ClrReg = HiReg = X86::AH; 1814198090Srdivacky ClrOpcode = 0; 1815198090Srdivacky SExtOpcode = X86::CBW; 1816198090Srdivacky break; 1817198090Srdivacky case MVT::i16: 1818198090Srdivacky LoReg = X86::AX; HiReg = X86::DX; 1819202375Srdivacky ClrOpcode = X86::MOV16r0; ClrReg = X86::DX; 1820198090Srdivacky SExtOpcode = X86::CWD; 1821198090Srdivacky break; 1822198090Srdivacky case MVT::i32: 1823201360Srdivacky LoReg = X86::EAX; ClrReg = HiReg = X86::EDX; 1824198090Srdivacky ClrOpcode = X86::MOV32r0; 1825198090Srdivacky SExtOpcode = X86::CDQ; 1826198090Srdivacky break; 1827198090Srdivacky case MVT::i64: 1828201360Srdivacky LoReg = X86::RAX; ClrReg = HiReg = X86::RDX; 1829202375Srdivacky ClrOpcode = X86::MOV64r0; 1830198090Srdivacky SExtOpcode = X86::CQO; 1831198090Srdivacky break; 1832198090Srdivacky } 1833193323Sed 1834198090Srdivacky SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 1835202375Srdivacky bool foldedLoad = TryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 1836198090Srdivacky bool signBitIsZero = CurDAG->SignBitIsZero(N0); 1837198090Srdivacky 1838198090Srdivacky SDValue InFlag; 1839198090Srdivacky if (NVT == MVT::i8 && (!isSigned || signBitIsZero)) { 1840198090Srdivacky // Special case for div8, just use a move with zero extension to AX to 1841198090Srdivacky // clear the upper 8 bits (AH). 1842198090Srdivacky SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Move, Chain; 1843202375Srdivacky if (TryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) { 1844198090Srdivacky SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0) }; 1845198090Srdivacky Move = 1846198090Srdivacky SDValue(CurDAG->getMachineNode(X86::MOVZX16rm8, dl, MVT::i16, 1847198090Srdivacky MVT::Other, Ops, 1848198090Srdivacky array_lengthof(Ops)), 0); 1849198090Srdivacky Chain = Move.getValue(1); 1850198090Srdivacky ReplaceUses(N0.getValue(1), Chain); 1851193323Sed } else { 1852198090Srdivacky Move = 1853198090Srdivacky SDValue(CurDAG->getMachineNode(X86::MOVZX16rr8, dl, MVT::i16, N0),0); 1854198090Srdivacky Chain = CurDAG->getEntryNode(); 1855198090Srdivacky } 1856198090Srdivacky Chain = CurDAG->getCopyToReg(Chain, dl, X86::AX, Move, SDValue()); 1857198090Srdivacky InFlag = Chain.getValue(1); 1858198090Srdivacky } else { 1859198090Srdivacky InFlag = 1860198090Srdivacky CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, 1861198090Srdivacky LoReg, N0, SDValue()).getValue(1); 1862198090Srdivacky if (isSigned && !signBitIsZero) { 1863198090Srdivacky // Sign extend the low part into the high part. 1864193323Sed InFlag = 1865218893Sdim SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InFlag),0); 1866198090Srdivacky } else { 1867198090Srdivacky // Zero out the high part, effectively zero extending the input. 1868202375Srdivacky SDValue ClrNode = 1869202375Srdivacky SDValue(CurDAG->getMachineNode(ClrOpcode, dl, NVT), 0); 1870201360Srdivacky InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg, 1871198090Srdivacky ClrNode, InFlag).getValue(1); 1872193323Sed } 1873198090Srdivacky } 1874193323Sed 1875198090Srdivacky if (foldedLoad) { 1876198090Srdivacky SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), 1877198090Srdivacky InFlag }; 1878198090Srdivacky SDNode *CNode = 1879218893Sdim CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops, 1880198090Srdivacky array_lengthof(Ops)); 1881198090Srdivacky InFlag = SDValue(CNode, 1); 1882198090Srdivacky // Update the chain. 1883198090Srdivacky ReplaceUses(N1.getValue(1), SDValue(CNode, 0)); 1884198090Srdivacky } else { 1885198090Srdivacky InFlag = 1886218893Sdim SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, N1, InFlag), 0); 1887198090Srdivacky } 1888198090Srdivacky 1889210299Sed // Prevent use of AH in a REX instruction by referencing AX instead. 1890210299Sed // Shift it down 8 bits. 1891210299Sed if (HiReg == X86::AH && Subtarget->is64Bit() && 1892210299Sed !SDValue(Node, 1).use_empty()) { 1893210299Sed SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1894210299Sed X86::AX, MVT::i16, InFlag); 1895210299Sed InFlag = Result.getValue(2); 1896210299Sed 1897210299Sed // If we also need AL (the quotient), get it by extracting a subreg from 1898210299Sed // Result. The fast register allocator does not like multiple CopyFromReg 1899210299Sed // nodes using aliasing registers. 1900210299Sed if (!SDValue(Node, 0).use_empty()) 1901210299Sed ReplaceUses(SDValue(Node, 0), 1902210299Sed CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); 1903210299Sed 1904210299Sed // Shift AX right by 8 bits instead of using AH. 1905210299Sed Result = SDValue(CurDAG->getMachineNode(X86::SHR16ri, dl, MVT::i16, 1906210299Sed Result, 1907210299Sed CurDAG->getTargetConstant(8, MVT::i8)), 1908210299Sed 0); 1909210299Sed ReplaceUses(SDValue(Node, 1), 1910210299Sed CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); 1911210299Sed } 1912198090Srdivacky // Copy the division (low) result, if it is needed. 1913202375Srdivacky if (!SDValue(Node, 0).use_empty()) { 1914198090Srdivacky SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1915198090Srdivacky LoReg, NVT, InFlag); 1916198090Srdivacky InFlag = Result.getValue(2); 1917202375Srdivacky ReplaceUses(SDValue(Node, 0), Result); 1918204642Srdivacky DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 1919198090Srdivacky } 1920198090Srdivacky // Copy the remainder (high) result, if it is needed. 1921202375Srdivacky if (!SDValue(Node, 1).use_empty()) { 1922210299Sed SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1923210299Sed HiReg, NVT, InFlag); 1924210299Sed InFlag = Result.getValue(2); 1925202375Srdivacky ReplaceUses(SDValue(Node, 1), Result); 1926204642Srdivacky DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 1927198090Srdivacky } 1928198090Srdivacky return NULL; 1929198090Srdivacky } 1930193323Sed 1931198090Srdivacky case X86ISD::CMP: { 1932198090Srdivacky SDValue N0 = Node->getOperand(0); 1933198090Srdivacky SDValue N1 = Node->getOperand(1); 1934198090Srdivacky 1935198090Srdivacky // Look for (X86cmp (and $op, $imm), 0) and see if we can convert it to 1936198090Srdivacky // use a smaller encoding. 1937212904Sdim if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && 1938212904Sdim HasNoSignedComparisonUses(Node)) 1939207618Srdivacky // Look past the truncate if CMP is the only use of it. 1940207618Srdivacky N0 = N0.getOperand(0); 1941198090Srdivacky if (N0.getNode()->getOpcode() == ISD::AND && N0.getNode()->hasOneUse() && 1942198090Srdivacky N0.getValueType() != MVT::i8 && 1943198090Srdivacky X86::isZeroNode(N1)) { 1944198090Srdivacky ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getNode()->getOperand(1)); 1945198090Srdivacky if (!C) break; 1946198090Srdivacky 1947198090Srdivacky // For example, convert "testl %eax, $8" to "testb %al, $8" 1948198090Srdivacky if ((C->getZExtValue() & ~UINT64_C(0xff)) == 0 && 1949198090Srdivacky (!(C->getZExtValue() & 0x80) || 1950198090Srdivacky HasNoSignedComparisonUses(Node))) { 1951198090Srdivacky SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), MVT::i8); 1952198090Srdivacky SDValue Reg = N0.getNode()->getOperand(0); 1953198090Srdivacky 1954198090Srdivacky // On x86-32, only the ABCD registers have 8-bit subregisters. 1955198090Srdivacky if (!Subtarget->is64Bit()) { 1956198090Srdivacky TargetRegisterClass *TRC = 0; 1957198090Srdivacky switch (N0.getValueType().getSimpleVT().SimpleTy) { 1958198090Srdivacky case MVT::i32: TRC = &X86::GR32_ABCDRegClass; break; 1959198090Srdivacky case MVT::i16: TRC = &X86::GR16_ABCDRegClass; break; 1960198090Srdivacky default: llvm_unreachable("Unsupported TEST operand type!"); 1961198090Srdivacky } 1962198090Srdivacky SDValue RC = CurDAG->getTargetConstant(TRC->getID(), MVT::i32); 1963198090Srdivacky Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl, 1964198090Srdivacky Reg.getValueType(), Reg, RC), 0); 1965198090Srdivacky } 1966198090Srdivacky 1967198090Srdivacky // Extract the l-register. 1968208599Srdivacky SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, 1969198090Srdivacky MVT::i8, Reg); 1970198090Srdivacky 1971198090Srdivacky // Emit a testb. 1972198090Srdivacky return CurDAG->getMachineNode(X86::TEST8ri, dl, MVT::i32, Subreg, Imm); 1973193323Sed } 1974198090Srdivacky 1975198090Srdivacky // For example, "testl %eax, $2048" to "testb %ah, $8". 1976198090Srdivacky if ((C->getZExtValue() & ~UINT64_C(0xff00)) == 0 && 1977198090Srdivacky (!(C->getZExtValue() & 0x8000) || 1978198090Srdivacky HasNoSignedComparisonUses(Node))) { 1979198090Srdivacky // Shift the immediate right by 8 bits. 1980198090Srdivacky SDValue ShiftedImm = CurDAG->getTargetConstant(C->getZExtValue() >> 8, 1981198090Srdivacky MVT::i8); 1982198090Srdivacky SDValue Reg = N0.getNode()->getOperand(0); 1983198090Srdivacky 1984198090Srdivacky // Put the value in an ABCD register. 1985198090Srdivacky TargetRegisterClass *TRC = 0; 1986198090Srdivacky switch (N0.getValueType().getSimpleVT().SimpleTy) { 1987198090Srdivacky case MVT::i64: TRC = &X86::GR64_ABCDRegClass; break; 1988198090Srdivacky case MVT::i32: TRC = &X86::GR32_ABCDRegClass; break; 1989198090Srdivacky case MVT::i16: TRC = &X86::GR16_ABCDRegClass; break; 1990198090Srdivacky default: llvm_unreachable("Unsupported TEST operand type!"); 1991198090Srdivacky } 1992198090Srdivacky SDValue RC = CurDAG->getTargetConstant(TRC->getID(), MVT::i32); 1993198090Srdivacky Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl, 1994198090Srdivacky Reg.getValueType(), Reg, RC), 0); 1995198090Srdivacky 1996198090Srdivacky // Extract the h-register. 1997208599Srdivacky SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl, 1998198090Srdivacky MVT::i8, Reg); 1999198090Srdivacky 2000198090Srdivacky // Emit a testb. No special NOREX tricks are needed since there's 2001198090Srdivacky // only one GPR operand! 2002198090Srdivacky return CurDAG->getMachineNode(X86::TEST8ri, dl, MVT::i32, 2003198090Srdivacky Subreg, ShiftedImm); 2004193323Sed } 2005198090Srdivacky 2006198090Srdivacky // For example, "testl %eax, $32776" to "testw %ax, $32776". 2007198090Srdivacky if ((C->getZExtValue() & ~UINT64_C(0xffff)) == 0 && 2008198090Srdivacky N0.getValueType() != MVT::i16 && 2009198090Srdivacky (!(C->getZExtValue() & 0x8000) || 2010198090Srdivacky HasNoSignedComparisonUses(Node))) { 2011198090Srdivacky SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), MVT::i16); 2012198090Srdivacky SDValue Reg = N0.getNode()->getOperand(0); 2013198090Srdivacky 2014198090Srdivacky // Extract the 16-bit subregister. 2015208599Srdivacky SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_16bit, dl, 2016198090Srdivacky MVT::i16, Reg); 2017198090Srdivacky 2018198090Srdivacky // Emit a testw. 2019198090Srdivacky return CurDAG->getMachineNode(X86::TEST16ri, dl, MVT::i32, Subreg, Imm); 2020193323Sed } 2021198090Srdivacky 2022198090Srdivacky // For example, "testq %rax, $268468232" to "testl %eax, $268468232". 2023198090Srdivacky if ((C->getZExtValue() & ~UINT64_C(0xffffffff)) == 0 && 2024198090Srdivacky N0.getValueType() == MVT::i64 && 2025198090Srdivacky (!(C->getZExtValue() & 0x80000000) || 2026198090Srdivacky HasNoSignedComparisonUses(Node))) { 2027198090Srdivacky SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), MVT::i32); 2028198090Srdivacky SDValue Reg = N0.getNode()->getOperand(0); 2029198090Srdivacky 2030198090Srdivacky // Extract the 32-bit subregister. 2031208599Srdivacky SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_32bit, dl, 2032198090Srdivacky MVT::i32, Reg); 2033198090Srdivacky 2034198090Srdivacky // Emit a testl. 2035198090Srdivacky return CurDAG->getMachineNode(X86::TEST32ri, dl, MVT::i32, Subreg, Imm); 2036198090Srdivacky } 2037193323Sed } 2038198090Srdivacky break; 2039193323Sed } 2040198090Srdivacky } 2041193323Sed 2042202375Srdivacky SDNode *ResNode = SelectCode(Node); 2043193323Sed 2044204642Srdivacky DEBUG(dbgs() << "=> "; 2045204642Srdivacky if (ResNode == NULL || ResNode == Node) 2046204642Srdivacky Node->dump(CurDAG); 2047204642Srdivacky else 2048204642Srdivacky ResNode->dump(CurDAG); 2049204642Srdivacky dbgs() << '\n'); 2050193323Sed 2051193323Sed return ResNode; 2052193323Sed} 2053193323Sed 2054193323Sedbool X86DAGToDAGISel:: 2055193323SedSelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode, 2056193323Sed std::vector<SDValue> &OutOps) { 2057193323Sed SDValue Op0, Op1, Op2, Op3, Op4; 2058193323Sed switch (ConstraintCode) { 2059193323Sed case 'o': // offsetable ?? 2060193323Sed case 'v': // not offsetable ?? 2061193323Sed default: return true; 2062193323Sed case 'm': // memory 2063218893Sdim if (!SelectAddr(0, Op, Op0, Op1, Op2, Op3, Op4)) 2064193323Sed return true; 2065193323Sed break; 2066193323Sed } 2067193323Sed 2068193323Sed OutOps.push_back(Op0); 2069193323Sed OutOps.push_back(Op1); 2070193323Sed OutOps.push_back(Op2); 2071193323Sed OutOps.push_back(Op3); 2072193323Sed OutOps.push_back(Op4); 2073193323Sed return false; 2074193323Sed} 2075193323Sed 2076193323Sed/// createX86ISelDag - This pass converts a legalized DAG into a 2077193323Sed/// X86-specific DAG, ready for instruction scheduling. 2078193323Sed/// 2079193323SedFunctionPass *llvm::createX86ISelDag(X86TargetMachine &TM, 2080193323Sed llvm::CodeGenOpt::Level OptLevel) { 2081193323Sed return new X86DAGToDAGISel(TM, OptLevel); 2082193323Sed} 2083