X86ISelDAGToDAG.cpp revision 198892
1//===- X86ISelDAGToDAG.cpp - A DAG pattern matching inst selector for X86 -===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines a DAG pattern matching instruction selector for X86, 11// converting from a legalized dag to a X86 dag. 12// 13//===----------------------------------------------------------------------===// 14 15#define DEBUG_TYPE "x86-isel" 16#include "X86.h" 17#include "X86InstrBuilder.h" 18#include "X86ISelLowering.h" 19#include "X86MachineFunctionInfo.h" 20#include "X86RegisterInfo.h" 21#include "X86Subtarget.h" 22#include "X86TargetMachine.h" 23#include "llvm/GlobalValue.h" 24#include "llvm/Instructions.h" 25#include "llvm/Intrinsics.h" 26#include "llvm/Support/CFG.h" 27#include "llvm/Type.h" 28#include "llvm/CodeGen/MachineConstantPool.h" 29#include "llvm/CodeGen/MachineFunction.h" 30#include "llvm/CodeGen/MachineFrameInfo.h" 31#include "llvm/CodeGen/MachineInstrBuilder.h" 32#include "llvm/CodeGen/MachineRegisterInfo.h" 33#include "llvm/CodeGen/SelectionDAGISel.h" 34#include "llvm/Target/TargetMachine.h" 35#include "llvm/Target/TargetOptions.h" 36#include "llvm/Support/Debug.h" 37#include "llvm/Support/ErrorHandling.h" 38#include "llvm/Support/MathExtras.h" 39#include "llvm/Support/raw_ostream.h" 40#include "llvm/ADT/SmallPtrSet.h" 41#include "llvm/ADT/Statistic.h" 42using namespace llvm; 43 44#include "llvm/Support/CommandLine.h" 45static cl::opt<bool> AvoidDupAddrCompute("x86-avoid-dup-address", cl::Hidden); 46 47STATISTIC(NumLoadMoved, "Number of loads moved below TokenFactor"); 48 49//===----------------------------------------------------------------------===// 50// Pattern Matcher Implementation 51//===----------------------------------------------------------------------===// 52 53namespace { 54 /// X86ISelAddressMode - This corresponds to X86AddressMode, but uses 55 /// SDValue's instead of register numbers for the leaves of the matched 56 /// tree. 57 struct X86ISelAddressMode { 58 enum { 59 RegBase, 60 FrameIndexBase 61 } BaseType; 62 63 struct { // This is really a union, discriminated by BaseType! 64 SDValue Reg; 65 int FrameIndex; 66 } Base; 67 68 unsigned Scale; 69 SDValue IndexReg; 70 int32_t Disp; 71 SDValue Segment; 72 GlobalValue *GV; 73 Constant *CP; 74 BlockAddress *BlockAddr; 75 const char *ES; 76 int JT; 77 unsigned Align; // CP alignment. 78 unsigned char SymbolFlags; // X86II::MO_* 79 80 X86ISelAddressMode() 81 : BaseType(RegBase), Scale(1), IndexReg(), Disp(0), 82 Segment(), GV(0), CP(0), BlockAddr(0), ES(0), JT(-1), Align(0), 83 SymbolFlags(X86II::MO_NO_FLAG) { 84 } 85 86 bool hasSymbolicDisplacement() const { 87 return GV != 0 || CP != 0 || ES != 0 || JT != -1 || BlockAddr != 0; 88 } 89 90 bool hasBaseOrIndexReg() const { 91 return IndexReg.getNode() != 0 || Base.Reg.getNode() != 0; 92 } 93 94 /// isRIPRelative - Return true if this addressing mode is already RIP 95 /// relative. 96 bool isRIPRelative() const { 97 if (BaseType != RegBase) return false; 98 if (RegisterSDNode *RegNode = 99 dyn_cast_or_null<RegisterSDNode>(Base.Reg.getNode())) 100 return RegNode->getReg() == X86::RIP; 101 return false; 102 } 103 104 void setBaseReg(SDValue Reg) { 105 BaseType = RegBase; 106 Base.Reg = Reg; 107 } 108 109 void dump() { 110 errs() << "X86ISelAddressMode " << this << '\n'; 111 errs() << "Base.Reg "; 112 if (Base.Reg.getNode() != 0) 113 Base.Reg.getNode()->dump(); 114 else 115 errs() << "nul"; 116 errs() << " Base.FrameIndex " << Base.FrameIndex << '\n' 117 << " Scale" << Scale << '\n' 118 << "IndexReg "; 119 if (IndexReg.getNode() != 0) 120 IndexReg.getNode()->dump(); 121 else 122 errs() << "nul"; 123 errs() << " Disp " << Disp << '\n' 124 << "GV "; 125 if (GV) 126 GV->dump(); 127 else 128 errs() << "nul"; 129 errs() << " CP "; 130 if (CP) 131 CP->dump(); 132 else 133 errs() << "nul"; 134 errs() << '\n' 135 << "ES "; 136 if (ES) 137 errs() << ES; 138 else 139 errs() << "nul"; 140 errs() << " JT" << JT << " Align" << Align << '\n'; 141 } 142 }; 143} 144 145namespace { 146 //===--------------------------------------------------------------------===// 147 /// ISel - X86 specific code to select X86 machine instructions for 148 /// SelectionDAG operations. 149 /// 150 class X86DAGToDAGISel : public SelectionDAGISel { 151 /// X86Lowering - This object fully describes how to lower LLVM code to an 152 /// X86-specific SelectionDAG. 153 X86TargetLowering &X86Lowering; 154 155 /// Subtarget - Keep a pointer to the X86Subtarget around so that we can 156 /// make the right decision when generating code for different targets. 157 const X86Subtarget *Subtarget; 158 159 /// OptForSize - If true, selector should try to optimize for code size 160 /// instead of performance. 161 bool OptForSize; 162 163 public: 164 explicit X86DAGToDAGISel(X86TargetMachine &tm, CodeGenOpt::Level OptLevel) 165 : SelectionDAGISel(tm, OptLevel), 166 X86Lowering(*tm.getTargetLowering()), 167 Subtarget(&tm.getSubtarget<X86Subtarget>()), 168 OptForSize(false) {} 169 170 virtual const char *getPassName() const { 171 return "X86 DAG->DAG Instruction Selection"; 172 } 173 174 /// InstructionSelect - This callback is invoked by 175 /// SelectionDAGISel when it has created a SelectionDAG for us to codegen. 176 virtual void InstructionSelect(); 177 178 virtual void EmitFunctionEntryCode(Function &Fn, MachineFunction &MF); 179 180 virtual 181 bool IsLegalAndProfitableToFold(SDNode *N, SDNode *U, SDNode *Root) const; 182 183// Include the pieces autogenerated from the target description. 184#include "X86GenDAGISel.inc" 185 186 private: 187 SDNode *Select(SDValue N); 188 SDNode *SelectAtomic64(SDNode *Node, unsigned Opc); 189 SDNode *SelectAtomicLoadAdd(SDNode *Node, EVT NVT); 190 191 bool MatchSegmentBaseAddress(SDValue N, X86ISelAddressMode &AM); 192 bool MatchLoad(SDValue N, X86ISelAddressMode &AM); 193 bool MatchWrapper(SDValue N, X86ISelAddressMode &AM); 194 bool MatchAddress(SDValue N, X86ISelAddressMode &AM); 195 bool MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM, 196 unsigned Depth); 197 bool MatchAddressBase(SDValue N, X86ISelAddressMode &AM); 198 bool SelectAddr(SDValue Op, SDValue N, SDValue &Base, 199 SDValue &Scale, SDValue &Index, SDValue &Disp, 200 SDValue &Segment); 201 bool SelectLEAAddr(SDValue Op, SDValue N, SDValue &Base, 202 SDValue &Scale, SDValue &Index, SDValue &Disp); 203 bool SelectTLSADDRAddr(SDValue Op, SDValue N, SDValue &Base, 204 SDValue &Scale, SDValue &Index, SDValue &Disp); 205 bool SelectScalarSSELoad(SDValue Op, SDValue Pred, 206 SDValue N, SDValue &Base, SDValue &Scale, 207 SDValue &Index, SDValue &Disp, 208 SDValue &Segment, 209 SDValue &InChain, SDValue &OutChain); 210 bool TryFoldLoad(SDValue P, SDValue N, 211 SDValue &Base, SDValue &Scale, 212 SDValue &Index, SDValue &Disp, 213 SDValue &Segment); 214 void PreprocessForRMW(); 215 void PreprocessForFPConvert(); 216 217 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 218 /// inline asm expressions. 219 virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, 220 char ConstraintCode, 221 std::vector<SDValue> &OutOps); 222 223 void EmitSpecialCodeForMain(MachineBasicBlock *BB, MachineFrameInfo *MFI); 224 225 inline void getAddressOperands(X86ISelAddressMode &AM, SDValue &Base, 226 SDValue &Scale, SDValue &Index, 227 SDValue &Disp, SDValue &Segment) { 228 Base = (AM.BaseType == X86ISelAddressMode::FrameIndexBase) ? 229 CurDAG->getTargetFrameIndex(AM.Base.FrameIndex, TLI.getPointerTy()) : 230 AM.Base.Reg; 231 Scale = getI8Imm(AM.Scale); 232 Index = AM.IndexReg; 233 // These are 32-bit even in 64-bit mode since RIP relative offset 234 // is 32-bit. 235 if (AM.GV) 236 Disp = CurDAG->getTargetGlobalAddress(AM.GV, MVT::i32, AM.Disp, 237 AM.SymbolFlags); 238 else if (AM.CP) 239 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, 240 AM.Align, AM.Disp, AM.SymbolFlags); 241 else if (AM.ES) 242 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags); 243 else if (AM.JT != -1) 244 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags); 245 else if (AM.BlockAddr) 246 Disp = CurDAG->getBlockAddress(AM.BlockAddr, DebugLoc()/*MVT::i32*/, 247 true /*AM.SymbolFlags*/); 248 else 249 Disp = CurDAG->getTargetConstant(AM.Disp, MVT::i32); 250 251 if (AM.Segment.getNode()) 252 Segment = AM.Segment; 253 else 254 Segment = CurDAG->getRegister(0, MVT::i32); 255 } 256 257 /// getI8Imm - Return a target constant with the specified value, of type 258 /// i8. 259 inline SDValue getI8Imm(unsigned Imm) { 260 return CurDAG->getTargetConstant(Imm, MVT::i8); 261 } 262 263 /// getI16Imm - Return a target constant with the specified value, of type 264 /// i16. 265 inline SDValue getI16Imm(unsigned Imm) { 266 return CurDAG->getTargetConstant(Imm, MVT::i16); 267 } 268 269 /// getI32Imm - Return a target constant with the specified value, of type 270 /// i32. 271 inline SDValue getI32Imm(unsigned Imm) { 272 return CurDAG->getTargetConstant(Imm, MVT::i32); 273 } 274 275 /// getGlobalBaseReg - Return an SDNode that returns the value of 276 /// the global base register. Output instructions required to 277 /// initialize the global base register, if necessary. 278 /// 279 SDNode *getGlobalBaseReg(); 280 281 /// getTargetMachine - Return a reference to the TargetMachine, casted 282 /// to the target-specific type. 283 const X86TargetMachine &getTargetMachine() { 284 return static_cast<const X86TargetMachine &>(TM); 285 } 286 287 /// getInstrInfo - Return a reference to the TargetInstrInfo, casted 288 /// to the target-specific type. 289 const X86InstrInfo *getInstrInfo() { 290 return getTargetMachine().getInstrInfo(); 291 } 292 293#ifndef NDEBUG 294 unsigned Indent; 295#endif 296 }; 297} 298 299 300bool X86DAGToDAGISel::IsLegalAndProfitableToFold(SDNode *N, SDNode *U, 301 SDNode *Root) const { 302 if (OptLevel == CodeGenOpt::None) return false; 303 304 if (U == Root) 305 switch (U->getOpcode()) { 306 default: break; 307 case ISD::ADD: 308 case ISD::ADDC: 309 case ISD::ADDE: 310 case ISD::AND: 311 case ISD::OR: 312 case ISD::XOR: { 313 SDValue Op1 = U->getOperand(1); 314 315 // If the other operand is a 8-bit immediate we should fold the immediate 316 // instead. This reduces code size. 317 // e.g. 318 // movl 4(%esp), %eax 319 // addl $4, %eax 320 // vs. 321 // movl $4, %eax 322 // addl 4(%esp), %eax 323 // The former is 2 bytes shorter. In case where the increment is 1, then 324 // the saving can be 4 bytes (by using incl %eax). 325 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Op1)) 326 if (Imm->getAPIntValue().isSignedIntN(8)) 327 return false; 328 329 // If the other operand is a TLS address, we should fold it instead. 330 // This produces 331 // movl %gs:0, %eax 332 // leal i@NTPOFF(%eax), %eax 333 // instead of 334 // movl $i@NTPOFF, %eax 335 // addl %gs:0, %eax 336 // if the block also has an access to a second TLS address this will save 337 // a load. 338 // FIXME: This is probably also true for non TLS addresses. 339 if (Op1.getOpcode() == X86ISD::Wrapper) { 340 SDValue Val = Op1.getOperand(0); 341 if (Val.getOpcode() == ISD::TargetGlobalTLSAddress) 342 return false; 343 } 344 } 345 } 346 347 // Proceed to 'generic' cycle finder code 348 return SelectionDAGISel::IsLegalAndProfitableToFold(N, U, Root); 349} 350 351/// MoveBelowTokenFactor - Replace TokenFactor operand with load's chain operand 352/// and move load below the TokenFactor. Replace store's chain operand with 353/// load's chain result. 354static void MoveBelowTokenFactor(SelectionDAG *CurDAG, SDValue Load, 355 SDValue Store, SDValue TF) { 356 SmallVector<SDValue, 4> Ops; 357 for (unsigned i = 0, e = TF.getNode()->getNumOperands(); i != e; ++i) 358 if (Load.getNode() == TF.getOperand(i).getNode()) 359 Ops.push_back(Load.getOperand(0)); 360 else 361 Ops.push_back(TF.getOperand(i)); 362 SDValue NewTF = CurDAG->UpdateNodeOperands(TF, &Ops[0], Ops.size()); 363 SDValue NewLoad = CurDAG->UpdateNodeOperands(Load, NewTF, 364 Load.getOperand(1), 365 Load.getOperand(2)); 366 CurDAG->UpdateNodeOperands(Store, NewLoad.getValue(1), Store.getOperand(1), 367 Store.getOperand(2), Store.getOperand(3)); 368} 369 370/// isRMWLoad - Return true if N is a load that's part of RMW sub-DAG. The 371/// chain produced by the load must only be used by the store's chain operand, 372/// otherwise this may produce a cycle in the DAG. 373/// 374static bool isRMWLoad(SDValue N, SDValue Chain, SDValue Address, 375 SDValue &Load) { 376 if (N.getOpcode() == ISD::BIT_CONVERT) 377 N = N.getOperand(0); 378 379 LoadSDNode *LD = dyn_cast<LoadSDNode>(N); 380 if (!LD || LD->isVolatile()) 381 return false; 382 if (LD->getAddressingMode() != ISD::UNINDEXED) 383 return false; 384 385 ISD::LoadExtType ExtType = LD->getExtensionType(); 386 if (ExtType != ISD::NON_EXTLOAD && ExtType != ISD::EXTLOAD) 387 return false; 388 389 if (N.hasOneUse() && 390 LD->hasNUsesOfValue(1, 1) && 391 N.getOperand(1) == Address && 392 LD->isOperandOf(Chain.getNode())) { 393 Load = N; 394 return true; 395 } 396 return false; 397} 398 399/// MoveBelowCallSeqStart - Replace CALLSEQ_START operand with load's chain 400/// operand and move load below the call's chain operand. 401static void MoveBelowCallSeqStart(SelectionDAG *CurDAG, SDValue Load, 402 SDValue Call, SDValue CallSeqStart) { 403 SmallVector<SDValue, 8> Ops; 404 SDValue Chain = CallSeqStart.getOperand(0); 405 if (Chain.getNode() == Load.getNode()) 406 Ops.push_back(Load.getOperand(0)); 407 else { 408 assert(Chain.getOpcode() == ISD::TokenFactor && 409 "Unexpected CallSeqStart chain operand"); 410 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) 411 if (Chain.getOperand(i).getNode() == Load.getNode()) 412 Ops.push_back(Load.getOperand(0)); 413 else 414 Ops.push_back(Chain.getOperand(i)); 415 SDValue NewChain = 416 CurDAG->getNode(ISD::TokenFactor, Load.getDebugLoc(), 417 MVT::Other, &Ops[0], Ops.size()); 418 Ops.clear(); 419 Ops.push_back(NewChain); 420 } 421 for (unsigned i = 1, e = CallSeqStart.getNumOperands(); i != e; ++i) 422 Ops.push_back(CallSeqStart.getOperand(i)); 423 CurDAG->UpdateNodeOperands(CallSeqStart, &Ops[0], Ops.size()); 424 CurDAG->UpdateNodeOperands(Load, Call.getOperand(0), 425 Load.getOperand(1), Load.getOperand(2)); 426 Ops.clear(); 427 Ops.push_back(SDValue(Load.getNode(), 1)); 428 for (unsigned i = 1, e = Call.getNode()->getNumOperands(); i != e; ++i) 429 Ops.push_back(Call.getOperand(i)); 430 CurDAG->UpdateNodeOperands(Call, &Ops[0], Ops.size()); 431} 432 433/// isCalleeLoad - Return true if call address is a load and it can be 434/// moved below CALLSEQ_START and the chains leading up to the call. 435/// Return the CALLSEQ_START by reference as a second output. 436static bool isCalleeLoad(SDValue Callee, SDValue &Chain) { 437 if (Callee.getNode() == Chain.getNode() || !Callee.hasOneUse()) 438 return false; 439 LoadSDNode *LD = dyn_cast<LoadSDNode>(Callee.getNode()); 440 if (!LD || 441 LD->isVolatile() || 442 LD->getAddressingMode() != ISD::UNINDEXED || 443 LD->getExtensionType() != ISD::NON_EXTLOAD) 444 return false; 445 446 // Now let's find the callseq_start. 447 while (Chain.getOpcode() != ISD::CALLSEQ_START) { 448 if (!Chain.hasOneUse()) 449 return false; 450 Chain = Chain.getOperand(0); 451 } 452 453 if (Chain.getOperand(0).getNode() == Callee.getNode()) 454 return true; 455 if (Chain.getOperand(0).getOpcode() == ISD::TokenFactor && 456 Callee.getValue(1).isOperandOf(Chain.getOperand(0).getNode()) && 457 Callee.getValue(1).hasOneUse()) 458 return true; 459 return false; 460} 461 462 463/// PreprocessForRMW - Preprocess the DAG to make instruction selection better. 464/// This is only run if not in -O0 mode. 465/// This allows the instruction selector to pick more read-modify-write 466/// instructions. This is a common case: 467/// 468/// [Load chain] 469/// ^ 470/// | 471/// [Load] 472/// ^ ^ 473/// | | 474/// / \- 475/// / | 476/// [TokenFactor] [Op] 477/// ^ ^ 478/// | | 479/// \ / 480/// \ / 481/// [Store] 482/// 483/// The fact the store's chain operand != load's chain will prevent the 484/// (store (op (load))) instruction from being selected. We can transform it to: 485/// 486/// [Load chain] 487/// ^ 488/// | 489/// [TokenFactor] 490/// ^ 491/// | 492/// [Load] 493/// ^ ^ 494/// | | 495/// | \- 496/// | | 497/// | [Op] 498/// | ^ 499/// | | 500/// \ / 501/// \ / 502/// [Store] 503void X86DAGToDAGISel::PreprocessForRMW() { 504 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 505 E = CurDAG->allnodes_end(); I != E; ++I) { 506 if (I->getOpcode() == X86ISD::CALL) { 507 /// Also try moving call address load from outside callseq_start to just 508 /// before the call to allow it to be folded. 509 /// 510 /// [Load chain] 511 /// ^ 512 /// | 513 /// [Load] 514 /// ^ ^ 515 /// | | 516 /// / \-- 517 /// / | 518 ///[CALLSEQ_START] | 519 /// ^ | 520 /// | | 521 /// [LOAD/C2Reg] | 522 /// | | 523 /// \ / 524 /// \ / 525 /// [CALL] 526 SDValue Chain = I->getOperand(0); 527 SDValue Load = I->getOperand(1); 528 if (!isCalleeLoad(Load, Chain)) 529 continue; 530 MoveBelowCallSeqStart(CurDAG, Load, SDValue(I, 0), Chain); 531 ++NumLoadMoved; 532 continue; 533 } 534 535 if (!ISD::isNON_TRUNCStore(I)) 536 continue; 537 SDValue Chain = I->getOperand(0); 538 539 if (Chain.getNode()->getOpcode() != ISD::TokenFactor) 540 continue; 541 542 SDValue N1 = I->getOperand(1); 543 SDValue N2 = I->getOperand(2); 544 if ((N1.getValueType().isFloatingPoint() && 545 !N1.getValueType().isVector()) || 546 !N1.hasOneUse()) 547 continue; 548 549 bool RModW = false; 550 SDValue Load; 551 unsigned Opcode = N1.getNode()->getOpcode(); 552 switch (Opcode) { 553 case ISD::ADD: 554 case ISD::MUL: 555 case ISD::AND: 556 case ISD::OR: 557 case ISD::XOR: 558 case ISD::ADDC: 559 case ISD::ADDE: 560 case ISD::VECTOR_SHUFFLE: { 561 SDValue N10 = N1.getOperand(0); 562 SDValue N11 = N1.getOperand(1); 563 RModW = isRMWLoad(N10, Chain, N2, Load); 564 if (!RModW) 565 RModW = isRMWLoad(N11, Chain, N2, Load); 566 break; 567 } 568 case ISD::SUB: 569 case ISD::SHL: 570 case ISD::SRA: 571 case ISD::SRL: 572 case ISD::ROTL: 573 case ISD::ROTR: 574 case ISD::SUBC: 575 case ISD::SUBE: 576 case X86ISD::SHLD: 577 case X86ISD::SHRD: { 578 SDValue N10 = N1.getOperand(0); 579 RModW = isRMWLoad(N10, Chain, N2, Load); 580 break; 581 } 582 } 583 584 if (RModW) { 585 MoveBelowTokenFactor(CurDAG, Load, SDValue(I, 0), Chain); 586 ++NumLoadMoved; 587 } 588 } 589} 590 591 592/// PreprocessForFPConvert - Walk over the dag lowering fpround and fpextend 593/// nodes that target the FP stack to be store and load to the stack. This is a 594/// gross hack. We would like to simply mark these as being illegal, but when 595/// we do that, legalize produces these when it expands calls, then expands 596/// these in the same legalize pass. We would like dag combine to be able to 597/// hack on these between the call expansion and the node legalization. As such 598/// this pass basically does "really late" legalization of these inline with the 599/// X86 isel pass. 600void X86DAGToDAGISel::PreprocessForFPConvert() { 601 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 602 E = CurDAG->allnodes_end(); I != E; ) { 603 SDNode *N = I++; // Preincrement iterator to avoid invalidation issues. 604 if (N->getOpcode() != ISD::FP_ROUND && N->getOpcode() != ISD::FP_EXTEND) 605 continue; 606 607 // If the source and destination are SSE registers, then this is a legal 608 // conversion that should not be lowered. 609 EVT SrcVT = N->getOperand(0).getValueType(); 610 EVT DstVT = N->getValueType(0); 611 bool SrcIsSSE = X86Lowering.isScalarFPTypeInSSEReg(SrcVT); 612 bool DstIsSSE = X86Lowering.isScalarFPTypeInSSEReg(DstVT); 613 if (SrcIsSSE && DstIsSSE) 614 continue; 615 616 if (!SrcIsSSE && !DstIsSSE) { 617 // If this is an FPStack extension, it is a noop. 618 if (N->getOpcode() == ISD::FP_EXTEND) 619 continue; 620 // If this is a value-preserving FPStack truncation, it is a noop. 621 if (N->getConstantOperandVal(1)) 622 continue; 623 } 624 625 // Here we could have an FP stack truncation or an FPStack <-> SSE convert. 626 // FPStack has extload and truncstore. SSE can fold direct loads into other 627 // operations. Based on this, decide what we want to do. 628 EVT MemVT; 629 if (N->getOpcode() == ISD::FP_ROUND) 630 MemVT = DstVT; // FP_ROUND must use DstVT, we can't do a 'trunc load'. 631 else 632 MemVT = SrcIsSSE ? SrcVT : DstVT; 633 634 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT); 635 DebugLoc dl = N->getDebugLoc(); 636 637 // FIXME: optimize the case where the src/dest is a load or store? 638 SDValue Store = CurDAG->getTruncStore(CurDAG->getEntryNode(), dl, 639 N->getOperand(0), 640 MemTmp, NULL, 0, MemVT); 641 SDValue Result = CurDAG->getExtLoad(ISD::EXTLOAD, dl, DstVT, Store, MemTmp, 642 NULL, 0, MemVT); 643 644 // We're about to replace all uses of the FP_ROUND/FP_EXTEND with the 645 // extload we created. This will cause general havok on the dag because 646 // anything below the conversion could be folded into other existing nodes. 647 // To avoid invalidating 'I', back it up to the convert node. 648 --I; 649 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 650 651 // Now that we did that, the node is dead. Increment the iterator to the 652 // next node to process, then delete N. 653 ++I; 654 CurDAG->DeleteNode(N); 655 } 656} 657 658/// InstructionSelectBasicBlock - This callback is invoked by SelectionDAGISel 659/// when it has created a SelectionDAG for us to codegen. 660void X86DAGToDAGISel::InstructionSelect() { 661 const Function *F = MF->getFunction(); 662 OptForSize = F->hasFnAttr(Attribute::OptimizeForSize); 663 664 DEBUG(BB->dump()); 665 if (OptLevel != CodeGenOpt::None) 666 PreprocessForRMW(); 667 668 // FIXME: This should only happen when not compiled with -O0. 669 PreprocessForFPConvert(); 670 671 // Codegen the basic block. 672#ifndef NDEBUG 673 DEBUG(errs() << "===== Instruction selection begins:\n"); 674 Indent = 0; 675#endif 676 SelectRoot(*CurDAG); 677#ifndef NDEBUG 678 DEBUG(errs() << "===== Instruction selection ends:\n"); 679#endif 680 681 CurDAG->RemoveDeadNodes(); 682} 683 684/// EmitSpecialCodeForMain - Emit any code that needs to be executed only in 685/// the main function. 686void X86DAGToDAGISel::EmitSpecialCodeForMain(MachineBasicBlock *BB, 687 MachineFrameInfo *MFI) { 688 const TargetInstrInfo *TII = TM.getInstrInfo(); 689 if (Subtarget->isTargetCygMing()) 690 BuildMI(BB, DebugLoc::getUnknownLoc(), 691 TII->get(X86::CALLpcrel32)).addExternalSymbol("__main"); 692} 693 694void X86DAGToDAGISel::EmitFunctionEntryCode(Function &Fn, MachineFunction &MF) { 695 // If this is main, emit special code for main. 696 MachineBasicBlock *BB = MF.begin(); 697 if (Fn.hasExternalLinkage() && Fn.getName() == "main") 698 EmitSpecialCodeForMain(BB, MF.getFrameInfo()); 699} 700 701 702bool X86DAGToDAGISel::MatchSegmentBaseAddress(SDValue N, 703 X86ISelAddressMode &AM) { 704 assert(N.getOpcode() == X86ISD::SegmentBaseAddress); 705 SDValue Segment = N.getOperand(0); 706 707 if (AM.Segment.getNode() == 0) { 708 AM.Segment = Segment; 709 return false; 710 } 711 712 return true; 713} 714 715bool X86DAGToDAGISel::MatchLoad(SDValue N, X86ISelAddressMode &AM) { 716 // This optimization is valid because the GNU TLS model defines that 717 // gs:0 (or fs:0 on X86-64) contains its own address. 718 // For more information see http://people.redhat.com/drepper/tls.pdf 719 720 SDValue Address = N.getOperand(1); 721 if (Address.getOpcode() == X86ISD::SegmentBaseAddress && 722 !MatchSegmentBaseAddress (Address, AM)) 723 return false; 724 725 return true; 726} 727 728/// MatchWrapper - Try to match X86ISD::Wrapper and X86ISD::WrapperRIP nodes 729/// into an addressing mode. These wrap things that will resolve down into a 730/// symbol reference. If no match is possible, this returns true, otherwise it 731/// returns false. 732bool X86DAGToDAGISel::MatchWrapper(SDValue N, X86ISelAddressMode &AM) { 733 // If the addressing mode already has a symbol as the displacement, we can 734 // never match another symbol. 735 if (AM.hasSymbolicDisplacement()) 736 return true; 737 738 SDValue N0 = N.getOperand(0); 739 CodeModel::Model M = TM.getCodeModel(); 740 741 // Handle X86-64 rip-relative addresses. We check this before checking direct 742 // folding because RIP is preferable to non-RIP accesses. 743 if (Subtarget->is64Bit() && 744 // Under X86-64 non-small code model, GV (and friends) are 64-bits, so 745 // they cannot be folded into immediate fields. 746 // FIXME: This can be improved for kernel and other models? 747 (M == CodeModel::Small || M == CodeModel::Kernel) && 748 // Base and index reg must be 0 in order to use %rip as base and lowering 749 // must allow RIP. 750 !AM.hasBaseOrIndexReg() && N.getOpcode() == X86ISD::WrapperRIP) { 751 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N0)) { 752 int64_t Offset = AM.Disp + G->getOffset(); 753 if (!X86::isOffsetSuitableForCodeModel(Offset, M)) return true; 754 AM.GV = G->getGlobal(); 755 AM.Disp = Offset; 756 AM.SymbolFlags = G->getTargetFlags(); 757 } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) { 758 int64_t Offset = AM.Disp + CP->getOffset(); 759 if (!X86::isOffsetSuitableForCodeModel(Offset, M)) return true; 760 AM.CP = CP->getConstVal(); 761 AM.Align = CP->getAlignment(); 762 AM.Disp = Offset; 763 AM.SymbolFlags = CP->getTargetFlags(); 764 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) { 765 AM.ES = S->getSymbol(); 766 AM.SymbolFlags = S->getTargetFlags(); 767 } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) { 768 AM.JT = J->getIndex(); 769 AM.SymbolFlags = J->getTargetFlags(); 770 } else { 771 AM.BlockAddr = cast<BlockAddressSDNode>(N0)->getBlockAddress(); 772 //AM.SymbolFlags = cast<BlockAddressSDNode>(N0)->getTargetFlags(); 773 } 774 775 if (N.getOpcode() == X86ISD::WrapperRIP) 776 AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64)); 777 return false; 778 } 779 780 // Handle the case when globals fit in our immediate field: This is true for 781 // X86-32 always and X86-64 when in -static -mcmodel=small mode. In 64-bit 782 // mode, this results in a non-RIP-relative computation. 783 if (!Subtarget->is64Bit() || 784 ((M == CodeModel::Small || M == CodeModel::Kernel) && 785 TM.getRelocationModel() == Reloc::Static)) { 786 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N0)) { 787 AM.GV = G->getGlobal(); 788 AM.Disp += G->getOffset(); 789 AM.SymbolFlags = G->getTargetFlags(); 790 } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) { 791 AM.CP = CP->getConstVal(); 792 AM.Align = CP->getAlignment(); 793 AM.Disp += CP->getOffset(); 794 AM.SymbolFlags = CP->getTargetFlags(); 795 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) { 796 AM.ES = S->getSymbol(); 797 AM.SymbolFlags = S->getTargetFlags(); 798 } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) { 799 AM.JT = J->getIndex(); 800 AM.SymbolFlags = J->getTargetFlags(); 801 } else { 802 AM.BlockAddr = cast<BlockAddressSDNode>(N0)->getBlockAddress(); 803 //AM.SymbolFlags = cast<BlockAddressSDNode>(N0)->getTargetFlags(); 804 } 805 return false; 806 } 807 808 return true; 809} 810 811/// MatchAddress - Add the specified node to the specified addressing mode, 812/// returning true if it cannot be done. This just pattern matches for the 813/// addressing mode. 814bool X86DAGToDAGISel::MatchAddress(SDValue N, X86ISelAddressMode &AM) { 815 if (MatchAddressRecursively(N, AM, 0)) 816 return true; 817 818 // Post-processing: Convert lea(,%reg,2) to lea(%reg,%reg), which has 819 // a smaller encoding and avoids a scaled-index. 820 if (AM.Scale == 2 && 821 AM.BaseType == X86ISelAddressMode::RegBase && 822 AM.Base.Reg.getNode() == 0) { 823 AM.Base.Reg = AM.IndexReg; 824 AM.Scale = 1; 825 } 826 827 // Post-processing: Convert foo to foo(%rip), even in non-PIC mode, 828 // because it has a smaller encoding. 829 // TODO: Which other code models can use this? 830 if (TM.getCodeModel() == CodeModel::Small && 831 Subtarget->is64Bit() && 832 AM.Scale == 1 && 833 AM.BaseType == X86ISelAddressMode::RegBase && 834 AM.Base.Reg.getNode() == 0 && 835 AM.IndexReg.getNode() == 0 && 836 AM.SymbolFlags == X86II::MO_NO_FLAG && 837 AM.hasSymbolicDisplacement()) 838 AM.Base.Reg = CurDAG->getRegister(X86::RIP, MVT::i64); 839 840 return false; 841} 842 843bool X86DAGToDAGISel::MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM, 844 unsigned Depth) { 845 bool is64Bit = Subtarget->is64Bit(); 846 DebugLoc dl = N.getDebugLoc(); 847 DEBUG({ 848 errs() << "MatchAddress: "; 849 AM.dump(); 850 }); 851 // Limit recursion. 852 if (Depth > 5) 853 return MatchAddressBase(N, AM); 854 855 CodeModel::Model M = TM.getCodeModel(); 856 857 // If this is already a %rip relative address, we can only merge immediates 858 // into it. Instead of handling this in every case, we handle it here. 859 // RIP relative addressing: %rip + 32-bit displacement! 860 if (AM.isRIPRelative()) { 861 // FIXME: JumpTable and ExternalSymbol address currently don't like 862 // displacements. It isn't very important, but this should be fixed for 863 // consistency. 864 if (!AM.ES && AM.JT != -1) return true; 865 866 if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N)) { 867 int64_t Val = AM.Disp + Cst->getSExtValue(); 868 if (X86::isOffsetSuitableForCodeModel(Val, M, 869 AM.hasSymbolicDisplacement())) { 870 AM.Disp = Val; 871 return false; 872 } 873 } 874 return true; 875 } 876 877 switch (N.getOpcode()) { 878 default: break; 879 case ISD::Constant: { 880 uint64_t Val = cast<ConstantSDNode>(N)->getSExtValue(); 881 if (!is64Bit || 882 X86::isOffsetSuitableForCodeModel(AM.Disp + Val, M, 883 AM.hasSymbolicDisplacement())) { 884 AM.Disp += Val; 885 return false; 886 } 887 break; 888 } 889 890 case X86ISD::SegmentBaseAddress: 891 if (!MatchSegmentBaseAddress(N, AM)) 892 return false; 893 break; 894 895 case X86ISD::Wrapper: 896 case X86ISD::WrapperRIP: 897 if (!MatchWrapper(N, AM)) 898 return false; 899 break; 900 901 case ISD::LOAD: 902 if (!MatchLoad(N, AM)) 903 return false; 904 break; 905 906 case ISD::FrameIndex: 907 if (AM.BaseType == X86ISelAddressMode::RegBase 908 && AM.Base.Reg.getNode() == 0) { 909 AM.BaseType = X86ISelAddressMode::FrameIndexBase; 910 AM.Base.FrameIndex = cast<FrameIndexSDNode>(N)->getIndex(); 911 return false; 912 } 913 break; 914 915 case ISD::SHL: 916 if (AM.IndexReg.getNode() != 0 || AM.Scale != 1) 917 break; 918 919 if (ConstantSDNode 920 *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1))) { 921 unsigned Val = CN->getZExtValue(); 922 // Note that we handle x<<1 as (,x,2) rather than (x,x) here so 923 // that the base operand remains free for further matching. If 924 // the base doesn't end up getting used, a post-processing step 925 // in MatchAddress turns (,x,2) into (x,x), which is cheaper. 926 if (Val == 1 || Val == 2 || Val == 3) { 927 AM.Scale = 1 << Val; 928 SDValue ShVal = N.getNode()->getOperand(0); 929 930 // Okay, we know that we have a scale by now. However, if the scaled 931 // value is an add of something and a constant, we can fold the 932 // constant into the disp field here. 933 if (ShVal.getNode()->getOpcode() == ISD::ADD && ShVal.hasOneUse() && 934 isa<ConstantSDNode>(ShVal.getNode()->getOperand(1))) { 935 AM.IndexReg = ShVal.getNode()->getOperand(0); 936 ConstantSDNode *AddVal = 937 cast<ConstantSDNode>(ShVal.getNode()->getOperand(1)); 938 uint64_t Disp = AM.Disp + (AddVal->getSExtValue() << Val); 939 if (!is64Bit || 940 X86::isOffsetSuitableForCodeModel(Disp, M, 941 AM.hasSymbolicDisplacement())) 942 AM.Disp = Disp; 943 else 944 AM.IndexReg = ShVal; 945 } else { 946 AM.IndexReg = ShVal; 947 } 948 return false; 949 } 950 break; 951 } 952 953 case ISD::SMUL_LOHI: 954 case ISD::UMUL_LOHI: 955 // A mul_lohi where we need the low part can be folded as a plain multiply. 956 if (N.getResNo() != 0) break; 957 // FALL THROUGH 958 case ISD::MUL: 959 case X86ISD::MUL_IMM: 960 // X*[3,5,9] -> X+X*[2,4,8] 961 if (AM.BaseType == X86ISelAddressMode::RegBase && 962 AM.Base.Reg.getNode() == 0 && 963 AM.IndexReg.getNode() == 0) { 964 if (ConstantSDNode 965 *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1))) 966 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 || 967 CN->getZExtValue() == 9) { 968 AM.Scale = unsigned(CN->getZExtValue())-1; 969 970 SDValue MulVal = N.getNode()->getOperand(0); 971 SDValue Reg; 972 973 // Okay, we know that we have a scale by now. However, if the scaled 974 // value is an add of something and a constant, we can fold the 975 // constant into the disp field here. 976 if (MulVal.getNode()->getOpcode() == ISD::ADD && MulVal.hasOneUse() && 977 isa<ConstantSDNode>(MulVal.getNode()->getOperand(1))) { 978 Reg = MulVal.getNode()->getOperand(0); 979 ConstantSDNode *AddVal = 980 cast<ConstantSDNode>(MulVal.getNode()->getOperand(1)); 981 uint64_t Disp = AM.Disp + AddVal->getSExtValue() * 982 CN->getZExtValue(); 983 if (!is64Bit || 984 X86::isOffsetSuitableForCodeModel(Disp, M, 985 AM.hasSymbolicDisplacement())) 986 AM.Disp = Disp; 987 else 988 Reg = N.getNode()->getOperand(0); 989 } else { 990 Reg = N.getNode()->getOperand(0); 991 } 992 993 AM.IndexReg = AM.Base.Reg = Reg; 994 return false; 995 } 996 } 997 break; 998 999 case ISD::SUB: { 1000 // Given A-B, if A can be completely folded into the address and 1001 // the index field with the index field unused, use -B as the index. 1002 // This is a win if a has multiple parts that can be folded into 1003 // the address. Also, this saves a mov if the base register has 1004 // other uses, since it avoids a two-address sub instruction, however 1005 // it costs an additional mov if the index register has other uses. 1006 1007 // Test if the LHS of the sub can be folded. 1008 X86ISelAddressMode Backup = AM; 1009 if (MatchAddressRecursively(N.getNode()->getOperand(0), AM, Depth+1)) { 1010 AM = Backup; 1011 break; 1012 } 1013 // Test if the index field is free for use. 1014 if (AM.IndexReg.getNode() || AM.isRIPRelative()) { 1015 AM = Backup; 1016 break; 1017 } 1018 int Cost = 0; 1019 SDValue RHS = N.getNode()->getOperand(1); 1020 // If the RHS involves a register with multiple uses, this 1021 // transformation incurs an extra mov, due to the neg instruction 1022 // clobbering its operand. 1023 if (!RHS.getNode()->hasOneUse() || 1024 RHS.getNode()->getOpcode() == ISD::CopyFromReg || 1025 RHS.getNode()->getOpcode() == ISD::TRUNCATE || 1026 RHS.getNode()->getOpcode() == ISD::ANY_EXTEND || 1027 (RHS.getNode()->getOpcode() == ISD::ZERO_EXTEND && 1028 RHS.getNode()->getOperand(0).getValueType() == MVT::i32)) 1029 ++Cost; 1030 // If the base is a register with multiple uses, this 1031 // transformation may save a mov. 1032 if ((AM.BaseType == X86ISelAddressMode::RegBase && 1033 AM.Base.Reg.getNode() && 1034 !AM.Base.Reg.getNode()->hasOneUse()) || 1035 AM.BaseType == X86ISelAddressMode::FrameIndexBase) 1036 --Cost; 1037 // If the folded LHS was interesting, this transformation saves 1038 // address arithmetic. 1039 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) + 1040 ((AM.Disp != 0) && (Backup.Disp == 0)) + 1041 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2) 1042 --Cost; 1043 // If it doesn't look like it may be an overall win, don't do it. 1044 if (Cost >= 0) { 1045 AM = Backup; 1046 break; 1047 } 1048 1049 // Ok, the transformation is legal and appears profitable. Go for it. 1050 SDValue Zero = CurDAG->getConstant(0, N.getValueType()); 1051 SDValue Neg = CurDAG->getNode(ISD::SUB, dl, N.getValueType(), Zero, RHS); 1052 AM.IndexReg = Neg; 1053 AM.Scale = 1; 1054 1055 // Insert the new nodes into the topological ordering. 1056 if (Zero.getNode()->getNodeId() == -1 || 1057 Zero.getNode()->getNodeId() > N.getNode()->getNodeId()) { 1058 CurDAG->RepositionNode(N.getNode(), Zero.getNode()); 1059 Zero.getNode()->setNodeId(N.getNode()->getNodeId()); 1060 } 1061 if (Neg.getNode()->getNodeId() == -1 || 1062 Neg.getNode()->getNodeId() > N.getNode()->getNodeId()) { 1063 CurDAG->RepositionNode(N.getNode(), Neg.getNode()); 1064 Neg.getNode()->setNodeId(N.getNode()->getNodeId()); 1065 } 1066 return false; 1067 } 1068 1069 case ISD::ADD: { 1070 X86ISelAddressMode Backup = AM; 1071 if (!MatchAddressRecursively(N.getNode()->getOperand(0), AM, Depth+1) && 1072 !MatchAddressRecursively(N.getNode()->getOperand(1), AM, Depth+1)) 1073 return false; 1074 AM = Backup; 1075 if (!MatchAddressRecursively(N.getNode()->getOperand(1), AM, Depth+1) && 1076 !MatchAddressRecursively(N.getNode()->getOperand(0), AM, Depth+1)) 1077 return false; 1078 AM = Backup; 1079 1080 // If we couldn't fold both operands into the address at the same time, 1081 // see if we can just put each operand into a register and fold at least 1082 // the add. 1083 if (AM.BaseType == X86ISelAddressMode::RegBase && 1084 !AM.Base.Reg.getNode() && 1085 !AM.IndexReg.getNode()) { 1086 AM.Base.Reg = N.getNode()->getOperand(0); 1087 AM.IndexReg = N.getNode()->getOperand(1); 1088 AM.Scale = 1; 1089 return false; 1090 } 1091 break; 1092 } 1093 1094 case ISD::OR: 1095 // Handle "X | C" as "X + C" iff X is known to have C bits clear. 1096 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1097 X86ISelAddressMode Backup = AM; 1098 uint64_t Offset = CN->getSExtValue(); 1099 // Start with the LHS as an addr mode. 1100 if (!MatchAddressRecursively(N.getOperand(0), AM, Depth+1) && 1101 // Address could not have picked a GV address for the displacement. 1102 AM.GV == NULL && 1103 // On x86-64, the resultant disp must fit in 32-bits. 1104 (!is64Bit || 1105 X86::isOffsetSuitableForCodeModel(AM.Disp + Offset, M, 1106 AM.hasSymbolicDisplacement())) && 1107 // Check to see if the LHS & C is zero. 1108 CurDAG->MaskedValueIsZero(N.getOperand(0), CN->getAPIntValue())) { 1109 AM.Disp += Offset; 1110 return false; 1111 } 1112 AM = Backup; 1113 } 1114 break; 1115 1116 case ISD::AND: { 1117 // Perform some heroic transforms on an and of a constant-count shift 1118 // with a constant to enable use of the scaled offset field. 1119 1120 SDValue Shift = N.getOperand(0); 1121 if (Shift.getNumOperands() != 2) break; 1122 1123 // Scale must not be used already. 1124 if (AM.IndexReg.getNode() != 0 || AM.Scale != 1) break; 1125 1126 SDValue X = Shift.getOperand(0); 1127 ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N.getOperand(1)); 1128 ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(Shift.getOperand(1)); 1129 if (!C1 || !C2) break; 1130 1131 // Handle "(X >> (8-C1)) & C2" as "(X >> 8) & 0xff)" if safe. This 1132 // allows us to convert the shift and and into an h-register extract and 1133 // a scaled index. 1134 if (Shift.getOpcode() == ISD::SRL && Shift.hasOneUse()) { 1135 unsigned ScaleLog = 8 - C1->getZExtValue(); 1136 if (ScaleLog > 0 && ScaleLog < 4 && 1137 C2->getZExtValue() == (UINT64_C(0xff) << ScaleLog)) { 1138 SDValue Eight = CurDAG->getConstant(8, MVT::i8); 1139 SDValue Mask = CurDAG->getConstant(0xff, N.getValueType()); 1140 SDValue Srl = CurDAG->getNode(ISD::SRL, dl, N.getValueType(), 1141 X, Eight); 1142 SDValue And = CurDAG->getNode(ISD::AND, dl, N.getValueType(), 1143 Srl, Mask); 1144 SDValue ShlCount = CurDAG->getConstant(ScaleLog, MVT::i8); 1145 SDValue Shl = CurDAG->getNode(ISD::SHL, dl, N.getValueType(), 1146 And, ShlCount); 1147 1148 // Insert the new nodes into the topological ordering. 1149 if (Eight.getNode()->getNodeId() == -1 || 1150 Eight.getNode()->getNodeId() > X.getNode()->getNodeId()) { 1151 CurDAG->RepositionNode(X.getNode(), Eight.getNode()); 1152 Eight.getNode()->setNodeId(X.getNode()->getNodeId()); 1153 } 1154 if (Mask.getNode()->getNodeId() == -1 || 1155 Mask.getNode()->getNodeId() > X.getNode()->getNodeId()) { 1156 CurDAG->RepositionNode(X.getNode(), Mask.getNode()); 1157 Mask.getNode()->setNodeId(X.getNode()->getNodeId()); 1158 } 1159 if (Srl.getNode()->getNodeId() == -1 || 1160 Srl.getNode()->getNodeId() > Shift.getNode()->getNodeId()) { 1161 CurDAG->RepositionNode(Shift.getNode(), Srl.getNode()); 1162 Srl.getNode()->setNodeId(Shift.getNode()->getNodeId()); 1163 } 1164 if (And.getNode()->getNodeId() == -1 || 1165 And.getNode()->getNodeId() > N.getNode()->getNodeId()) { 1166 CurDAG->RepositionNode(N.getNode(), And.getNode()); 1167 And.getNode()->setNodeId(N.getNode()->getNodeId()); 1168 } 1169 if (ShlCount.getNode()->getNodeId() == -1 || 1170 ShlCount.getNode()->getNodeId() > X.getNode()->getNodeId()) { 1171 CurDAG->RepositionNode(X.getNode(), ShlCount.getNode()); 1172 ShlCount.getNode()->setNodeId(N.getNode()->getNodeId()); 1173 } 1174 if (Shl.getNode()->getNodeId() == -1 || 1175 Shl.getNode()->getNodeId() > N.getNode()->getNodeId()) { 1176 CurDAG->RepositionNode(N.getNode(), Shl.getNode()); 1177 Shl.getNode()->setNodeId(N.getNode()->getNodeId()); 1178 } 1179 CurDAG->ReplaceAllUsesWith(N, Shl); 1180 AM.IndexReg = And; 1181 AM.Scale = (1 << ScaleLog); 1182 return false; 1183 } 1184 } 1185 1186 // Handle "(X << C1) & C2" as "(X & (C2>>C1)) << C1" if safe and if this 1187 // allows us to fold the shift into this addressing mode. 1188 if (Shift.getOpcode() != ISD::SHL) break; 1189 1190 // Not likely to be profitable if either the AND or SHIFT node has more 1191 // than one use (unless all uses are for address computation). Besides, 1192 // isel mechanism requires their node ids to be reused. 1193 if (!N.hasOneUse() || !Shift.hasOneUse()) 1194 break; 1195 1196 // Verify that the shift amount is something we can fold. 1197 unsigned ShiftCst = C1->getZExtValue(); 1198 if (ShiftCst != 1 && ShiftCst != 2 && ShiftCst != 3) 1199 break; 1200 1201 // Get the new AND mask, this folds to a constant. 1202 SDValue NewANDMask = CurDAG->getNode(ISD::SRL, dl, N.getValueType(), 1203 SDValue(C2, 0), SDValue(C1, 0)); 1204 SDValue NewAND = CurDAG->getNode(ISD::AND, dl, N.getValueType(), X, 1205 NewANDMask); 1206 SDValue NewSHIFT = CurDAG->getNode(ISD::SHL, dl, N.getValueType(), 1207 NewAND, SDValue(C1, 0)); 1208 1209 // Insert the new nodes into the topological ordering. 1210 if (C1->getNodeId() > X.getNode()->getNodeId()) { 1211 CurDAG->RepositionNode(X.getNode(), C1); 1212 C1->setNodeId(X.getNode()->getNodeId()); 1213 } 1214 if (NewANDMask.getNode()->getNodeId() == -1 || 1215 NewANDMask.getNode()->getNodeId() > X.getNode()->getNodeId()) { 1216 CurDAG->RepositionNode(X.getNode(), NewANDMask.getNode()); 1217 NewANDMask.getNode()->setNodeId(X.getNode()->getNodeId()); 1218 } 1219 if (NewAND.getNode()->getNodeId() == -1 || 1220 NewAND.getNode()->getNodeId() > Shift.getNode()->getNodeId()) { 1221 CurDAG->RepositionNode(Shift.getNode(), NewAND.getNode()); 1222 NewAND.getNode()->setNodeId(Shift.getNode()->getNodeId()); 1223 } 1224 if (NewSHIFT.getNode()->getNodeId() == -1 || 1225 NewSHIFT.getNode()->getNodeId() > N.getNode()->getNodeId()) { 1226 CurDAG->RepositionNode(N.getNode(), NewSHIFT.getNode()); 1227 NewSHIFT.getNode()->setNodeId(N.getNode()->getNodeId()); 1228 } 1229 1230 CurDAG->ReplaceAllUsesWith(N, NewSHIFT); 1231 1232 AM.Scale = 1 << ShiftCst; 1233 AM.IndexReg = NewAND; 1234 return false; 1235 } 1236 } 1237 1238 return MatchAddressBase(N, AM); 1239} 1240 1241/// MatchAddressBase - Helper for MatchAddress. Add the specified node to the 1242/// specified addressing mode without any further recursion. 1243bool X86DAGToDAGISel::MatchAddressBase(SDValue N, X86ISelAddressMode &AM) { 1244 // Is the base register already occupied? 1245 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base.Reg.getNode()) { 1246 // If so, check to see if the scale index register is set. 1247 if (AM.IndexReg.getNode() == 0) { 1248 AM.IndexReg = N; 1249 AM.Scale = 1; 1250 return false; 1251 } 1252 1253 // Otherwise, we cannot select it. 1254 return true; 1255 } 1256 1257 // Default, generate it as a register. 1258 AM.BaseType = X86ISelAddressMode::RegBase; 1259 AM.Base.Reg = N; 1260 return false; 1261} 1262 1263/// SelectAddr - returns true if it is able pattern match an addressing mode. 1264/// It returns the operands which make up the maximal addressing mode it can 1265/// match by reference. 1266bool X86DAGToDAGISel::SelectAddr(SDValue Op, SDValue N, SDValue &Base, 1267 SDValue &Scale, SDValue &Index, 1268 SDValue &Disp, SDValue &Segment) { 1269 X86ISelAddressMode AM; 1270 bool Done = false; 1271 if (AvoidDupAddrCompute && !N.hasOneUse()) { 1272 unsigned Opcode = N.getOpcode(); 1273 if (Opcode != ISD::Constant && Opcode != ISD::FrameIndex && 1274 Opcode != X86ISD::Wrapper && Opcode != X86ISD::WrapperRIP) { 1275 // If we are able to fold N into addressing mode, then we'll allow it even 1276 // if N has multiple uses. In general, addressing computation is used as 1277 // addresses by all of its uses. But watch out for CopyToReg uses, that 1278 // means the address computation is liveout. It will be computed by a LEA 1279 // so we want to avoid computing the address twice. 1280 for (SDNode::use_iterator UI = N.getNode()->use_begin(), 1281 UE = N.getNode()->use_end(); UI != UE; ++UI) { 1282 if (UI->getOpcode() == ISD::CopyToReg) { 1283 MatchAddressBase(N, AM); 1284 Done = true; 1285 break; 1286 } 1287 } 1288 } 1289 } 1290 1291 if (!Done && MatchAddress(N, AM)) 1292 return false; 1293 1294 EVT VT = N.getValueType(); 1295 if (AM.BaseType == X86ISelAddressMode::RegBase) { 1296 if (!AM.Base.Reg.getNode()) 1297 AM.Base.Reg = CurDAG->getRegister(0, VT); 1298 } 1299 1300 if (!AM.IndexReg.getNode()) 1301 AM.IndexReg = CurDAG->getRegister(0, VT); 1302 1303 getAddressOperands(AM, Base, Scale, Index, Disp, Segment); 1304 return true; 1305} 1306 1307/// SelectScalarSSELoad - Match a scalar SSE load. In particular, we want to 1308/// match a load whose top elements are either undef or zeros. The load flavor 1309/// is derived from the type of N, which is either v4f32 or v2f64. 1310bool X86DAGToDAGISel::SelectScalarSSELoad(SDValue Op, SDValue Pred, 1311 SDValue N, SDValue &Base, 1312 SDValue &Scale, SDValue &Index, 1313 SDValue &Disp, SDValue &Segment, 1314 SDValue &InChain, 1315 SDValue &OutChain) { 1316 if (N.getOpcode() == ISD::SCALAR_TO_VECTOR) { 1317 InChain = N.getOperand(0).getValue(1); 1318 if (ISD::isNON_EXTLoad(InChain.getNode()) && 1319 InChain.getValue(0).hasOneUse() && 1320 N.hasOneUse() && 1321 IsLegalAndProfitableToFold(N.getNode(), Pred.getNode(), Op.getNode())) { 1322 LoadSDNode *LD = cast<LoadSDNode>(InChain); 1323 if (!SelectAddr(Op, LD->getBasePtr(), Base, Scale, Index, Disp, Segment)) 1324 return false; 1325 OutChain = LD->getChain(); 1326 return true; 1327 } 1328 } 1329 1330 // Also handle the case where we explicitly require zeros in the top 1331 // elements. This is a vector shuffle from the zero vector. 1332 if (N.getOpcode() == X86ISD::VZEXT_MOVL && N.getNode()->hasOneUse() && 1333 // Check to see if the top elements are all zeros (or bitcast of zeros). 1334 N.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR && 1335 N.getOperand(0).getNode()->hasOneUse() && 1336 ISD::isNON_EXTLoad(N.getOperand(0).getOperand(0).getNode()) && 1337 N.getOperand(0).getOperand(0).hasOneUse()) { 1338 // Okay, this is a zero extending load. Fold it. 1339 LoadSDNode *LD = cast<LoadSDNode>(N.getOperand(0).getOperand(0)); 1340 if (!SelectAddr(Op, LD->getBasePtr(), Base, Scale, Index, Disp, Segment)) 1341 return false; 1342 OutChain = LD->getChain(); 1343 InChain = SDValue(LD, 1); 1344 return true; 1345 } 1346 return false; 1347} 1348 1349 1350/// SelectLEAAddr - it calls SelectAddr and determines if the maximal addressing 1351/// mode it matches can be cost effectively emitted as an LEA instruction. 1352bool X86DAGToDAGISel::SelectLEAAddr(SDValue Op, SDValue N, 1353 SDValue &Base, SDValue &Scale, 1354 SDValue &Index, SDValue &Disp) { 1355 X86ISelAddressMode AM; 1356 1357 // Set AM.Segment to prevent MatchAddress from using one. LEA doesn't support 1358 // segments. 1359 SDValue Copy = AM.Segment; 1360 SDValue T = CurDAG->getRegister(0, MVT::i32); 1361 AM.Segment = T; 1362 if (MatchAddress(N, AM)) 1363 return false; 1364 assert (T == AM.Segment); 1365 AM.Segment = Copy; 1366 1367 EVT VT = N.getValueType(); 1368 unsigned Complexity = 0; 1369 if (AM.BaseType == X86ISelAddressMode::RegBase) 1370 if (AM.Base.Reg.getNode()) 1371 Complexity = 1; 1372 else 1373 AM.Base.Reg = CurDAG->getRegister(0, VT); 1374 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase) 1375 Complexity = 4; 1376 1377 if (AM.IndexReg.getNode()) 1378 Complexity++; 1379 else 1380 AM.IndexReg = CurDAG->getRegister(0, VT); 1381 1382 // Don't match just leal(,%reg,2). It's cheaper to do addl %reg, %reg, or with 1383 // a simple shift. 1384 if (AM.Scale > 1) 1385 Complexity++; 1386 1387 // FIXME: We are artificially lowering the criteria to turn ADD %reg, $GA 1388 // to a LEA. This is determined with some expermentation but is by no means 1389 // optimal (especially for code size consideration). LEA is nice because of 1390 // its three-address nature. Tweak the cost function again when we can run 1391 // convertToThreeAddress() at register allocation time. 1392 if (AM.hasSymbolicDisplacement()) { 1393 // For X86-64, we should always use lea to materialize RIP relative 1394 // addresses. 1395 if (Subtarget->is64Bit()) 1396 Complexity = 4; 1397 else 1398 Complexity += 2; 1399 } 1400 1401 if (AM.Disp && (AM.Base.Reg.getNode() || AM.IndexReg.getNode())) 1402 Complexity++; 1403 1404 // If it isn't worth using an LEA, reject it. 1405 if (Complexity <= 2) 1406 return false; 1407 1408 SDValue Segment; 1409 getAddressOperands(AM, Base, Scale, Index, Disp, Segment); 1410 return true; 1411} 1412 1413/// SelectTLSADDRAddr - This is only run on TargetGlobalTLSAddress nodes. 1414bool X86DAGToDAGISel::SelectTLSADDRAddr(SDValue Op, SDValue N, SDValue &Base, 1415 SDValue &Scale, SDValue &Index, 1416 SDValue &Disp) { 1417 assert(Op.getOpcode() == X86ISD::TLSADDR); 1418 assert(N.getOpcode() == ISD::TargetGlobalTLSAddress); 1419 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 1420 1421 X86ISelAddressMode AM; 1422 AM.GV = GA->getGlobal(); 1423 AM.Disp += GA->getOffset(); 1424 AM.Base.Reg = CurDAG->getRegister(0, N.getValueType()); 1425 AM.SymbolFlags = GA->getTargetFlags(); 1426 1427 if (N.getValueType() == MVT::i32) { 1428 AM.Scale = 1; 1429 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32); 1430 } else { 1431 AM.IndexReg = CurDAG->getRegister(0, MVT::i64); 1432 } 1433 1434 SDValue Segment; 1435 getAddressOperands(AM, Base, Scale, Index, Disp, Segment); 1436 return true; 1437} 1438 1439 1440bool X86DAGToDAGISel::TryFoldLoad(SDValue P, SDValue N, 1441 SDValue &Base, SDValue &Scale, 1442 SDValue &Index, SDValue &Disp, 1443 SDValue &Segment) { 1444 if (ISD::isNON_EXTLoad(N.getNode()) && 1445 N.hasOneUse() && 1446 IsLegalAndProfitableToFold(N.getNode(), P.getNode(), P.getNode())) 1447 return SelectAddr(P, N.getOperand(1), Base, Scale, Index, Disp, Segment); 1448 return false; 1449} 1450 1451/// getGlobalBaseReg - Return an SDNode that returns the value of 1452/// the global base register. Output instructions required to 1453/// initialize the global base register, if necessary. 1454/// 1455SDNode *X86DAGToDAGISel::getGlobalBaseReg() { 1456 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF); 1457 return CurDAG->getRegister(GlobalBaseReg, TLI.getPointerTy()).getNode(); 1458} 1459 1460static SDNode *FindCallStartFromCall(SDNode *Node) { 1461 if (Node->getOpcode() == ISD::CALLSEQ_START) return Node; 1462 assert(Node->getOperand(0).getValueType() == MVT::Other && 1463 "Node doesn't have a token chain argument!"); 1464 return FindCallStartFromCall(Node->getOperand(0).getNode()); 1465} 1466 1467SDNode *X86DAGToDAGISel::SelectAtomic64(SDNode *Node, unsigned Opc) { 1468 SDValue Chain = Node->getOperand(0); 1469 SDValue In1 = Node->getOperand(1); 1470 SDValue In2L = Node->getOperand(2); 1471 SDValue In2H = Node->getOperand(3); 1472 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 1473 if (!SelectAddr(In1, In1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) 1474 return NULL; 1475 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1476 MemOp[0] = cast<MemSDNode>(Node)->getMemOperand(); 1477 const SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, In2L, In2H, Chain}; 1478 SDNode *ResNode = CurDAG->getMachineNode(Opc, Node->getDebugLoc(), 1479 MVT::i32, MVT::i32, MVT::Other, Ops, 1480 array_lengthof(Ops)); 1481 cast<MachineSDNode>(ResNode)->setMemRefs(MemOp, MemOp + 1); 1482 return ResNode; 1483} 1484 1485SDNode *X86DAGToDAGISel::SelectAtomicLoadAdd(SDNode *Node, EVT NVT) { 1486 if (Node->hasAnyUseOfValue(0)) 1487 return 0; 1488 1489 // Optimize common patterns for __sync_add_and_fetch and 1490 // __sync_sub_and_fetch where the result is not used. This allows us 1491 // to use "lock" version of add, sub, inc, dec instructions. 1492 // FIXME: Do not use special instructions but instead add the "lock" 1493 // prefix to the target node somehow. The extra information will then be 1494 // transferred to machine instruction and it denotes the prefix. 1495 SDValue Chain = Node->getOperand(0); 1496 SDValue Ptr = Node->getOperand(1); 1497 SDValue Val = Node->getOperand(2); 1498 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 1499 if (!SelectAddr(Ptr, Ptr, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) 1500 return 0; 1501 1502 bool isInc = false, isDec = false, isSub = false, isCN = false; 1503 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val); 1504 if (CN) { 1505 isCN = true; 1506 int64_t CNVal = CN->getSExtValue(); 1507 if (CNVal == 1) 1508 isInc = true; 1509 else if (CNVal == -1) 1510 isDec = true; 1511 else if (CNVal >= 0) 1512 Val = CurDAG->getTargetConstant(CNVal, NVT); 1513 else { 1514 isSub = true; 1515 Val = CurDAG->getTargetConstant(-CNVal, NVT); 1516 } 1517 } else if (Val.hasOneUse() && 1518 Val.getOpcode() == ISD::SUB && 1519 X86::isZeroNode(Val.getOperand(0))) { 1520 isSub = true; 1521 Val = Val.getOperand(1); 1522 } 1523 1524 unsigned Opc = 0; 1525 switch (NVT.getSimpleVT().SimpleTy) { 1526 default: return 0; 1527 case MVT::i8: 1528 if (isInc) 1529 Opc = X86::LOCK_INC8m; 1530 else if (isDec) 1531 Opc = X86::LOCK_DEC8m; 1532 else if (isSub) { 1533 if (isCN) 1534 Opc = X86::LOCK_SUB8mi; 1535 else 1536 Opc = X86::LOCK_SUB8mr; 1537 } else { 1538 if (isCN) 1539 Opc = X86::LOCK_ADD8mi; 1540 else 1541 Opc = X86::LOCK_ADD8mr; 1542 } 1543 break; 1544 case MVT::i16: 1545 if (isInc) 1546 Opc = X86::LOCK_INC16m; 1547 else if (isDec) 1548 Opc = X86::LOCK_DEC16m; 1549 else if (isSub) { 1550 if (isCN) { 1551 if (Predicate_i16immSExt8(Val.getNode())) 1552 Opc = X86::LOCK_SUB16mi8; 1553 else 1554 Opc = X86::LOCK_SUB16mi; 1555 } else 1556 Opc = X86::LOCK_SUB16mr; 1557 } else { 1558 if (isCN) { 1559 if (Predicate_i16immSExt8(Val.getNode())) 1560 Opc = X86::LOCK_ADD16mi8; 1561 else 1562 Opc = X86::LOCK_ADD16mi; 1563 } else 1564 Opc = X86::LOCK_ADD16mr; 1565 } 1566 break; 1567 case MVT::i32: 1568 if (isInc) 1569 Opc = X86::LOCK_INC32m; 1570 else if (isDec) 1571 Opc = X86::LOCK_DEC32m; 1572 else if (isSub) { 1573 if (isCN) { 1574 if (Predicate_i32immSExt8(Val.getNode())) 1575 Opc = X86::LOCK_SUB32mi8; 1576 else 1577 Opc = X86::LOCK_SUB32mi; 1578 } else 1579 Opc = X86::LOCK_SUB32mr; 1580 } else { 1581 if (isCN) { 1582 if (Predicate_i32immSExt8(Val.getNode())) 1583 Opc = X86::LOCK_ADD32mi8; 1584 else 1585 Opc = X86::LOCK_ADD32mi; 1586 } else 1587 Opc = X86::LOCK_ADD32mr; 1588 } 1589 break; 1590 case MVT::i64: 1591 if (isInc) 1592 Opc = X86::LOCK_INC64m; 1593 else if (isDec) 1594 Opc = X86::LOCK_DEC64m; 1595 else if (isSub) { 1596 Opc = X86::LOCK_SUB64mr; 1597 if (isCN) { 1598 if (Predicate_i64immSExt8(Val.getNode())) 1599 Opc = X86::LOCK_SUB64mi8; 1600 else if (Predicate_i64immSExt32(Val.getNode())) 1601 Opc = X86::LOCK_SUB64mi32; 1602 } 1603 } else { 1604 Opc = X86::LOCK_ADD64mr; 1605 if (isCN) { 1606 if (Predicate_i64immSExt8(Val.getNode())) 1607 Opc = X86::LOCK_ADD64mi8; 1608 else if (Predicate_i64immSExt32(Val.getNode())) 1609 Opc = X86::LOCK_ADD64mi32; 1610 } 1611 } 1612 break; 1613 } 1614 1615 DebugLoc dl = Node->getDebugLoc(); 1616 SDValue Undef = SDValue(CurDAG->getMachineNode(TargetInstrInfo::IMPLICIT_DEF, 1617 dl, NVT), 0); 1618 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1619 MemOp[0] = cast<MemSDNode>(Node)->getMemOperand(); 1620 if (isInc || isDec) { 1621 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain }; 1622 SDValue Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 6), 0); 1623 cast<MachineSDNode>(Ret)->setMemRefs(MemOp, MemOp + 1); 1624 SDValue RetVals[] = { Undef, Ret }; 1625 return CurDAG->getMergeValues(RetVals, 2, dl).getNode(); 1626 } else { 1627 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Val, Chain }; 1628 SDValue Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 7), 0); 1629 cast<MachineSDNode>(Ret)->setMemRefs(MemOp, MemOp + 1); 1630 SDValue RetVals[] = { Undef, Ret }; 1631 return CurDAG->getMergeValues(RetVals, 2, dl).getNode(); 1632 } 1633} 1634 1635/// HasNoSignedComparisonUses - Test whether the given X86ISD::CMP node has 1636/// any uses which require the SF or OF bits to be accurate. 1637static bool HasNoSignedComparisonUses(SDNode *N) { 1638 // Examine each user of the node. 1639 for (SDNode::use_iterator UI = N->use_begin(), 1640 UE = N->use_end(); UI != UE; ++UI) { 1641 // Only examine CopyToReg uses. 1642 if (UI->getOpcode() != ISD::CopyToReg) 1643 return false; 1644 // Only examine CopyToReg uses that copy to EFLAGS. 1645 if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != 1646 X86::EFLAGS) 1647 return false; 1648 // Examine each user of the CopyToReg use. 1649 for (SDNode::use_iterator FlagUI = UI->use_begin(), 1650 FlagUE = UI->use_end(); FlagUI != FlagUE; ++FlagUI) { 1651 // Only examine the Flag result. 1652 if (FlagUI.getUse().getResNo() != 1) continue; 1653 // Anything unusual: assume conservatively. 1654 if (!FlagUI->isMachineOpcode()) return false; 1655 // Examine the opcode of the user. 1656 switch (FlagUI->getMachineOpcode()) { 1657 // These comparisons don't treat the most significant bit specially. 1658 case X86::SETAr: case X86::SETAEr: case X86::SETBr: case X86::SETBEr: 1659 case X86::SETEr: case X86::SETNEr: case X86::SETPr: case X86::SETNPr: 1660 case X86::SETAm: case X86::SETAEm: case X86::SETBm: case X86::SETBEm: 1661 case X86::SETEm: case X86::SETNEm: case X86::SETPm: case X86::SETNPm: 1662 case X86::JA: case X86::JAE: case X86::JB: case X86::JBE: 1663 case X86::JE: case X86::JNE: case X86::JP: case X86::JNP: 1664 case X86::CMOVA16rr: case X86::CMOVA16rm: 1665 case X86::CMOVA32rr: case X86::CMOVA32rm: 1666 case X86::CMOVA64rr: case X86::CMOVA64rm: 1667 case X86::CMOVAE16rr: case X86::CMOVAE16rm: 1668 case X86::CMOVAE32rr: case X86::CMOVAE32rm: 1669 case X86::CMOVAE64rr: case X86::CMOVAE64rm: 1670 case X86::CMOVB16rr: case X86::CMOVB16rm: 1671 case X86::CMOVB32rr: case X86::CMOVB32rm: 1672 case X86::CMOVB64rr: case X86::CMOVB64rm: 1673 case X86::CMOVBE16rr: case X86::CMOVBE16rm: 1674 case X86::CMOVBE32rr: case X86::CMOVBE32rm: 1675 case X86::CMOVBE64rr: case X86::CMOVBE64rm: 1676 case X86::CMOVE16rr: case X86::CMOVE16rm: 1677 case X86::CMOVE32rr: case X86::CMOVE32rm: 1678 case X86::CMOVE64rr: case X86::CMOVE64rm: 1679 case X86::CMOVNE16rr: case X86::CMOVNE16rm: 1680 case X86::CMOVNE32rr: case X86::CMOVNE32rm: 1681 case X86::CMOVNE64rr: case X86::CMOVNE64rm: 1682 case X86::CMOVNP16rr: case X86::CMOVNP16rm: 1683 case X86::CMOVNP32rr: case X86::CMOVNP32rm: 1684 case X86::CMOVNP64rr: case X86::CMOVNP64rm: 1685 case X86::CMOVP16rr: case X86::CMOVP16rm: 1686 case X86::CMOVP32rr: case X86::CMOVP32rm: 1687 case X86::CMOVP64rr: case X86::CMOVP64rm: 1688 continue; 1689 // Anything else: assume conservatively. 1690 default: return false; 1691 } 1692 } 1693 } 1694 return true; 1695} 1696 1697SDNode *X86DAGToDAGISel::Select(SDValue N) { 1698 SDNode *Node = N.getNode(); 1699 EVT NVT = Node->getValueType(0); 1700 unsigned Opc, MOpc; 1701 unsigned Opcode = Node->getOpcode(); 1702 DebugLoc dl = Node->getDebugLoc(); 1703 1704#ifndef NDEBUG 1705 DEBUG({ 1706 errs() << std::string(Indent, ' ') << "Selecting: "; 1707 Node->dump(CurDAG); 1708 errs() << '\n'; 1709 }); 1710 Indent += 2; 1711#endif 1712 1713 if (Node->isMachineOpcode()) { 1714#ifndef NDEBUG 1715 DEBUG({ 1716 errs() << std::string(Indent-2, ' ') << "== "; 1717 Node->dump(CurDAG); 1718 errs() << '\n'; 1719 }); 1720 Indent -= 2; 1721#endif 1722 return NULL; // Already selected. 1723 } 1724 1725 switch (Opcode) { 1726 default: break; 1727 case X86ISD::GlobalBaseReg: 1728 return getGlobalBaseReg(); 1729 1730 case X86ISD::ATOMOR64_DAG: 1731 return SelectAtomic64(Node, X86::ATOMOR6432); 1732 case X86ISD::ATOMXOR64_DAG: 1733 return SelectAtomic64(Node, X86::ATOMXOR6432); 1734 case X86ISD::ATOMADD64_DAG: 1735 return SelectAtomic64(Node, X86::ATOMADD6432); 1736 case X86ISD::ATOMSUB64_DAG: 1737 return SelectAtomic64(Node, X86::ATOMSUB6432); 1738 case X86ISD::ATOMNAND64_DAG: 1739 return SelectAtomic64(Node, X86::ATOMNAND6432); 1740 case X86ISD::ATOMAND64_DAG: 1741 return SelectAtomic64(Node, X86::ATOMAND6432); 1742 case X86ISD::ATOMSWAP64_DAG: 1743 return SelectAtomic64(Node, X86::ATOMSWAP6432); 1744 1745 case ISD::ATOMIC_LOAD_ADD: { 1746 SDNode *RetVal = SelectAtomicLoadAdd(Node, NVT); 1747 if (RetVal) 1748 return RetVal; 1749 break; 1750 } 1751 1752 case ISD::SMUL_LOHI: 1753 case ISD::UMUL_LOHI: { 1754 SDValue N0 = Node->getOperand(0); 1755 SDValue N1 = Node->getOperand(1); 1756 1757 bool isSigned = Opcode == ISD::SMUL_LOHI; 1758 if (!isSigned) { 1759 switch (NVT.getSimpleVT().SimpleTy) { 1760 default: llvm_unreachable("Unsupported VT!"); 1761 case MVT::i8: Opc = X86::MUL8r; MOpc = X86::MUL8m; break; 1762 case MVT::i16: Opc = X86::MUL16r; MOpc = X86::MUL16m; break; 1763 case MVT::i32: Opc = X86::MUL32r; MOpc = X86::MUL32m; break; 1764 case MVT::i64: Opc = X86::MUL64r; MOpc = X86::MUL64m; break; 1765 } 1766 } else { 1767 switch (NVT.getSimpleVT().SimpleTy) { 1768 default: llvm_unreachable("Unsupported VT!"); 1769 case MVT::i8: Opc = X86::IMUL8r; MOpc = X86::IMUL8m; break; 1770 case MVT::i16: Opc = X86::IMUL16r; MOpc = X86::IMUL16m; break; 1771 case MVT::i32: Opc = X86::IMUL32r; MOpc = X86::IMUL32m; break; 1772 case MVT::i64: Opc = X86::IMUL64r; MOpc = X86::IMUL64m; break; 1773 } 1774 } 1775 1776 unsigned LoReg, HiReg; 1777 switch (NVT.getSimpleVT().SimpleTy) { 1778 default: llvm_unreachable("Unsupported VT!"); 1779 case MVT::i8: LoReg = X86::AL; HiReg = X86::AH; break; 1780 case MVT::i16: LoReg = X86::AX; HiReg = X86::DX; break; 1781 case MVT::i32: LoReg = X86::EAX; HiReg = X86::EDX; break; 1782 case MVT::i64: LoReg = X86::RAX; HiReg = X86::RDX; break; 1783 } 1784 1785 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 1786 bool foldedLoad = TryFoldLoad(N, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 1787 // Multiply is commmutative. 1788 if (!foldedLoad) { 1789 foldedLoad = TryFoldLoad(N, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 1790 if (foldedLoad) 1791 std::swap(N0, N1); 1792 } 1793 1794 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg, 1795 N0, SDValue()).getValue(1); 1796 1797 if (foldedLoad) { 1798 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), 1799 InFlag }; 1800 SDNode *CNode = 1801 CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Flag, Ops, 1802 array_lengthof(Ops)); 1803 InFlag = SDValue(CNode, 1); 1804 // Update the chain. 1805 ReplaceUses(N1.getValue(1), SDValue(CNode, 0)); 1806 } else { 1807 InFlag = 1808 SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Flag, N1, InFlag), 0); 1809 } 1810 1811 // Copy the low half of the result, if it is needed. 1812 if (!N.getValue(0).use_empty()) { 1813 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1814 LoReg, NVT, InFlag); 1815 InFlag = Result.getValue(2); 1816 ReplaceUses(N.getValue(0), Result); 1817#ifndef NDEBUG 1818 DEBUG({ 1819 errs() << std::string(Indent-2, ' ') << "=> "; 1820 Result.getNode()->dump(CurDAG); 1821 errs() << '\n'; 1822 }); 1823#endif 1824 } 1825 // Copy the high half of the result, if it is needed. 1826 if (!N.getValue(1).use_empty()) { 1827 SDValue Result; 1828 if (HiReg == X86::AH && Subtarget->is64Bit()) { 1829 // Prevent use of AH in a REX instruction by referencing AX instead. 1830 // Shift it down 8 bits. 1831 Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1832 X86::AX, MVT::i16, InFlag); 1833 InFlag = Result.getValue(2); 1834 Result = SDValue(CurDAG->getMachineNode(X86::SHR16ri, dl, MVT::i16, 1835 Result, 1836 CurDAG->getTargetConstant(8, MVT::i8)), 0); 1837 // Then truncate it down to i8. 1838 Result = CurDAG->getTargetExtractSubreg(X86::SUBREG_8BIT, dl, 1839 MVT::i8, Result); 1840 } else { 1841 Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1842 HiReg, NVT, InFlag); 1843 InFlag = Result.getValue(2); 1844 } 1845 ReplaceUses(N.getValue(1), Result); 1846#ifndef NDEBUG 1847 DEBUG({ 1848 errs() << std::string(Indent-2, ' ') << "=> "; 1849 Result.getNode()->dump(CurDAG); 1850 errs() << '\n'; 1851 }); 1852#endif 1853 } 1854 1855#ifndef NDEBUG 1856 Indent -= 2; 1857#endif 1858 1859 return NULL; 1860 } 1861 1862 case ISD::SDIVREM: 1863 case ISD::UDIVREM: { 1864 SDValue N0 = Node->getOperand(0); 1865 SDValue N1 = Node->getOperand(1); 1866 1867 bool isSigned = Opcode == ISD::SDIVREM; 1868 if (!isSigned) { 1869 switch (NVT.getSimpleVT().SimpleTy) { 1870 default: llvm_unreachable("Unsupported VT!"); 1871 case MVT::i8: Opc = X86::DIV8r; MOpc = X86::DIV8m; break; 1872 case MVT::i16: Opc = X86::DIV16r; MOpc = X86::DIV16m; break; 1873 case MVT::i32: Opc = X86::DIV32r; MOpc = X86::DIV32m; break; 1874 case MVT::i64: Opc = X86::DIV64r; MOpc = X86::DIV64m; break; 1875 } 1876 } else { 1877 switch (NVT.getSimpleVT().SimpleTy) { 1878 default: llvm_unreachable("Unsupported VT!"); 1879 case MVT::i8: Opc = X86::IDIV8r; MOpc = X86::IDIV8m; break; 1880 case MVT::i16: Opc = X86::IDIV16r; MOpc = X86::IDIV16m; break; 1881 case MVT::i32: Opc = X86::IDIV32r; MOpc = X86::IDIV32m; break; 1882 case MVT::i64: Opc = X86::IDIV64r; MOpc = X86::IDIV64m; break; 1883 } 1884 } 1885 1886 unsigned LoReg, HiReg; 1887 unsigned ClrOpcode, SExtOpcode; 1888 switch (NVT.getSimpleVT().SimpleTy) { 1889 default: llvm_unreachable("Unsupported VT!"); 1890 case MVT::i8: 1891 LoReg = X86::AL; HiReg = X86::AH; 1892 ClrOpcode = 0; 1893 SExtOpcode = X86::CBW; 1894 break; 1895 case MVT::i16: 1896 LoReg = X86::AX; HiReg = X86::DX; 1897 ClrOpcode = X86::MOV16r0; 1898 SExtOpcode = X86::CWD; 1899 break; 1900 case MVT::i32: 1901 LoReg = X86::EAX; HiReg = X86::EDX; 1902 ClrOpcode = X86::MOV32r0; 1903 SExtOpcode = X86::CDQ; 1904 break; 1905 case MVT::i64: 1906 LoReg = X86::RAX; HiReg = X86::RDX; 1907 ClrOpcode = ~0U; // NOT USED. 1908 SExtOpcode = X86::CQO; 1909 break; 1910 } 1911 1912 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 1913 bool foldedLoad = TryFoldLoad(N, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 1914 bool signBitIsZero = CurDAG->SignBitIsZero(N0); 1915 1916 SDValue InFlag; 1917 if (NVT == MVT::i8 && (!isSigned || signBitIsZero)) { 1918 // Special case for div8, just use a move with zero extension to AX to 1919 // clear the upper 8 bits (AH). 1920 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Move, Chain; 1921 if (TryFoldLoad(N, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) { 1922 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0) }; 1923 Move = 1924 SDValue(CurDAG->getMachineNode(X86::MOVZX16rm8, dl, MVT::i16, 1925 MVT::Other, Ops, 1926 array_lengthof(Ops)), 0); 1927 Chain = Move.getValue(1); 1928 ReplaceUses(N0.getValue(1), Chain); 1929 } else { 1930 Move = 1931 SDValue(CurDAG->getMachineNode(X86::MOVZX16rr8, dl, MVT::i16, N0),0); 1932 Chain = CurDAG->getEntryNode(); 1933 } 1934 Chain = CurDAG->getCopyToReg(Chain, dl, X86::AX, Move, SDValue()); 1935 InFlag = Chain.getValue(1); 1936 } else { 1937 InFlag = 1938 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, 1939 LoReg, N0, SDValue()).getValue(1); 1940 if (isSigned && !signBitIsZero) { 1941 // Sign extend the low part into the high part. 1942 InFlag = 1943 SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Flag, InFlag),0); 1944 } else { 1945 // Zero out the high part, effectively zero extending the input. 1946 SDValue ClrNode; 1947 1948 if (NVT.getSimpleVT() == MVT::i64) { 1949 ClrNode = SDValue(CurDAG->getMachineNode(X86::MOV32r0, dl, MVT::i32), 1950 0); 1951 // We just did a 32-bit clear, insert it into a 64-bit register to 1952 // clear the whole 64-bit reg. 1953 SDValue Undef = 1954 SDValue(CurDAG->getMachineNode(TargetInstrInfo::IMPLICIT_DEF, 1955 dl, MVT::i64), 0); 1956 SDValue SubRegNo = 1957 CurDAG->getTargetConstant(X86::SUBREG_32BIT, MVT::i32); 1958 ClrNode = 1959 SDValue(CurDAG->getMachineNode(TargetInstrInfo::INSERT_SUBREG, dl, 1960 MVT::i64, Undef, ClrNode, SubRegNo), 1961 0); 1962 } else { 1963 ClrNode = SDValue(CurDAG->getMachineNode(ClrOpcode, dl, NVT), 0); 1964 } 1965 1966 InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, HiReg, 1967 ClrNode, InFlag).getValue(1); 1968 } 1969 } 1970 1971 if (foldedLoad) { 1972 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), 1973 InFlag }; 1974 SDNode *CNode = 1975 CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Flag, Ops, 1976 array_lengthof(Ops)); 1977 InFlag = SDValue(CNode, 1); 1978 // Update the chain. 1979 ReplaceUses(N1.getValue(1), SDValue(CNode, 0)); 1980 } else { 1981 InFlag = 1982 SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Flag, N1, InFlag), 0); 1983 } 1984 1985 // Copy the division (low) result, if it is needed. 1986 if (!N.getValue(0).use_empty()) { 1987 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1988 LoReg, NVT, InFlag); 1989 InFlag = Result.getValue(2); 1990 ReplaceUses(N.getValue(0), Result); 1991#ifndef NDEBUG 1992 DEBUG({ 1993 errs() << std::string(Indent-2, ' ') << "=> "; 1994 Result.getNode()->dump(CurDAG); 1995 errs() << '\n'; 1996 }); 1997#endif 1998 } 1999 // Copy the remainder (high) result, if it is needed. 2000 if (!N.getValue(1).use_empty()) { 2001 SDValue Result; 2002 if (HiReg == X86::AH && Subtarget->is64Bit()) { 2003 // Prevent use of AH in a REX instruction by referencing AX instead. 2004 // Shift it down 8 bits. 2005 Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 2006 X86::AX, MVT::i16, InFlag); 2007 InFlag = Result.getValue(2); 2008 Result = SDValue(CurDAG->getMachineNode(X86::SHR16ri, dl, MVT::i16, 2009 Result, 2010 CurDAG->getTargetConstant(8, MVT::i8)), 2011 0); 2012 // Then truncate it down to i8. 2013 Result = CurDAG->getTargetExtractSubreg(X86::SUBREG_8BIT, dl, 2014 MVT::i8, Result); 2015 } else { 2016 Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 2017 HiReg, NVT, InFlag); 2018 InFlag = Result.getValue(2); 2019 } 2020 ReplaceUses(N.getValue(1), Result); 2021#ifndef NDEBUG 2022 DEBUG({ 2023 errs() << std::string(Indent-2, ' ') << "=> "; 2024 Result.getNode()->dump(CurDAG); 2025 errs() << '\n'; 2026 }); 2027#endif 2028 } 2029 2030#ifndef NDEBUG 2031 Indent -= 2; 2032#endif 2033 2034 return NULL; 2035 } 2036 2037 case X86ISD::CMP: { 2038 SDValue N0 = Node->getOperand(0); 2039 SDValue N1 = Node->getOperand(1); 2040 2041 // Look for (X86cmp (and $op, $imm), 0) and see if we can convert it to 2042 // use a smaller encoding. 2043 if (N0.getNode()->getOpcode() == ISD::AND && N0.getNode()->hasOneUse() && 2044 N0.getValueType() != MVT::i8 && 2045 X86::isZeroNode(N1)) { 2046 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getNode()->getOperand(1)); 2047 if (!C) break; 2048 2049 // For example, convert "testl %eax, $8" to "testb %al, $8" 2050 if ((C->getZExtValue() & ~UINT64_C(0xff)) == 0 && 2051 (!(C->getZExtValue() & 0x80) || 2052 HasNoSignedComparisonUses(Node))) { 2053 SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), MVT::i8); 2054 SDValue Reg = N0.getNode()->getOperand(0); 2055 2056 // On x86-32, only the ABCD registers have 8-bit subregisters. 2057 if (!Subtarget->is64Bit()) { 2058 TargetRegisterClass *TRC = 0; 2059 switch (N0.getValueType().getSimpleVT().SimpleTy) { 2060 case MVT::i32: TRC = &X86::GR32_ABCDRegClass; break; 2061 case MVT::i16: TRC = &X86::GR16_ABCDRegClass; break; 2062 default: llvm_unreachable("Unsupported TEST operand type!"); 2063 } 2064 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), MVT::i32); 2065 Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl, 2066 Reg.getValueType(), Reg, RC), 0); 2067 } 2068 2069 // Extract the l-register. 2070 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::SUBREG_8BIT, dl, 2071 MVT::i8, Reg); 2072 2073 // Emit a testb. 2074 return CurDAG->getMachineNode(X86::TEST8ri, dl, MVT::i32, Subreg, Imm); 2075 } 2076 2077 // For example, "testl %eax, $2048" to "testb %ah, $8". 2078 if ((C->getZExtValue() & ~UINT64_C(0xff00)) == 0 && 2079 (!(C->getZExtValue() & 0x8000) || 2080 HasNoSignedComparisonUses(Node))) { 2081 // Shift the immediate right by 8 bits. 2082 SDValue ShiftedImm = CurDAG->getTargetConstant(C->getZExtValue() >> 8, 2083 MVT::i8); 2084 SDValue Reg = N0.getNode()->getOperand(0); 2085 2086 // Put the value in an ABCD register. 2087 TargetRegisterClass *TRC = 0; 2088 switch (N0.getValueType().getSimpleVT().SimpleTy) { 2089 case MVT::i64: TRC = &X86::GR64_ABCDRegClass; break; 2090 case MVT::i32: TRC = &X86::GR32_ABCDRegClass; break; 2091 case MVT::i16: TRC = &X86::GR16_ABCDRegClass; break; 2092 default: llvm_unreachable("Unsupported TEST operand type!"); 2093 } 2094 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), MVT::i32); 2095 Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl, 2096 Reg.getValueType(), Reg, RC), 0); 2097 2098 // Extract the h-register. 2099 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::SUBREG_8BIT_HI, dl, 2100 MVT::i8, Reg); 2101 2102 // Emit a testb. No special NOREX tricks are needed since there's 2103 // only one GPR operand! 2104 return CurDAG->getMachineNode(X86::TEST8ri, dl, MVT::i32, 2105 Subreg, ShiftedImm); 2106 } 2107 2108 // For example, "testl %eax, $32776" to "testw %ax, $32776". 2109 if ((C->getZExtValue() & ~UINT64_C(0xffff)) == 0 && 2110 N0.getValueType() != MVT::i16 && 2111 (!(C->getZExtValue() & 0x8000) || 2112 HasNoSignedComparisonUses(Node))) { 2113 SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), MVT::i16); 2114 SDValue Reg = N0.getNode()->getOperand(0); 2115 2116 // Extract the 16-bit subregister. 2117 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::SUBREG_16BIT, dl, 2118 MVT::i16, Reg); 2119 2120 // Emit a testw. 2121 return CurDAG->getMachineNode(X86::TEST16ri, dl, MVT::i32, Subreg, Imm); 2122 } 2123 2124 // For example, "testq %rax, $268468232" to "testl %eax, $268468232". 2125 if ((C->getZExtValue() & ~UINT64_C(0xffffffff)) == 0 && 2126 N0.getValueType() == MVT::i64 && 2127 (!(C->getZExtValue() & 0x80000000) || 2128 HasNoSignedComparisonUses(Node))) { 2129 SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), MVT::i32); 2130 SDValue Reg = N0.getNode()->getOperand(0); 2131 2132 // Extract the 32-bit subregister. 2133 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::SUBREG_32BIT, dl, 2134 MVT::i32, Reg); 2135 2136 // Emit a testl. 2137 return CurDAG->getMachineNode(X86::TEST32ri, dl, MVT::i32, Subreg, Imm); 2138 } 2139 } 2140 break; 2141 } 2142 } 2143 2144 SDNode *ResNode = SelectCode(N); 2145 2146#ifndef NDEBUG 2147 DEBUG({ 2148 errs() << std::string(Indent-2, ' ') << "=> "; 2149 if (ResNode == NULL || ResNode == N.getNode()) 2150 N.getNode()->dump(CurDAG); 2151 else 2152 ResNode->dump(CurDAG); 2153 errs() << '\n'; 2154 }); 2155 Indent -= 2; 2156#endif 2157 2158 return ResNode; 2159} 2160 2161bool X86DAGToDAGISel:: 2162SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode, 2163 std::vector<SDValue> &OutOps) { 2164 SDValue Op0, Op1, Op2, Op3, Op4; 2165 switch (ConstraintCode) { 2166 case 'o': // offsetable ?? 2167 case 'v': // not offsetable ?? 2168 default: return true; 2169 case 'm': // memory 2170 if (!SelectAddr(Op, Op, Op0, Op1, Op2, Op3, Op4)) 2171 return true; 2172 break; 2173 } 2174 2175 OutOps.push_back(Op0); 2176 OutOps.push_back(Op1); 2177 OutOps.push_back(Op2); 2178 OutOps.push_back(Op3); 2179 OutOps.push_back(Op4); 2180 return false; 2181} 2182 2183/// createX86ISelDag - This pass converts a legalized DAG into a 2184/// X86-specific DAG, ready for instruction scheduling. 2185/// 2186FunctionPass *llvm::createX86ISelDag(X86TargetMachine &TM, 2187 llvm::CodeGenOpt::Level OptLevel) { 2188 return new X86DAGToDAGISel(TM, OptLevel); 2189} 2190