X86ISelDAGToDAG.cpp revision 296417
1//===- X86ISelDAGToDAG.cpp - A DAG pattern matching inst selector for X86 -===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines a DAG pattern matching instruction selector for X86, 11// converting from a legalized dag to a X86 dag. 12// 13//===----------------------------------------------------------------------===// 14 15#include "X86.h" 16#include "X86InstrBuilder.h" 17#include "X86MachineFunctionInfo.h" 18#include "X86RegisterInfo.h" 19#include "X86Subtarget.h" 20#include "X86TargetMachine.h" 21#include "llvm/ADT/Statistic.h" 22#include "llvm/CodeGen/MachineFrameInfo.h" 23#include "llvm/CodeGen/MachineFunction.h" 24#include "llvm/CodeGen/MachineInstrBuilder.h" 25#include "llvm/CodeGen/MachineRegisterInfo.h" 26#include "llvm/CodeGen/SelectionDAGISel.h" 27#include "llvm/IR/Function.h" 28#include "llvm/IR/Instructions.h" 29#include "llvm/IR/Intrinsics.h" 30#include "llvm/IR/Type.h" 31#include "llvm/Support/Debug.h" 32#include "llvm/Support/ErrorHandling.h" 33#include "llvm/Support/MathExtras.h" 34#include "llvm/Support/raw_ostream.h" 35#include "llvm/Target/TargetMachine.h" 36#include "llvm/Target/TargetOptions.h" 37#include <stdint.h> 38using namespace llvm; 39 40#define DEBUG_TYPE "x86-isel" 41 42STATISTIC(NumLoadMoved, "Number of loads moved below TokenFactor"); 43 44//===----------------------------------------------------------------------===// 45// Pattern Matcher Implementation 46//===----------------------------------------------------------------------===// 47 48namespace { 49 /// This corresponds to X86AddressMode, but uses SDValue's instead of register 50 /// numbers for the leaves of the matched tree. 51 struct X86ISelAddressMode { 52 enum { 53 RegBase, 54 FrameIndexBase 55 } BaseType; 56 57 // This is really a union, discriminated by BaseType! 58 SDValue Base_Reg; 59 int Base_FrameIndex; 60 61 unsigned Scale; 62 SDValue IndexReg; 63 int32_t Disp; 64 SDValue Segment; 65 const GlobalValue *GV; 66 const Constant *CP; 67 const BlockAddress *BlockAddr; 68 const char *ES; 69 MCSymbol *MCSym; 70 int JT; 71 unsigned Align; // CP alignment. 72 unsigned char SymbolFlags; // X86II::MO_* 73 74 X86ISelAddressMode() 75 : BaseType(RegBase), Base_FrameIndex(0), Scale(1), IndexReg(), Disp(0), 76 Segment(), GV(nullptr), CP(nullptr), BlockAddr(nullptr), ES(nullptr), 77 MCSym(nullptr), JT(-1), Align(0), SymbolFlags(X86II::MO_NO_FLAG) {} 78 79 bool hasSymbolicDisplacement() const { 80 return GV != nullptr || CP != nullptr || ES != nullptr || 81 MCSym != nullptr || JT != -1 || BlockAddr != nullptr; 82 } 83 84 bool hasBaseOrIndexReg() const { 85 return BaseType == FrameIndexBase || 86 IndexReg.getNode() != nullptr || Base_Reg.getNode() != nullptr; 87 } 88 89 /// Return true if this addressing mode is already RIP-relative. 90 bool isRIPRelative() const { 91 if (BaseType != RegBase) return false; 92 if (RegisterSDNode *RegNode = 93 dyn_cast_or_null<RegisterSDNode>(Base_Reg.getNode())) 94 return RegNode->getReg() == X86::RIP; 95 return false; 96 } 97 98 void setBaseReg(SDValue Reg) { 99 BaseType = RegBase; 100 Base_Reg = Reg; 101 } 102 103#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 104 void dump() { 105 dbgs() << "X86ISelAddressMode " << this << '\n'; 106 dbgs() << "Base_Reg "; 107 if (Base_Reg.getNode()) 108 Base_Reg.getNode()->dump(); 109 else 110 dbgs() << "nul"; 111 dbgs() << " Base.FrameIndex " << Base_FrameIndex << '\n' 112 << " Scale" << Scale << '\n' 113 << "IndexReg "; 114 if (IndexReg.getNode()) 115 IndexReg.getNode()->dump(); 116 else 117 dbgs() << "nul"; 118 dbgs() << " Disp " << Disp << '\n' 119 << "GV "; 120 if (GV) 121 GV->dump(); 122 else 123 dbgs() << "nul"; 124 dbgs() << " CP "; 125 if (CP) 126 CP->dump(); 127 else 128 dbgs() << "nul"; 129 dbgs() << '\n' 130 << "ES "; 131 if (ES) 132 dbgs() << ES; 133 else 134 dbgs() << "nul"; 135 dbgs() << " MCSym "; 136 if (MCSym) 137 dbgs() << MCSym; 138 else 139 dbgs() << "nul"; 140 dbgs() << " JT" << JT << " Align" << Align << '\n'; 141 } 142#endif 143 }; 144} 145 146namespace { 147 //===--------------------------------------------------------------------===// 148 /// ISel - X86-specific code to select X86 machine instructions for 149 /// SelectionDAG operations. 150 /// 151 class X86DAGToDAGISel final : public SelectionDAGISel { 152 /// Keep a pointer to the X86Subtarget around so that we can 153 /// make the right decision when generating code for different targets. 154 const X86Subtarget *Subtarget; 155 156 /// If true, selector should try to optimize for code size instead of 157 /// performance. 158 bool OptForSize; 159 160 public: 161 explicit X86DAGToDAGISel(X86TargetMachine &tm, CodeGenOpt::Level OptLevel) 162 : SelectionDAGISel(tm, OptLevel), OptForSize(false) {} 163 164 const char *getPassName() const override { 165 return "X86 DAG->DAG Instruction Selection"; 166 } 167 168 bool runOnMachineFunction(MachineFunction &MF) override { 169 // Reset the subtarget each time through. 170 Subtarget = &MF.getSubtarget<X86Subtarget>(); 171 SelectionDAGISel::runOnMachineFunction(MF); 172 return true; 173 } 174 175 void EmitFunctionEntryCode() override; 176 177 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override; 178 179 void PreprocessISelDAG() override; 180 181 inline bool immSext8(SDNode *N) const { 182 return isInt<8>(cast<ConstantSDNode>(N)->getSExtValue()); 183 } 184 185 // True if the 64-bit immediate fits in a 32-bit sign-extended field. 186 inline bool i64immSExt32(SDNode *N) const { 187 uint64_t v = cast<ConstantSDNode>(N)->getZExtValue(); 188 return (int64_t)v == (int32_t)v; 189 } 190 191// Include the pieces autogenerated from the target description. 192#include "X86GenDAGISel.inc" 193 194 private: 195 SDNode *Select(SDNode *N) override; 196 SDNode *selectGather(SDNode *N, unsigned Opc); 197 SDNode *selectAtomicLoadArith(SDNode *Node, MVT NVT); 198 199 bool foldOffsetIntoAddress(uint64_t Offset, X86ISelAddressMode &AM); 200 bool matchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM); 201 bool matchWrapper(SDValue N, X86ISelAddressMode &AM); 202 bool matchAddress(SDValue N, X86ISelAddressMode &AM); 203 bool matchAdd(SDValue N, X86ISelAddressMode &AM, unsigned Depth); 204 bool matchAddressRecursively(SDValue N, X86ISelAddressMode &AM, 205 unsigned Depth); 206 bool matchAddressBase(SDValue N, X86ISelAddressMode &AM); 207 bool selectAddr(SDNode *Parent, SDValue N, SDValue &Base, 208 SDValue &Scale, SDValue &Index, SDValue &Disp, 209 SDValue &Segment); 210 bool selectVectorAddr(SDNode *Parent, SDValue N, SDValue &Base, 211 SDValue &Scale, SDValue &Index, SDValue &Disp, 212 SDValue &Segment); 213 bool selectMOV64Imm32(SDValue N, SDValue &Imm); 214 bool selectLEAAddr(SDValue N, SDValue &Base, 215 SDValue &Scale, SDValue &Index, SDValue &Disp, 216 SDValue &Segment); 217 bool selectLEA64_32Addr(SDValue N, SDValue &Base, 218 SDValue &Scale, SDValue &Index, SDValue &Disp, 219 SDValue &Segment); 220 bool selectTLSADDRAddr(SDValue N, SDValue &Base, 221 SDValue &Scale, SDValue &Index, SDValue &Disp, 222 SDValue &Segment); 223 bool selectScalarSSELoad(SDNode *Root, SDValue N, 224 SDValue &Base, SDValue &Scale, 225 SDValue &Index, SDValue &Disp, 226 SDValue &Segment, 227 SDValue &NodeWithChain); 228 229 bool tryFoldLoad(SDNode *P, SDValue N, 230 SDValue &Base, SDValue &Scale, 231 SDValue &Index, SDValue &Disp, 232 SDValue &Segment); 233 234 /// Implement addressing mode selection for inline asm expressions. 235 bool SelectInlineAsmMemoryOperand(const SDValue &Op, 236 unsigned ConstraintID, 237 std::vector<SDValue> &OutOps) override; 238 239 void emitSpecialCodeForMain(); 240 241 inline void getAddressOperands(X86ISelAddressMode &AM, SDLoc DL, 242 SDValue &Base, SDValue &Scale, 243 SDValue &Index, SDValue &Disp, 244 SDValue &Segment) { 245 Base = (AM.BaseType == X86ISelAddressMode::FrameIndexBase) 246 ? CurDAG->getTargetFrameIndex( 247 AM.Base_FrameIndex, 248 TLI->getPointerTy(CurDAG->getDataLayout())) 249 : AM.Base_Reg; 250 Scale = getI8Imm(AM.Scale, DL); 251 Index = AM.IndexReg; 252 // These are 32-bit even in 64-bit mode since RIP-relative offset 253 // is 32-bit. 254 if (AM.GV) 255 Disp = CurDAG->getTargetGlobalAddress(AM.GV, SDLoc(), 256 MVT::i32, AM.Disp, 257 AM.SymbolFlags); 258 else if (AM.CP) 259 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, 260 AM.Align, AM.Disp, AM.SymbolFlags); 261 else if (AM.ES) { 262 assert(!AM.Disp && "Non-zero displacement is ignored with ES."); 263 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags); 264 } else if (AM.MCSym) { 265 assert(!AM.Disp && "Non-zero displacement is ignored with MCSym."); 266 assert(AM.SymbolFlags == 0 && "oo"); 267 Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32); 268 } else if (AM.JT != -1) { 269 assert(!AM.Disp && "Non-zero displacement is ignored with JT."); 270 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags); 271 } else if (AM.BlockAddr) 272 Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp, 273 AM.SymbolFlags); 274 else 275 Disp = CurDAG->getTargetConstant(AM.Disp, DL, MVT::i32); 276 277 if (AM.Segment.getNode()) 278 Segment = AM.Segment; 279 else 280 Segment = CurDAG->getRegister(0, MVT::i32); 281 } 282 283 // Utility function to determine whether we should avoid selecting 284 // immediate forms of instructions for better code size or not. 285 // At a high level, we'd like to avoid such instructions when 286 // we have similar constants used within the same basic block 287 // that can be kept in a register. 288 // 289 bool shouldAvoidImmediateInstFormsForSize(SDNode *N) const { 290 uint32_t UseCount = 0; 291 292 // Do not want to hoist if we're not optimizing for size. 293 // TODO: We'd like to remove this restriction. 294 // See the comment in X86InstrInfo.td for more info. 295 if (!OptForSize) 296 return false; 297 298 // Walk all the users of the immediate. 299 for (SDNode::use_iterator UI = N->use_begin(), 300 UE = N->use_end(); (UI != UE) && (UseCount < 2); ++UI) { 301 302 SDNode *User = *UI; 303 304 // This user is already selected. Count it as a legitimate use and 305 // move on. 306 if (User->isMachineOpcode()) { 307 UseCount++; 308 continue; 309 } 310 311 // We want to count stores of immediates as real uses. 312 if (User->getOpcode() == ISD::STORE && 313 User->getOperand(1).getNode() == N) { 314 UseCount++; 315 continue; 316 } 317 318 // We don't currently match users that have > 2 operands (except 319 // for stores, which are handled above) 320 // Those instruction won't match in ISEL, for now, and would 321 // be counted incorrectly. 322 // This may change in the future as we add additional instruction 323 // types. 324 if (User->getNumOperands() != 2) 325 continue; 326 327 // Immediates that are used for offsets as part of stack 328 // manipulation should be left alone. These are typically 329 // used to indicate SP offsets for argument passing and 330 // will get pulled into stores/pushes (implicitly). 331 if (User->getOpcode() == X86ISD::ADD || 332 User->getOpcode() == ISD::ADD || 333 User->getOpcode() == X86ISD::SUB || 334 User->getOpcode() == ISD::SUB) { 335 336 // Find the other operand of the add/sub. 337 SDValue OtherOp = User->getOperand(0); 338 if (OtherOp.getNode() == N) 339 OtherOp = User->getOperand(1); 340 341 // Don't count if the other operand is SP. 342 RegisterSDNode *RegNode; 343 if (OtherOp->getOpcode() == ISD::CopyFromReg && 344 (RegNode = dyn_cast_or_null<RegisterSDNode>( 345 OtherOp->getOperand(1).getNode()))) 346 if ((RegNode->getReg() == X86::ESP) || 347 (RegNode->getReg() == X86::RSP)) 348 continue; 349 } 350 351 // ... otherwise, count this and move on. 352 UseCount++; 353 } 354 355 // If we have more than 1 use, then recommend for hoisting. 356 return (UseCount > 1); 357 } 358 359 /// Return a target constant with the specified value of type i8. 360 inline SDValue getI8Imm(unsigned Imm, SDLoc DL) { 361 return CurDAG->getTargetConstant(Imm, DL, MVT::i8); 362 } 363 364 /// Return a target constant with the specified value, of type i32. 365 inline SDValue getI32Imm(unsigned Imm, SDLoc DL) { 366 return CurDAG->getTargetConstant(Imm, DL, MVT::i32); 367 } 368 369 /// Return an SDNode that returns the value of the global base register. 370 /// Output instructions required to initialize the global base register, 371 /// if necessary. 372 SDNode *getGlobalBaseReg(); 373 374 /// Return a reference to the TargetMachine, casted to the target-specific 375 /// type. 376 const X86TargetMachine &getTargetMachine() const { 377 return static_cast<const X86TargetMachine &>(TM); 378 } 379 380 /// Return a reference to the TargetInstrInfo, casted to the target-specific 381 /// type. 382 const X86InstrInfo *getInstrInfo() const { 383 return Subtarget->getInstrInfo(); 384 } 385 386 /// \brief Address-mode matching performs shift-of-and to and-of-shift 387 /// reassociation in order to expose more scaled addressing 388 /// opportunities. 389 bool ComplexPatternFuncMutatesDAG() const override { 390 return true; 391 } 392 }; 393} 394 395 396bool 397X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const { 398 if (OptLevel == CodeGenOpt::None) return false; 399 400 if (!N.hasOneUse()) 401 return false; 402 403 if (N.getOpcode() != ISD::LOAD) 404 return true; 405 406 // If N is a load, do additional profitability checks. 407 if (U == Root) { 408 switch (U->getOpcode()) { 409 default: break; 410 case X86ISD::ADD: 411 case X86ISD::SUB: 412 case X86ISD::AND: 413 case X86ISD::XOR: 414 case X86ISD::OR: 415 case ISD::ADD: 416 case ISD::ADDC: 417 case ISD::ADDE: 418 case ISD::AND: 419 case ISD::OR: 420 case ISD::XOR: { 421 SDValue Op1 = U->getOperand(1); 422 423 // If the other operand is a 8-bit immediate we should fold the immediate 424 // instead. This reduces code size. 425 // e.g. 426 // movl 4(%esp), %eax 427 // addl $4, %eax 428 // vs. 429 // movl $4, %eax 430 // addl 4(%esp), %eax 431 // The former is 2 bytes shorter. In case where the increment is 1, then 432 // the saving can be 4 bytes (by using incl %eax). 433 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Op1)) 434 if (Imm->getAPIntValue().isSignedIntN(8)) 435 return false; 436 437 // If the other operand is a TLS address, we should fold it instead. 438 // This produces 439 // movl %gs:0, %eax 440 // leal i@NTPOFF(%eax), %eax 441 // instead of 442 // movl $i@NTPOFF, %eax 443 // addl %gs:0, %eax 444 // if the block also has an access to a second TLS address this will save 445 // a load. 446 // FIXME: This is probably also true for non-TLS addresses. 447 if (Op1.getOpcode() == X86ISD::Wrapper) { 448 SDValue Val = Op1.getOperand(0); 449 if (Val.getOpcode() == ISD::TargetGlobalTLSAddress) 450 return false; 451 } 452 } 453 } 454 } 455 456 return true; 457} 458 459/// Replace the original chain operand of the call with 460/// load's chain operand and move load below the call's chain operand. 461static void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load, 462 SDValue Call, SDValue OrigChain) { 463 SmallVector<SDValue, 8> Ops; 464 SDValue Chain = OrigChain.getOperand(0); 465 if (Chain.getNode() == Load.getNode()) 466 Ops.push_back(Load.getOperand(0)); 467 else { 468 assert(Chain.getOpcode() == ISD::TokenFactor && 469 "Unexpected chain operand"); 470 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) 471 if (Chain.getOperand(i).getNode() == Load.getNode()) 472 Ops.push_back(Load.getOperand(0)); 473 else 474 Ops.push_back(Chain.getOperand(i)); 475 SDValue NewChain = 476 CurDAG->getNode(ISD::TokenFactor, SDLoc(Load), MVT::Other, Ops); 477 Ops.clear(); 478 Ops.push_back(NewChain); 479 } 480 Ops.append(OrigChain->op_begin() + 1, OrigChain->op_end()); 481 CurDAG->UpdateNodeOperands(OrigChain.getNode(), Ops); 482 CurDAG->UpdateNodeOperands(Load.getNode(), Call.getOperand(0), 483 Load.getOperand(1), Load.getOperand(2)); 484 485 Ops.clear(); 486 Ops.push_back(SDValue(Load.getNode(), 1)); 487 Ops.append(Call->op_begin() + 1, Call->op_end()); 488 CurDAG->UpdateNodeOperands(Call.getNode(), Ops); 489} 490 491/// Return true if call address is a load and it can be 492/// moved below CALLSEQ_START and the chains leading up to the call. 493/// Return the CALLSEQ_START by reference as a second output. 494/// In the case of a tail call, there isn't a callseq node between the call 495/// chain and the load. 496static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq) { 497 // The transformation is somewhat dangerous if the call's chain was glued to 498 // the call. After MoveBelowOrigChain the load is moved between the call and 499 // the chain, this can create a cycle if the load is not folded. So it is 500 // *really* important that we are sure the load will be folded. 501 if (Callee.getNode() == Chain.getNode() || !Callee.hasOneUse()) 502 return false; 503 LoadSDNode *LD = dyn_cast<LoadSDNode>(Callee.getNode()); 504 if (!LD || 505 LD->isVolatile() || 506 LD->getAddressingMode() != ISD::UNINDEXED || 507 LD->getExtensionType() != ISD::NON_EXTLOAD) 508 return false; 509 510 // Now let's find the callseq_start. 511 while (HasCallSeq && Chain.getOpcode() != ISD::CALLSEQ_START) { 512 if (!Chain.hasOneUse()) 513 return false; 514 Chain = Chain.getOperand(0); 515 } 516 517 if (!Chain.getNumOperands()) 518 return false; 519 // Since we are not checking for AA here, conservatively abort if the chain 520 // writes to memory. It's not safe to move the callee (a load) across a store. 521 if (isa<MemSDNode>(Chain.getNode()) && 522 cast<MemSDNode>(Chain.getNode())->writeMem()) 523 return false; 524 if (Chain.getOperand(0).getNode() == Callee.getNode()) 525 return true; 526 if (Chain.getOperand(0).getOpcode() == ISD::TokenFactor && 527 Callee.getValue(1).isOperandOf(Chain.getOperand(0).getNode()) && 528 Callee.getValue(1).hasOneUse()) 529 return true; 530 return false; 531} 532 533void X86DAGToDAGISel::PreprocessISelDAG() { 534 // OptForSize is used in pattern predicates that isel is matching. 535 OptForSize = MF->getFunction()->optForSize(); 536 537 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 538 E = CurDAG->allnodes_end(); I != E; ) { 539 SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues. 540 541 if (OptLevel != CodeGenOpt::None && 542 // Only does this when target favors doesn't favor register indirect 543 // call. 544 ((N->getOpcode() == X86ISD::CALL && !Subtarget->callRegIndirect()) || 545 (N->getOpcode() == X86ISD::TC_RETURN && 546 // Only does this if load can be folded into TC_RETURN. 547 (Subtarget->is64Bit() || 548 getTargetMachine().getRelocationModel() != Reloc::PIC_)))) { 549 /// Also try moving call address load from outside callseq_start to just 550 /// before the call to allow it to be folded. 551 /// 552 /// [Load chain] 553 /// ^ 554 /// | 555 /// [Load] 556 /// ^ ^ 557 /// | | 558 /// / \-- 559 /// / | 560 ///[CALLSEQ_START] | 561 /// ^ | 562 /// | | 563 /// [LOAD/C2Reg] | 564 /// | | 565 /// \ / 566 /// \ / 567 /// [CALL] 568 bool HasCallSeq = N->getOpcode() == X86ISD::CALL; 569 SDValue Chain = N->getOperand(0); 570 SDValue Load = N->getOperand(1); 571 if (!isCalleeLoad(Load, Chain, HasCallSeq)) 572 continue; 573 moveBelowOrigChain(CurDAG, Load, SDValue(N, 0), Chain); 574 ++NumLoadMoved; 575 continue; 576 } 577 578 // Lower fpround and fpextend nodes that target the FP stack to be store and 579 // load to the stack. This is a gross hack. We would like to simply mark 580 // these as being illegal, but when we do that, legalize produces these when 581 // it expands calls, then expands these in the same legalize pass. We would 582 // like dag combine to be able to hack on these between the call expansion 583 // and the node legalization. As such this pass basically does "really 584 // late" legalization of these inline with the X86 isel pass. 585 // FIXME: This should only happen when not compiled with -O0. 586 if (N->getOpcode() != ISD::FP_ROUND && N->getOpcode() != ISD::FP_EXTEND) 587 continue; 588 589 MVT SrcVT = N->getOperand(0).getSimpleValueType(); 590 MVT DstVT = N->getSimpleValueType(0); 591 592 // If any of the sources are vectors, no fp stack involved. 593 if (SrcVT.isVector() || DstVT.isVector()) 594 continue; 595 596 // If the source and destination are SSE registers, then this is a legal 597 // conversion that should not be lowered. 598 const X86TargetLowering *X86Lowering = 599 static_cast<const X86TargetLowering *>(TLI); 600 bool SrcIsSSE = X86Lowering->isScalarFPTypeInSSEReg(SrcVT); 601 bool DstIsSSE = X86Lowering->isScalarFPTypeInSSEReg(DstVT); 602 if (SrcIsSSE && DstIsSSE) 603 continue; 604 605 if (!SrcIsSSE && !DstIsSSE) { 606 // If this is an FPStack extension, it is a noop. 607 if (N->getOpcode() == ISD::FP_EXTEND) 608 continue; 609 // If this is a value-preserving FPStack truncation, it is a noop. 610 if (N->getConstantOperandVal(1)) 611 continue; 612 } 613 614 // Here we could have an FP stack truncation or an FPStack <-> SSE convert. 615 // FPStack has extload and truncstore. SSE can fold direct loads into other 616 // operations. Based on this, decide what we want to do. 617 MVT MemVT; 618 if (N->getOpcode() == ISD::FP_ROUND) 619 MemVT = DstVT; // FP_ROUND must use DstVT, we can't do a 'trunc load'. 620 else 621 MemVT = SrcIsSSE ? SrcVT : DstVT; 622 623 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT); 624 SDLoc dl(N); 625 626 // FIXME: optimize the case where the src/dest is a load or store? 627 SDValue Store = CurDAG->getTruncStore(CurDAG->getEntryNode(), dl, 628 N->getOperand(0), 629 MemTmp, MachinePointerInfo(), MemVT, 630 false, false, 0); 631 SDValue Result = CurDAG->getExtLoad(ISD::EXTLOAD, dl, DstVT, Store, MemTmp, 632 MachinePointerInfo(), 633 MemVT, false, false, false, 0); 634 635 // We're about to replace all uses of the FP_ROUND/FP_EXTEND with the 636 // extload we created. This will cause general havok on the dag because 637 // anything below the conversion could be folded into other existing nodes. 638 // To avoid invalidating 'I', back it up to the convert node. 639 --I; 640 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 641 642 // Now that we did that, the node is dead. Increment the iterator to the 643 // next node to process, then delete N. 644 ++I; 645 CurDAG->DeleteNode(N); 646 } 647} 648 649 650/// Emit any code that needs to be executed only in the main function. 651void X86DAGToDAGISel::emitSpecialCodeForMain() { 652 if (Subtarget->isTargetCygMing()) { 653 TargetLowering::ArgListTy Args; 654 auto &DL = CurDAG->getDataLayout(); 655 656 TargetLowering::CallLoweringInfo CLI(*CurDAG); 657 CLI.setChain(CurDAG->getRoot()) 658 .setCallee(CallingConv::C, Type::getVoidTy(*CurDAG->getContext()), 659 CurDAG->getExternalSymbol("__main", TLI->getPointerTy(DL)), 660 std::move(Args), 0); 661 const TargetLowering &TLI = CurDAG->getTargetLoweringInfo(); 662 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI); 663 CurDAG->setRoot(Result.second); 664 } 665} 666 667void X86DAGToDAGISel::EmitFunctionEntryCode() { 668 // If this is main, emit special code for main. 669 if (const Function *Fn = MF->getFunction()) 670 if (Fn->hasExternalLinkage() && Fn->getName() == "main") 671 emitSpecialCodeForMain(); 672} 673 674static bool isDispSafeForFrameIndex(int64_t Val) { 675 // On 64-bit platforms, we can run into an issue where a frame index 676 // includes a displacement that, when added to the explicit displacement, 677 // will overflow the displacement field. Assuming that the frame index 678 // displacement fits into a 31-bit integer (which is only slightly more 679 // aggressive than the current fundamental assumption that it fits into 680 // a 32-bit integer), a 31-bit disp should always be safe. 681 return isInt<31>(Val); 682} 683 684bool X86DAGToDAGISel::foldOffsetIntoAddress(uint64_t Offset, 685 X86ISelAddressMode &AM) { 686 // Cannot combine ExternalSymbol displacements with integer offsets. 687 if (Offset != 0 && (AM.ES || AM.MCSym)) 688 return true; 689 int64_t Val = AM.Disp + Offset; 690 CodeModel::Model M = TM.getCodeModel(); 691 if (Subtarget->is64Bit()) { 692 if (!X86::isOffsetSuitableForCodeModel(Val, M, 693 AM.hasSymbolicDisplacement())) 694 return true; 695 // In addition to the checks required for a register base, check that 696 // we do not try to use an unsafe Disp with a frame index. 697 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase && 698 !isDispSafeForFrameIndex(Val)) 699 return true; 700 } 701 AM.Disp = Val; 702 return false; 703 704} 705 706bool X86DAGToDAGISel::matchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM){ 707 SDValue Address = N->getOperand(1); 708 709 // load gs:0 -> GS segment register. 710 // load fs:0 -> FS segment register. 711 // 712 // This optimization is valid because the GNU TLS model defines that 713 // gs:0 (or fs:0 on X86-64) contains its own address. 714 // For more information see http://people.redhat.com/drepper/tls.pdf 715 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Address)) 716 if (C->getSExtValue() == 0 && AM.Segment.getNode() == nullptr && 717 Subtarget->isTargetLinux()) 718 switch (N->getPointerInfo().getAddrSpace()) { 719 case 256: 720 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); 721 return false; 722 case 257: 723 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); 724 return false; 725 } 726 727 return true; 728} 729 730/// Try to match X86ISD::Wrapper and X86ISD::WrapperRIP nodes into an addressing 731/// mode. These wrap things that will resolve down into a symbol reference. 732/// If no match is possible, this returns true, otherwise it returns false. 733bool X86DAGToDAGISel::matchWrapper(SDValue N, X86ISelAddressMode &AM) { 734 // If the addressing mode already has a symbol as the displacement, we can 735 // never match another symbol. 736 if (AM.hasSymbolicDisplacement()) 737 return true; 738 739 SDValue N0 = N.getOperand(0); 740 CodeModel::Model M = TM.getCodeModel(); 741 742 // Handle X86-64 rip-relative addresses. We check this before checking direct 743 // folding because RIP is preferable to non-RIP accesses. 744 if (Subtarget->is64Bit() && N.getOpcode() == X86ISD::WrapperRIP && 745 // Under X86-64 non-small code model, GV (and friends) are 64-bits, so 746 // they cannot be folded into immediate fields. 747 // FIXME: This can be improved for kernel and other models? 748 (M == CodeModel::Small || M == CodeModel::Kernel)) { 749 // Base and index reg must be 0 in order to use %rip as base. 750 if (AM.hasBaseOrIndexReg()) 751 return true; 752 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N0)) { 753 X86ISelAddressMode Backup = AM; 754 AM.GV = G->getGlobal(); 755 AM.SymbolFlags = G->getTargetFlags(); 756 if (foldOffsetIntoAddress(G->getOffset(), AM)) { 757 AM = Backup; 758 return true; 759 } 760 } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) { 761 X86ISelAddressMode Backup = AM; 762 AM.CP = CP->getConstVal(); 763 AM.Align = CP->getAlignment(); 764 AM.SymbolFlags = CP->getTargetFlags(); 765 if (foldOffsetIntoAddress(CP->getOffset(), AM)) { 766 AM = Backup; 767 return true; 768 } 769 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) { 770 AM.ES = S->getSymbol(); 771 AM.SymbolFlags = S->getTargetFlags(); 772 } else if (auto *S = dyn_cast<MCSymbolSDNode>(N0)) { 773 AM.MCSym = S->getMCSymbol(); 774 } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) { 775 AM.JT = J->getIndex(); 776 AM.SymbolFlags = J->getTargetFlags(); 777 } else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(N0)) { 778 X86ISelAddressMode Backup = AM; 779 AM.BlockAddr = BA->getBlockAddress(); 780 AM.SymbolFlags = BA->getTargetFlags(); 781 if (foldOffsetIntoAddress(BA->getOffset(), AM)) { 782 AM = Backup; 783 return true; 784 } 785 } else 786 llvm_unreachable("Unhandled symbol reference node."); 787 788 if (N.getOpcode() == X86ISD::WrapperRIP) 789 AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64)); 790 return false; 791 } 792 793 // Handle the case when globals fit in our immediate field: This is true for 794 // X86-32 always and X86-64 when in -mcmodel=small mode. In 64-bit 795 // mode, this only applies to a non-RIP-relative computation. 796 if (!Subtarget->is64Bit() || 797 M == CodeModel::Small || M == CodeModel::Kernel) { 798 assert(N.getOpcode() != X86ISD::WrapperRIP && 799 "RIP-relative addressing already handled"); 800 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N0)) { 801 AM.GV = G->getGlobal(); 802 AM.Disp += G->getOffset(); 803 AM.SymbolFlags = G->getTargetFlags(); 804 } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) { 805 AM.CP = CP->getConstVal(); 806 AM.Align = CP->getAlignment(); 807 AM.Disp += CP->getOffset(); 808 AM.SymbolFlags = CP->getTargetFlags(); 809 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) { 810 AM.ES = S->getSymbol(); 811 AM.SymbolFlags = S->getTargetFlags(); 812 } else if (auto *S = dyn_cast<MCSymbolSDNode>(N0)) { 813 AM.MCSym = S->getMCSymbol(); 814 } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) { 815 AM.JT = J->getIndex(); 816 AM.SymbolFlags = J->getTargetFlags(); 817 } else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(N0)) { 818 AM.BlockAddr = BA->getBlockAddress(); 819 AM.Disp += BA->getOffset(); 820 AM.SymbolFlags = BA->getTargetFlags(); 821 } else 822 llvm_unreachable("Unhandled symbol reference node."); 823 return false; 824 } 825 826 return true; 827} 828 829/// Add the specified node to the specified addressing mode, returning true if 830/// it cannot be done. This just pattern matches for the addressing mode. 831bool X86DAGToDAGISel::matchAddress(SDValue N, X86ISelAddressMode &AM) { 832 if (matchAddressRecursively(N, AM, 0)) 833 return true; 834 835 // Post-processing: Convert lea(,%reg,2) to lea(%reg,%reg), which has 836 // a smaller encoding and avoids a scaled-index. 837 if (AM.Scale == 2 && 838 AM.BaseType == X86ISelAddressMode::RegBase && 839 AM.Base_Reg.getNode() == nullptr) { 840 AM.Base_Reg = AM.IndexReg; 841 AM.Scale = 1; 842 } 843 844 // Post-processing: Convert foo to foo(%rip), even in non-PIC mode, 845 // because it has a smaller encoding. 846 // TODO: Which other code models can use this? 847 if (TM.getCodeModel() == CodeModel::Small && 848 Subtarget->is64Bit() && 849 AM.Scale == 1 && 850 AM.BaseType == X86ISelAddressMode::RegBase && 851 AM.Base_Reg.getNode() == nullptr && 852 AM.IndexReg.getNode() == nullptr && 853 AM.SymbolFlags == X86II::MO_NO_FLAG && 854 AM.hasSymbolicDisplacement()) 855 AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64); 856 857 return false; 858} 859 860bool X86DAGToDAGISel::matchAdd(SDValue N, X86ISelAddressMode &AM, 861 unsigned Depth) { 862 // Add an artificial use to this node so that we can keep track of 863 // it if it gets CSE'd with a different node. 864 HandleSDNode Handle(N); 865 866 X86ISelAddressMode Backup = AM; 867 if (!matchAddressRecursively(N.getOperand(0), AM, Depth+1) && 868 !matchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1)) 869 return false; 870 AM = Backup; 871 872 // Try again after commuting the operands. 873 if (!matchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1) && 874 !matchAddressRecursively(Handle.getValue().getOperand(0), AM, Depth+1)) 875 return false; 876 AM = Backup; 877 878 // If we couldn't fold both operands into the address at the same time, 879 // see if we can just put each operand into a register and fold at least 880 // the add. 881 if (AM.BaseType == X86ISelAddressMode::RegBase && 882 !AM.Base_Reg.getNode() && 883 !AM.IndexReg.getNode()) { 884 N = Handle.getValue(); 885 AM.Base_Reg = N.getOperand(0); 886 AM.IndexReg = N.getOperand(1); 887 AM.Scale = 1; 888 return false; 889 } 890 N = Handle.getValue(); 891 return true; 892} 893 894// Insert a node into the DAG at least before the Pos node's position. This 895// will reposition the node as needed, and will assign it a node ID that is <= 896// the Pos node's ID. Note that this does *not* preserve the uniqueness of node 897// IDs! The selection DAG must no longer depend on their uniqueness when this 898// is used. 899static void insertDAGNode(SelectionDAG &DAG, SDValue Pos, SDValue N) { 900 if (N.getNode()->getNodeId() == -1 || 901 N.getNode()->getNodeId() > Pos.getNode()->getNodeId()) { 902 DAG.RepositionNode(Pos.getNode()->getIterator(), N.getNode()); 903 N.getNode()->setNodeId(Pos.getNode()->getNodeId()); 904 } 905} 906 907// Transform "(X >> (8-C1)) & (0xff << C1)" to "((X >> 8) & 0xff) << C1" if 908// safe. This allows us to convert the shift and and into an h-register 909// extract and a scaled index. Returns false if the simplification is 910// performed. 911static bool foldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N, 912 uint64_t Mask, 913 SDValue Shift, SDValue X, 914 X86ISelAddressMode &AM) { 915 if (Shift.getOpcode() != ISD::SRL || 916 !isa<ConstantSDNode>(Shift.getOperand(1)) || 917 !Shift.hasOneUse()) 918 return true; 919 920 int ScaleLog = 8 - Shift.getConstantOperandVal(1); 921 if (ScaleLog <= 0 || ScaleLog >= 4 || 922 Mask != (0xffu << ScaleLog)) 923 return true; 924 925 MVT VT = N.getSimpleValueType(); 926 SDLoc DL(N); 927 SDValue Eight = DAG.getConstant(8, DL, MVT::i8); 928 SDValue NewMask = DAG.getConstant(0xff, DL, VT); 929 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, X, Eight); 930 SDValue And = DAG.getNode(ISD::AND, DL, VT, Srl, NewMask); 931 SDValue ShlCount = DAG.getConstant(ScaleLog, DL, MVT::i8); 932 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And, ShlCount); 933 934 // Insert the new nodes into the topological ordering. We must do this in 935 // a valid topological ordering as nothing is going to go back and re-sort 936 // these nodes. We continually insert before 'N' in sequence as this is 937 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no 938 // hierarchy left to express. 939 insertDAGNode(DAG, N, Eight); 940 insertDAGNode(DAG, N, Srl); 941 insertDAGNode(DAG, N, NewMask); 942 insertDAGNode(DAG, N, And); 943 insertDAGNode(DAG, N, ShlCount); 944 insertDAGNode(DAG, N, Shl); 945 DAG.ReplaceAllUsesWith(N, Shl); 946 AM.IndexReg = And; 947 AM.Scale = (1 << ScaleLog); 948 return false; 949} 950 951// Transforms "(X << C1) & C2" to "(X & (C2>>C1)) << C1" if safe and if this 952// allows us to fold the shift into this addressing mode. Returns false if the 953// transform succeeded. 954static bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N, 955 uint64_t Mask, 956 SDValue Shift, SDValue X, 957 X86ISelAddressMode &AM) { 958 if (Shift.getOpcode() != ISD::SHL || 959 !isa<ConstantSDNode>(Shift.getOperand(1))) 960 return true; 961 962 // Not likely to be profitable if either the AND or SHIFT node has more 963 // than one use (unless all uses are for address computation). Besides, 964 // isel mechanism requires their node ids to be reused. 965 if (!N.hasOneUse() || !Shift.hasOneUse()) 966 return true; 967 968 // Verify that the shift amount is something we can fold. 969 unsigned ShiftAmt = Shift.getConstantOperandVal(1); 970 if (ShiftAmt != 1 && ShiftAmt != 2 && ShiftAmt != 3) 971 return true; 972 973 MVT VT = N.getSimpleValueType(); 974 SDLoc DL(N); 975 SDValue NewMask = DAG.getConstant(Mask >> ShiftAmt, DL, VT); 976 SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, NewMask); 977 SDValue NewShift = DAG.getNode(ISD::SHL, DL, VT, NewAnd, Shift.getOperand(1)); 978 979 // Insert the new nodes into the topological ordering. We must do this in 980 // a valid topological ordering as nothing is going to go back and re-sort 981 // these nodes. We continually insert before 'N' in sequence as this is 982 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no 983 // hierarchy left to express. 984 insertDAGNode(DAG, N, NewMask); 985 insertDAGNode(DAG, N, NewAnd); 986 insertDAGNode(DAG, N, NewShift); 987 DAG.ReplaceAllUsesWith(N, NewShift); 988 989 AM.Scale = 1 << ShiftAmt; 990 AM.IndexReg = NewAnd; 991 return false; 992} 993 994// Implement some heroics to detect shifts of masked values where the mask can 995// be replaced by extending the shift and undoing that in the addressing mode 996// scale. Patterns such as (shl (srl x, c1), c2) are canonicalized into (and 997// (srl x, SHIFT), MASK) by DAGCombines that don't know the shl can be done in 998// the addressing mode. This results in code such as: 999// 1000// int f(short *y, int *lookup_table) { 1001// ... 1002// return *y + lookup_table[*y >> 11]; 1003// } 1004// 1005// Turning into: 1006// movzwl (%rdi), %eax 1007// movl %eax, %ecx 1008// shrl $11, %ecx 1009// addl (%rsi,%rcx,4), %eax 1010// 1011// Instead of: 1012// movzwl (%rdi), %eax 1013// movl %eax, %ecx 1014// shrl $9, %ecx 1015// andl $124, %rcx 1016// addl (%rsi,%rcx), %eax 1017// 1018// Note that this function assumes the mask is provided as a mask *after* the 1019// value is shifted. The input chain may or may not match that, but computing 1020// such a mask is trivial. 1021static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N, 1022 uint64_t Mask, 1023 SDValue Shift, SDValue X, 1024 X86ISelAddressMode &AM) { 1025 if (Shift.getOpcode() != ISD::SRL || !Shift.hasOneUse() || 1026 !isa<ConstantSDNode>(Shift.getOperand(1))) 1027 return true; 1028 1029 unsigned ShiftAmt = Shift.getConstantOperandVal(1); 1030 unsigned MaskLZ = countLeadingZeros(Mask); 1031 unsigned MaskTZ = countTrailingZeros(Mask); 1032 1033 // The amount of shift we're trying to fit into the addressing mode is taken 1034 // from the trailing zeros of the mask. 1035 unsigned AMShiftAmt = MaskTZ; 1036 1037 // There is nothing we can do here unless the mask is removing some bits. 1038 // Also, the addressing mode can only represent shifts of 1, 2, or 3 bits. 1039 if (AMShiftAmt <= 0 || AMShiftAmt > 3) return true; 1040 1041 // We also need to ensure that mask is a continuous run of bits. 1042 if (countTrailingOnes(Mask >> MaskTZ) + MaskTZ + MaskLZ != 64) return true; 1043 1044 // Scale the leading zero count down based on the actual size of the value. 1045 // Also scale it down based on the size of the shift. 1046 MaskLZ -= (64 - X.getSimpleValueType().getSizeInBits()) + ShiftAmt; 1047 1048 // The final check is to ensure that any masked out high bits of X are 1049 // already known to be zero. Otherwise, the mask has a semantic impact 1050 // other than masking out a couple of low bits. Unfortunately, because of 1051 // the mask, zero extensions will be removed from operands in some cases. 1052 // This code works extra hard to look through extensions because we can 1053 // replace them with zero extensions cheaply if necessary. 1054 bool ReplacingAnyExtend = false; 1055 if (X.getOpcode() == ISD::ANY_EXTEND) { 1056 unsigned ExtendBits = X.getSimpleValueType().getSizeInBits() - 1057 X.getOperand(0).getSimpleValueType().getSizeInBits(); 1058 // Assume that we'll replace the any-extend with a zero-extend, and 1059 // narrow the search to the extended value. 1060 X = X.getOperand(0); 1061 MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits; 1062 ReplacingAnyExtend = true; 1063 } 1064 APInt MaskedHighBits = 1065 APInt::getHighBitsSet(X.getSimpleValueType().getSizeInBits(), MaskLZ); 1066 APInt KnownZero, KnownOne; 1067 DAG.computeKnownBits(X, KnownZero, KnownOne); 1068 if (MaskedHighBits != KnownZero) return true; 1069 1070 // We've identified a pattern that can be transformed into a single shift 1071 // and an addressing mode. Make it so. 1072 MVT VT = N.getSimpleValueType(); 1073 if (ReplacingAnyExtend) { 1074 assert(X.getValueType() != VT); 1075 // We looked through an ANY_EXTEND node, insert a ZERO_EXTEND. 1076 SDValue NewX = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(X), VT, X); 1077 insertDAGNode(DAG, N, NewX); 1078 X = NewX; 1079 } 1080 SDLoc DL(N); 1081 SDValue NewSRLAmt = DAG.getConstant(ShiftAmt + AMShiftAmt, DL, MVT::i8); 1082 SDValue NewSRL = DAG.getNode(ISD::SRL, DL, VT, X, NewSRLAmt); 1083 SDValue NewSHLAmt = DAG.getConstant(AMShiftAmt, DL, MVT::i8); 1084 SDValue NewSHL = DAG.getNode(ISD::SHL, DL, VT, NewSRL, NewSHLAmt); 1085 1086 // Insert the new nodes into the topological ordering. We must do this in 1087 // a valid topological ordering as nothing is going to go back and re-sort 1088 // these nodes. We continually insert before 'N' in sequence as this is 1089 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no 1090 // hierarchy left to express. 1091 insertDAGNode(DAG, N, NewSRLAmt); 1092 insertDAGNode(DAG, N, NewSRL); 1093 insertDAGNode(DAG, N, NewSHLAmt); 1094 insertDAGNode(DAG, N, NewSHL); 1095 DAG.ReplaceAllUsesWith(N, NewSHL); 1096 1097 AM.Scale = 1 << AMShiftAmt; 1098 AM.IndexReg = NewSRL; 1099 return false; 1100} 1101 1102bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM, 1103 unsigned Depth) { 1104 SDLoc dl(N); 1105 DEBUG({ 1106 dbgs() << "MatchAddress: "; 1107 AM.dump(); 1108 }); 1109 // Limit recursion. 1110 if (Depth > 5) 1111 return matchAddressBase(N, AM); 1112 1113 // If this is already a %rip relative address, we can only merge immediates 1114 // into it. Instead of handling this in every case, we handle it here. 1115 // RIP relative addressing: %rip + 32-bit displacement! 1116 if (AM.isRIPRelative()) { 1117 // FIXME: JumpTable and ExternalSymbol address currently don't like 1118 // displacements. It isn't very important, but this should be fixed for 1119 // consistency. 1120 if (!(AM.ES || AM.MCSym) && AM.JT != -1) 1121 return true; 1122 1123 if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N)) 1124 if (!foldOffsetIntoAddress(Cst->getSExtValue(), AM)) 1125 return false; 1126 return true; 1127 } 1128 1129 switch (N.getOpcode()) { 1130 default: break; 1131 case ISD::LOCAL_RECOVER: { 1132 if (!AM.hasSymbolicDisplacement() && AM.Disp == 0) 1133 if (const auto *ESNode = dyn_cast<MCSymbolSDNode>(N.getOperand(0))) { 1134 // Use the symbol and don't prefix it. 1135 AM.MCSym = ESNode->getMCSymbol(); 1136 return false; 1137 } 1138 break; 1139 } 1140 case ISD::Constant: { 1141 uint64_t Val = cast<ConstantSDNode>(N)->getSExtValue(); 1142 if (!foldOffsetIntoAddress(Val, AM)) 1143 return false; 1144 break; 1145 } 1146 1147 case X86ISD::Wrapper: 1148 case X86ISD::WrapperRIP: 1149 if (!matchWrapper(N, AM)) 1150 return false; 1151 break; 1152 1153 case ISD::LOAD: 1154 if (!matchLoadInAddress(cast<LoadSDNode>(N), AM)) 1155 return false; 1156 break; 1157 1158 case ISD::FrameIndex: 1159 if (AM.BaseType == X86ISelAddressMode::RegBase && 1160 AM.Base_Reg.getNode() == nullptr && 1161 (!Subtarget->is64Bit() || isDispSafeForFrameIndex(AM.Disp))) { 1162 AM.BaseType = X86ISelAddressMode::FrameIndexBase; 1163 AM.Base_FrameIndex = cast<FrameIndexSDNode>(N)->getIndex(); 1164 return false; 1165 } 1166 break; 1167 1168 case ISD::SHL: 1169 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) 1170 break; 1171 1172 if (ConstantSDNode 1173 *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1))) { 1174 unsigned Val = CN->getZExtValue(); 1175 // Note that we handle x<<1 as (,x,2) rather than (x,x) here so 1176 // that the base operand remains free for further matching. If 1177 // the base doesn't end up getting used, a post-processing step 1178 // in MatchAddress turns (,x,2) into (x,x), which is cheaper. 1179 if (Val == 1 || Val == 2 || Val == 3) { 1180 AM.Scale = 1 << Val; 1181 SDValue ShVal = N.getNode()->getOperand(0); 1182 1183 // Okay, we know that we have a scale by now. However, if the scaled 1184 // value is an add of something and a constant, we can fold the 1185 // constant into the disp field here. 1186 if (CurDAG->isBaseWithConstantOffset(ShVal)) { 1187 AM.IndexReg = ShVal.getNode()->getOperand(0); 1188 ConstantSDNode *AddVal = 1189 cast<ConstantSDNode>(ShVal.getNode()->getOperand(1)); 1190 uint64_t Disp = (uint64_t)AddVal->getSExtValue() << Val; 1191 if (!foldOffsetIntoAddress(Disp, AM)) 1192 return false; 1193 } 1194 1195 AM.IndexReg = ShVal; 1196 return false; 1197 } 1198 } 1199 break; 1200 1201 case ISD::SRL: { 1202 // Scale must not be used already. 1203 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) break; 1204 1205 SDValue And = N.getOperand(0); 1206 if (And.getOpcode() != ISD::AND) break; 1207 SDValue X = And.getOperand(0); 1208 1209 // We only handle up to 64-bit values here as those are what matter for 1210 // addressing mode optimizations. 1211 if (X.getSimpleValueType().getSizeInBits() > 64) break; 1212 1213 // The mask used for the transform is expected to be post-shift, but we 1214 // found the shift first so just apply the shift to the mask before passing 1215 // it down. 1216 if (!isa<ConstantSDNode>(N.getOperand(1)) || 1217 !isa<ConstantSDNode>(And.getOperand(1))) 1218 break; 1219 uint64_t Mask = And.getConstantOperandVal(1) >> N.getConstantOperandVal(1); 1220 1221 // Try to fold the mask and shift into the scale, and return false if we 1222 // succeed. 1223 if (!foldMaskAndShiftToScale(*CurDAG, N, Mask, N, X, AM)) 1224 return false; 1225 break; 1226 } 1227 1228 case ISD::SMUL_LOHI: 1229 case ISD::UMUL_LOHI: 1230 // A mul_lohi where we need the low part can be folded as a plain multiply. 1231 if (N.getResNo() != 0) break; 1232 // FALL THROUGH 1233 case ISD::MUL: 1234 case X86ISD::MUL_IMM: 1235 // X*[3,5,9] -> X+X*[2,4,8] 1236 if (AM.BaseType == X86ISelAddressMode::RegBase && 1237 AM.Base_Reg.getNode() == nullptr && 1238 AM.IndexReg.getNode() == nullptr) { 1239 if (ConstantSDNode 1240 *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1))) 1241 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 || 1242 CN->getZExtValue() == 9) { 1243 AM.Scale = unsigned(CN->getZExtValue())-1; 1244 1245 SDValue MulVal = N.getNode()->getOperand(0); 1246 SDValue Reg; 1247 1248 // Okay, we know that we have a scale by now. However, if the scaled 1249 // value is an add of something and a constant, we can fold the 1250 // constant into the disp field here. 1251 if (MulVal.getNode()->getOpcode() == ISD::ADD && MulVal.hasOneUse() && 1252 isa<ConstantSDNode>(MulVal.getNode()->getOperand(1))) { 1253 Reg = MulVal.getNode()->getOperand(0); 1254 ConstantSDNode *AddVal = 1255 cast<ConstantSDNode>(MulVal.getNode()->getOperand(1)); 1256 uint64_t Disp = AddVal->getSExtValue() * CN->getZExtValue(); 1257 if (foldOffsetIntoAddress(Disp, AM)) 1258 Reg = N.getNode()->getOperand(0); 1259 } else { 1260 Reg = N.getNode()->getOperand(0); 1261 } 1262 1263 AM.IndexReg = AM.Base_Reg = Reg; 1264 return false; 1265 } 1266 } 1267 break; 1268 1269 case ISD::SUB: { 1270 // Given A-B, if A can be completely folded into the address and 1271 // the index field with the index field unused, use -B as the index. 1272 // This is a win if a has multiple parts that can be folded into 1273 // the address. Also, this saves a mov if the base register has 1274 // other uses, since it avoids a two-address sub instruction, however 1275 // it costs an additional mov if the index register has other uses. 1276 1277 // Add an artificial use to this node so that we can keep track of 1278 // it if it gets CSE'd with a different node. 1279 HandleSDNode Handle(N); 1280 1281 // Test if the LHS of the sub can be folded. 1282 X86ISelAddressMode Backup = AM; 1283 if (matchAddressRecursively(N.getNode()->getOperand(0), AM, Depth+1)) { 1284 AM = Backup; 1285 break; 1286 } 1287 // Test if the index field is free for use. 1288 if (AM.IndexReg.getNode() || AM.isRIPRelative()) { 1289 AM = Backup; 1290 break; 1291 } 1292 1293 int Cost = 0; 1294 SDValue RHS = Handle.getValue().getNode()->getOperand(1); 1295 // If the RHS involves a register with multiple uses, this 1296 // transformation incurs an extra mov, due to the neg instruction 1297 // clobbering its operand. 1298 if (!RHS.getNode()->hasOneUse() || 1299 RHS.getNode()->getOpcode() == ISD::CopyFromReg || 1300 RHS.getNode()->getOpcode() == ISD::TRUNCATE || 1301 RHS.getNode()->getOpcode() == ISD::ANY_EXTEND || 1302 (RHS.getNode()->getOpcode() == ISD::ZERO_EXTEND && 1303 RHS.getNode()->getOperand(0).getValueType() == MVT::i32)) 1304 ++Cost; 1305 // If the base is a register with multiple uses, this 1306 // transformation may save a mov. 1307 if ((AM.BaseType == X86ISelAddressMode::RegBase && 1308 AM.Base_Reg.getNode() && 1309 !AM.Base_Reg.getNode()->hasOneUse()) || 1310 AM.BaseType == X86ISelAddressMode::FrameIndexBase) 1311 --Cost; 1312 // If the folded LHS was interesting, this transformation saves 1313 // address arithmetic. 1314 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) + 1315 ((AM.Disp != 0) && (Backup.Disp == 0)) + 1316 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2) 1317 --Cost; 1318 // If it doesn't look like it may be an overall win, don't do it. 1319 if (Cost >= 0) { 1320 AM = Backup; 1321 break; 1322 } 1323 1324 // Ok, the transformation is legal and appears profitable. Go for it. 1325 SDValue Zero = CurDAG->getConstant(0, dl, N.getValueType()); 1326 SDValue Neg = CurDAG->getNode(ISD::SUB, dl, N.getValueType(), Zero, RHS); 1327 AM.IndexReg = Neg; 1328 AM.Scale = 1; 1329 1330 // Insert the new nodes into the topological ordering. 1331 insertDAGNode(*CurDAG, N, Zero); 1332 insertDAGNode(*CurDAG, N, Neg); 1333 return false; 1334 } 1335 1336 case ISD::ADD: 1337 if (!matchAdd(N, AM, Depth)) 1338 return false; 1339 break; 1340 1341 case ISD::OR: 1342 // We want to look through a transform in InstCombine and DAGCombiner that 1343 // turns 'add' into 'or', so we can treat this 'or' exactly like an 'add'. 1344 // Example: (or (and x, 1), (shl y, 3)) --> (add (and x, 1), (shl y, 3)) 1345 // An 'lea' can then be used to match the shift (multiply) and add: 1346 // and $1, %esi 1347 // lea (%rsi, %rdi, 8), %rax 1348 if (CurDAG->haveNoCommonBitsSet(N.getOperand(0), N.getOperand(1)) && 1349 !matchAdd(N, AM, Depth)) 1350 return false; 1351 break; 1352 1353 case ISD::AND: { 1354 // Perform some heroic transforms on an and of a constant-count shift 1355 // with a constant to enable use of the scaled offset field. 1356 1357 // Scale must not be used already. 1358 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) break; 1359 1360 SDValue Shift = N.getOperand(0); 1361 if (Shift.getOpcode() != ISD::SRL && Shift.getOpcode() != ISD::SHL) break; 1362 SDValue X = Shift.getOperand(0); 1363 1364 // We only handle up to 64-bit values here as those are what matter for 1365 // addressing mode optimizations. 1366 if (X.getSimpleValueType().getSizeInBits() > 64) break; 1367 1368 if (!isa<ConstantSDNode>(N.getOperand(1))) 1369 break; 1370 uint64_t Mask = N.getConstantOperandVal(1); 1371 1372 // Try to fold the mask and shift into an extract and scale. 1373 if (!foldMaskAndShiftToExtract(*CurDAG, N, Mask, Shift, X, AM)) 1374 return false; 1375 1376 // Try to fold the mask and shift directly into the scale. 1377 if (!foldMaskAndShiftToScale(*CurDAG, N, Mask, Shift, X, AM)) 1378 return false; 1379 1380 // Try to swap the mask and shift to place shifts which can be done as 1381 // a scale on the outside of the mask. 1382 if (!foldMaskedShiftToScaledMask(*CurDAG, N, Mask, Shift, X, AM)) 1383 return false; 1384 break; 1385 } 1386 } 1387 1388 return matchAddressBase(N, AM); 1389} 1390 1391/// Helper for MatchAddress. Add the specified node to the 1392/// specified addressing mode without any further recursion. 1393bool X86DAGToDAGISel::matchAddressBase(SDValue N, X86ISelAddressMode &AM) { 1394 // Is the base register already occupied? 1395 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) { 1396 // If so, check to see if the scale index register is set. 1397 if (!AM.IndexReg.getNode()) { 1398 AM.IndexReg = N; 1399 AM.Scale = 1; 1400 return false; 1401 } 1402 1403 // Otherwise, we cannot select it. 1404 return true; 1405 } 1406 1407 // Default, generate it as a register. 1408 AM.BaseType = X86ISelAddressMode::RegBase; 1409 AM.Base_Reg = N; 1410 return false; 1411} 1412 1413bool X86DAGToDAGISel::selectVectorAddr(SDNode *Parent, SDValue N, SDValue &Base, 1414 SDValue &Scale, SDValue &Index, 1415 SDValue &Disp, SDValue &Segment) { 1416 1417 MaskedGatherScatterSDNode *Mgs = dyn_cast<MaskedGatherScatterSDNode>(Parent); 1418 if (!Mgs) 1419 return false; 1420 X86ISelAddressMode AM; 1421 unsigned AddrSpace = Mgs->getPointerInfo().getAddrSpace(); 1422 // AddrSpace 256 -> GS, 257 -> FS. 1423 if (AddrSpace == 256) 1424 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); 1425 if (AddrSpace == 257) 1426 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); 1427 1428 SDLoc DL(N); 1429 Base = Mgs->getBasePtr(); 1430 Index = Mgs->getIndex(); 1431 unsigned ScalarSize = Mgs->getValue().getValueType().getScalarSizeInBits(); 1432 Scale = getI8Imm(ScalarSize/8, DL); 1433 1434 // If Base is 0, the whole address is in index and the Scale is 1 1435 if (isa<ConstantSDNode>(Base)) { 1436 assert(cast<ConstantSDNode>(Base)->isNullValue() && 1437 "Unexpected base in gather/scatter"); 1438 Scale = getI8Imm(1, DL); 1439 Base = CurDAG->getRegister(0, MVT::i32); 1440 } 1441 if (AM.Segment.getNode()) 1442 Segment = AM.Segment; 1443 else 1444 Segment = CurDAG->getRegister(0, MVT::i32); 1445 Disp = CurDAG->getTargetConstant(0, DL, MVT::i32); 1446 return true; 1447} 1448 1449/// Returns true if it is able to pattern match an addressing mode. 1450/// It returns the operands which make up the maximal addressing mode it can 1451/// match by reference. 1452/// 1453/// Parent is the parent node of the addr operand that is being matched. It 1454/// is always a load, store, atomic node, or null. It is only null when 1455/// checking memory operands for inline asm nodes. 1456bool X86DAGToDAGISel::selectAddr(SDNode *Parent, SDValue N, SDValue &Base, 1457 SDValue &Scale, SDValue &Index, 1458 SDValue &Disp, SDValue &Segment) { 1459 X86ISelAddressMode AM; 1460 1461 if (Parent && 1462 // This list of opcodes are all the nodes that have an "addr:$ptr" operand 1463 // that are not a MemSDNode, and thus don't have proper addrspace info. 1464 Parent->getOpcode() != ISD::INTRINSIC_W_CHAIN && // unaligned loads, fixme 1465 Parent->getOpcode() != ISD::INTRINSIC_VOID && // nontemporal stores 1466 Parent->getOpcode() != X86ISD::TLSCALL && // Fixme 1467 Parent->getOpcode() != X86ISD::EH_SJLJ_SETJMP && // setjmp 1468 Parent->getOpcode() != X86ISD::EH_SJLJ_LONGJMP) { // longjmp 1469 unsigned AddrSpace = 1470 cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace(); 1471 // AddrSpace 256 -> GS, 257 -> FS. 1472 if (AddrSpace == 256) 1473 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); 1474 if (AddrSpace == 257) 1475 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); 1476 } 1477 1478 if (matchAddress(N, AM)) 1479 return false; 1480 1481 MVT VT = N.getSimpleValueType(); 1482 if (AM.BaseType == X86ISelAddressMode::RegBase) { 1483 if (!AM.Base_Reg.getNode()) 1484 AM.Base_Reg = CurDAG->getRegister(0, VT); 1485 } 1486 1487 if (!AM.IndexReg.getNode()) 1488 AM.IndexReg = CurDAG->getRegister(0, VT); 1489 1490 getAddressOperands(AM, SDLoc(N), Base, Scale, Index, Disp, Segment); 1491 return true; 1492} 1493 1494/// Match a scalar SSE load. In particular, we want to match a load whose top 1495/// elements are either undef or zeros. The load flavor is derived from the 1496/// type of N, which is either v4f32 or v2f64. 1497/// 1498/// We also return: 1499/// PatternChainNode: this is the matched node that has a chain input and 1500/// output. 1501bool X86DAGToDAGISel::selectScalarSSELoad(SDNode *Root, 1502 SDValue N, SDValue &Base, 1503 SDValue &Scale, SDValue &Index, 1504 SDValue &Disp, SDValue &Segment, 1505 SDValue &PatternNodeWithChain) { 1506 if (N.getOpcode() == ISD::SCALAR_TO_VECTOR) { 1507 PatternNodeWithChain = N.getOperand(0); 1508 if (ISD::isNON_EXTLoad(PatternNodeWithChain.getNode()) && 1509 PatternNodeWithChain.hasOneUse() && 1510 IsProfitableToFold(N.getOperand(0), N.getNode(), Root) && 1511 IsLegalToFold(N.getOperand(0), N.getNode(), Root, OptLevel)) { 1512 LoadSDNode *LD = cast<LoadSDNode>(PatternNodeWithChain); 1513 if (!selectAddr(LD, LD->getBasePtr(), Base, Scale, Index, Disp, Segment)) 1514 return false; 1515 return true; 1516 } 1517 } 1518 1519 // Also handle the case where we explicitly require zeros in the top 1520 // elements. This is a vector shuffle from the zero vector. 1521 if (N.getOpcode() == X86ISD::VZEXT_MOVL && N.getNode()->hasOneUse() && 1522 // Check to see if the top elements are all zeros (or bitcast of zeros). 1523 N.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR && 1524 N.getOperand(0).getNode()->hasOneUse() && 1525 ISD::isNON_EXTLoad(N.getOperand(0).getOperand(0).getNode()) && 1526 N.getOperand(0).getOperand(0).hasOneUse() && 1527 IsProfitableToFold(N.getOperand(0), N.getNode(), Root) && 1528 IsLegalToFold(N.getOperand(0), N.getNode(), Root, OptLevel)) { 1529 // Okay, this is a zero extending load. Fold it. 1530 LoadSDNode *LD = cast<LoadSDNode>(N.getOperand(0).getOperand(0)); 1531 if (!selectAddr(LD, LD->getBasePtr(), Base, Scale, Index, Disp, Segment)) 1532 return false; 1533 PatternNodeWithChain = SDValue(LD, 0); 1534 return true; 1535 } 1536 return false; 1537} 1538 1539 1540bool X86DAGToDAGISel::selectMOV64Imm32(SDValue N, SDValue &Imm) { 1541 if (const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) { 1542 uint64_t ImmVal = CN->getZExtValue(); 1543 if ((uint32_t)ImmVal != (uint64_t)ImmVal) 1544 return false; 1545 1546 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), MVT::i64); 1547 return true; 1548 } 1549 1550 // In static codegen with small code model, we can get the address of a label 1551 // into a register with 'movl'. TableGen has already made sure we're looking 1552 // at a label of some kind. 1553 assert(N->getOpcode() == X86ISD::Wrapper && 1554 "Unexpected node type for MOV32ri64"); 1555 N = N.getOperand(0); 1556 1557 if (N->getOpcode() != ISD::TargetConstantPool && 1558 N->getOpcode() != ISD::TargetJumpTable && 1559 N->getOpcode() != ISD::TargetGlobalAddress && 1560 N->getOpcode() != ISD::TargetExternalSymbol && 1561 N->getOpcode() != ISD::MCSymbol && 1562 N->getOpcode() != ISD::TargetBlockAddress) 1563 return false; 1564 1565 Imm = N; 1566 return TM.getCodeModel() == CodeModel::Small; 1567} 1568 1569bool X86DAGToDAGISel::selectLEA64_32Addr(SDValue N, SDValue &Base, 1570 SDValue &Scale, SDValue &Index, 1571 SDValue &Disp, SDValue &Segment) { 1572 if (!selectLEAAddr(N, Base, Scale, Index, Disp, Segment)) 1573 return false; 1574 1575 SDLoc DL(N); 1576 RegisterSDNode *RN = dyn_cast<RegisterSDNode>(Base); 1577 if (RN && RN->getReg() == 0) 1578 Base = CurDAG->getRegister(0, MVT::i64); 1579 else if (Base.getValueType() == MVT::i32 && !dyn_cast<FrameIndexSDNode>(Base)) { 1580 // Base could already be %rip, particularly in the x32 ABI. 1581 Base = SDValue(CurDAG->getMachineNode( 1582 TargetOpcode::SUBREG_TO_REG, DL, MVT::i64, 1583 CurDAG->getTargetConstant(0, DL, MVT::i64), 1584 Base, 1585 CurDAG->getTargetConstant(X86::sub_32bit, DL, MVT::i32)), 1586 0); 1587 } 1588 1589 RN = dyn_cast<RegisterSDNode>(Index); 1590 if (RN && RN->getReg() == 0) 1591 Index = CurDAG->getRegister(0, MVT::i64); 1592 else { 1593 assert(Index.getValueType() == MVT::i32 && 1594 "Expect to be extending 32-bit registers for use in LEA"); 1595 Index = SDValue(CurDAG->getMachineNode( 1596 TargetOpcode::SUBREG_TO_REG, DL, MVT::i64, 1597 CurDAG->getTargetConstant(0, DL, MVT::i64), 1598 Index, 1599 CurDAG->getTargetConstant(X86::sub_32bit, DL, 1600 MVT::i32)), 1601 0); 1602 } 1603 1604 return true; 1605} 1606 1607/// Calls SelectAddr and determines if the maximal addressing 1608/// mode it matches can be cost effectively emitted as an LEA instruction. 1609bool X86DAGToDAGISel::selectLEAAddr(SDValue N, 1610 SDValue &Base, SDValue &Scale, 1611 SDValue &Index, SDValue &Disp, 1612 SDValue &Segment) { 1613 X86ISelAddressMode AM; 1614 1615 // Set AM.Segment to prevent MatchAddress from using one. LEA doesn't support 1616 // segments. 1617 SDValue Copy = AM.Segment; 1618 SDValue T = CurDAG->getRegister(0, MVT::i32); 1619 AM.Segment = T; 1620 if (matchAddress(N, AM)) 1621 return false; 1622 assert (T == AM.Segment); 1623 AM.Segment = Copy; 1624 1625 MVT VT = N.getSimpleValueType(); 1626 unsigned Complexity = 0; 1627 if (AM.BaseType == X86ISelAddressMode::RegBase) 1628 if (AM.Base_Reg.getNode()) 1629 Complexity = 1; 1630 else 1631 AM.Base_Reg = CurDAG->getRegister(0, VT); 1632 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase) 1633 Complexity = 4; 1634 1635 if (AM.IndexReg.getNode()) 1636 Complexity++; 1637 else 1638 AM.IndexReg = CurDAG->getRegister(0, VT); 1639 1640 // Don't match just leal(,%reg,2). It's cheaper to do addl %reg, %reg, or with 1641 // a simple shift. 1642 if (AM.Scale > 1) 1643 Complexity++; 1644 1645 // FIXME: We are artificially lowering the criteria to turn ADD %reg, $GA 1646 // to a LEA. This is determined with some experimentation but is by no means 1647 // optimal (especially for code size consideration). LEA is nice because of 1648 // its three-address nature. Tweak the cost function again when we can run 1649 // convertToThreeAddress() at register allocation time. 1650 if (AM.hasSymbolicDisplacement()) { 1651 // For X86-64, always use LEA to materialize RIP-relative addresses. 1652 if (Subtarget->is64Bit()) 1653 Complexity = 4; 1654 else 1655 Complexity += 2; 1656 } 1657 1658 if (AM.Disp && (AM.Base_Reg.getNode() || AM.IndexReg.getNode())) 1659 Complexity++; 1660 1661 // If it isn't worth using an LEA, reject it. 1662 if (Complexity <= 2) 1663 return false; 1664 1665 getAddressOperands(AM, SDLoc(N), Base, Scale, Index, Disp, Segment); 1666 return true; 1667} 1668 1669/// This is only run on TargetGlobalTLSAddress nodes. 1670bool X86DAGToDAGISel::selectTLSADDRAddr(SDValue N, SDValue &Base, 1671 SDValue &Scale, SDValue &Index, 1672 SDValue &Disp, SDValue &Segment) { 1673 assert(N.getOpcode() == ISD::TargetGlobalTLSAddress); 1674 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 1675 1676 X86ISelAddressMode AM; 1677 AM.GV = GA->getGlobal(); 1678 AM.Disp += GA->getOffset(); 1679 AM.Base_Reg = CurDAG->getRegister(0, N.getValueType()); 1680 AM.SymbolFlags = GA->getTargetFlags(); 1681 1682 if (N.getValueType() == MVT::i32) { 1683 AM.Scale = 1; 1684 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32); 1685 } else { 1686 AM.IndexReg = CurDAG->getRegister(0, MVT::i64); 1687 } 1688 1689 getAddressOperands(AM, SDLoc(N), Base, Scale, Index, Disp, Segment); 1690 return true; 1691} 1692 1693 1694bool X86DAGToDAGISel::tryFoldLoad(SDNode *P, SDValue N, 1695 SDValue &Base, SDValue &Scale, 1696 SDValue &Index, SDValue &Disp, 1697 SDValue &Segment) { 1698 if (!ISD::isNON_EXTLoad(N.getNode()) || 1699 !IsProfitableToFold(N, P, P) || 1700 !IsLegalToFold(N, P, P, OptLevel)) 1701 return false; 1702 1703 return selectAddr(N.getNode(), 1704 N.getOperand(1), Base, Scale, Index, Disp, Segment); 1705} 1706 1707/// Return an SDNode that returns the value of the global base register. 1708/// Output instructions required to initialize the global base register, 1709/// if necessary. 1710SDNode *X86DAGToDAGISel::getGlobalBaseReg() { 1711 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF); 1712 auto &DL = MF->getDataLayout(); 1713 return CurDAG->getRegister(GlobalBaseReg, TLI->getPointerTy(DL)).getNode(); 1714} 1715 1716/// Atomic opcode table 1717/// 1718enum AtomicOpc { 1719 ADD, 1720 SUB, 1721 INC, 1722 DEC, 1723 OR, 1724 AND, 1725 XOR, 1726 AtomicOpcEnd 1727}; 1728 1729enum AtomicSz { 1730 ConstantI8, 1731 I8, 1732 SextConstantI16, 1733 ConstantI16, 1734 I16, 1735 SextConstantI32, 1736 ConstantI32, 1737 I32, 1738 SextConstantI64, 1739 ConstantI64, 1740 I64, 1741 AtomicSzEnd 1742}; 1743 1744static const uint16_t AtomicOpcTbl[AtomicOpcEnd][AtomicSzEnd] = { 1745 { 1746 X86::LOCK_ADD8mi, 1747 X86::LOCK_ADD8mr, 1748 X86::LOCK_ADD16mi8, 1749 X86::LOCK_ADD16mi, 1750 X86::LOCK_ADD16mr, 1751 X86::LOCK_ADD32mi8, 1752 X86::LOCK_ADD32mi, 1753 X86::LOCK_ADD32mr, 1754 X86::LOCK_ADD64mi8, 1755 X86::LOCK_ADD64mi32, 1756 X86::LOCK_ADD64mr, 1757 }, 1758 { 1759 X86::LOCK_SUB8mi, 1760 X86::LOCK_SUB8mr, 1761 X86::LOCK_SUB16mi8, 1762 X86::LOCK_SUB16mi, 1763 X86::LOCK_SUB16mr, 1764 X86::LOCK_SUB32mi8, 1765 X86::LOCK_SUB32mi, 1766 X86::LOCK_SUB32mr, 1767 X86::LOCK_SUB64mi8, 1768 X86::LOCK_SUB64mi32, 1769 X86::LOCK_SUB64mr, 1770 }, 1771 { 1772 0, 1773 X86::LOCK_INC8m, 1774 0, 1775 0, 1776 X86::LOCK_INC16m, 1777 0, 1778 0, 1779 X86::LOCK_INC32m, 1780 0, 1781 0, 1782 X86::LOCK_INC64m, 1783 }, 1784 { 1785 0, 1786 X86::LOCK_DEC8m, 1787 0, 1788 0, 1789 X86::LOCK_DEC16m, 1790 0, 1791 0, 1792 X86::LOCK_DEC32m, 1793 0, 1794 0, 1795 X86::LOCK_DEC64m, 1796 }, 1797 { 1798 X86::LOCK_OR8mi, 1799 X86::LOCK_OR8mr, 1800 X86::LOCK_OR16mi8, 1801 X86::LOCK_OR16mi, 1802 X86::LOCK_OR16mr, 1803 X86::LOCK_OR32mi8, 1804 X86::LOCK_OR32mi, 1805 X86::LOCK_OR32mr, 1806 X86::LOCK_OR64mi8, 1807 X86::LOCK_OR64mi32, 1808 X86::LOCK_OR64mr, 1809 }, 1810 { 1811 X86::LOCK_AND8mi, 1812 X86::LOCK_AND8mr, 1813 X86::LOCK_AND16mi8, 1814 X86::LOCK_AND16mi, 1815 X86::LOCK_AND16mr, 1816 X86::LOCK_AND32mi8, 1817 X86::LOCK_AND32mi, 1818 X86::LOCK_AND32mr, 1819 X86::LOCK_AND64mi8, 1820 X86::LOCK_AND64mi32, 1821 X86::LOCK_AND64mr, 1822 }, 1823 { 1824 X86::LOCK_XOR8mi, 1825 X86::LOCK_XOR8mr, 1826 X86::LOCK_XOR16mi8, 1827 X86::LOCK_XOR16mi, 1828 X86::LOCK_XOR16mr, 1829 X86::LOCK_XOR32mi8, 1830 X86::LOCK_XOR32mi, 1831 X86::LOCK_XOR32mr, 1832 X86::LOCK_XOR64mi8, 1833 X86::LOCK_XOR64mi32, 1834 X86::LOCK_XOR64mr, 1835 } 1836}; 1837 1838// Return the target constant operand for atomic-load-op and do simple 1839// translations, such as from atomic-load-add to lock-sub. The return value is 1840// one of the following 3 cases: 1841// + target-constant, the operand could be supported as a target constant. 1842// + empty, the operand is not needed any more with the new op selected. 1843// + non-empty, otherwise. 1844static SDValue getAtomicLoadArithTargetConstant(SelectionDAG *CurDAG, 1845 SDLoc dl, 1846 enum AtomicOpc &Op, MVT NVT, 1847 SDValue Val, 1848 const X86Subtarget *Subtarget) { 1849 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val)) { 1850 int64_t CNVal = CN->getSExtValue(); 1851 // Quit if not 32-bit imm. 1852 if ((int32_t)CNVal != CNVal) 1853 return Val; 1854 // Quit if INT32_MIN: it would be negated as it is negative and overflow, 1855 // producing an immediate that does not fit in the 32 bits available for 1856 // an immediate operand to sub. However, it still fits in 32 bits for the 1857 // add (since it is not negated) so we can return target-constant. 1858 if (CNVal == INT32_MIN) 1859 return CurDAG->getTargetConstant(CNVal, dl, NVT); 1860 // For atomic-load-add, we could do some optimizations. 1861 if (Op == ADD) { 1862 // Translate to INC/DEC if ADD by 1 or -1. 1863 if (((CNVal == 1) || (CNVal == -1)) && !Subtarget->slowIncDec()) { 1864 Op = (CNVal == 1) ? INC : DEC; 1865 // No more constant operand after being translated into INC/DEC. 1866 return SDValue(); 1867 } 1868 // Translate to SUB if ADD by negative value. 1869 if (CNVal < 0) { 1870 Op = SUB; 1871 CNVal = -CNVal; 1872 } 1873 } 1874 return CurDAG->getTargetConstant(CNVal, dl, NVT); 1875 } 1876 1877 // If the value operand is single-used, try to optimize it. 1878 if (Op == ADD && Val.hasOneUse()) { 1879 // Translate (atomic-load-add ptr (sub 0 x)) back to (lock-sub x). 1880 if (Val.getOpcode() == ISD::SUB && X86::isZeroNode(Val.getOperand(0))) { 1881 Op = SUB; 1882 return Val.getOperand(1); 1883 } 1884 // A special case for i16, which needs truncating as, in most cases, it's 1885 // promoted to i32. We will translate 1886 // (atomic-load-add (truncate (sub 0 x))) to (lock-sub (EXTRACT_SUBREG x)) 1887 if (Val.getOpcode() == ISD::TRUNCATE && NVT == MVT::i16 && 1888 Val.getOperand(0).getOpcode() == ISD::SUB && 1889 X86::isZeroNode(Val.getOperand(0).getOperand(0))) { 1890 Op = SUB; 1891 Val = Val.getOperand(0); 1892 return CurDAG->getTargetExtractSubreg(X86::sub_16bit, dl, NVT, 1893 Val.getOperand(1)); 1894 } 1895 } 1896 1897 return Val; 1898} 1899 1900SDNode *X86DAGToDAGISel::selectAtomicLoadArith(SDNode *Node, MVT NVT) { 1901 if (Node->hasAnyUseOfValue(0)) 1902 return nullptr; 1903 1904 SDLoc dl(Node); 1905 1906 // Optimize common patterns for __sync_or_and_fetch and similar arith 1907 // operations where the result is not used. This allows us to use the "lock" 1908 // version of the arithmetic instruction. 1909 SDValue Chain = Node->getOperand(0); 1910 SDValue Ptr = Node->getOperand(1); 1911 SDValue Val = Node->getOperand(2); 1912 SDValue Base, Scale, Index, Disp, Segment; 1913 if (!selectAddr(Node, Ptr, Base, Scale, Index, Disp, Segment)) 1914 return nullptr; 1915 1916 // Which index into the table. 1917 enum AtomicOpc Op; 1918 switch (Node->getOpcode()) { 1919 default: 1920 return nullptr; 1921 case ISD::ATOMIC_LOAD_OR: 1922 Op = OR; 1923 break; 1924 case ISD::ATOMIC_LOAD_AND: 1925 Op = AND; 1926 break; 1927 case ISD::ATOMIC_LOAD_XOR: 1928 Op = XOR; 1929 break; 1930 case ISD::ATOMIC_LOAD_ADD: 1931 Op = ADD; 1932 break; 1933 } 1934 1935 Val = getAtomicLoadArithTargetConstant(CurDAG, dl, Op, NVT, Val, Subtarget); 1936 bool isUnOp = !Val.getNode(); 1937 bool isCN = Val.getNode() && (Val.getOpcode() == ISD::TargetConstant); 1938 1939 unsigned Opc = 0; 1940 switch (NVT.SimpleTy) { 1941 default: return nullptr; 1942 case MVT::i8: 1943 if (isCN) 1944 Opc = AtomicOpcTbl[Op][ConstantI8]; 1945 else 1946 Opc = AtomicOpcTbl[Op][I8]; 1947 break; 1948 case MVT::i16: 1949 if (isCN) { 1950 if (immSext8(Val.getNode())) 1951 Opc = AtomicOpcTbl[Op][SextConstantI16]; 1952 else 1953 Opc = AtomicOpcTbl[Op][ConstantI16]; 1954 } else 1955 Opc = AtomicOpcTbl[Op][I16]; 1956 break; 1957 case MVT::i32: 1958 if (isCN) { 1959 if (immSext8(Val.getNode())) 1960 Opc = AtomicOpcTbl[Op][SextConstantI32]; 1961 else 1962 Opc = AtomicOpcTbl[Op][ConstantI32]; 1963 } else 1964 Opc = AtomicOpcTbl[Op][I32]; 1965 break; 1966 case MVT::i64: 1967 if (isCN) { 1968 if (immSext8(Val.getNode())) 1969 Opc = AtomicOpcTbl[Op][SextConstantI64]; 1970 else if (i64immSExt32(Val.getNode())) 1971 Opc = AtomicOpcTbl[Op][ConstantI64]; 1972 else 1973 llvm_unreachable("True 64 bits constant in SelectAtomicLoadArith"); 1974 } else 1975 Opc = AtomicOpcTbl[Op][I64]; 1976 break; 1977 } 1978 1979 assert(Opc != 0 && "Invalid arith lock transform!"); 1980 1981 // Building the new node. 1982 SDValue Ret; 1983 if (isUnOp) { 1984 SDValue Ops[] = { Base, Scale, Index, Disp, Segment, Chain }; 1985 Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops), 0); 1986 } else { 1987 SDValue Ops[] = { Base, Scale, Index, Disp, Segment, Val, Chain }; 1988 Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops), 0); 1989 } 1990 1991 // Copying the MachineMemOperand. 1992 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1993 MemOp[0] = cast<MemSDNode>(Node)->getMemOperand(); 1994 cast<MachineSDNode>(Ret)->setMemRefs(MemOp, MemOp + 1); 1995 1996 // We need to have two outputs as that is what the original instruction had. 1997 // So we add a dummy, undefined output. This is safe as we checked first 1998 // that no-one uses our output anyway. 1999 SDValue Undef = SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, 2000 dl, NVT), 0); 2001 SDValue RetVals[] = { Undef, Ret }; 2002 return CurDAG->getMergeValues(RetVals, dl).getNode(); 2003} 2004 2005/// Test whether the given X86ISD::CMP node has any uses which require the SF 2006/// or OF bits to be accurate. 2007static bool hasNoSignedComparisonUses(SDNode *N) { 2008 // Examine each user of the node. 2009 for (SDNode::use_iterator UI = N->use_begin(), 2010 UE = N->use_end(); UI != UE; ++UI) { 2011 // Only examine CopyToReg uses. 2012 if (UI->getOpcode() != ISD::CopyToReg) 2013 return false; 2014 // Only examine CopyToReg uses that copy to EFLAGS. 2015 if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != 2016 X86::EFLAGS) 2017 return false; 2018 // Examine each user of the CopyToReg use. 2019 for (SDNode::use_iterator FlagUI = UI->use_begin(), 2020 FlagUE = UI->use_end(); FlagUI != FlagUE; ++FlagUI) { 2021 // Only examine the Flag result. 2022 if (FlagUI.getUse().getResNo() != 1) continue; 2023 // Anything unusual: assume conservatively. 2024 if (!FlagUI->isMachineOpcode()) return false; 2025 // Examine the opcode of the user. 2026 switch (FlagUI->getMachineOpcode()) { 2027 // These comparisons don't treat the most significant bit specially. 2028 case X86::SETAr: case X86::SETAEr: case X86::SETBr: case X86::SETBEr: 2029 case X86::SETEr: case X86::SETNEr: case X86::SETPr: case X86::SETNPr: 2030 case X86::SETAm: case X86::SETAEm: case X86::SETBm: case X86::SETBEm: 2031 case X86::SETEm: case X86::SETNEm: case X86::SETPm: case X86::SETNPm: 2032 case X86::JA_1: case X86::JAE_1: case X86::JB_1: case X86::JBE_1: 2033 case X86::JE_1: case X86::JNE_1: case X86::JP_1: case X86::JNP_1: 2034 case X86::CMOVA16rr: case X86::CMOVA16rm: 2035 case X86::CMOVA32rr: case X86::CMOVA32rm: 2036 case X86::CMOVA64rr: case X86::CMOVA64rm: 2037 case X86::CMOVAE16rr: case X86::CMOVAE16rm: 2038 case X86::CMOVAE32rr: case X86::CMOVAE32rm: 2039 case X86::CMOVAE64rr: case X86::CMOVAE64rm: 2040 case X86::CMOVB16rr: case X86::CMOVB16rm: 2041 case X86::CMOVB32rr: case X86::CMOVB32rm: 2042 case X86::CMOVB64rr: case X86::CMOVB64rm: 2043 case X86::CMOVBE16rr: case X86::CMOVBE16rm: 2044 case X86::CMOVBE32rr: case X86::CMOVBE32rm: 2045 case X86::CMOVBE64rr: case X86::CMOVBE64rm: 2046 case X86::CMOVE16rr: case X86::CMOVE16rm: 2047 case X86::CMOVE32rr: case X86::CMOVE32rm: 2048 case X86::CMOVE64rr: case X86::CMOVE64rm: 2049 case X86::CMOVNE16rr: case X86::CMOVNE16rm: 2050 case X86::CMOVNE32rr: case X86::CMOVNE32rm: 2051 case X86::CMOVNE64rr: case X86::CMOVNE64rm: 2052 case X86::CMOVNP16rr: case X86::CMOVNP16rm: 2053 case X86::CMOVNP32rr: case X86::CMOVNP32rm: 2054 case X86::CMOVNP64rr: case X86::CMOVNP64rm: 2055 case X86::CMOVP16rr: case X86::CMOVP16rm: 2056 case X86::CMOVP32rr: case X86::CMOVP32rm: 2057 case X86::CMOVP64rr: case X86::CMOVP64rm: 2058 continue; 2059 // Anything else: assume conservatively. 2060 default: return false; 2061 } 2062 } 2063 } 2064 return true; 2065} 2066 2067/// Check whether or not the chain ending in StoreNode is suitable for doing 2068/// the {load; increment or decrement; store} to modify transformation. 2069static bool isLoadIncOrDecStore(StoreSDNode *StoreNode, unsigned Opc, 2070 SDValue StoredVal, SelectionDAG *CurDAG, 2071 LoadSDNode* &LoadNode, SDValue &InputChain) { 2072 2073 // is the value stored the result of a DEC or INC? 2074 if (!(Opc == X86ISD::DEC || Opc == X86ISD::INC)) return false; 2075 2076 // is the stored value result 0 of the load? 2077 if (StoredVal.getResNo() != 0) return false; 2078 2079 // are there other uses of the loaded value than the inc or dec? 2080 if (!StoredVal.getNode()->hasNUsesOfValue(1, 0)) return false; 2081 2082 // is the store non-extending and non-indexed? 2083 if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal()) 2084 return false; 2085 2086 SDValue Load = StoredVal->getOperand(0); 2087 // Is the stored value a non-extending and non-indexed load? 2088 if (!ISD::isNormalLoad(Load.getNode())) return false; 2089 2090 // Return LoadNode by reference. 2091 LoadNode = cast<LoadSDNode>(Load); 2092 // is the size of the value one that we can handle? (i.e. 64, 32, 16, or 8) 2093 EVT LdVT = LoadNode->getMemoryVT(); 2094 if (LdVT != MVT::i64 && LdVT != MVT::i32 && LdVT != MVT::i16 && 2095 LdVT != MVT::i8) 2096 return false; 2097 2098 // Is store the only read of the loaded value? 2099 if (!Load.hasOneUse()) 2100 return false; 2101 2102 // Is the address of the store the same as the load? 2103 if (LoadNode->getBasePtr() != StoreNode->getBasePtr() || 2104 LoadNode->getOffset() != StoreNode->getOffset()) 2105 return false; 2106 2107 // Check if the chain is produced by the load or is a TokenFactor with 2108 // the load output chain as an operand. Return InputChain by reference. 2109 SDValue Chain = StoreNode->getChain(); 2110 2111 bool ChainCheck = false; 2112 if (Chain == Load.getValue(1)) { 2113 ChainCheck = true; 2114 InputChain = LoadNode->getChain(); 2115 } else if (Chain.getOpcode() == ISD::TokenFactor) { 2116 SmallVector<SDValue, 4> ChainOps; 2117 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) { 2118 SDValue Op = Chain.getOperand(i); 2119 if (Op == Load.getValue(1)) { 2120 ChainCheck = true; 2121 continue; 2122 } 2123 2124 // Make sure using Op as part of the chain would not cause a cycle here. 2125 // In theory, we could check whether the chain node is a predecessor of 2126 // the load. But that can be very expensive. Instead visit the uses and 2127 // make sure they all have smaller node id than the load. 2128 int LoadId = LoadNode->getNodeId(); 2129 for (SDNode::use_iterator UI = Op.getNode()->use_begin(), 2130 UE = UI->use_end(); UI != UE; ++UI) { 2131 if (UI.getUse().getResNo() != 0) 2132 continue; 2133 if (UI->getNodeId() > LoadId) 2134 return false; 2135 } 2136 2137 ChainOps.push_back(Op); 2138 } 2139 2140 if (ChainCheck) 2141 // Make a new TokenFactor with all the other input chains except 2142 // for the load. 2143 InputChain = CurDAG->getNode(ISD::TokenFactor, SDLoc(Chain), 2144 MVT::Other, ChainOps); 2145 } 2146 if (!ChainCheck) 2147 return false; 2148 2149 return true; 2150} 2151 2152/// Get the appropriate X86 opcode for an in-memory increment or decrement. 2153/// Opc should be X86ISD::DEC or X86ISD::INC. 2154static unsigned getFusedLdStOpcode(EVT &LdVT, unsigned Opc) { 2155 if (Opc == X86ISD::DEC) { 2156 if (LdVT == MVT::i64) return X86::DEC64m; 2157 if (LdVT == MVT::i32) return X86::DEC32m; 2158 if (LdVT == MVT::i16) return X86::DEC16m; 2159 if (LdVT == MVT::i8) return X86::DEC8m; 2160 } else { 2161 assert(Opc == X86ISD::INC && "unrecognized opcode"); 2162 if (LdVT == MVT::i64) return X86::INC64m; 2163 if (LdVT == MVT::i32) return X86::INC32m; 2164 if (LdVT == MVT::i16) return X86::INC16m; 2165 if (LdVT == MVT::i8) return X86::INC8m; 2166 } 2167 llvm_unreachable("unrecognized size for LdVT"); 2168} 2169 2170/// Customized ISel for GATHER operations. 2171SDNode *X86DAGToDAGISel::selectGather(SDNode *Node, unsigned Opc) { 2172 // Operands of Gather: VSrc, Base, VIdx, VMask, Scale 2173 SDValue Chain = Node->getOperand(0); 2174 SDValue VSrc = Node->getOperand(2); 2175 SDValue Base = Node->getOperand(3); 2176 SDValue VIdx = Node->getOperand(4); 2177 SDValue VMask = Node->getOperand(5); 2178 ConstantSDNode *Scale = dyn_cast<ConstantSDNode>(Node->getOperand(6)); 2179 if (!Scale) 2180 return nullptr; 2181 2182 SDVTList VTs = CurDAG->getVTList(VSrc.getValueType(), VSrc.getValueType(), 2183 MVT::Other); 2184 2185 SDLoc DL(Node); 2186 2187 // Memory Operands: Base, Scale, Index, Disp, Segment 2188 SDValue Disp = CurDAG->getTargetConstant(0, DL, MVT::i32); 2189 SDValue Segment = CurDAG->getRegister(0, MVT::i32); 2190 const SDValue Ops[] = { VSrc, Base, getI8Imm(Scale->getSExtValue(), DL), VIdx, 2191 Disp, Segment, VMask, Chain}; 2192 SDNode *ResNode = CurDAG->getMachineNode(Opc, DL, VTs, Ops); 2193 // Node has 2 outputs: VDst and MVT::Other. 2194 // ResNode has 3 outputs: VDst, VMask_wb, and MVT::Other. 2195 // We replace VDst of Node with VDst of ResNode, and Other of Node with Other 2196 // of ResNode. 2197 ReplaceUses(SDValue(Node, 0), SDValue(ResNode, 0)); 2198 ReplaceUses(SDValue(Node, 1), SDValue(ResNode, 2)); 2199 return ResNode; 2200} 2201 2202SDNode *X86DAGToDAGISel::Select(SDNode *Node) { 2203 MVT NVT = Node->getSimpleValueType(0); 2204 unsigned Opc, MOpc; 2205 unsigned Opcode = Node->getOpcode(); 2206 SDLoc dl(Node); 2207 2208 DEBUG(dbgs() << "Selecting: "; Node->dump(CurDAG); dbgs() << '\n'); 2209 2210 if (Node->isMachineOpcode()) { 2211 DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << '\n'); 2212 Node->setNodeId(-1); 2213 return nullptr; // Already selected. 2214 } 2215 2216 switch (Opcode) { 2217 default: break; 2218 case ISD::BRIND: { 2219 if (Subtarget->isTargetNaCl()) 2220 // NaCl has its own pass where jmp %r32 are converted to jmp %r64. We 2221 // leave the instruction alone. 2222 break; 2223 if (Subtarget->isTarget64BitILP32()) { 2224 // Converts a 32-bit register to a 64-bit, zero-extended version of 2225 // it. This is needed because x86-64 can do many things, but jmp %r32 2226 // ain't one of them. 2227 const SDValue &Target = Node->getOperand(1); 2228 assert(Target.getSimpleValueType() == llvm::MVT::i32); 2229 SDValue ZextTarget = CurDAG->getZExtOrTrunc(Target, dl, EVT(MVT::i64)); 2230 SDValue Brind = CurDAG->getNode(ISD::BRIND, dl, MVT::Other, 2231 Node->getOperand(0), ZextTarget); 2232 ReplaceUses(SDValue(Node, 0), Brind); 2233 SelectCode(ZextTarget.getNode()); 2234 SelectCode(Brind.getNode()); 2235 return nullptr; 2236 } 2237 break; 2238 } 2239 case ISD::INTRINSIC_W_CHAIN: { 2240 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue(); 2241 switch (IntNo) { 2242 default: break; 2243 case Intrinsic::x86_avx2_gather_d_pd: 2244 case Intrinsic::x86_avx2_gather_d_pd_256: 2245 case Intrinsic::x86_avx2_gather_q_pd: 2246 case Intrinsic::x86_avx2_gather_q_pd_256: 2247 case Intrinsic::x86_avx2_gather_d_ps: 2248 case Intrinsic::x86_avx2_gather_d_ps_256: 2249 case Intrinsic::x86_avx2_gather_q_ps: 2250 case Intrinsic::x86_avx2_gather_q_ps_256: 2251 case Intrinsic::x86_avx2_gather_d_q: 2252 case Intrinsic::x86_avx2_gather_d_q_256: 2253 case Intrinsic::x86_avx2_gather_q_q: 2254 case Intrinsic::x86_avx2_gather_q_q_256: 2255 case Intrinsic::x86_avx2_gather_d_d: 2256 case Intrinsic::x86_avx2_gather_d_d_256: 2257 case Intrinsic::x86_avx2_gather_q_d: 2258 case Intrinsic::x86_avx2_gather_q_d_256: { 2259 if (!Subtarget->hasAVX2()) 2260 break; 2261 unsigned Opc; 2262 switch (IntNo) { 2263 default: llvm_unreachable("Impossible intrinsic"); 2264 case Intrinsic::x86_avx2_gather_d_pd: Opc = X86::VGATHERDPDrm; break; 2265 case Intrinsic::x86_avx2_gather_d_pd_256: Opc = X86::VGATHERDPDYrm; break; 2266 case Intrinsic::x86_avx2_gather_q_pd: Opc = X86::VGATHERQPDrm; break; 2267 case Intrinsic::x86_avx2_gather_q_pd_256: Opc = X86::VGATHERQPDYrm; break; 2268 case Intrinsic::x86_avx2_gather_d_ps: Opc = X86::VGATHERDPSrm; break; 2269 case Intrinsic::x86_avx2_gather_d_ps_256: Opc = X86::VGATHERDPSYrm; break; 2270 case Intrinsic::x86_avx2_gather_q_ps: Opc = X86::VGATHERQPSrm; break; 2271 case Intrinsic::x86_avx2_gather_q_ps_256: Opc = X86::VGATHERQPSYrm; break; 2272 case Intrinsic::x86_avx2_gather_d_q: Opc = X86::VPGATHERDQrm; break; 2273 case Intrinsic::x86_avx2_gather_d_q_256: Opc = X86::VPGATHERDQYrm; break; 2274 case Intrinsic::x86_avx2_gather_q_q: Opc = X86::VPGATHERQQrm; break; 2275 case Intrinsic::x86_avx2_gather_q_q_256: Opc = X86::VPGATHERQQYrm; break; 2276 case Intrinsic::x86_avx2_gather_d_d: Opc = X86::VPGATHERDDrm; break; 2277 case Intrinsic::x86_avx2_gather_d_d_256: Opc = X86::VPGATHERDDYrm; break; 2278 case Intrinsic::x86_avx2_gather_q_d: Opc = X86::VPGATHERQDrm; break; 2279 case Intrinsic::x86_avx2_gather_q_d_256: Opc = X86::VPGATHERQDYrm; break; 2280 } 2281 SDNode *RetVal = selectGather(Node, Opc); 2282 if (RetVal) 2283 // We already called ReplaceUses inside SelectGather. 2284 return nullptr; 2285 break; 2286 } 2287 } 2288 break; 2289 } 2290 case X86ISD::GlobalBaseReg: 2291 return getGlobalBaseReg(); 2292 2293 case X86ISD::SHRUNKBLEND: { 2294 // SHRUNKBLEND selects like a regular VSELECT. 2295 SDValue VSelect = CurDAG->getNode( 2296 ISD::VSELECT, SDLoc(Node), Node->getValueType(0), Node->getOperand(0), 2297 Node->getOperand(1), Node->getOperand(2)); 2298 ReplaceUses(SDValue(Node, 0), VSelect); 2299 SelectCode(VSelect.getNode()); 2300 // We already called ReplaceUses. 2301 return nullptr; 2302 } 2303 2304 case ISD::ATOMIC_LOAD_XOR: 2305 case ISD::ATOMIC_LOAD_AND: 2306 case ISD::ATOMIC_LOAD_OR: 2307 case ISD::ATOMIC_LOAD_ADD: { 2308 SDNode *RetVal = selectAtomicLoadArith(Node, NVT); 2309 if (RetVal) 2310 return RetVal; 2311 break; 2312 } 2313 case ISD::AND: 2314 case ISD::OR: 2315 case ISD::XOR: { 2316 // For operations of the form (x << C1) op C2, check if we can use a smaller 2317 // encoding for C2 by transforming it into (x op (C2>>C1)) << C1. 2318 SDValue N0 = Node->getOperand(0); 2319 SDValue N1 = Node->getOperand(1); 2320 2321 if (N0->getOpcode() != ISD::SHL || !N0->hasOneUse()) 2322 break; 2323 2324 // i8 is unshrinkable, i16 should be promoted to i32. 2325 if (NVT != MVT::i32 && NVT != MVT::i64) 2326 break; 2327 2328 ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N1); 2329 ConstantSDNode *ShlCst = dyn_cast<ConstantSDNode>(N0->getOperand(1)); 2330 if (!Cst || !ShlCst) 2331 break; 2332 2333 int64_t Val = Cst->getSExtValue(); 2334 uint64_t ShlVal = ShlCst->getZExtValue(); 2335 2336 // Make sure that we don't change the operation by removing bits. 2337 // This only matters for OR and XOR, AND is unaffected. 2338 uint64_t RemovedBitsMask = (1ULL << ShlVal) - 1; 2339 if (Opcode != ISD::AND && (Val & RemovedBitsMask) != 0) 2340 break; 2341 2342 unsigned ShlOp, AddOp, Op; 2343 MVT CstVT = NVT; 2344 2345 // Check the minimum bitwidth for the new constant. 2346 // TODO: AND32ri is the same as AND64ri32 with zext imm. 2347 // TODO: MOV32ri+OR64r is cheaper than MOV64ri64+OR64rr 2348 // TODO: Using 16 and 8 bit operations is also possible for or32 & xor32. 2349 if (!isInt<8>(Val) && isInt<8>(Val >> ShlVal)) 2350 CstVT = MVT::i8; 2351 else if (!isInt<32>(Val) && isInt<32>(Val >> ShlVal)) 2352 CstVT = MVT::i32; 2353 2354 // Bail if there is no smaller encoding. 2355 if (NVT == CstVT) 2356 break; 2357 2358 switch (NVT.SimpleTy) { 2359 default: llvm_unreachable("Unsupported VT!"); 2360 case MVT::i32: 2361 assert(CstVT == MVT::i8); 2362 ShlOp = X86::SHL32ri; 2363 AddOp = X86::ADD32rr; 2364 2365 switch (Opcode) { 2366 default: llvm_unreachable("Impossible opcode"); 2367 case ISD::AND: Op = X86::AND32ri8; break; 2368 case ISD::OR: Op = X86::OR32ri8; break; 2369 case ISD::XOR: Op = X86::XOR32ri8; break; 2370 } 2371 break; 2372 case MVT::i64: 2373 assert(CstVT == MVT::i8 || CstVT == MVT::i32); 2374 ShlOp = X86::SHL64ri; 2375 AddOp = X86::ADD64rr; 2376 2377 switch (Opcode) { 2378 default: llvm_unreachable("Impossible opcode"); 2379 case ISD::AND: Op = CstVT==MVT::i8? X86::AND64ri8 : X86::AND64ri32; break; 2380 case ISD::OR: Op = CstVT==MVT::i8? X86::OR64ri8 : X86::OR64ri32; break; 2381 case ISD::XOR: Op = CstVT==MVT::i8? X86::XOR64ri8 : X86::XOR64ri32; break; 2382 } 2383 break; 2384 } 2385 2386 // Emit the smaller op and the shift. 2387 SDValue NewCst = CurDAG->getTargetConstant(Val >> ShlVal, dl, CstVT); 2388 SDNode *New = CurDAG->getMachineNode(Op, dl, NVT, N0->getOperand(0),NewCst); 2389 if (ShlVal == 1) 2390 return CurDAG->SelectNodeTo(Node, AddOp, NVT, SDValue(New, 0), 2391 SDValue(New, 0)); 2392 return CurDAG->SelectNodeTo(Node, ShlOp, NVT, SDValue(New, 0), 2393 getI8Imm(ShlVal, dl)); 2394 } 2395 case X86ISD::UMUL8: 2396 case X86ISD::SMUL8: { 2397 SDValue N0 = Node->getOperand(0); 2398 SDValue N1 = Node->getOperand(1); 2399 2400 Opc = (Opcode == X86ISD::SMUL8 ? X86::IMUL8r : X86::MUL8r); 2401 2402 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::AL, 2403 N0, SDValue()).getValue(1); 2404 2405 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32); 2406 SDValue Ops[] = {N1, InFlag}; 2407 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); 2408 2409 ReplaceUses(SDValue(Node, 0), SDValue(CNode, 0)); 2410 ReplaceUses(SDValue(Node, 1), SDValue(CNode, 1)); 2411 return nullptr; 2412 } 2413 2414 case X86ISD::UMUL: { 2415 SDValue N0 = Node->getOperand(0); 2416 SDValue N1 = Node->getOperand(1); 2417 2418 unsigned LoReg; 2419 switch (NVT.SimpleTy) { 2420 default: llvm_unreachable("Unsupported VT!"); 2421 case MVT::i8: LoReg = X86::AL; Opc = X86::MUL8r; break; 2422 case MVT::i16: LoReg = X86::AX; Opc = X86::MUL16r; break; 2423 case MVT::i32: LoReg = X86::EAX; Opc = X86::MUL32r; break; 2424 case MVT::i64: LoReg = X86::RAX; Opc = X86::MUL64r; break; 2425 } 2426 2427 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg, 2428 N0, SDValue()).getValue(1); 2429 2430 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::i32); 2431 SDValue Ops[] = {N1, InFlag}; 2432 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); 2433 2434 ReplaceUses(SDValue(Node, 0), SDValue(CNode, 0)); 2435 ReplaceUses(SDValue(Node, 1), SDValue(CNode, 1)); 2436 ReplaceUses(SDValue(Node, 2), SDValue(CNode, 2)); 2437 return nullptr; 2438 } 2439 2440 case ISD::SMUL_LOHI: 2441 case ISD::UMUL_LOHI: { 2442 SDValue N0 = Node->getOperand(0); 2443 SDValue N1 = Node->getOperand(1); 2444 2445 bool isSigned = Opcode == ISD::SMUL_LOHI; 2446 bool hasBMI2 = Subtarget->hasBMI2(); 2447 if (!isSigned) { 2448 switch (NVT.SimpleTy) { 2449 default: llvm_unreachable("Unsupported VT!"); 2450 case MVT::i8: Opc = X86::MUL8r; MOpc = X86::MUL8m; break; 2451 case MVT::i16: Opc = X86::MUL16r; MOpc = X86::MUL16m; break; 2452 case MVT::i32: Opc = hasBMI2 ? X86::MULX32rr : X86::MUL32r; 2453 MOpc = hasBMI2 ? X86::MULX32rm : X86::MUL32m; break; 2454 case MVT::i64: Opc = hasBMI2 ? X86::MULX64rr : X86::MUL64r; 2455 MOpc = hasBMI2 ? X86::MULX64rm : X86::MUL64m; break; 2456 } 2457 } else { 2458 switch (NVT.SimpleTy) { 2459 default: llvm_unreachable("Unsupported VT!"); 2460 case MVT::i8: Opc = X86::IMUL8r; MOpc = X86::IMUL8m; break; 2461 case MVT::i16: Opc = X86::IMUL16r; MOpc = X86::IMUL16m; break; 2462 case MVT::i32: Opc = X86::IMUL32r; MOpc = X86::IMUL32m; break; 2463 case MVT::i64: Opc = X86::IMUL64r; MOpc = X86::IMUL64m; break; 2464 } 2465 } 2466 2467 unsigned SrcReg, LoReg, HiReg; 2468 switch (Opc) { 2469 default: llvm_unreachable("Unknown MUL opcode!"); 2470 case X86::IMUL8r: 2471 case X86::MUL8r: 2472 SrcReg = LoReg = X86::AL; HiReg = X86::AH; 2473 break; 2474 case X86::IMUL16r: 2475 case X86::MUL16r: 2476 SrcReg = LoReg = X86::AX; HiReg = X86::DX; 2477 break; 2478 case X86::IMUL32r: 2479 case X86::MUL32r: 2480 SrcReg = LoReg = X86::EAX; HiReg = X86::EDX; 2481 break; 2482 case X86::IMUL64r: 2483 case X86::MUL64r: 2484 SrcReg = LoReg = X86::RAX; HiReg = X86::RDX; 2485 break; 2486 case X86::MULX32rr: 2487 SrcReg = X86::EDX; LoReg = HiReg = 0; 2488 break; 2489 case X86::MULX64rr: 2490 SrcReg = X86::RDX; LoReg = HiReg = 0; 2491 break; 2492 } 2493 2494 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 2495 bool foldedLoad = tryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 2496 // Multiply is commmutative. 2497 if (!foldedLoad) { 2498 foldedLoad = tryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 2499 if (foldedLoad) 2500 std::swap(N0, N1); 2501 } 2502 2503 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, SrcReg, 2504 N0, SDValue()).getValue(1); 2505 SDValue ResHi, ResLo; 2506 2507 if (foldedLoad) { 2508 SDValue Chain; 2509 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), 2510 InFlag }; 2511 if (MOpc == X86::MULX32rm || MOpc == X86::MULX64rm) { 2512 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::Other, MVT::Glue); 2513 SDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); 2514 ResHi = SDValue(CNode, 0); 2515 ResLo = SDValue(CNode, 1); 2516 Chain = SDValue(CNode, 2); 2517 InFlag = SDValue(CNode, 3); 2518 } else { 2519 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue); 2520 SDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); 2521 Chain = SDValue(CNode, 0); 2522 InFlag = SDValue(CNode, 1); 2523 } 2524 2525 // Update the chain. 2526 ReplaceUses(N1.getValue(1), Chain); 2527 } else { 2528 SDValue Ops[] = { N1, InFlag }; 2529 if (Opc == X86::MULX32rr || Opc == X86::MULX64rr) { 2530 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::Glue); 2531 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); 2532 ResHi = SDValue(CNode, 0); 2533 ResLo = SDValue(CNode, 1); 2534 InFlag = SDValue(CNode, 2); 2535 } else { 2536 SDVTList VTs = CurDAG->getVTList(MVT::Glue); 2537 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); 2538 InFlag = SDValue(CNode, 0); 2539 } 2540 } 2541 2542 // Prevent use of AH in a REX instruction by referencing AX instead. 2543 if (HiReg == X86::AH && Subtarget->is64Bit() && 2544 !SDValue(Node, 1).use_empty()) { 2545 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 2546 X86::AX, MVT::i16, InFlag); 2547 InFlag = Result.getValue(2); 2548 // Get the low part if needed. Don't use getCopyFromReg for aliasing 2549 // registers. 2550 if (!SDValue(Node, 0).use_empty()) 2551 ReplaceUses(SDValue(Node, 1), 2552 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); 2553 2554 // Shift AX down 8 bits. 2555 Result = SDValue(CurDAG->getMachineNode(X86::SHR16ri, dl, MVT::i16, 2556 Result, 2557 CurDAG->getTargetConstant(8, dl, MVT::i8)), 2558 0); 2559 // Then truncate it down to i8. 2560 ReplaceUses(SDValue(Node, 1), 2561 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); 2562 } 2563 // Copy the low half of the result, if it is needed. 2564 if (!SDValue(Node, 0).use_empty()) { 2565 if (!ResLo.getNode()) { 2566 assert(LoReg && "Register for low half is not defined!"); 2567 ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg, NVT, 2568 InFlag); 2569 InFlag = ResLo.getValue(2); 2570 } 2571 ReplaceUses(SDValue(Node, 0), ResLo); 2572 DEBUG(dbgs() << "=> "; ResLo.getNode()->dump(CurDAG); dbgs() << '\n'); 2573 } 2574 // Copy the high half of the result, if it is needed. 2575 if (!SDValue(Node, 1).use_empty()) { 2576 if (!ResHi.getNode()) { 2577 assert(HiReg && "Register for high half is not defined!"); 2578 ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg, NVT, 2579 InFlag); 2580 InFlag = ResHi.getValue(2); 2581 } 2582 ReplaceUses(SDValue(Node, 1), ResHi); 2583 DEBUG(dbgs() << "=> "; ResHi.getNode()->dump(CurDAG); dbgs() << '\n'); 2584 } 2585 2586 return nullptr; 2587 } 2588 2589 case ISD::SDIVREM: 2590 case ISD::UDIVREM: 2591 case X86ISD::SDIVREM8_SEXT_HREG: 2592 case X86ISD::UDIVREM8_ZEXT_HREG: { 2593 SDValue N0 = Node->getOperand(0); 2594 SDValue N1 = Node->getOperand(1); 2595 2596 bool isSigned = (Opcode == ISD::SDIVREM || 2597 Opcode == X86ISD::SDIVREM8_SEXT_HREG); 2598 if (!isSigned) { 2599 switch (NVT.SimpleTy) { 2600 default: llvm_unreachable("Unsupported VT!"); 2601 case MVT::i8: Opc = X86::DIV8r; MOpc = X86::DIV8m; break; 2602 case MVT::i16: Opc = X86::DIV16r; MOpc = X86::DIV16m; break; 2603 case MVT::i32: Opc = X86::DIV32r; MOpc = X86::DIV32m; break; 2604 case MVT::i64: Opc = X86::DIV64r; MOpc = X86::DIV64m; break; 2605 } 2606 } else { 2607 switch (NVT.SimpleTy) { 2608 default: llvm_unreachable("Unsupported VT!"); 2609 case MVT::i8: Opc = X86::IDIV8r; MOpc = X86::IDIV8m; break; 2610 case MVT::i16: Opc = X86::IDIV16r; MOpc = X86::IDIV16m; break; 2611 case MVT::i32: Opc = X86::IDIV32r; MOpc = X86::IDIV32m; break; 2612 case MVT::i64: Opc = X86::IDIV64r; MOpc = X86::IDIV64m; break; 2613 } 2614 } 2615 2616 unsigned LoReg, HiReg, ClrReg; 2617 unsigned SExtOpcode; 2618 switch (NVT.SimpleTy) { 2619 default: llvm_unreachable("Unsupported VT!"); 2620 case MVT::i8: 2621 LoReg = X86::AL; ClrReg = HiReg = X86::AH; 2622 SExtOpcode = X86::CBW; 2623 break; 2624 case MVT::i16: 2625 LoReg = X86::AX; HiReg = X86::DX; 2626 ClrReg = X86::DX; 2627 SExtOpcode = X86::CWD; 2628 break; 2629 case MVT::i32: 2630 LoReg = X86::EAX; ClrReg = HiReg = X86::EDX; 2631 SExtOpcode = X86::CDQ; 2632 break; 2633 case MVT::i64: 2634 LoReg = X86::RAX; ClrReg = HiReg = X86::RDX; 2635 SExtOpcode = X86::CQO; 2636 break; 2637 } 2638 2639 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 2640 bool foldedLoad = tryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 2641 bool signBitIsZero = CurDAG->SignBitIsZero(N0); 2642 2643 SDValue InFlag; 2644 if (NVT == MVT::i8 && (!isSigned || signBitIsZero)) { 2645 // Special case for div8, just use a move with zero extension to AX to 2646 // clear the upper 8 bits (AH). 2647 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Move, Chain; 2648 if (tryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) { 2649 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0) }; 2650 Move = 2651 SDValue(CurDAG->getMachineNode(X86::MOVZX32rm8, dl, MVT::i32, 2652 MVT::Other, Ops), 0); 2653 Chain = Move.getValue(1); 2654 ReplaceUses(N0.getValue(1), Chain); 2655 } else { 2656 Move = 2657 SDValue(CurDAG->getMachineNode(X86::MOVZX32rr8, dl, MVT::i32, N0),0); 2658 Chain = CurDAG->getEntryNode(); 2659 } 2660 Chain = CurDAG->getCopyToReg(Chain, dl, X86::EAX, Move, SDValue()); 2661 InFlag = Chain.getValue(1); 2662 } else { 2663 InFlag = 2664 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, 2665 LoReg, N0, SDValue()).getValue(1); 2666 if (isSigned && !signBitIsZero) { 2667 // Sign extend the low part into the high part. 2668 InFlag = 2669 SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InFlag),0); 2670 } else { 2671 // Zero out the high part, effectively zero extending the input. 2672 SDValue ClrNode = SDValue(CurDAG->getMachineNode(X86::MOV32r0, dl, NVT), 0); 2673 switch (NVT.SimpleTy) { 2674 case MVT::i16: 2675 ClrNode = 2676 SDValue(CurDAG->getMachineNode( 2677 TargetOpcode::EXTRACT_SUBREG, dl, MVT::i16, ClrNode, 2678 CurDAG->getTargetConstant(X86::sub_16bit, dl, 2679 MVT::i32)), 2680 0); 2681 break; 2682 case MVT::i32: 2683 break; 2684 case MVT::i64: 2685 ClrNode = 2686 SDValue(CurDAG->getMachineNode( 2687 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64, 2688 CurDAG->getTargetConstant(0, dl, MVT::i64), ClrNode, 2689 CurDAG->getTargetConstant(X86::sub_32bit, dl, 2690 MVT::i32)), 2691 0); 2692 break; 2693 default: 2694 llvm_unreachable("Unexpected division source"); 2695 } 2696 2697 InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg, 2698 ClrNode, InFlag).getValue(1); 2699 } 2700 } 2701 2702 if (foldedLoad) { 2703 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), 2704 InFlag }; 2705 SDNode *CNode = 2706 CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops); 2707 InFlag = SDValue(CNode, 1); 2708 // Update the chain. 2709 ReplaceUses(N1.getValue(1), SDValue(CNode, 0)); 2710 } else { 2711 InFlag = 2712 SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, N1, InFlag), 0); 2713 } 2714 2715 // Prevent use of AH in a REX instruction by explicitly copying it to 2716 // an ABCD_L register. 2717 // 2718 // The current assumption of the register allocator is that isel 2719 // won't generate explicit references to the GR8_ABCD_H registers. If 2720 // the allocator and/or the backend get enhanced to be more robust in 2721 // that regard, this can be, and should be, removed. 2722 if (HiReg == X86::AH && !SDValue(Node, 1).use_empty()) { 2723 SDValue AHCopy = CurDAG->getRegister(X86::AH, MVT::i8); 2724 unsigned AHExtOpcode = 2725 isSigned ? X86::MOVSX32_NOREXrr8 : X86::MOVZX32_NOREXrr8; 2726 2727 SDNode *RNode = CurDAG->getMachineNode(AHExtOpcode, dl, MVT::i32, 2728 MVT::Glue, AHCopy, InFlag); 2729 SDValue Result(RNode, 0); 2730 InFlag = SDValue(RNode, 1); 2731 2732 if (Opcode == X86ISD::UDIVREM8_ZEXT_HREG || 2733 Opcode == X86ISD::SDIVREM8_SEXT_HREG) { 2734 if (Node->getValueType(1) == MVT::i64) { 2735 // It's not possible to directly movsx AH to a 64bit register, because 2736 // the latter needs the REX prefix, but the former can't have it. 2737 assert(Opcode != X86ISD::SDIVREM8_SEXT_HREG && 2738 "Unexpected i64 sext of h-register"); 2739 Result = 2740 SDValue(CurDAG->getMachineNode( 2741 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64, 2742 CurDAG->getTargetConstant(0, dl, MVT::i64), Result, 2743 CurDAG->getTargetConstant(X86::sub_32bit, dl, 2744 MVT::i32)), 2745 0); 2746 } 2747 } else { 2748 Result = 2749 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result); 2750 } 2751 ReplaceUses(SDValue(Node, 1), Result); 2752 DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 2753 } 2754 // Copy the division (low) result, if it is needed. 2755 if (!SDValue(Node, 0).use_empty()) { 2756 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 2757 LoReg, NVT, InFlag); 2758 InFlag = Result.getValue(2); 2759 ReplaceUses(SDValue(Node, 0), Result); 2760 DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 2761 } 2762 // Copy the remainder (high) result, if it is needed. 2763 if (!SDValue(Node, 1).use_empty()) { 2764 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 2765 HiReg, NVT, InFlag); 2766 InFlag = Result.getValue(2); 2767 ReplaceUses(SDValue(Node, 1), Result); 2768 DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 2769 } 2770 return nullptr; 2771 } 2772 2773 case X86ISD::CMP: 2774 case X86ISD::SUB: { 2775 // Sometimes a SUB is used to perform comparison. 2776 if (Opcode == X86ISD::SUB && Node->hasAnyUseOfValue(0)) 2777 // This node is not a CMP. 2778 break; 2779 SDValue N0 = Node->getOperand(0); 2780 SDValue N1 = Node->getOperand(1); 2781 2782 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && 2783 hasNoSignedComparisonUses(Node)) 2784 N0 = N0.getOperand(0); 2785 2786 // Look for (X86cmp (and $op, $imm), 0) and see if we can convert it to 2787 // use a smaller encoding. 2788 // Look past the truncate if CMP is the only use of it. 2789 if ((N0.getNode()->getOpcode() == ISD::AND || 2790 (N0.getResNo() == 0 && N0.getNode()->getOpcode() == X86ISD::AND)) && 2791 N0.getNode()->hasOneUse() && 2792 N0.getValueType() != MVT::i8 && 2793 X86::isZeroNode(N1)) { 2794 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getNode()->getOperand(1)); 2795 if (!C) break; 2796 2797 // For example, convert "testl %eax, $8" to "testb %al, $8" 2798 if ((C->getZExtValue() & ~UINT64_C(0xff)) == 0 && 2799 (!(C->getZExtValue() & 0x80) || 2800 hasNoSignedComparisonUses(Node))) { 2801 SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), dl, MVT::i8); 2802 SDValue Reg = N0.getNode()->getOperand(0); 2803 2804 // On x86-32, only the ABCD registers have 8-bit subregisters. 2805 if (!Subtarget->is64Bit()) { 2806 const TargetRegisterClass *TRC; 2807 switch (N0.getSimpleValueType().SimpleTy) { 2808 case MVT::i32: TRC = &X86::GR32_ABCDRegClass; break; 2809 case MVT::i16: TRC = &X86::GR16_ABCDRegClass; break; 2810 default: llvm_unreachable("Unsupported TEST operand type!"); 2811 } 2812 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32); 2813 Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl, 2814 Reg.getValueType(), Reg, RC), 0); 2815 } 2816 2817 // Extract the l-register. 2818 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, 2819 MVT::i8, Reg); 2820 2821 // Emit a testb. 2822 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST8ri, dl, MVT::i32, 2823 Subreg, Imm); 2824 // Replace SUB|CMP with TEST, since SUB has two outputs while TEST has 2825 // one, do not call ReplaceAllUsesWith. 2826 ReplaceUses(SDValue(Node, (Opcode == X86ISD::SUB ? 1 : 0)), 2827 SDValue(NewNode, 0)); 2828 return nullptr; 2829 } 2830 2831 // For example, "testl %eax, $2048" to "testb %ah, $8". 2832 if ((C->getZExtValue() & ~UINT64_C(0xff00)) == 0 && 2833 (!(C->getZExtValue() & 0x8000) || 2834 hasNoSignedComparisonUses(Node))) { 2835 // Shift the immediate right by 8 bits. 2836 SDValue ShiftedImm = CurDAG->getTargetConstant(C->getZExtValue() >> 8, 2837 dl, MVT::i8); 2838 SDValue Reg = N0.getNode()->getOperand(0); 2839 2840 // Put the value in an ABCD register. 2841 const TargetRegisterClass *TRC; 2842 switch (N0.getSimpleValueType().SimpleTy) { 2843 case MVT::i64: TRC = &X86::GR64_ABCDRegClass; break; 2844 case MVT::i32: TRC = &X86::GR32_ABCDRegClass; break; 2845 case MVT::i16: TRC = &X86::GR16_ABCDRegClass; break; 2846 default: llvm_unreachable("Unsupported TEST operand type!"); 2847 } 2848 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32); 2849 Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl, 2850 Reg.getValueType(), Reg, RC), 0); 2851 2852 // Extract the h-register. 2853 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl, 2854 MVT::i8, Reg); 2855 2856 // Emit a testb. The EXTRACT_SUBREG becomes a COPY that can only 2857 // target GR8_NOREX registers, so make sure the register class is 2858 // forced. 2859 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST8ri_NOREX, dl, 2860 MVT::i32, Subreg, ShiftedImm); 2861 // Replace SUB|CMP with TEST, since SUB has two outputs while TEST has 2862 // one, do not call ReplaceAllUsesWith. 2863 ReplaceUses(SDValue(Node, (Opcode == X86ISD::SUB ? 1 : 0)), 2864 SDValue(NewNode, 0)); 2865 return nullptr; 2866 } 2867 2868 // For example, "testl %eax, $32776" to "testw %ax, $32776". 2869 if ((C->getZExtValue() & ~UINT64_C(0xffff)) == 0 && 2870 N0.getValueType() != MVT::i16 && 2871 (!(C->getZExtValue() & 0x8000) || 2872 hasNoSignedComparisonUses(Node))) { 2873 SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), dl, 2874 MVT::i16); 2875 SDValue Reg = N0.getNode()->getOperand(0); 2876 2877 // Extract the 16-bit subregister. 2878 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_16bit, dl, 2879 MVT::i16, Reg); 2880 2881 // Emit a testw. 2882 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST16ri, dl, MVT::i32, 2883 Subreg, Imm); 2884 // Replace SUB|CMP with TEST, since SUB has two outputs while TEST has 2885 // one, do not call ReplaceAllUsesWith. 2886 ReplaceUses(SDValue(Node, (Opcode == X86ISD::SUB ? 1 : 0)), 2887 SDValue(NewNode, 0)); 2888 return nullptr; 2889 } 2890 2891 // For example, "testq %rax, $268468232" to "testl %eax, $268468232". 2892 if ((C->getZExtValue() & ~UINT64_C(0xffffffff)) == 0 && 2893 N0.getValueType() == MVT::i64 && 2894 (!(C->getZExtValue() & 0x80000000) || 2895 hasNoSignedComparisonUses(Node))) { 2896 SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), dl, 2897 MVT::i32); 2898 SDValue Reg = N0.getNode()->getOperand(0); 2899 2900 // Extract the 32-bit subregister. 2901 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_32bit, dl, 2902 MVT::i32, Reg); 2903 2904 // Emit a testl. 2905 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST32ri, dl, MVT::i32, 2906 Subreg, Imm); 2907 // Replace SUB|CMP with TEST, since SUB has two outputs while TEST has 2908 // one, do not call ReplaceAllUsesWith. 2909 ReplaceUses(SDValue(Node, (Opcode == X86ISD::SUB ? 1 : 0)), 2910 SDValue(NewNode, 0)); 2911 return nullptr; 2912 } 2913 } 2914 break; 2915 } 2916 case ISD::STORE: { 2917 // Change a chain of {load; incr or dec; store} of the same value into 2918 // a simple increment or decrement through memory of that value, if the 2919 // uses of the modified value and its address are suitable. 2920 // The DEC64m tablegen pattern is currently not able to match the case where 2921 // the EFLAGS on the original DEC are used. (This also applies to 2922 // {INC,DEC}X{64,32,16,8}.) 2923 // We'll need to improve tablegen to allow flags to be transferred from a 2924 // node in the pattern to the result node. probably with a new keyword 2925 // for example, we have this 2926 // def DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst), "dec{q}\t$dst", 2927 // [(store (add (loadi64 addr:$dst), -1), addr:$dst), 2928 // (implicit EFLAGS)]>; 2929 // but maybe need something like this 2930 // def DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst), "dec{q}\t$dst", 2931 // [(store (add (loadi64 addr:$dst), -1), addr:$dst), 2932 // (transferrable EFLAGS)]>; 2933 2934 StoreSDNode *StoreNode = cast<StoreSDNode>(Node); 2935 SDValue StoredVal = StoreNode->getOperand(1); 2936 unsigned Opc = StoredVal->getOpcode(); 2937 2938 LoadSDNode *LoadNode = nullptr; 2939 SDValue InputChain; 2940 if (!isLoadIncOrDecStore(StoreNode, Opc, StoredVal, CurDAG, 2941 LoadNode, InputChain)) 2942 break; 2943 2944 SDValue Base, Scale, Index, Disp, Segment; 2945 if (!selectAddr(LoadNode, LoadNode->getBasePtr(), 2946 Base, Scale, Index, Disp, Segment)) 2947 break; 2948 2949 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(2); 2950 MemOp[0] = StoreNode->getMemOperand(); 2951 MemOp[1] = LoadNode->getMemOperand(); 2952 const SDValue Ops[] = { Base, Scale, Index, Disp, Segment, InputChain }; 2953 EVT LdVT = LoadNode->getMemoryVT(); 2954 unsigned newOpc = getFusedLdStOpcode(LdVT, Opc); 2955 MachineSDNode *Result = CurDAG->getMachineNode(newOpc, 2956 SDLoc(Node), 2957 MVT::i32, MVT::Other, Ops); 2958 Result->setMemRefs(MemOp, MemOp + 2); 2959 2960 ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1)); 2961 ReplaceUses(SDValue(StoredVal.getNode(), 1), SDValue(Result, 0)); 2962 2963 return Result; 2964 } 2965 } 2966 2967 SDNode *ResNode = SelectCode(Node); 2968 2969 DEBUG(dbgs() << "=> "; 2970 if (ResNode == nullptr || ResNode == Node) 2971 Node->dump(CurDAG); 2972 else 2973 ResNode->dump(CurDAG); 2974 dbgs() << '\n'); 2975 2976 return ResNode; 2977} 2978 2979bool X86DAGToDAGISel:: 2980SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, 2981 std::vector<SDValue> &OutOps) { 2982 SDValue Op0, Op1, Op2, Op3, Op4; 2983 switch (ConstraintID) { 2984 default: 2985 llvm_unreachable("Unexpected asm memory constraint"); 2986 case InlineAsm::Constraint_i: 2987 // FIXME: It seems strange that 'i' is needed here since it's supposed to 2988 // be an immediate and not a memory constraint. 2989 // Fallthrough. 2990 case InlineAsm::Constraint_o: // offsetable ?? 2991 case InlineAsm::Constraint_v: // not offsetable ?? 2992 case InlineAsm::Constraint_m: // memory 2993 case InlineAsm::Constraint_X: 2994 if (!selectAddr(nullptr, Op, Op0, Op1, Op2, Op3, Op4)) 2995 return true; 2996 break; 2997 } 2998 2999 OutOps.push_back(Op0); 3000 OutOps.push_back(Op1); 3001 OutOps.push_back(Op2); 3002 OutOps.push_back(Op3); 3003 OutOps.push_back(Op4); 3004 return false; 3005} 3006 3007/// This pass converts a legalized DAG into a X86-specific DAG, 3008/// ready for instruction scheduling. 3009FunctionPass *llvm::createX86ISelDag(X86TargetMachine &TM, 3010 CodeGenOpt::Level OptLevel) { 3011 return new X86DAGToDAGISel(TM, OptLevel); 3012} 3013