SystemZISelDAGToDAG.cpp revision 314564
1251607Sdim//===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===// 2251607Sdim// 3251607Sdim// The LLVM Compiler Infrastructure 4251607Sdim// 5251607Sdim// This file is distributed under the University of Illinois Open Source 6251607Sdim// License. See LICENSE.TXT for details. 7251607Sdim// 8251607Sdim//===----------------------------------------------------------------------===// 9251607Sdim// 10251607Sdim// This file defines an instruction selector for the SystemZ target. 11251607Sdim// 12251607Sdim//===----------------------------------------------------------------------===// 13251607Sdim 14251607Sdim#include "SystemZTargetMachine.h" 15261991Sdim#include "llvm/Analysis/AliasAnalysis.h" 16251607Sdim#include "llvm/CodeGen/SelectionDAGISel.h" 17251607Sdim#include "llvm/Support/Debug.h" 18251607Sdim#include "llvm/Support/raw_ostream.h" 19251607Sdim 20251607Sdimusing namespace llvm; 21251607Sdim 22276479Sdim#define DEBUG_TYPE "systemz-isel" 23276479Sdim 24251607Sdimnamespace { 25251607Sdim// Used to build addressing modes. 26251607Sdimstruct SystemZAddressingMode { 27251607Sdim // The shape of the address. 28251607Sdim enum AddrForm { 29251607Sdim // base+displacement 30251607Sdim FormBD, 31251607Sdim 32251607Sdim // base+displacement+index for load and store operands 33251607Sdim FormBDXNormal, 34251607Sdim 35251607Sdim // base+displacement+index for load address operands 36251607Sdim FormBDXLA, 37251607Sdim 38251607Sdim // base+displacement+index+ADJDYNALLOC 39251607Sdim FormBDXDynAlloc 40251607Sdim }; 41251607Sdim AddrForm Form; 42251607Sdim 43251607Sdim // The type of displacement. The enum names here correspond directly 44251607Sdim // to the definitions in SystemZOperand.td. We could split them into 45251607Sdim // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it. 46251607Sdim enum DispRange { 47251607Sdim Disp12Only, 48251607Sdim Disp12Pair, 49251607Sdim Disp20Only, 50251607Sdim Disp20Only128, 51251607Sdim Disp20Pair 52251607Sdim }; 53251607Sdim DispRange DR; 54251607Sdim 55251607Sdim // The parts of the address. The address is equivalent to: 56251607Sdim // 57251607Sdim // Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0) 58251607Sdim SDValue Base; 59251607Sdim int64_t Disp; 60251607Sdim SDValue Index; 61251607Sdim bool IncludesDynAlloc; 62251607Sdim 63251607Sdim SystemZAddressingMode(AddrForm form, DispRange dr) 64251607Sdim : Form(form), DR(dr), Base(), Disp(0), Index(), 65251607Sdim IncludesDynAlloc(false) {} 66251607Sdim 67251607Sdim // True if the address can have an index register. 68251607Sdim bool hasIndexField() { return Form != FormBD; } 69251607Sdim 70251607Sdim // True if the address can (and must) include ADJDYNALLOC. 71251607Sdim bool isDynAlloc() { return Form == FormBDXDynAlloc; } 72251607Sdim 73251607Sdim void dump() { 74251607Sdim errs() << "SystemZAddressingMode " << this << '\n'; 75251607Sdim 76251607Sdim errs() << " Base "; 77276479Sdim if (Base.getNode()) 78251607Sdim Base.getNode()->dump(); 79251607Sdim else 80251607Sdim errs() << "null\n"; 81251607Sdim 82251607Sdim if (hasIndexField()) { 83251607Sdim errs() << " Index "; 84276479Sdim if (Index.getNode()) 85251607Sdim Index.getNode()->dump(); 86251607Sdim else 87251607Sdim errs() << "null\n"; 88251607Sdim } 89251607Sdim 90251607Sdim errs() << " Disp " << Disp; 91251607Sdim if (IncludesDynAlloc) 92251607Sdim errs() << " + ADJDYNALLOC"; 93251607Sdim errs() << '\n'; 94251607Sdim } 95251607Sdim}; 96251607Sdim 97261991Sdim// Return a mask with Count low bits set. 98261991Sdimstatic uint64_t allOnes(unsigned int Count) { 99288943Sdim assert(Count <= 64); 100288943Sdim if (Count > 63) 101288943Sdim return UINT64_MAX; 102288943Sdim return (uint64_t(1) << Count) - 1; 103261991Sdim} 104261991Sdim 105261991Sdim// Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation 106261991Sdim// given by Opcode. The operands are: Input (R2), Start (I3), End (I4) and 107261991Sdim// Rotate (I5). The combined operand value is effectively: 108261991Sdim// 109261991Sdim// (or (rotl Input, Rotate), ~Mask) 110261991Sdim// 111261991Sdim// for RNSBG and: 112261991Sdim// 113261991Sdim// (and (rotl Input, Rotate), Mask) 114261991Sdim// 115261991Sdim// otherwise. The output value has BitSize bits, although Input may be 116309124Sdim// narrower (in which case the upper bits are don't care), or wider (in which 117309124Sdim// case the result will be truncated as part of the operation). 118261991Sdimstruct RxSBGOperands { 119261991Sdim RxSBGOperands(unsigned Op, SDValue N) 120314564Sdim : Opcode(Op), BitSize(N.getValueSizeInBits()), 121261991Sdim Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63), 122261991Sdim Rotate(0) {} 123261991Sdim 124261991Sdim unsigned Opcode; 125261991Sdim unsigned BitSize; 126261991Sdim uint64_t Mask; 127261991Sdim SDValue Input; 128261991Sdim unsigned Start; 129261991Sdim unsigned End; 130261991Sdim unsigned Rotate; 131261991Sdim}; 132261991Sdim 133251607Sdimclass SystemZDAGToDAGISel : public SelectionDAGISel { 134288943Sdim const SystemZSubtarget *Subtarget; 135251607Sdim 136251607Sdim // Used by SystemZOperands.td to create integer constants. 137261991Sdim inline SDValue getImm(const SDNode *Node, uint64_t Imm) const { 138288943Sdim return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0)); 139251607Sdim } 140251607Sdim 141261991Sdim const SystemZTargetMachine &getTargetMachine() const { 142261991Sdim return static_cast<const SystemZTargetMachine &>(TM); 143261991Sdim } 144261991Sdim 145261991Sdim const SystemZInstrInfo *getInstrInfo() const { 146288943Sdim return Subtarget->getInstrInfo(); 147261991Sdim } 148261991Sdim 149251607Sdim // Try to fold more of the base or index of AM into AM, where IsBase 150251607Sdim // selects between the base and index. 151261991Sdim bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const; 152251607Sdim 153251607Sdim // Try to describe N in AM, returning true on success. 154261991Sdim bool selectAddress(SDValue N, SystemZAddressingMode &AM) const; 155251607Sdim 156251607Sdim // Extract individual target operands from matched address AM. 157251607Sdim void getAddressOperands(const SystemZAddressingMode &AM, EVT VT, 158261991Sdim SDValue &Base, SDValue &Disp) const; 159251607Sdim void getAddressOperands(const SystemZAddressingMode &AM, EVT VT, 160261991Sdim SDValue &Base, SDValue &Disp, SDValue &Index) const; 161251607Sdim 162251607Sdim // Try to match Addr as a FormBD address with displacement type DR. 163251607Sdim // Return true on success, storing the base and displacement in 164251607Sdim // Base and Disp respectively. 165251607Sdim bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr, 166261991Sdim SDValue &Base, SDValue &Disp) const; 167251607Sdim 168261991Sdim // Try to match Addr as a FormBDX address with displacement type DR. 169261991Sdim // Return true on success and if the result had no index. Store the 170261991Sdim // base and displacement in Base and Disp respectively. 171261991Sdim bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr, 172261991Sdim SDValue &Base, SDValue &Disp) const; 173261991Sdim 174251607Sdim // Try to match Addr as a FormBDX* address of form Form with 175251607Sdim // displacement type DR. Return true on success, storing the base, 176251607Sdim // displacement and index in Base, Disp and Index respectively. 177251607Sdim bool selectBDXAddr(SystemZAddressingMode::AddrForm Form, 178251607Sdim SystemZAddressingMode::DispRange DR, SDValue Addr, 179261991Sdim SDValue &Base, SDValue &Disp, SDValue &Index) const; 180251607Sdim 181251607Sdim // PC-relative address matching routines used by SystemZOperands.td. 182261991Sdim bool selectPCRelAddress(SDValue Addr, SDValue &Target) const { 183261991Sdim if (SystemZISD::isPCREL(Addr.getOpcode())) { 184251607Sdim Target = Addr.getOperand(0); 185251607Sdim return true; 186251607Sdim } 187251607Sdim return false; 188251607Sdim } 189251607Sdim 190251607Sdim // BD matching routines used by SystemZOperands.td. 191261991Sdim bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const { 192251607Sdim return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp); 193251607Sdim } 194261991Sdim bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const { 195251607Sdim return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp); 196251607Sdim } 197261991Sdim bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const { 198251607Sdim return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp); 199251607Sdim } 200261991Sdim bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const { 201251607Sdim return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp); 202251607Sdim } 203251607Sdim 204261991Sdim // MVI matching routines used by SystemZOperands.td. 205261991Sdim bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const { 206261991Sdim return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp); 207261991Sdim } 208261991Sdim bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const { 209261991Sdim return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp); 210261991Sdim } 211261991Sdim 212251607Sdim // BDX matching routines used by SystemZOperands.td. 213251607Sdim bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp, 214261991Sdim SDValue &Index) const { 215251607Sdim return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 216251607Sdim SystemZAddressingMode::Disp12Only, 217251607Sdim Addr, Base, Disp, Index); 218251607Sdim } 219251607Sdim bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp, 220261991Sdim SDValue &Index) const { 221251607Sdim return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 222251607Sdim SystemZAddressingMode::Disp12Pair, 223251607Sdim Addr, Base, Disp, Index); 224251607Sdim } 225251607Sdim bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp, 226261991Sdim SDValue &Index) const { 227251607Sdim return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc, 228251607Sdim SystemZAddressingMode::Disp12Only, 229251607Sdim Addr, Base, Disp, Index); 230251607Sdim } 231251607Sdim bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp, 232261991Sdim SDValue &Index) const { 233251607Sdim return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 234251607Sdim SystemZAddressingMode::Disp20Only, 235251607Sdim Addr, Base, Disp, Index); 236251607Sdim } 237251607Sdim bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp, 238261991Sdim SDValue &Index) const { 239251607Sdim return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 240251607Sdim SystemZAddressingMode::Disp20Only128, 241251607Sdim Addr, Base, Disp, Index); 242251607Sdim } 243251607Sdim bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp, 244261991Sdim SDValue &Index) const { 245251607Sdim return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 246251607Sdim SystemZAddressingMode::Disp20Pair, 247251607Sdim Addr, Base, Disp, Index); 248251607Sdim } 249251607Sdim bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp, 250261991Sdim SDValue &Index) const { 251251607Sdim return selectBDXAddr(SystemZAddressingMode::FormBDXLA, 252251607Sdim SystemZAddressingMode::Disp12Pair, 253251607Sdim Addr, Base, Disp, Index); 254251607Sdim } 255251607Sdim bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp, 256261991Sdim SDValue &Index) const { 257251607Sdim return selectBDXAddr(SystemZAddressingMode::FormBDXLA, 258251607Sdim SystemZAddressingMode::Disp20Pair, 259251607Sdim Addr, Base, Disp, Index); 260251607Sdim } 261251607Sdim 262288943Sdim // Try to match Addr as an address with a base, 12-bit displacement 263288943Sdim // and index, where the index is element Elem of a vector. 264288943Sdim // Return true on success, storing the base, displacement and vector 265288943Sdim // in Base, Disp and Index respectively. 266288943Sdim bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base, 267288943Sdim SDValue &Disp, SDValue &Index) const; 268288943Sdim 269261991Sdim // Check whether (or Op (and X InsertMask)) is effectively an insertion 270261991Sdim // of X into bits InsertMask of some Y != Op. Return true if so and 271261991Sdim // set Op to that Y. 272261991Sdim bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const; 273261991Sdim 274261991Sdim // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used. 275261991Sdim // Return true on success. 276261991Sdim bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const; 277261991Sdim 278261991Sdim // Try to fold some of RxSBG.Input into other fields of RxSBG. 279261991Sdim // Return true on success. 280261991Sdim bool expandRxSBG(RxSBGOperands &RxSBG) const; 281261991Sdim 282261991Sdim // Return an undefined value of type VT. 283309124Sdim SDValue getUNDEF(const SDLoc &DL, EVT VT) const; 284261991Sdim 285261991Sdim // Convert N to VT, if it isn't already. 286309124Sdim SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const; 287261991Sdim 288261991Sdim // Try to implement AND or shift node N using RISBG with the zero flag set. 289261991Sdim // Return the selected node on success, otherwise return null. 290309124Sdim bool tryRISBGZero(SDNode *N); 291261991Sdim 292261991Sdim // Try to use RISBG or Opcode to implement OR or XOR node N. 293261991Sdim // Return the selected node on success, otherwise return null. 294309124Sdim bool tryRxSBG(SDNode *N, unsigned Opcode); 295261991Sdim 296251607Sdim // If Op0 is null, then Node is a constant that can be loaded using: 297251607Sdim // 298251607Sdim // (Opcode UpperVal LowerVal) 299251607Sdim // 300251607Sdim // If Op0 is nonnull, then Node can be implemented using: 301251607Sdim // 302251607Sdim // (Opcode (Opcode Op0 UpperVal) LowerVal) 303309124Sdim void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0, 304309124Sdim uint64_t UpperVal, uint64_t LowerVal); 305251607Sdim 306288943Sdim // Try to use gather instruction Opcode to implement vector insertion N. 307309124Sdim bool tryGather(SDNode *N, unsigned Opcode); 308288943Sdim 309288943Sdim // Try to use scatter instruction Opcode to implement store Store. 310309124Sdim bool tryScatter(StoreSDNode *Store, unsigned Opcode); 311288943Sdim 312261991Sdim // Return true if Load and Store are loads and stores of the same size 313261991Sdim // and are guaranteed not to overlap. Such operations can be implemented 314261991Sdim // using block (SS-format) instructions. 315261991Sdim // 316261991Sdim // Partial overlap would lead to incorrect code, since the block operations 317261991Sdim // are logically bytewise, even though they have a fast path for the 318261991Sdim // non-overlapping case. We also need to avoid full overlap (i.e. two 319261991Sdim // addresses that might be equal at run time) because although that case 320261991Sdim // would be handled correctly, it might be implemented by millicode. 321261991Sdim bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const; 322261991Sdim 323261991Sdim // N is a (store (load Y), X) pattern. Return true if it can use an MVC 324261991Sdim // from Y to X. 325261991Sdim bool storeLoadCanUseMVC(SDNode *N) const; 326261991Sdim 327261991Sdim // N is a (store (op (load A[0]), (load A[1])), X) pattern. Return true 328261991Sdim // if A[1 - I] == X and if N can use a block operation like NC from A[I] 329261991Sdim // to X. 330261991Sdim bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const; 331261991Sdim 332251607Sdimpublic: 333251607Sdim SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel) 334288943Sdim : SelectionDAGISel(TM, OptLevel) {} 335251607Sdim 336288943Sdim bool runOnMachineFunction(MachineFunction &MF) override { 337288943Sdim Subtarget = &MF.getSubtarget<SystemZSubtarget>(); 338288943Sdim return SelectionDAGISel::runOnMachineFunction(MF); 339288943Sdim } 340288943Sdim 341251607Sdim // Override MachineFunctionPass. 342314564Sdim StringRef getPassName() const override { 343251607Sdim return "SystemZ DAG->DAG Pattern Instruction Selection"; 344251607Sdim } 345251607Sdim 346251607Sdim // Override SelectionDAGISel. 347309124Sdim void Select(SDNode *Node) override; 348288943Sdim bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, 349276479Sdim std::vector<SDValue> &OutOps) override; 350251607Sdim 351251607Sdim // Include the pieces autogenerated from the target description. 352251607Sdim #include "SystemZGenDAGISel.inc" 353251607Sdim}; 354251607Sdim} // end anonymous namespace 355251607Sdim 356251607SdimFunctionPass *llvm::createSystemZISelDag(SystemZTargetMachine &TM, 357251607Sdim CodeGenOpt::Level OptLevel) { 358251607Sdim return new SystemZDAGToDAGISel(TM, OptLevel); 359251607Sdim} 360251607Sdim 361251607Sdim// Return true if Val should be selected as a displacement for an address 362251607Sdim// with range DR. Here we're interested in the range of both the instruction 363251607Sdim// described by DR and of any pairing instruction. 364251607Sdimstatic bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) { 365251607Sdim switch (DR) { 366251607Sdim case SystemZAddressingMode::Disp12Only: 367251607Sdim return isUInt<12>(Val); 368251607Sdim 369251607Sdim case SystemZAddressingMode::Disp12Pair: 370251607Sdim case SystemZAddressingMode::Disp20Only: 371251607Sdim case SystemZAddressingMode::Disp20Pair: 372251607Sdim return isInt<20>(Val); 373251607Sdim 374251607Sdim case SystemZAddressingMode::Disp20Only128: 375251607Sdim return isInt<20>(Val) && isInt<20>(Val + 8); 376251607Sdim } 377251607Sdim llvm_unreachable("Unhandled displacement range"); 378251607Sdim} 379251607Sdim 380251607Sdim// Change the base or index in AM to Value, where IsBase selects 381251607Sdim// between the base and index. 382251607Sdimstatic void changeComponent(SystemZAddressingMode &AM, bool IsBase, 383251607Sdim SDValue Value) { 384251607Sdim if (IsBase) 385251607Sdim AM.Base = Value; 386251607Sdim else 387251607Sdim AM.Index = Value; 388251607Sdim} 389251607Sdim 390251607Sdim// The base or index of AM is equivalent to Value + ADJDYNALLOC, 391251607Sdim// where IsBase selects between the base and index. Try to fold the 392251607Sdim// ADJDYNALLOC into AM. 393251607Sdimstatic bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, 394251607Sdim SDValue Value) { 395251607Sdim if (AM.isDynAlloc() && !AM.IncludesDynAlloc) { 396251607Sdim changeComponent(AM, IsBase, Value); 397251607Sdim AM.IncludesDynAlloc = true; 398251607Sdim return true; 399251607Sdim } 400251607Sdim return false; 401251607Sdim} 402251607Sdim 403251607Sdim// The base of AM is equivalent to Base + Index. Try to use Index as 404251607Sdim// the index register. 405251607Sdimstatic bool expandIndex(SystemZAddressingMode &AM, SDValue Base, 406251607Sdim SDValue Index) { 407251607Sdim if (AM.hasIndexField() && !AM.Index.getNode()) { 408251607Sdim AM.Base = Base; 409251607Sdim AM.Index = Index; 410251607Sdim return true; 411251607Sdim } 412251607Sdim return false; 413251607Sdim} 414251607Sdim 415251607Sdim// The base or index of AM is equivalent to Op0 + Op1, where IsBase selects 416251607Sdim// between the base and index. Try to fold Op1 into AM's displacement. 417251607Sdimstatic bool expandDisp(SystemZAddressingMode &AM, bool IsBase, 418261991Sdim SDValue Op0, uint64_t Op1) { 419251607Sdim // First try adjusting the displacement. 420261991Sdim int64_t TestDisp = AM.Disp + Op1; 421251607Sdim if (selectDisp(AM.DR, TestDisp)) { 422251607Sdim changeComponent(AM, IsBase, Op0); 423251607Sdim AM.Disp = TestDisp; 424251607Sdim return true; 425251607Sdim } 426251607Sdim 427251607Sdim // We could consider forcing the displacement into a register and 428251607Sdim // using it as an index, but it would need to be carefully tuned. 429251607Sdim return false; 430251607Sdim} 431251607Sdim 432251607Sdimbool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM, 433261991Sdim bool IsBase) const { 434251607Sdim SDValue N = IsBase ? AM.Base : AM.Index; 435251607Sdim unsigned Opcode = N.getOpcode(); 436251607Sdim if (Opcode == ISD::TRUNCATE) { 437251607Sdim N = N.getOperand(0); 438251607Sdim Opcode = N.getOpcode(); 439251607Sdim } 440251607Sdim if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(N)) { 441251607Sdim SDValue Op0 = N.getOperand(0); 442251607Sdim SDValue Op1 = N.getOperand(1); 443251607Sdim 444251607Sdim unsigned Op0Code = Op0->getOpcode(); 445251607Sdim unsigned Op1Code = Op1->getOpcode(); 446251607Sdim 447251607Sdim if (Op0Code == SystemZISD::ADJDYNALLOC) 448251607Sdim return expandAdjDynAlloc(AM, IsBase, Op1); 449251607Sdim if (Op1Code == SystemZISD::ADJDYNALLOC) 450251607Sdim return expandAdjDynAlloc(AM, IsBase, Op0); 451251607Sdim 452251607Sdim if (Op0Code == ISD::Constant) 453261991Sdim return expandDisp(AM, IsBase, Op1, 454261991Sdim cast<ConstantSDNode>(Op0)->getSExtValue()); 455251607Sdim if (Op1Code == ISD::Constant) 456261991Sdim return expandDisp(AM, IsBase, Op0, 457261991Sdim cast<ConstantSDNode>(Op1)->getSExtValue()); 458251607Sdim 459251607Sdim if (IsBase && expandIndex(AM, Op0, Op1)) 460251607Sdim return true; 461251607Sdim } 462261991Sdim if (Opcode == SystemZISD::PCREL_OFFSET) { 463261991Sdim SDValue Full = N.getOperand(0); 464261991Sdim SDValue Base = N.getOperand(1); 465261991Sdim SDValue Anchor = Base.getOperand(0); 466261991Sdim uint64_t Offset = (cast<GlobalAddressSDNode>(Full)->getOffset() - 467261991Sdim cast<GlobalAddressSDNode>(Anchor)->getOffset()); 468261991Sdim return expandDisp(AM, IsBase, Base, Offset); 469261991Sdim } 470251607Sdim return false; 471251607Sdim} 472251607Sdim 473251607Sdim// Return true if an instruction with displacement range DR should be 474251607Sdim// used for displacement value Val. selectDisp(DR, Val) must already hold. 475251607Sdimstatic bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) { 476251607Sdim assert(selectDisp(DR, Val) && "Invalid displacement"); 477251607Sdim switch (DR) { 478251607Sdim case SystemZAddressingMode::Disp12Only: 479251607Sdim case SystemZAddressingMode::Disp20Only: 480251607Sdim case SystemZAddressingMode::Disp20Only128: 481251607Sdim return true; 482251607Sdim 483251607Sdim case SystemZAddressingMode::Disp12Pair: 484251607Sdim // Use the other instruction if the displacement is too large. 485251607Sdim return isUInt<12>(Val); 486251607Sdim 487251607Sdim case SystemZAddressingMode::Disp20Pair: 488251607Sdim // Use the other instruction if the displacement is small enough. 489251607Sdim return !isUInt<12>(Val); 490251607Sdim } 491251607Sdim llvm_unreachable("Unhandled displacement range"); 492251607Sdim} 493251607Sdim 494251607Sdim// Return true if Base + Disp + Index should be performed by LA(Y). 495251607Sdimstatic bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) { 496251607Sdim // Don't use LA(Y) for constants. 497251607Sdim if (!Base) 498251607Sdim return false; 499251607Sdim 500251607Sdim // Always use LA(Y) for frame addresses, since we know that the destination 501251607Sdim // register is almost always (perhaps always) going to be different from 502251607Sdim // the frame register. 503251607Sdim if (Base->getOpcode() == ISD::FrameIndex) 504251607Sdim return true; 505251607Sdim 506251607Sdim if (Disp) { 507251607Sdim // Always use LA(Y) if there is a base, displacement and index. 508251607Sdim if (Index) 509251607Sdim return true; 510251607Sdim 511251607Sdim // Always use LA if the displacement is small enough. It should always 512251607Sdim // be no worse than AGHI (and better if it avoids a move). 513251607Sdim if (isUInt<12>(Disp)) 514251607Sdim return true; 515251607Sdim 516251607Sdim // For similar reasons, always use LAY if the constant is too big for AGHI. 517251607Sdim // LAY should be no worse than AGFI. 518251607Sdim if (!isInt<16>(Disp)) 519251607Sdim return true; 520251607Sdim } else { 521251607Sdim // Don't use LA for plain registers. 522251607Sdim if (!Index) 523251607Sdim return false; 524251607Sdim 525251607Sdim // Don't use LA for plain addition if the index operand is only used 526251607Sdim // once. It should be a natural two-operand addition in that case. 527251607Sdim if (Index->hasOneUse()) 528251607Sdim return false; 529251607Sdim 530251607Sdim // Prefer addition if the second operation is sign-extended, in the 531251607Sdim // hope of using AGF. 532251607Sdim unsigned IndexOpcode = Index->getOpcode(); 533251607Sdim if (IndexOpcode == ISD::SIGN_EXTEND || 534251607Sdim IndexOpcode == ISD::SIGN_EXTEND_INREG) 535251607Sdim return false; 536251607Sdim } 537251607Sdim 538251607Sdim // Don't use LA for two-operand addition if either operand is only 539251607Sdim // used once. The addition instructions are better in that case. 540251607Sdim if (Base->hasOneUse()) 541251607Sdim return false; 542251607Sdim 543251607Sdim return true; 544251607Sdim} 545251607Sdim 546251607Sdim// Return true if Addr is suitable for AM, updating AM if so. 547251607Sdimbool SystemZDAGToDAGISel::selectAddress(SDValue Addr, 548261991Sdim SystemZAddressingMode &AM) const { 549251607Sdim // Start out assuming that the address will need to be loaded separately, 550251607Sdim // then try to extend it as much as we can. 551251607Sdim AM.Base = Addr; 552251607Sdim 553251607Sdim // First try treating the address as a constant. 554251607Sdim if (Addr.getOpcode() == ISD::Constant && 555261991Sdim expandDisp(AM, true, SDValue(), 556261991Sdim cast<ConstantSDNode>(Addr)->getSExtValue())) 557251607Sdim ; 558309124Sdim // Also see if it's a bare ADJDYNALLOC. 559309124Sdim else if (Addr.getOpcode() == SystemZISD::ADJDYNALLOC && 560309124Sdim expandAdjDynAlloc(AM, true, SDValue())) 561309124Sdim ; 562251607Sdim else 563251607Sdim // Otherwise try expanding each component. 564251607Sdim while (expandAddress(AM, true) || 565251607Sdim (AM.Index.getNode() && expandAddress(AM, false))) 566251607Sdim continue; 567251607Sdim 568251607Sdim // Reject cases where it isn't profitable to use LA(Y). 569251607Sdim if (AM.Form == SystemZAddressingMode::FormBDXLA && 570251607Sdim !shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode())) 571251607Sdim return false; 572251607Sdim 573251607Sdim // Reject cases where the other instruction in a pair should be used. 574251607Sdim if (!isValidDisp(AM.DR, AM.Disp)) 575251607Sdim return false; 576251607Sdim 577251607Sdim // Make sure that ADJDYNALLOC is included where necessary. 578251607Sdim if (AM.isDynAlloc() && !AM.IncludesDynAlloc) 579251607Sdim return false; 580251607Sdim 581251607Sdim DEBUG(AM.dump()); 582251607Sdim return true; 583251607Sdim} 584251607Sdim 585251607Sdim// Insert a node into the DAG at least before Pos. This will reposition 586251607Sdim// the node as needed, and will assign it a node ID that is <= Pos's ID. 587251607Sdim// Note that this does *not* preserve the uniqueness of node IDs! 588251607Sdim// The selection DAG must no longer depend on their uniqueness when this 589251607Sdim// function is used. 590251607Sdimstatic void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) { 591251607Sdim if (N.getNode()->getNodeId() == -1 || 592251607Sdim N.getNode()->getNodeId() > Pos->getNodeId()) { 593296417Sdim DAG->RepositionNode(Pos->getIterator(), N.getNode()); 594251607Sdim N.getNode()->setNodeId(Pos->getNodeId()); 595251607Sdim } 596251607Sdim} 597251607Sdim 598251607Sdimvoid SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM, 599251607Sdim EVT VT, SDValue &Base, 600261991Sdim SDValue &Disp) const { 601251607Sdim Base = AM.Base; 602251607Sdim if (!Base.getNode()) 603251607Sdim // Register 0 means "no base". This is mostly useful for shifts. 604251607Sdim Base = CurDAG->getRegister(0, VT); 605251607Sdim else if (Base.getOpcode() == ISD::FrameIndex) { 606251607Sdim // Lower a FrameIndex to a TargetFrameIndex. 607251607Sdim int64_t FrameIndex = cast<FrameIndexSDNode>(Base)->getIndex(); 608251607Sdim Base = CurDAG->getTargetFrameIndex(FrameIndex, VT); 609251607Sdim } else if (Base.getValueType() != VT) { 610251607Sdim // Truncate values from i64 to i32, for shifts. 611251607Sdim assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 && 612251607Sdim "Unexpected truncation"); 613261991Sdim SDLoc DL(Base); 614251607Sdim SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base); 615251607Sdim insertDAGNode(CurDAG, Base.getNode(), Trunc); 616251607Sdim Base = Trunc; 617251607Sdim } 618251607Sdim 619251607Sdim // Lower the displacement to a TargetConstant. 620288943Sdim Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT); 621251607Sdim} 622251607Sdim 623251607Sdimvoid SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM, 624251607Sdim EVT VT, SDValue &Base, 625261991Sdim SDValue &Disp, 626261991Sdim SDValue &Index) const { 627251607Sdim getAddressOperands(AM, VT, Base, Disp); 628251607Sdim 629251607Sdim Index = AM.Index; 630251607Sdim if (!Index.getNode()) 631251607Sdim // Register 0 means "no index". 632251607Sdim Index = CurDAG->getRegister(0, VT); 633251607Sdim} 634251607Sdim 635251607Sdimbool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR, 636251607Sdim SDValue Addr, SDValue &Base, 637261991Sdim SDValue &Disp) const { 638251607Sdim SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR); 639251607Sdim if (!selectAddress(Addr, AM)) 640251607Sdim return false; 641251607Sdim 642251607Sdim getAddressOperands(AM, Addr.getValueType(), Base, Disp); 643251607Sdim return true; 644251607Sdim} 645251607Sdim 646261991Sdimbool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR, 647261991Sdim SDValue Addr, SDValue &Base, 648261991Sdim SDValue &Disp) const { 649261991Sdim SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR); 650261991Sdim if (!selectAddress(Addr, AM) || AM.Index.getNode()) 651261991Sdim return false; 652261991Sdim 653261991Sdim getAddressOperands(AM, Addr.getValueType(), Base, Disp); 654261991Sdim return true; 655261991Sdim} 656261991Sdim 657251607Sdimbool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form, 658251607Sdim SystemZAddressingMode::DispRange DR, 659251607Sdim SDValue Addr, SDValue &Base, 660261991Sdim SDValue &Disp, SDValue &Index) const { 661251607Sdim SystemZAddressingMode AM(Form, DR); 662251607Sdim if (!selectAddress(Addr, AM)) 663251607Sdim return false; 664251607Sdim 665251607Sdim getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index); 666251607Sdim return true; 667251607Sdim} 668251607Sdim 669288943Sdimbool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem, 670288943Sdim SDValue &Base, 671288943Sdim SDValue &Disp, 672288943Sdim SDValue &Index) const { 673288943Sdim SDValue Regs[2]; 674288943Sdim if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) && 675288943Sdim Regs[0].getNode() && Regs[1].getNode()) { 676288943Sdim for (unsigned int I = 0; I < 2; ++I) { 677288943Sdim Base = Regs[I]; 678288943Sdim Index = Regs[1 - I]; 679288943Sdim // We can't tell here whether the index vector has the right type 680288943Sdim // for the access; the caller needs to do that instead. 681288943Sdim if (Index.getOpcode() == ISD::ZERO_EXTEND) 682288943Sdim Index = Index.getOperand(0); 683288943Sdim if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT && 684288943Sdim Index.getOperand(1) == Elem) { 685288943Sdim Index = Index.getOperand(0); 686288943Sdim return true; 687288943Sdim } 688288943Sdim } 689288943Sdim } 690288943Sdim return false; 691288943Sdim} 692288943Sdim 693261991Sdimbool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op, 694261991Sdim uint64_t InsertMask) const { 695261991Sdim // We're only interested in cases where the insertion is into some operand 696261991Sdim // of Op, rather than into Op itself. The only useful case is an AND. 697261991Sdim if (Op.getOpcode() != ISD::AND) 698261991Sdim return false; 699261991Sdim 700261991Sdim // We need a constant mask. 701276479Sdim auto *MaskNode = dyn_cast<ConstantSDNode>(Op.getOperand(1).getNode()); 702261991Sdim if (!MaskNode) 703261991Sdim return false; 704261991Sdim 705261991Sdim // It's not an insertion of Op.getOperand(0) if the two masks overlap. 706261991Sdim uint64_t AndMask = MaskNode->getZExtValue(); 707261991Sdim if (InsertMask & AndMask) 708261991Sdim return false; 709261991Sdim 710261991Sdim // It's only an insertion if all bits are covered or are known to be zero. 711261991Sdim // The inner check covers all cases but is more expensive. 712314564Sdim uint64_t Used = allOnes(Op.getValueSizeInBits()); 713261991Sdim if (Used != (AndMask | InsertMask)) { 714261991Sdim APInt KnownZero, KnownOne; 715276479Sdim CurDAG->computeKnownBits(Op.getOperand(0), KnownZero, KnownOne); 716261991Sdim if (Used != (AndMask | InsertMask | KnownZero.getZExtValue())) 717261991Sdim return false; 718261991Sdim } 719261991Sdim 720261991Sdim Op = Op.getOperand(0); 721261991Sdim return true; 722261991Sdim} 723261991Sdim 724261991Sdimbool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG, 725261991Sdim uint64_t Mask) const { 726261991Sdim const SystemZInstrInfo *TII = getInstrInfo(); 727261991Sdim if (RxSBG.Rotate != 0) 728261991Sdim Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)); 729261991Sdim Mask &= RxSBG.Mask; 730261991Sdim if (TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) { 731261991Sdim RxSBG.Mask = Mask; 732261991Sdim return true; 733261991Sdim } 734261991Sdim return false; 735261991Sdim} 736261991Sdim 737261991Sdim// Return true if any bits of (RxSBG.Input & Mask) are significant. 738261991Sdimstatic bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) { 739261991Sdim // Rotate the mask in the same way as RxSBG.Input is rotated. 740261991Sdim if (RxSBG.Rotate != 0) 741261991Sdim Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate))); 742261991Sdim return (Mask & RxSBG.Mask) != 0; 743261991Sdim} 744261991Sdim 745261991Sdimbool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const { 746261991Sdim SDValue N = RxSBG.Input; 747261991Sdim unsigned Opcode = N.getOpcode(); 748261991Sdim switch (Opcode) { 749309124Sdim case ISD::TRUNCATE: { 750309124Sdim if (RxSBG.Opcode == SystemZ::RNSBG) 751309124Sdim return false; 752314564Sdim uint64_t BitSize = N.getValueSizeInBits(); 753309124Sdim uint64_t Mask = allOnes(BitSize); 754309124Sdim if (!refineRxSBGMask(RxSBG, Mask)) 755309124Sdim return false; 756309124Sdim RxSBG.Input = N.getOperand(0); 757309124Sdim return true; 758309124Sdim } 759261991Sdim case ISD::AND: { 760261991Sdim if (RxSBG.Opcode == SystemZ::RNSBG) 761261991Sdim return false; 762261991Sdim 763276479Sdim auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 764261991Sdim if (!MaskNode) 765261991Sdim return false; 766261991Sdim 767261991Sdim SDValue Input = N.getOperand(0); 768261991Sdim uint64_t Mask = MaskNode->getZExtValue(); 769261991Sdim if (!refineRxSBGMask(RxSBG, Mask)) { 770261991Sdim // If some bits of Input are already known zeros, those bits will have 771261991Sdim // been removed from the mask. See if adding them back in makes the 772261991Sdim // mask suitable. 773261991Sdim APInt KnownZero, KnownOne; 774276479Sdim CurDAG->computeKnownBits(Input, KnownZero, KnownOne); 775261991Sdim Mask |= KnownZero.getZExtValue(); 776261991Sdim if (!refineRxSBGMask(RxSBG, Mask)) 777261991Sdim return false; 778261991Sdim } 779261991Sdim RxSBG.Input = Input; 780261991Sdim return true; 781261991Sdim } 782261991Sdim 783261991Sdim case ISD::OR: { 784261991Sdim if (RxSBG.Opcode != SystemZ::RNSBG) 785261991Sdim return false; 786261991Sdim 787276479Sdim auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 788261991Sdim if (!MaskNode) 789261991Sdim return false; 790261991Sdim 791261991Sdim SDValue Input = N.getOperand(0); 792261991Sdim uint64_t Mask = ~MaskNode->getZExtValue(); 793261991Sdim if (!refineRxSBGMask(RxSBG, Mask)) { 794261991Sdim // If some bits of Input are already known ones, those bits will have 795261991Sdim // been removed from the mask. See if adding them back in makes the 796261991Sdim // mask suitable. 797261991Sdim APInt KnownZero, KnownOne; 798276479Sdim CurDAG->computeKnownBits(Input, KnownZero, KnownOne); 799261991Sdim Mask &= ~KnownOne.getZExtValue(); 800261991Sdim if (!refineRxSBGMask(RxSBG, Mask)) 801261991Sdim return false; 802261991Sdim } 803261991Sdim RxSBG.Input = Input; 804261991Sdim return true; 805261991Sdim } 806261991Sdim 807261991Sdim case ISD::ROTL: { 808261991Sdim // Any 64-bit rotate left can be merged into the RxSBG. 809261991Sdim if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64) 810261991Sdim return false; 811276479Sdim auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 812261991Sdim if (!CountNode) 813261991Sdim return false; 814261991Sdim 815261991Sdim RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63; 816261991Sdim RxSBG.Input = N.getOperand(0); 817261991Sdim return true; 818261991Sdim } 819296417Sdim 820276479Sdim case ISD::ANY_EXTEND: 821276479Sdim // Bits above the extended operand are don't-care. 822276479Sdim RxSBG.Input = N.getOperand(0); 823276479Sdim return true; 824276479Sdim 825261991Sdim case ISD::ZERO_EXTEND: 826276479Sdim if (RxSBG.Opcode != SystemZ::RNSBG) { 827276479Sdim // Restrict the mask to the extended operand. 828314564Sdim unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits(); 829276479Sdim if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize))) 830276479Sdim return false; 831276479Sdim 832276479Sdim RxSBG.Input = N.getOperand(0); 833276479Sdim return true; 834276479Sdim } 835314564Sdim LLVM_FALLTHROUGH; 836296417Sdim 837276479Sdim case ISD::SIGN_EXTEND: { 838261991Sdim // Check that the extension bits are don't-care (i.e. are masked out 839261991Sdim // by the final mask). 840314564Sdim unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits(); 841261991Sdim if (maskMatters(RxSBG, allOnes(RxSBG.BitSize) - allOnes(InnerBitSize))) 842261991Sdim return false; 843261991Sdim 844261991Sdim RxSBG.Input = N.getOperand(0); 845261991Sdim return true; 846261991Sdim } 847261991Sdim 848261991Sdim case ISD::SHL: { 849276479Sdim auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 850261991Sdim if (!CountNode) 851261991Sdim return false; 852261991Sdim 853261991Sdim uint64_t Count = CountNode->getZExtValue(); 854314564Sdim unsigned BitSize = N.getValueSizeInBits(); 855261991Sdim if (Count < 1 || Count >= BitSize) 856261991Sdim return false; 857261991Sdim 858261991Sdim if (RxSBG.Opcode == SystemZ::RNSBG) { 859261991Sdim // Treat (shl X, count) as (rotl X, size-count) as long as the bottom 860261991Sdim // count bits from RxSBG.Input are ignored. 861261991Sdim if (maskMatters(RxSBG, allOnes(Count))) 862261991Sdim return false; 863261991Sdim } else { 864261991Sdim // Treat (shl X, count) as (and (rotl X, count), ~0<<count). 865261991Sdim if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count) << Count)) 866261991Sdim return false; 867261991Sdim } 868261991Sdim 869261991Sdim RxSBG.Rotate = (RxSBG.Rotate + Count) & 63; 870261991Sdim RxSBG.Input = N.getOperand(0); 871261991Sdim return true; 872261991Sdim } 873261991Sdim 874261991Sdim case ISD::SRL: 875261991Sdim case ISD::SRA: { 876276479Sdim auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 877261991Sdim if (!CountNode) 878261991Sdim return false; 879261991Sdim 880261991Sdim uint64_t Count = CountNode->getZExtValue(); 881314564Sdim unsigned BitSize = N.getValueSizeInBits(); 882261991Sdim if (Count < 1 || Count >= BitSize) 883261991Sdim return false; 884261991Sdim 885261991Sdim if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) { 886261991Sdim // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top 887261991Sdim // count bits from RxSBG.Input are ignored. 888261991Sdim if (maskMatters(RxSBG, allOnes(Count) << (BitSize - Count))) 889261991Sdim return false; 890261991Sdim } else { 891261991Sdim // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count), 892261991Sdim // which is similar to SLL above. 893261991Sdim if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count))) 894261991Sdim return false; 895261991Sdim } 896261991Sdim 897261991Sdim RxSBG.Rotate = (RxSBG.Rotate - Count) & 63; 898261991Sdim RxSBG.Input = N.getOperand(0); 899261991Sdim return true; 900261991Sdim } 901261991Sdim default: 902261991Sdim return false; 903261991Sdim } 904261991Sdim} 905261991Sdim 906309124SdimSDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const { 907261991Sdim SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT); 908261991Sdim return SDValue(N, 0); 909261991Sdim} 910261991Sdim 911309124SdimSDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT, 912309124Sdim SDValue N) const { 913261991Sdim if (N.getValueType() == MVT::i32 && VT == MVT::i64) 914261991Sdim return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32, 915261991Sdim DL, VT, getUNDEF(DL, MVT::i64), N); 916261991Sdim if (N.getValueType() == MVT::i64 && VT == MVT::i32) 917261991Sdim return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N); 918261991Sdim assert(N.getValueType() == VT && "Unexpected value types"); 919261991Sdim return N; 920261991Sdim} 921261991Sdim 922309124Sdimbool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) { 923288943Sdim SDLoc DL(N); 924261991Sdim EVT VT = N->getValueType(0); 925288943Sdim if (!VT.isInteger() || VT.getSizeInBits() > 64) 926309124Sdim return false; 927261991Sdim RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0)); 928261991Sdim unsigned Count = 0; 929261991Sdim while (expandRxSBG(RISBG)) 930309124Sdim // The widening or narrowing is expected to be free. 931309124Sdim // Counting widening or narrowing as a saved operation will result in 932309124Sdim // preferring an R*SBG over a simple shift/logical instruction. 933309124Sdim if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND && 934309124Sdim RISBG.Input.getOpcode() != ISD::TRUNCATE) 935261991Sdim Count += 1; 936261991Sdim if (Count == 0) 937309124Sdim return false; 938261991Sdim 939314564Sdim // Prefer to use normal shift instructions over RISBG, since they can handle 940314564Sdim // all cases and are sometimes shorter. 941314564Sdim if (Count == 1 && N->getOpcode() != ISD::AND) 942314564Sdim return false; 943314564Sdim 944314564Sdim // Prefer register extensions like LLC over RISBG. Also prefer to start 945314564Sdim // out with normal ANDs if one instruction would be enough. We can convert 946314564Sdim // these ANDs into an RISBG later if a three-address instruction is useful. 947314564Sdim if (RISBG.Rotate == 0) { 948314564Sdim bool PreferAnd = false; 949314564Sdim // Prefer AND for any 32-bit and-immediate operation. 950314564Sdim if (VT == MVT::i32) 951314564Sdim PreferAnd = true; 952314564Sdim // As well as for any 64-bit operation that can be implemented via LLC(R), 953314564Sdim // LLH(R), LLGT(R), or one of the and-immediate instructions. 954314564Sdim else if (RISBG.Mask == 0xff || 955314564Sdim RISBG.Mask == 0xffff || 956314564Sdim RISBG.Mask == 0x7fffffff || 957314564Sdim SystemZ::isImmLF(~RISBG.Mask) || 958314564Sdim SystemZ::isImmHF(~RISBG.Mask)) 959314564Sdim PreferAnd = true; 960314564Sdim // And likewise for the LLZRGF instruction, which doesn't have a register 961314564Sdim // to register version. 962314564Sdim else if (auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) { 963314564Sdim if (Load->getMemoryVT() == MVT::i32 && 964314564Sdim (Load->getExtensionType() == ISD::EXTLOAD || 965314564Sdim Load->getExtensionType() == ISD::ZEXTLOAD) && 966314564Sdim RISBG.Mask == 0xffffff00 && 967314564Sdim Subtarget->hasLoadAndZeroRightmostByte()) 968314564Sdim PreferAnd = true; 969314564Sdim } 970314564Sdim if (PreferAnd) { 971314564Sdim // Replace the current node with an AND. Note that the current node 972314564Sdim // might already be that same AND, in which case it is already CSE'd 973314564Sdim // with it, and we must not call ReplaceNode. 974314564Sdim SDValue In = convertTo(DL, VT, RISBG.Input); 975314564Sdim SDValue Mask = CurDAG->getConstant(RISBG.Mask, DL, VT); 976314564Sdim SDValue New = CurDAG->getNode(ISD::AND, DL, VT, In, Mask); 977314564Sdim if (N != New.getNode()) { 978314564Sdim insertDAGNode(CurDAG, N, Mask); 979314564Sdim insertDAGNode(CurDAG, N, New); 980314564Sdim ReplaceNode(N, New.getNode()); 981314564Sdim N = New.getNode(); 982261991Sdim } 983314564Sdim // Now, select the machine opcode to implement this operation. 984314564Sdim SelectCode(N); 985314564Sdim return true; 986261991Sdim } 987296417Sdim } 988261991Sdim 989261991Sdim unsigned Opcode = SystemZ::RISBG; 990288943Sdim // Prefer RISBGN if available, since it does not clobber CC. 991288943Sdim if (Subtarget->hasMiscellaneousExtensions()) 992288943Sdim Opcode = SystemZ::RISBGN; 993261991Sdim EVT OpcodeVT = MVT::i64; 994288943Sdim if (VT == MVT::i32 && Subtarget->hasHighWord()) { 995261991Sdim Opcode = SystemZ::RISBMux; 996261991Sdim OpcodeVT = MVT::i32; 997261991Sdim RISBG.Start &= 31; 998261991Sdim RISBG.End &= 31; 999261991Sdim } 1000261991Sdim SDValue Ops[5] = { 1001288943Sdim getUNDEF(DL, OpcodeVT), 1002288943Sdim convertTo(DL, OpcodeVT, RISBG.Input), 1003288943Sdim CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32), 1004288943Sdim CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32), 1005288943Sdim CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32) 1006261991Sdim }; 1007309124Sdim SDValue New = convertTo( 1008309124Sdim DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops), 0)); 1009309124Sdim ReplaceUses(N, New.getNode()); 1010309124Sdim CurDAG->RemoveDeadNode(N); 1011309124Sdim return true; 1012261991Sdim} 1013261991Sdim 1014309124Sdimbool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) { 1015288943Sdim SDLoc DL(N); 1016288943Sdim EVT VT = N->getValueType(0); 1017288943Sdim if (!VT.isInteger() || VT.getSizeInBits() > 64) 1018309124Sdim return false; 1019261991Sdim // Try treating each operand of N as the second operand of the RxSBG 1020261991Sdim // and see which goes deepest. 1021261991Sdim RxSBGOperands RxSBG[] = { 1022261991Sdim RxSBGOperands(Opcode, N->getOperand(0)), 1023261991Sdim RxSBGOperands(Opcode, N->getOperand(1)) 1024261991Sdim }; 1025261991Sdim unsigned Count[] = { 0, 0 }; 1026261991Sdim for (unsigned I = 0; I < 2; ++I) 1027261991Sdim while (expandRxSBG(RxSBG[I])) 1028309124Sdim // The widening or narrowing is expected to be free. 1029309124Sdim // Counting widening or narrowing as a saved operation will result in 1030309124Sdim // preferring an R*SBG over a simple shift/logical instruction. 1031309124Sdim if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND && 1032309124Sdim RxSBG[I].Input.getOpcode() != ISD::TRUNCATE) 1033261991Sdim Count[I] += 1; 1034261991Sdim 1035261991Sdim // Do nothing if neither operand is suitable. 1036261991Sdim if (Count[0] == 0 && Count[1] == 0) 1037309124Sdim return false; 1038261991Sdim 1039261991Sdim // Pick the deepest second operand. 1040261991Sdim unsigned I = Count[0] > Count[1] ? 0 : 1; 1041261991Sdim SDValue Op0 = N->getOperand(I ^ 1); 1042261991Sdim 1043261991Sdim // Prefer IC for character insertions from memory. 1044261991Sdim if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0) 1045276479Sdim if (auto *Load = dyn_cast<LoadSDNode>(Op0.getNode())) 1046261991Sdim if (Load->getMemoryVT() == MVT::i8) 1047309124Sdim return false; 1048261991Sdim 1049261991Sdim // See whether we can avoid an AND in the first operand by converting 1050261991Sdim // ROSBG to RISBG. 1051288943Sdim if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[I].Mask)) { 1052261991Sdim Opcode = SystemZ::RISBG; 1053288943Sdim // Prefer RISBGN if available, since it does not clobber CC. 1054288943Sdim if (Subtarget->hasMiscellaneousExtensions()) 1055288943Sdim Opcode = SystemZ::RISBGN; 1056288943Sdim } 1057288943Sdim 1058261991Sdim SDValue Ops[5] = { 1059288943Sdim convertTo(DL, MVT::i64, Op0), 1060288943Sdim convertTo(DL, MVT::i64, RxSBG[I].Input), 1061288943Sdim CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32), 1062288943Sdim CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32), 1063288943Sdim CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32) 1064261991Sdim }; 1065309124Sdim SDValue New = convertTo( 1066309124Sdim DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops), 0)); 1067309124Sdim ReplaceNode(N, New.getNode()); 1068309124Sdim return true; 1069261991Sdim} 1070261991Sdim 1071309124Sdimvoid SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node, 1072309124Sdim SDValue Op0, uint64_t UpperVal, 1073309124Sdim uint64_t LowerVal) { 1074251607Sdim EVT VT = Node->getValueType(0); 1075261991Sdim SDLoc DL(Node); 1076288943Sdim SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT); 1077251607Sdim if (Op0.getNode()) 1078251607Sdim Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper); 1079251607Sdim 1080309124Sdim { 1081309124Sdim // When we haven't passed in Op0, Upper will be a constant. In order to 1082309124Sdim // prevent folding back to the large immediate in `Or = getNode(...)` we run 1083309124Sdim // SelectCode first and end up with an opaque machine node. This means that 1084309124Sdim // we need to use a handle to keep track of Upper in case it gets CSE'd by 1085309124Sdim // SelectCode. 1086309124Sdim // 1087309124Sdim // Note that in the case where Op0 is passed in we could just call 1088309124Sdim // SelectCode(Upper) later, along with the SelectCode(Or), and avoid needing 1089309124Sdim // the handle at all, but it's fine to do it here. 1090309124Sdim // 1091309124Sdim // TODO: This is a pretty hacky way to do this. Can we do something that 1092309124Sdim // doesn't require a two paragraph explanation? 1093309124Sdim HandleSDNode Handle(Upper); 1094309124Sdim SelectCode(Upper.getNode()); 1095309124Sdim Upper = Handle.getValue(); 1096309124Sdim } 1097309124Sdim 1098288943Sdim SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT); 1099251607Sdim SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower); 1100309124Sdim 1101309124Sdim ReplaceUses(Node, Or.getNode()); 1102309124Sdim CurDAG->RemoveDeadNode(Node); 1103309124Sdim 1104309124Sdim SelectCode(Or.getNode()); 1105251607Sdim} 1106251607Sdim 1107309124Sdimbool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) { 1108288943Sdim SDValue ElemV = N->getOperand(2); 1109288943Sdim auto *ElemN = dyn_cast<ConstantSDNode>(ElemV); 1110288943Sdim if (!ElemN) 1111309124Sdim return false; 1112288943Sdim 1113288943Sdim unsigned Elem = ElemN->getZExtValue(); 1114288943Sdim EVT VT = N->getValueType(0); 1115288943Sdim if (Elem >= VT.getVectorNumElements()) 1116309124Sdim return false; 1117288943Sdim 1118288943Sdim auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1)); 1119288943Sdim if (!Load || !Load->hasOneUse()) 1120309124Sdim return false; 1121288943Sdim if (Load->getMemoryVT().getSizeInBits() != 1122288943Sdim Load->getValueType(0).getSizeInBits()) 1123309124Sdim return false; 1124288943Sdim 1125288943Sdim SDValue Base, Disp, Index; 1126288943Sdim if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) || 1127288943Sdim Index.getValueType() != VT.changeVectorElementTypeToInteger()) 1128309124Sdim return false; 1129288943Sdim 1130288943Sdim SDLoc DL(Load); 1131288943Sdim SDValue Ops[] = { 1132288943Sdim N->getOperand(0), Base, Disp, Index, 1133288943Sdim CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain() 1134288943Sdim }; 1135288943Sdim SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops); 1136288943Sdim ReplaceUses(SDValue(Load, 1), SDValue(Res, 1)); 1137309124Sdim ReplaceNode(N, Res); 1138309124Sdim return true; 1139288943Sdim} 1140288943Sdim 1141309124Sdimbool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) { 1142288943Sdim SDValue Value = Store->getValue(); 1143288943Sdim if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT) 1144309124Sdim return false; 1145314564Sdim if (Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits()) 1146309124Sdim return false; 1147288943Sdim 1148288943Sdim SDValue ElemV = Value.getOperand(1); 1149288943Sdim auto *ElemN = dyn_cast<ConstantSDNode>(ElemV); 1150288943Sdim if (!ElemN) 1151309124Sdim return false; 1152288943Sdim 1153288943Sdim SDValue Vec = Value.getOperand(0); 1154288943Sdim EVT VT = Vec.getValueType(); 1155288943Sdim unsigned Elem = ElemN->getZExtValue(); 1156288943Sdim if (Elem >= VT.getVectorNumElements()) 1157309124Sdim return false; 1158288943Sdim 1159288943Sdim SDValue Base, Disp, Index; 1160288943Sdim if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) || 1161288943Sdim Index.getValueType() != VT.changeVectorElementTypeToInteger()) 1162309124Sdim return false; 1163288943Sdim 1164288943Sdim SDLoc DL(Store); 1165288943Sdim SDValue Ops[] = { 1166288943Sdim Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32), 1167288943Sdim Store->getChain() 1168288943Sdim }; 1169309124Sdim ReplaceNode(Store, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops)); 1170309124Sdim return true; 1171288943Sdim} 1172288943Sdim 1173261991Sdimbool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store, 1174261991Sdim LoadSDNode *Load) const { 1175261991Sdim // Check that the two memory operands have the same size. 1176261991Sdim if (Load->getMemoryVT() != Store->getMemoryVT()) 1177261991Sdim return false; 1178261991Sdim 1179261991Sdim // Volatility stops an access from being decomposed. 1180261991Sdim if (Load->isVolatile() || Store->isVolatile()) 1181261991Sdim return false; 1182261991Sdim 1183261991Sdim // There's no chance of overlap if the load is invariant. 1184314564Sdim if (Load->isInvariant() && Load->isDereferenceable()) 1185261991Sdim return true; 1186261991Sdim 1187261991Sdim // Otherwise we need to check whether there's an alias. 1188276479Sdim const Value *V1 = Load->getMemOperand()->getValue(); 1189276479Sdim const Value *V2 = Store->getMemOperand()->getValue(); 1190261991Sdim if (!V1 || !V2) 1191261991Sdim return false; 1192261991Sdim 1193261991Sdim // Reject equality. 1194261991Sdim uint64_t Size = Load->getMemoryVT().getStoreSize(); 1195261991Sdim int64_t End1 = Load->getSrcValueOffset() + Size; 1196261991Sdim int64_t End2 = Store->getSrcValueOffset() + Size; 1197261991Sdim if (V1 == V2 && End1 == End2) 1198261991Sdim return false; 1199261991Sdim 1200288943Sdim return !AA->alias(MemoryLocation(V1, End1, Load->getAAInfo()), 1201288943Sdim MemoryLocation(V2, End2, Store->getAAInfo())); 1202261991Sdim} 1203261991Sdim 1204261991Sdimbool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const { 1205276479Sdim auto *Store = cast<StoreSDNode>(N); 1206276479Sdim auto *Load = cast<LoadSDNode>(Store->getValue()); 1207261991Sdim 1208261991Sdim // Prefer not to use MVC if either address can use ... RELATIVE LONG 1209261991Sdim // instructions. 1210261991Sdim uint64_t Size = Load->getMemoryVT().getStoreSize(); 1211261991Sdim if (Size > 1 && Size <= 8) { 1212261991Sdim // Prefer LHRL, LRL and LGRL. 1213261991Sdim if (SystemZISD::isPCREL(Load->getBasePtr().getOpcode())) 1214261991Sdim return false; 1215261991Sdim // Prefer STHRL, STRL and STGRL. 1216261991Sdim if (SystemZISD::isPCREL(Store->getBasePtr().getOpcode())) 1217261991Sdim return false; 1218261991Sdim } 1219261991Sdim 1220261991Sdim return canUseBlockOperation(Store, Load); 1221261991Sdim} 1222261991Sdim 1223261991Sdimbool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N, 1224261991Sdim unsigned I) const { 1225276479Sdim auto *StoreA = cast<StoreSDNode>(N); 1226276479Sdim auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 - I)); 1227276479Sdim auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(I)); 1228261991Sdim return !LoadA->isVolatile() && canUseBlockOperation(StoreA, LoadB); 1229261991Sdim} 1230261991Sdim 1231309124Sdimvoid SystemZDAGToDAGISel::Select(SDNode *Node) { 1232251607Sdim // Dump information about the Node being selected 1233251607Sdim DEBUG(errs() << "Selecting: "; Node->dump(CurDAG); errs() << "\n"); 1234251607Sdim 1235251607Sdim // If we have a custom node, we already have selected! 1236251607Sdim if (Node->isMachineOpcode()) { 1237251607Sdim DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n"); 1238255804Sdim Node->setNodeId(-1); 1239309124Sdim return; 1240251607Sdim } 1241251607Sdim 1242251607Sdim unsigned Opcode = Node->getOpcode(); 1243251607Sdim switch (Opcode) { 1244251607Sdim case ISD::OR: 1245261991Sdim if (Node->getOperand(1).getOpcode() != ISD::Constant) 1246309124Sdim if (tryRxSBG(Node, SystemZ::ROSBG)) 1247309124Sdim return; 1248261991Sdim goto or_xor; 1249261991Sdim 1250251607Sdim case ISD::XOR: 1251261991Sdim if (Node->getOperand(1).getOpcode() != ISD::Constant) 1252309124Sdim if (tryRxSBG(Node, SystemZ::RXSBG)) 1253309124Sdim return; 1254261991Sdim // Fall through. 1255261991Sdim or_xor: 1256251607Sdim // If this is a 64-bit operation in which both 32-bit halves are nonzero, 1257251607Sdim // split the operation into two. 1258309124Sdim if (Node->getValueType(0) == MVT::i64) 1259276479Sdim if (auto *Op1 = dyn_cast<ConstantSDNode>(Node->getOperand(1))) { 1260251607Sdim uint64_t Val = Op1->getZExtValue(); 1261309124Sdim if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) { 1262309124Sdim splitLargeImmediate(Opcode, Node, Node->getOperand(0), 1263309124Sdim Val - uint32_t(Val), uint32_t(Val)); 1264309124Sdim return; 1265309124Sdim } 1266251607Sdim } 1267251607Sdim break; 1268251607Sdim 1269261991Sdim case ISD::AND: 1270261991Sdim if (Node->getOperand(1).getOpcode() != ISD::Constant) 1271309124Sdim if (tryRxSBG(Node, SystemZ::RNSBG)) 1272309124Sdim return; 1273314564Sdim LLVM_FALLTHROUGH; 1274261991Sdim case ISD::ROTL: 1275261991Sdim case ISD::SHL: 1276261991Sdim case ISD::SRL: 1277276479Sdim case ISD::ZERO_EXTEND: 1278309124Sdim if (tryRISBGZero(Node)) 1279309124Sdim return; 1280261991Sdim break; 1281261991Sdim 1282251607Sdim case ISD::Constant: 1283251607Sdim // If this is a 64-bit constant that is out of the range of LLILF, 1284251607Sdim // LLIHF and LGFI, split it into two 32-bit pieces. 1285251607Sdim if (Node->getValueType(0) == MVT::i64) { 1286251607Sdim uint64_t Val = cast<ConstantSDNode>(Node)->getZExtValue(); 1287309124Sdim if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val) && !isInt<32>(Val)) { 1288309124Sdim splitLargeImmediate(ISD::OR, Node, SDValue(), Val - uint32_t(Val), 1289309124Sdim uint32_t(Val)); 1290309124Sdim return; 1291309124Sdim } 1292251607Sdim } 1293251607Sdim break; 1294251607Sdim 1295261991Sdim case SystemZISD::SELECT_CCMASK: { 1296261991Sdim SDValue Op0 = Node->getOperand(0); 1297261991Sdim SDValue Op1 = Node->getOperand(1); 1298261991Sdim // Prefer to put any load first, so that it can be matched as a 1299314564Sdim // conditional load. Likewise for constants in range for LOCHI. 1300314564Sdim if ((Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) || 1301314564Sdim (Subtarget->hasLoadStoreOnCond2() && 1302314564Sdim Node->getValueType(0).isInteger() && 1303314564Sdim Op1.getOpcode() == ISD::Constant && 1304314564Sdim isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) && 1305314564Sdim !(Op0.getOpcode() == ISD::Constant && 1306314564Sdim isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) { 1307261991Sdim SDValue CCValid = Node->getOperand(2); 1308261991Sdim SDValue CCMask = Node->getOperand(3); 1309261991Sdim uint64_t ConstCCValid = 1310261991Sdim cast<ConstantSDNode>(CCValid.getNode())->getZExtValue(); 1311261991Sdim uint64_t ConstCCMask = 1312261991Sdim cast<ConstantSDNode>(CCMask.getNode())->getZExtValue(); 1313261991Sdim // Invert the condition. 1314288943Sdim CCMask = CurDAG->getConstant(ConstCCValid ^ ConstCCMask, SDLoc(Node), 1315261991Sdim CCMask.getValueType()); 1316261991Sdim SDValue Op4 = Node->getOperand(4); 1317261991Sdim Node = CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4); 1318261991Sdim } 1319261991Sdim break; 1320251607Sdim } 1321288943Sdim 1322288943Sdim case ISD::INSERT_VECTOR_ELT: { 1323288943Sdim EVT VT = Node->getValueType(0); 1324314564Sdim unsigned ElemBitSize = VT.getScalarSizeInBits(); 1325309124Sdim if (ElemBitSize == 32) { 1326309124Sdim if (tryGather(Node, SystemZ::VGEF)) 1327309124Sdim return; 1328309124Sdim } else if (ElemBitSize == 64) { 1329309124Sdim if (tryGather(Node, SystemZ::VGEG)) 1330309124Sdim return; 1331309124Sdim } 1332288943Sdim break; 1333261991Sdim } 1334251607Sdim 1335288943Sdim case ISD::STORE: { 1336288943Sdim auto *Store = cast<StoreSDNode>(Node); 1337314564Sdim unsigned ElemBitSize = Store->getValue().getValueSizeInBits(); 1338309124Sdim if (ElemBitSize == 32) { 1339309124Sdim if (tryScatter(Store, SystemZ::VSCEF)) 1340309124Sdim return; 1341309124Sdim } else if (ElemBitSize == 64) { 1342309124Sdim if (tryScatter(Store, SystemZ::VSCEG)) 1343309124Sdim return; 1344309124Sdim } 1345288943Sdim break; 1346288943Sdim } 1347288943Sdim } 1348288943Sdim 1349309124Sdim SelectCode(Node); 1350251607Sdim} 1351251607Sdim 1352251607Sdimbool SystemZDAGToDAGISel:: 1353251607SdimSelectInlineAsmMemoryOperand(const SDValue &Op, 1354288943Sdim unsigned ConstraintID, 1355251607Sdim std::vector<SDValue> &OutOps) { 1356309124Sdim SystemZAddressingMode::AddrForm Form; 1357309124Sdim SystemZAddressingMode::DispRange DispRange; 1358309124Sdim SDValue Base, Disp, Index; 1359309124Sdim 1360288943Sdim switch(ConstraintID) { 1361288943Sdim default: 1362288943Sdim llvm_unreachable("Unexpected asm memory constraint"); 1363288943Sdim case InlineAsm::Constraint_i: 1364288943Sdim case InlineAsm::Constraint_Q: 1365309124Sdim // Accept an address with a short displacement, but no index. 1366309124Sdim Form = SystemZAddressingMode::FormBD; 1367309124Sdim DispRange = SystemZAddressingMode::Disp12Only; 1368309124Sdim break; 1369288943Sdim case InlineAsm::Constraint_R: 1370309124Sdim // Accept an address with a short displacement and an index. 1371309124Sdim Form = SystemZAddressingMode::FormBDXNormal; 1372309124Sdim DispRange = SystemZAddressingMode::Disp12Only; 1373309124Sdim break; 1374288943Sdim case InlineAsm::Constraint_S: 1375309124Sdim // Accept an address with a long displacement, but no index. 1376309124Sdim Form = SystemZAddressingMode::FormBD; 1377309124Sdim DispRange = SystemZAddressingMode::Disp20Only; 1378309124Sdim break; 1379288943Sdim case InlineAsm::Constraint_T: 1380309124Sdim case InlineAsm::Constraint_m: 1381309124Sdim // Accept an address with a long displacement and an index. 1382309124Sdim // m works the same as T, as this is the most general case. 1383309124Sdim Form = SystemZAddressingMode::FormBDXNormal; 1384309124Sdim DispRange = SystemZAddressingMode::Disp20Only; 1385288943Sdim break; 1386288943Sdim } 1387309124Sdim 1388309124Sdim if (selectBDXAddr(Form, DispRange, Op, Base, Disp, Index)) { 1389314564Sdim const TargetRegisterClass *TRC = 1390314564Sdim Subtarget->getRegisterInfo()->getPointerRegClass(*MF); 1391314564Sdim SDLoc DL(Base); 1392314564Sdim SDValue RC = CurDAG->getTargetConstant(TRC->getID(), DL, MVT::i32); 1393314564Sdim 1394314564Sdim // Make sure that the base address doesn't go into %r0. 1395314564Sdim // If it's a TargetFrameIndex or a fixed register, we shouldn't do anything. 1396314564Sdim if (Base.getOpcode() != ISD::TargetFrameIndex && 1397314564Sdim Base.getOpcode() != ISD::Register) { 1398314564Sdim Base = 1399314564Sdim SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, 1400314564Sdim DL, Base.getValueType(), 1401314564Sdim Base, RC), 0); 1402314564Sdim } 1403314564Sdim 1404314564Sdim // Make sure that the index register isn't assigned to %r0 either. 1405314564Sdim if (Index.getOpcode() != ISD::Register) { 1406314564Sdim Index = 1407314564Sdim SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, 1408314564Sdim DL, Index.getValueType(), 1409314564Sdim Index, RC), 0); 1410314564Sdim } 1411314564Sdim 1412309124Sdim OutOps.push_back(Base); 1413309124Sdim OutOps.push_back(Disp); 1414309124Sdim OutOps.push_back(Index); 1415309124Sdim return false; 1416309124Sdim } 1417309124Sdim 1418288943Sdim return true; 1419251607Sdim} 1420