1249259Sdim//===-- AArch64ISelDAGToDAG.cpp - A dag to dag inst selector for AArch64 --===// 2249259Sdim// 3249259Sdim// The LLVM Compiler Infrastructure 4249259Sdim// 5249259Sdim// This file is distributed under the University of Illinois Open Source 6249259Sdim// License. See LICENSE.TXT for details. 7249259Sdim// 8249259Sdim//===----------------------------------------------------------------------===// 9249259Sdim// 10249259Sdim// This file defines an instruction selector for the AArch64 target. 11249259Sdim// 12249259Sdim//===----------------------------------------------------------------------===// 13249259Sdim 14249259Sdim#define DEBUG_TYPE "aarch64-isel" 15249259Sdim#include "AArch64.h" 16249259Sdim#include "AArch64InstrInfo.h" 17249259Sdim#include "AArch64Subtarget.h" 18249259Sdim#include "AArch64TargetMachine.h" 19249259Sdim#include "Utils/AArch64BaseInfo.h" 20249259Sdim#include "llvm/ADT/APSInt.h" 21249259Sdim#include "llvm/CodeGen/SelectionDAGISel.h" 22249259Sdim#include "llvm/IR/GlobalValue.h" 23249259Sdim#include "llvm/Support/Debug.h" 24249259Sdim#include "llvm/Support/raw_ostream.h" 25249259Sdim 26249259Sdimusing namespace llvm; 27249259Sdim 28249259Sdim//===--------------------------------------------------------------------===// 29249259Sdim/// AArch64 specific code to select AArch64 machine instructions for 30249259Sdim/// SelectionDAG operations. 31249259Sdim/// 32249259Sdimnamespace { 33249259Sdim 34249259Sdimclass AArch64DAGToDAGISel : public SelectionDAGISel { 35249259Sdim AArch64TargetMachine &TM; 36249259Sdim 37249259Sdim /// Keep a pointer to the AArch64Subtarget around so that we can 38249259Sdim /// make the right decision when generating code for different targets. 39249259Sdim const AArch64Subtarget *Subtarget; 40249259Sdim 41249259Sdimpublic: 42249259Sdim explicit AArch64DAGToDAGISel(AArch64TargetMachine &tm, 43249259Sdim CodeGenOpt::Level OptLevel) 44249259Sdim : SelectionDAGISel(tm, OptLevel), TM(tm), 45249259Sdim Subtarget(&TM.getSubtarget<AArch64Subtarget>()) { 46249259Sdim } 47249259Sdim 48249259Sdim virtual const char *getPassName() const { 49249259Sdim return "AArch64 Instruction Selection"; 50249259Sdim } 51249259Sdim 52249259Sdim // Include the pieces autogenerated from the target description. 53249259Sdim#include "AArch64GenDAGISel.inc" 54249259Sdim 55249259Sdim template<unsigned MemSize> 56249259Sdim bool SelectOffsetUImm12(SDValue N, SDValue &UImm12) { 57249259Sdim const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N); 58249259Sdim if (!CN || CN->getZExtValue() % MemSize != 0 59249259Sdim || CN->getZExtValue() / MemSize > 0xfff) 60249259Sdim return false; 61249259Sdim 62249259Sdim UImm12 = CurDAG->getTargetConstant(CN->getZExtValue() / MemSize, MVT::i64); 63249259Sdim return true; 64249259Sdim } 65249259Sdim 66249259Sdim template<unsigned RegWidth> 67249259Sdim bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos) { 68249259Sdim return SelectCVTFixedPosOperand(N, FixedPos, RegWidth); 69249259Sdim } 70249259Sdim 71252723Sdim /// Used for pre-lowered address-reference nodes, so we already know 72252723Sdim /// the fields match. This operand's job is simply to add an 73263509Sdim /// appropriate shift operand to the MOVZ/MOVK instruction. 74263509Sdim template<unsigned LogShift> 75252723Sdim bool SelectMOVWAddressRef(SDValue N, SDValue &Imm, SDValue &Shift) { 76252723Sdim Imm = N; 77263509Sdim Shift = CurDAG->getTargetConstant(LogShift, MVT::i32); 78252723Sdim return true; 79252723Sdim } 80252723Sdim 81249259Sdim bool SelectFPZeroOperand(SDValue N, SDValue &Dummy); 82249259Sdim 83249259Sdim bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, 84249259Sdim unsigned RegWidth); 85249259Sdim 86249259Sdim bool SelectInlineAsmMemoryOperand(const SDValue &Op, 87249259Sdim char ConstraintCode, 88249259Sdim std::vector<SDValue> &OutOps); 89249259Sdim 90249259Sdim bool SelectLogicalImm(SDValue N, SDValue &Imm); 91249259Sdim 92249259Sdim template<unsigned RegWidth> 93249259Sdim bool SelectTSTBOperand(SDValue N, SDValue &FixedPos) { 94249259Sdim return SelectTSTBOperand(N, FixedPos, RegWidth); 95249259Sdim } 96249259Sdim 97249259Sdim bool SelectTSTBOperand(SDValue N, SDValue &FixedPos, unsigned RegWidth); 98249259Sdim 99252723Sdim SDNode *SelectAtomic(SDNode *N, unsigned Op8, unsigned Op16, unsigned Op32, 100252723Sdim unsigned Op64); 101252723Sdim 102252723Sdim /// Put the given constant into a pool and return a DAG which will give its 103252723Sdim /// address. 104263509Sdim SDValue getConstantPoolItemAddress(SDLoc DL, const Constant *CV); 105252723Sdim 106249259Sdim SDNode *TrySelectToMoveImm(SDNode *N); 107249259Sdim SDNode *LowerToFPLitPool(SDNode *Node); 108249259Sdim SDNode *SelectToLitPool(SDNode *N); 109249259Sdim 110249259Sdim SDNode* Select(SDNode*); 111249259Sdimprivate: 112263509Sdim /// Get the opcode for table lookup instruction 113263509Sdim unsigned getTBLOpc(bool IsExt, bool Is64Bit, unsigned NumOfVec); 114263509Sdim 115263509Sdim /// Select NEON table lookup intrinsics. NumVecs should be 1, 2, 3 or 4. 116263509Sdim /// IsExt is to indicate if the result will be extended with an argument. 117263509Sdim SDNode *SelectVTBL(SDNode *N, unsigned NumVecs, bool IsExt); 118263509Sdim 119263509Sdim /// Select NEON load intrinsics. NumVecs should be 1, 2, 3 or 4. 120263509Sdim SDNode *SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs, 121263509Sdim const uint16_t *Opcode); 122263509Sdim 123263509Sdim /// Select NEON store intrinsics. NumVecs should be 1, 2, 3 or 4. 124263509Sdim SDNode *SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, 125263509Sdim const uint16_t *Opcodes); 126263509Sdim 127263509Sdim /// Form sequences of consecutive 64/128-bit registers for use in NEON 128263509Sdim /// instructions making use of a vector-list (e.g. ldN, tbl). Vecs must have 129263509Sdim /// between 1 and 4 elements. If it contains a single element that is returned 130263509Sdim /// unchanged; otherwise a REG_SEQUENCE value is returned. 131263509Sdim SDValue createDTuple(ArrayRef<SDValue> Vecs); 132263509Sdim SDValue createQTuple(ArrayRef<SDValue> Vecs); 133263509Sdim 134263509Sdim /// Generic helper for the createDTuple/createQTuple 135263509Sdim /// functions. Those should almost always be called instead. 136263509Sdim SDValue createTuple(ArrayRef<SDValue> Vecs, unsigned RegClassIDs[], 137263509Sdim unsigned SubRegs[]); 138263509Sdim 139263509Sdim /// Select NEON load-duplicate intrinsics. NumVecs should be 2, 3 or 4. 140263509Sdim /// The opcode array specifies the instructions used for load. 141263509Sdim SDNode *SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs, 142263509Sdim const uint16_t *Opcodes); 143263509Sdim 144263509Sdim /// Select NEON load/store lane intrinsics. NumVecs should be 2, 3 or 4. 145263509Sdim /// The opcode arrays specify the instructions used for load/store. 146263509Sdim SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating, 147263509Sdim unsigned NumVecs, const uint16_t *Opcodes); 148263509Sdim 149263509Sdim SDValue getTargetSubregToReg(int SRIdx, SDLoc DL, EVT VT, EVT VTD, 150263509Sdim SDValue Operand); 151249259Sdim}; 152249259Sdim} 153249259Sdim 154249259Sdimbool 155249259SdimAArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, 156249259Sdim unsigned RegWidth) { 157249259Sdim const ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N); 158249259Sdim if (!CN) return false; 159249259Sdim 160249259Sdim // An FCVT[SU] instruction performs: convertToInt(Val * 2^fbits) where fbits 161249259Sdim // is between 1 and 32 for a destination w-register, or 1 and 64 for an 162249259Sdim // x-register. 163249259Sdim // 164249259Sdim // By this stage, we've detected (fp_to_[su]int (fmul Val, THIS_NODE)) so we 165249259Sdim // want THIS_NODE to be 2^fbits. This is much easier to deal with using 166249259Sdim // integers. 167249259Sdim bool IsExact; 168249259Sdim 169249259Sdim // fbits is between 1 and 64 in the worst-case, which means the fmul 170249259Sdim // could have 2^64 as an actual operand. Need 65 bits of precision. 171249259Sdim APSInt IntVal(65, true); 172249259Sdim CN->getValueAPF().convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact); 173249259Sdim 174249259Sdim // N.b. isPowerOf2 also checks for > 0. 175249259Sdim if (!IsExact || !IntVal.isPowerOf2()) return false; 176249259Sdim unsigned FBits = IntVal.logBase2(); 177249259Sdim 178249259Sdim // Checks above should have guaranteed that we haven't lost information in 179249259Sdim // finding FBits, but it must still be in range. 180249259Sdim if (FBits == 0 || FBits > RegWidth) return false; 181249259Sdim 182249259Sdim FixedPos = CurDAG->getTargetConstant(64 - FBits, MVT::i32); 183249259Sdim return true; 184249259Sdim} 185249259Sdim 186249259Sdimbool 187249259SdimAArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(const SDValue &Op, 188249259Sdim char ConstraintCode, 189249259Sdim std::vector<SDValue> &OutOps) { 190249259Sdim switch (ConstraintCode) { 191249259Sdim default: llvm_unreachable("Unrecognised AArch64 memory constraint"); 192249259Sdim case 'm': 193249259Sdim // FIXME: more freedom is actually permitted for 'm'. We can go 194249259Sdim // hunting for a base and an offset if we want. Of course, since 195249259Sdim // we don't really know how the operand is going to be used we're 196249259Sdim // probably restricted to the load/store pair's simm7 as an offset 197249259Sdim // range anyway. 198249259Sdim case 'Q': 199249259Sdim OutOps.push_back(Op); 200249259Sdim } 201249259Sdim 202249259Sdim return false; 203249259Sdim} 204249259Sdim 205249259Sdimbool 206249259SdimAArch64DAGToDAGISel::SelectFPZeroOperand(SDValue N, SDValue &Dummy) { 207249259Sdim ConstantFPSDNode *Imm = dyn_cast<ConstantFPSDNode>(N); 208249259Sdim if (!Imm || !Imm->getValueAPF().isPosZero()) 209249259Sdim return false; 210249259Sdim 211249259Sdim // Doesn't actually carry any information, but keeps TableGen quiet. 212249259Sdim Dummy = CurDAG->getTargetConstant(0, MVT::i32); 213249259Sdim return true; 214249259Sdim} 215249259Sdim 216249259Sdimbool AArch64DAGToDAGISel::SelectLogicalImm(SDValue N, SDValue &Imm) { 217249259Sdim uint32_t Bits; 218249259Sdim uint32_t RegWidth = N.getValueType().getSizeInBits(); 219249259Sdim 220249259Sdim ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N); 221249259Sdim if (!CN) return false; 222249259Sdim 223249259Sdim if (!A64Imms::isLogicalImm(RegWidth, CN->getZExtValue(), Bits)) 224249259Sdim return false; 225249259Sdim 226249259Sdim Imm = CurDAG->getTargetConstant(Bits, MVT::i32); 227249259Sdim return true; 228249259Sdim} 229249259Sdim 230249259SdimSDNode *AArch64DAGToDAGISel::TrySelectToMoveImm(SDNode *Node) { 231249259Sdim SDNode *ResNode; 232263509Sdim SDLoc dl(Node); 233249259Sdim EVT DestType = Node->getValueType(0); 234249259Sdim unsigned DestWidth = DestType.getSizeInBits(); 235249259Sdim 236249259Sdim unsigned MOVOpcode; 237249259Sdim EVT MOVType; 238249259Sdim int UImm16, Shift; 239249259Sdim uint32_t LogicalBits; 240249259Sdim 241249259Sdim uint64_t BitPat = cast<ConstantSDNode>(Node)->getZExtValue(); 242249259Sdim if (A64Imms::isMOVZImm(DestWidth, BitPat, UImm16, Shift)) { 243249259Sdim MOVType = DestType; 244249259Sdim MOVOpcode = DestWidth == 64 ? AArch64::MOVZxii : AArch64::MOVZwii; 245249259Sdim } else if (A64Imms::isMOVNImm(DestWidth, BitPat, UImm16, Shift)) { 246249259Sdim MOVType = DestType; 247249259Sdim MOVOpcode = DestWidth == 64 ? AArch64::MOVNxii : AArch64::MOVNwii; 248249259Sdim } else if (DestWidth == 64 && A64Imms::isMOVNImm(32, BitPat, UImm16, Shift)) { 249249259Sdim // To get something like 0x0000_0000_ffff_1234 into a 64-bit register we can 250249259Sdim // use a 32-bit instruction: "movn w0, 0xedbc". 251249259Sdim MOVType = MVT::i32; 252249259Sdim MOVOpcode = AArch64::MOVNwii; 253249259Sdim } else if (A64Imms::isLogicalImm(DestWidth, BitPat, LogicalBits)) { 254249259Sdim MOVOpcode = DestWidth == 64 ? AArch64::ORRxxi : AArch64::ORRwwi; 255249259Sdim uint16_t ZR = DestWidth == 64 ? AArch64::XZR : AArch64::WZR; 256249259Sdim 257249259Sdim return CurDAG->getMachineNode(MOVOpcode, dl, DestType, 258249259Sdim CurDAG->getRegister(ZR, DestType), 259249259Sdim CurDAG->getTargetConstant(LogicalBits, MVT::i32)); 260249259Sdim } else { 261249259Sdim // Can't handle it in one instruction. There's scope for permitting two (or 262249259Sdim // more) instructions, but that'll need more thought. 263249259Sdim return NULL; 264249259Sdim } 265249259Sdim 266249259Sdim ResNode = CurDAG->getMachineNode(MOVOpcode, dl, MOVType, 267249259Sdim CurDAG->getTargetConstant(UImm16, MVT::i32), 268249259Sdim CurDAG->getTargetConstant(Shift, MVT::i32)); 269249259Sdim 270249259Sdim if (MOVType != DestType) { 271249259Sdim ResNode = CurDAG->getMachineNode(TargetOpcode::SUBREG_TO_REG, dl, 272249259Sdim MVT::i64, MVT::i32, MVT::Other, 273249259Sdim CurDAG->getTargetConstant(0, MVT::i64), 274249259Sdim SDValue(ResNode, 0), 275249259Sdim CurDAG->getTargetConstant(AArch64::sub_32, MVT::i32)); 276249259Sdim } 277249259Sdim 278249259Sdim return ResNode; 279249259Sdim} 280249259Sdim 281252723SdimSDValue 282263509SdimAArch64DAGToDAGISel::getConstantPoolItemAddress(SDLoc DL, 283252723Sdim const Constant *CV) { 284263509Sdim EVT PtrVT = getTargetLowering()->getPointerTy(); 285252723Sdim 286263509Sdim switch (getTargetLowering()->getTargetMachine().getCodeModel()) { 287252723Sdim case CodeModel::Small: { 288252723Sdim unsigned Alignment = 289263509Sdim getTargetLowering()->getDataLayout()->getABITypeAlignment(CV->getType()); 290252723Sdim return CurDAG->getNode( 291252723Sdim AArch64ISD::WrapperSmall, DL, PtrVT, 292252723Sdim CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_NO_FLAG), 293252723Sdim CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_LO12), 294252723Sdim CurDAG->getConstant(Alignment, MVT::i32)); 295252723Sdim } 296252723Sdim case CodeModel::Large: { 297252723Sdim SDNode *LitAddr; 298252723Sdim LitAddr = CurDAG->getMachineNode( 299252723Sdim AArch64::MOVZxii, DL, PtrVT, 300252723Sdim CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_ABS_G3), 301263509Sdim CurDAG->getTargetConstant(3, MVT::i32)); 302252723Sdim LitAddr = CurDAG->getMachineNode( 303252723Sdim AArch64::MOVKxii, DL, PtrVT, SDValue(LitAddr, 0), 304252723Sdim CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_ABS_G2_NC), 305263509Sdim CurDAG->getTargetConstant(2, MVT::i32)); 306252723Sdim LitAddr = CurDAG->getMachineNode( 307252723Sdim AArch64::MOVKxii, DL, PtrVT, SDValue(LitAddr, 0), 308252723Sdim CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_ABS_G1_NC), 309263509Sdim CurDAG->getTargetConstant(1, MVT::i32)); 310252723Sdim LitAddr = CurDAG->getMachineNode( 311252723Sdim AArch64::MOVKxii, DL, PtrVT, SDValue(LitAddr, 0), 312252723Sdim CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_ABS_G0_NC), 313252723Sdim CurDAG->getTargetConstant(0, MVT::i32)); 314252723Sdim return SDValue(LitAddr, 0); 315252723Sdim } 316252723Sdim default: 317252723Sdim llvm_unreachable("Only small and large code models supported now"); 318252723Sdim } 319252723Sdim} 320252723Sdim 321249259SdimSDNode *AArch64DAGToDAGISel::SelectToLitPool(SDNode *Node) { 322263509Sdim SDLoc DL(Node); 323249259Sdim uint64_t UnsignedVal = cast<ConstantSDNode>(Node)->getZExtValue(); 324249259Sdim int64_t SignedVal = cast<ConstantSDNode>(Node)->getSExtValue(); 325249259Sdim EVT DestType = Node->getValueType(0); 326249259Sdim 327249259Sdim // Since we may end up loading a 64-bit constant from a 32-bit entry the 328249259Sdim // constant in the pool may have a different type to the eventual node. 329249259Sdim ISD::LoadExtType Extension; 330249259Sdim EVT MemType; 331249259Sdim 332249259Sdim assert((DestType == MVT::i64 || DestType == MVT::i32) 333249259Sdim && "Only expect integer constants at the moment"); 334249259Sdim 335249259Sdim if (DestType == MVT::i32) { 336249259Sdim Extension = ISD::NON_EXTLOAD; 337249259Sdim MemType = MVT::i32; 338249259Sdim } else if (UnsignedVal <= UINT32_MAX) { 339249259Sdim Extension = ISD::ZEXTLOAD; 340249259Sdim MemType = MVT::i32; 341249259Sdim } else if (SignedVal >= INT32_MIN && SignedVal <= INT32_MAX) { 342249259Sdim Extension = ISD::SEXTLOAD; 343249259Sdim MemType = MVT::i32; 344249259Sdim } else { 345249259Sdim Extension = ISD::NON_EXTLOAD; 346249259Sdim MemType = MVT::i64; 347249259Sdim } 348249259Sdim 349249259Sdim Constant *CV = ConstantInt::get(Type::getIntNTy(*CurDAG->getContext(), 350249259Sdim MemType.getSizeInBits()), 351249259Sdim UnsignedVal); 352252723Sdim SDValue PoolAddr = getConstantPoolItemAddress(DL, CV); 353263509Sdim unsigned Alignment = 354263509Sdim getTargetLowering()->getDataLayout()->getABITypeAlignment(CV->getType()); 355249259Sdim 356249259Sdim return CurDAG->getExtLoad(Extension, DL, DestType, CurDAG->getEntryNode(), 357249259Sdim PoolAddr, 358249259Sdim MachinePointerInfo::getConstantPool(), MemType, 359249259Sdim /* isVolatile = */ false, 360249259Sdim /* isNonTemporal = */ false, 361249259Sdim Alignment).getNode(); 362249259Sdim} 363249259Sdim 364249259SdimSDNode *AArch64DAGToDAGISel::LowerToFPLitPool(SDNode *Node) { 365263509Sdim SDLoc DL(Node); 366249259Sdim const ConstantFP *FV = cast<ConstantFPSDNode>(Node)->getConstantFPValue(); 367249259Sdim EVT DestType = Node->getValueType(0); 368249259Sdim 369263509Sdim unsigned Alignment = 370263509Sdim getTargetLowering()->getDataLayout()->getABITypeAlignment(FV->getType()); 371252723Sdim SDValue PoolAddr = getConstantPoolItemAddress(DL, FV); 372249259Sdim 373249259Sdim return CurDAG->getLoad(DestType, DL, CurDAG->getEntryNode(), PoolAddr, 374249259Sdim MachinePointerInfo::getConstantPool(), 375249259Sdim /* isVolatile = */ false, 376249259Sdim /* isNonTemporal = */ false, 377249259Sdim /* isInvariant = */ true, 378249259Sdim Alignment).getNode(); 379249259Sdim} 380249259Sdim 381249259Sdimbool 382249259SdimAArch64DAGToDAGISel::SelectTSTBOperand(SDValue N, SDValue &FixedPos, 383249259Sdim unsigned RegWidth) { 384249259Sdim const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N); 385249259Sdim if (!CN) return false; 386249259Sdim 387249259Sdim uint64_t Val = CN->getZExtValue(); 388249259Sdim 389249259Sdim if (!isPowerOf2_64(Val)) return false; 390249259Sdim 391249259Sdim unsigned TestedBit = Log2_64(Val); 392249259Sdim // Checks above should have guaranteed that we haven't lost information in 393249259Sdim // finding TestedBit, but it must still be in range. 394249259Sdim if (TestedBit >= RegWidth) return false; 395249259Sdim 396249259Sdim FixedPos = CurDAG->getTargetConstant(TestedBit, MVT::i64); 397249259Sdim return true; 398249259Sdim} 399249259Sdim 400252723SdimSDNode *AArch64DAGToDAGISel::SelectAtomic(SDNode *Node, unsigned Op8, 401252723Sdim unsigned Op16,unsigned Op32, 402252723Sdim unsigned Op64) { 403252723Sdim // Mostly direct translation to the given operations, except that we preserve 404252723Sdim // the AtomicOrdering for use later on. 405252723Sdim AtomicSDNode *AN = cast<AtomicSDNode>(Node); 406252723Sdim EVT VT = AN->getMemoryVT(); 407252723Sdim 408252723Sdim unsigned Op; 409252723Sdim if (VT == MVT::i8) 410252723Sdim Op = Op8; 411252723Sdim else if (VT == MVT::i16) 412252723Sdim Op = Op16; 413252723Sdim else if (VT == MVT::i32) 414252723Sdim Op = Op32; 415252723Sdim else if (VT == MVT::i64) 416252723Sdim Op = Op64; 417252723Sdim else 418252723Sdim llvm_unreachable("Unexpected atomic operation"); 419252723Sdim 420252723Sdim SmallVector<SDValue, 4> Ops; 421252723Sdim for (unsigned i = 1; i < AN->getNumOperands(); ++i) 422252723Sdim Ops.push_back(AN->getOperand(i)); 423252723Sdim 424252723Sdim Ops.push_back(CurDAG->getTargetConstant(AN->getOrdering(), MVT::i32)); 425252723Sdim Ops.push_back(AN->getOperand(0)); // Chain moves to the end 426252723Sdim 427252723Sdim return CurDAG->SelectNodeTo(Node, Op, 428252723Sdim AN->getValueType(0), MVT::Other, 429252723Sdim &Ops[0], Ops.size()); 430252723Sdim} 431252723Sdim 432263509SdimSDValue AArch64DAGToDAGISel::createDTuple(ArrayRef<SDValue> Regs) { 433263509Sdim static unsigned RegClassIDs[] = { AArch64::DPairRegClassID, 434263509Sdim AArch64::DTripleRegClassID, 435263509Sdim AArch64::DQuadRegClassID }; 436263509Sdim static unsigned SubRegs[] = { AArch64::dsub_0, AArch64::dsub_1, 437263509Sdim AArch64::dsub_2, AArch64::dsub_3 }; 438263509Sdim 439263509Sdim return createTuple(Regs, RegClassIDs, SubRegs); 440263509Sdim} 441263509Sdim 442263509SdimSDValue AArch64DAGToDAGISel::createQTuple(ArrayRef<SDValue> Regs) { 443263509Sdim static unsigned RegClassIDs[] = { AArch64::QPairRegClassID, 444263509Sdim AArch64::QTripleRegClassID, 445263509Sdim AArch64::QQuadRegClassID }; 446263509Sdim static unsigned SubRegs[] = { AArch64::qsub_0, AArch64::qsub_1, 447263509Sdim AArch64::qsub_2, AArch64::qsub_3 }; 448263509Sdim 449263509Sdim return createTuple(Regs, RegClassIDs, SubRegs); 450263509Sdim} 451263509Sdim 452263509SdimSDValue AArch64DAGToDAGISel::createTuple(ArrayRef<SDValue> Regs, 453263509Sdim unsigned RegClassIDs[], 454263509Sdim unsigned SubRegs[]) { 455263509Sdim // There's no special register-class for a vector-list of 1 element: it's just 456263509Sdim // a vector. 457263509Sdim if (Regs.size() == 1) 458263509Sdim return Regs[0]; 459263509Sdim 460263509Sdim assert(Regs.size() >= 2 && Regs.size() <= 4); 461263509Sdim 462263509Sdim SDLoc DL(Regs[0].getNode()); 463263509Sdim 464263509Sdim SmallVector<SDValue, 4> Ops; 465263509Sdim 466263509Sdim // First operand of REG_SEQUENCE is the desired RegClass. 467263509Sdim Ops.push_back( 468263509Sdim CurDAG->getTargetConstant(RegClassIDs[Regs.size() - 2], MVT::i32)); 469263509Sdim 470263509Sdim // Then we get pairs of source & subregister-position for the components. 471263509Sdim for (unsigned i = 0; i < Regs.size(); ++i) { 472263509Sdim Ops.push_back(Regs[i]); 473263509Sdim Ops.push_back(CurDAG->getTargetConstant(SubRegs[i], MVT::i32)); 474263509Sdim } 475263509Sdim 476263509Sdim SDNode *N = 477263509Sdim CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops); 478263509Sdim return SDValue(N, 0); 479263509Sdim} 480263509Sdim 481263509Sdim 482263509Sdim// Get the register stride update opcode of a VLD/VST instruction that 483263509Sdim// is otherwise equivalent to the given fixed stride updating instruction. 484263509Sdimstatic unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) { 485263509Sdim switch (Opc) { 486263509Sdim default: break; 487263509Sdim case AArch64::LD1WB_8B_fixed: return AArch64::LD1WB_8B_register; 488263509Sdim case AArch64::LD1WB_4H_fixed: return AArch64::LD1WB_4H_register; 489263509Sdim case AArch64::LD1WB_2S_fixed: return AArch64::LD1WB_2S_register; 490263509Sdim case AArch64::LD1WB_1D_fixed: return AArch64::LD1WB_1D_register; 491263509Sdim case AArch64::LD1WB_16B_fixed: return AArch64::LD1WB_16B_register; 492263509Sdim case AArch64::LD1WB_8H_fixed: return AArch64::LD1WB_8H_register; 493263509Sdim case AArch64::LD1WB_4S_fixed: return AArch64::LD1WB_4S_register; 494263509Sdim case AArch64::LD1WB_2D_fixed: return AArch64::LD1WB_2D_register; 495263509Sdim 496263509Sdim case AArch64::LD2WB_8B_fixed: return AArch64::LD2WB_8B_register; 497263509Sdim case AArch64::LD2WB_4H_fixed: return AArch64::LD2WB_4H_register; 498263509Sdim case AArch64::LD2WB_2S_fixed: return AArch64::LD2WB_2S_register; 499263509Sdim case AArch64::LD2WB_16B_fixed: return AArch64::LD2WB_16B_register; 500263509Sdim case AArch64::LD2WB_8H_fixed: return AArch64::LD2WB_8H_register; 501263509Sdim case AArch64::LD2WB_4S_fixed: return AArch64::LD2WB_4S_register; 502263509Sdim case AArch64::LD2WB_2D_fixed: return AArch64::LD2WB_2D_register; 503263509Sdim 504263509Sdim case AArch64::LD3WB_8B_fixed: return AArch64::LD3WB_8B_register; 505263509Sdim case AArch64::LD3WB_4H_fixed: return AArch64::LD3WB_4H_register; 506263509Sdim case AArch64::LD3WB_2S_fixed: return AArch64::LD3WB_2S_register; 507263509Sdim case AArch64::LD3WB_16B_fixed: return AArch64::LD3WB_16B_register; 508263509Sdim case AArch64::LD3WB_8H_fixed: return AArch64::LD3WB_8H_register; 509263509Sdim case AArch64::LD3WB_4S_fixed: return AArch64::LD3WB_4S_register; 510263509Sdim case AArch64::LD3WB_2D_fixed: return AArch64::LD3WB_2D_register; 511263509Sdim 512263509Sdim case AArch64::LD4WB_8B_fixed: return AArch64::LD4WB_8B_register; 513263509Sdim case AArch64::LD4WB_4H_fixed: return AArch64::LD4WB_4H_register; 514263509Sdim case AArch64::LD4WB_2S_fixed: return AArch64::LD4WB_2S_register; 515263509Sdim case AArch64::LD4WB_16B_fixed: return AArch64::LD4WB_16B_register; 516263509Sdim case AArch64::LD4WB_8H_fixed: return AArch64::LD4WB_8H_register; 517263509Sdim case AArch64::LD4WB_4S_fixed: return AArch64::LD4WB_4S_register; 518263509Sdim case AArch64::LD4WB_2D_fixed: return AArch64::LD4WB_2D_register; 519263509Sdim 520263509Sdim case AArch64::LD1x2WB_8B_fixed: return AArch64::LD1x2WB_8B_register; 521263509Sdim case AArch64::LD1x2WB_4H_fixed: return AArch64::LD1x2WB_4H_register; 522263509Sdim case AArch64::LD1x2WB_2S_fixed: return AArch64::LD1x2WB_2S_register; 523263509Sdim case AArch64::LD1x2WB_1D_fixed: return AArch64::LD1x2WB_1D_register; 524263509Sdim case AArch64::LD1x2WB_16B_fixed: return AArch64::LD1x2WB_16B_register; 525263509Sdim case AArch64::LD1x2WB_8H_fixed: return AArch64::LD1x2WB_8H_register; 526263509Sdim case AArch64::LD1x2WB_4S_fixed: return AArch64::LD1x2WB_4S_register; 527263509Sdim case AArch64::LD1x2WB_2D_fixed: return AArch64::LD1x2WB_2D_register; 528263509Sdim 529263509Sdim case AArch64::LD1x3WB_8B_fixed: return AArch64::LD1x3WB_8B_register; 530263509Sdim case AArch64::LD1x3WB_4H_fixed: return AArch64::LD1x3WB_4H_register; 531263509Sdim case AArch64::LD1x3WB_2S_fixed: return AArch64::LD1x3WB_2S_register; 532263509Sdim case AArch64::LD1x3WB_1D_fixed: return AArch64::LD1x3WB_1D_register; 533263509Sdim case AArch64::LD1x3WB_16B_fixed: return AArch64::LD1x3WB_16B_register; 534263509Sdim case AArch64::LD1x3WB_8H_fixed: return AArch64::LD1x3WB_8H_register; 535263509Sdim case AArch64::LD1x3WB_4S_fixed: return AArch64::LD1x3WB_4S_register; 536263509Sdim case AArch64::LD1x3WB_2D_fixed: return AArch64::LD1x3WB_2D_register; 537263509Sdim 538263509Sdim case AArch64::LD1x4WB_8B_fixed: return AArch64::LD1x4WB_8B_register; 539263509Sdim case AArch64::LD1x4WB_4H_fixed: return AArch64::LD1x4WB_4H_register; 540263509Sdim case AArch64::LD1x4WB_2S_fixed: return AArch64::LD1x4WB_2S_register; 541263509Sdim case AArch64::LD1x4WB_1D_fixed: return AArch64::LD1x4WB_1D_register; 542263509Sdim case AArch64::LD1x4WB_16B_fixed: return AArch64::LD1x4WB_16B_register; 543263509Sdim case AArch64::LD1x4WB_8H_fixed: return AArch64::LD1x4WB_8H_register; 544263509Sdim case AArch64::LD1x4WB_4S_fixed: return AArch64::LD1x4WB_4S_register; 545263509Sdim case AArch64::LD1x4WB_2D_fixed: return AArch64::LD1x4WB_2D_register; 546263509Sdim 547263509Sdim case AArch64::ST1WB_8B_fixed: return AArch64::ST1WB_8B_register; 548263509Sdim case AArch64::ST1WB_4H_fixed: return AArch64::ST1WB_4H_register; 549263509Sdim case AArch64::ST1WB_2S_fixed: return AArch64::ST1WB_2S_register; 550263509Sdim case AArch64::ST1WB_1D_fixed: return AArch64::ST1WB_1D_register; 551263509Sdim case AArch64::ST1WB_16B_fixed: return AArch64::ST1WB_16B_register; 552263509Sdim case AArch64::ST1WB_8H_fixed: return AArch64::ST1WB_8H_register; 553263509Sdim case AArch64::ST1WB_4S_fixed: return AArch64::ST1WB_4S_register; 554263509Sdim case AArch64::ST1WB_2D_fixed: return AArch64::ST1WB_2D_register; 555263509Sdim 556263509Sdim case AArch64::ST2WB_8B_fixed: return AArch64::ST2WB_8B_register; 557263509Sdim case AArch64::ST2WB_4H_fixed: return AArch64::ST2WB_4H_register; 558263509Sdim case AArch64::ST2WB_2S_fixed: return AArch64::ST2WB_2S_register; 559263509Sdim case AArch64::ST2WB_16B_fixed: return AArch64::ST2WB_16B_register; 560263509Sdim case AArch64::ST2WB_8H_fixed: return AArch64::ST2WB_8H_register; 561263509Sdim case AArch64::ST2WB_4S_fixed: return AArch64::ST2WB_4S_register; 562263509Sdim case AArch64::ST2WB_2D_fixed: return AArch64::ST2WB_2D_register; 563263509Sdim 564263509Sdim case AArch64::ST3WB_8B_fixed: return AArch64::ST3WB_8B_register; 565263509Sdim case AArch64::ST3WB_4H_fixed: return AArch64::ST3WB_4H_register; 566263509Sdim case AArch64::ST3WB_2S_fixed: return AArch64::ST3WB_2S_register; 567263509Sdim case AArch64::ST3WB_16B_fixed: return AArch64::ST3WB_16B_register; 568263509Sdim case AArch64::ST3WB_8H_fixed: return AArch64::ST3WB_8H_register; 569263509Sdim case AArch64::ST3WB_4S_fixed: return AArch64::ST3WB_4S_register; 570263509Sdim case AArch64::ST3WB_2D_fixed: return AArch64::ST3WB_2D_register; 571263509Sdim 572263509Sdim case AArch64::ST4WB_8B_fixed: return AArch64::ST4WB_8B_register; 573263509Sdim case AArch64::ST4WB_4H_fixed: return AArch64::ST4WB_4H_register; 574263509Sdim case AArch64::ST4WB_2S_fixed: return AArch64::ST4WB_2S_register; 575263509Sdim case AArch64::ST4WB_16B_fixed: return AArch64::ST4WB_16B_register; 576263509Sdim case AArch64::ST4WB_8H_fixed: return AArch64::ST4WB_8H_register; 577263509Sdim case AArch64::ST4WB_4S_fixed: return AArch64::ST4WB_4S_register; 578263509Sdim case AArch64::ST4WB_2D_fixed: return AArch64::ST4WB_2D_register; 579263509Sdim 580263509Sdim case AArch64::ST1x2WB_8B_fixed: return AArch64::ST1x2WB_8B_register; 581263509Sdim case AArch64::ST1x2WB_4H_fixed: return AArch64::ST1x2WB_4H_register; 582263509Sdim case AArch64::ST1x2WB_2S_fixed: return AArch64::ST1x2WB_2S_register; 583263509Sdim case AArch64::ST1x2WB_1D_fixed: return AArch64::ST1x2WB_1D_register; 584263509Sdim case AArch64::ST1x2WB_16B_fixed: return AArch64::ST1x2WB_16B_register; 585263509Sdim case AArch64::ST1x2WB_8H_fixed: return AArch64::ST1x2WB_8H_register; 586263509Sdim case AArch64::ST1x2WB_4S_fixed: return AArch64::ST1x2WB_4S_register; 587263509Sdim case AArch64::ST1x2WB_2D_fixed: return AArch64::ST1x2WB_2D_register; 588263509Sdim 589263509Sdim case AArch64::ST1x3WB_8B_fixed: return AArch64::ST1x3WB_8B_register; 590263509Sdim case AArch64::ST1x3WB_4H_fixed: return AArch64::ST1x3WB_4H_register; 591263509Sdim case AArch64::ST1x3WB_2S_fixed: return AArch64::ST1x3WB_2S_register; 592263509Sdim case AArch64::ST1x3WB_1D_fixed: return AArch64::ST1x3WB_1D_register; 593263509Sdim case AArch64::ST1x3WB_16B_fixed: return AArch64::ST1x3WB_16B_register; 594263509Sdim case AArch64::ST1x3WB_8H_fixed: return AArch64::ST1x3WB_8H_register; 595263509Sdim case AArch64::ST1x3WB_4S_fixed: return AArch64::ST1x3WB_4S_register; 596263509Sdim case AArch64::ST1x3WB_2D_fixed: return AArch64::ST1x3WB_2D_register; 597263509Sdim 598263509Sdim case AArch64::ST1x4WB_8B_fixed: return AArch64::ST1x4WB_8B_register; 599263509Sdim case AArch64::ST1x4WB_4H_fixed: return AArch64::ST1x4WB_4H_register; 600263509Sdim case AArch64::ST1x4WB_2S_fixed: return AArch64::ST1x4WB_2S_register; 601263509Sdim case AArch64::ST1x4WB_1D_fixed: return AArch64::ST1x4WB_1D_register; 602263509Sdim case AArch64::ST1x4WB_16B_fixed: return AArch64::ST1x4WB_16B_register; 603263509Sdim case AArch64::ST1x4WB_8H_fixed: return AArch64::ST1x4WB_8H_register; 604263509Sdim case AArch64::ST1x4WB_4S_fixed: return AArch64::ST1x4WB_4S_register; 605263509Sdim case AArch64::ST1x4WB_2D_fixed: return AArch64::ST1x4WB_2D_register; 606263509Sdim 607263509Sdim // Post-index of duplicate loads 608263509Sdim case AArch64::LD2R_WB_8B_fixed: return AArch64::LD2R_WB_8B_register; 609263509Sdim case AArch64::LD2R_WB_4H_fixed: return AArch64::LD2R_WB_4H_register; 610263509Sdim case AArch64::LD2R_WB_2S_fixed: return AArch64::LD2R_WB_2S_register; 611263509Sdim case AArch64::LD2R_WB_1D_fixed: return AArch64::LD2R_WB_1D_register; 612263509Sdim case AArch64::LD2R_WB_16B_fixed: return AArch64::LD2R_WB_16B_register; 613263509Sdim case AArch64::LD2R_WB_8H_fixed: return AArch64::LD2R_WB_8H_register; 614263509Sdim case AArch64::LD2R_WB_4S_fixed: return AArch64::LD2R_WB_4S_register; 615263509Sdim case AArch64::LD2R_WB_2D_fixed: return AArch64::LD2R_WB_2D_register; 616263509Sdim 617263509Sdim case AArch64::LD3R_WB_8B_fixed: return AArch64::LD3R_WB_8B_register; 618263509Sdim case AArch64::LD3R_WB_4H_fixed: return AArch64::LD3R_WB_4H_register; 619263509Sdim case AArch64::LD3R_WB_2S_fixed: return AArch64::LD3R_WB_2S_register; 620263509Sdim case AArch64::LD3R_WB_1D_fixed: return AArch64::LD3R_WB_1D_register; 621263509Sdim case AArch64::LD3R_WB_16B_fixed: return AArch64::LD3R_WB_16B_register; 622263509Sdim case AArch64::LD3R_WB_8H_fixed: return AArch64::LD3R_WB_8H_register; 623263509Sdim case AArch64::LD3R_WB_4S_fixed: return AArch64::LD3R_WB_4S_register; 624263509Sdim case AArch64::LD3R_WB_2D_fixed: return AArch64::LD3R_WB_2D_register; 625263509Sdim 626263509Sdim case AArch64::LD4R_WB_8B_fixed: return AArch64::LD4R_WB_8B_register; 627263509Sdim case AArch64::LD4R_WB_4H_fixed: return AArch64::LD4R_WB_4H_register; 628263509Sdim case AArch64::LD4R_WB_2S_fixed: return AArch64::LD4R_WB_2S_register; 629263509Sdim case AArch64::LD4R_WB_1D_fixed: return AArch64::LD4R_WB_1D_register; 630263509Sdim case AArch64::LD4R_WB_16B_fixed: return AArch64::LD4R_WB_16B_register; 631263509Sdim case AArch64::LD4R_WB_8H_fixed: return AArch64::LD4R_WB_8H_register; 632263509Sdim case AArch64::LD4R_WB_4S_fixed: return AArch64::LD4R_WB_4S_register; 633263509Sdim case AArch64::LD4R_WB_2D_fixed: return AArch64::LD4R_WB_2D_register; 634263509Sdim 635263509Sdim // Post-index of lane loads 636263509Sdim case AArch64::LD2LN_WB_B_fixed: return AArch64::LD2LN_WB_B_register; 637263509Sdim case AArch64::LD2LN_WB_H_fixed: return AArch64::LD2LN_WB_H_register; 638263509Sdim case AArch64::LD2LN_WB_S_fixed: return AArch64::LD2LN_WB_S_register; 639263509Sdim case AArch64::LD2LN_WB_D_fixed: return AArch64::LD2LN_WB_D_register; 640263509Sdim 641263509Sdim case AArch64::LD3LN_WB_B_fixed: return AArch64::LD3LN_WB_B_register; 642263509Sdim case AArch64::LD3LN_WB_H_fixed: return AArch64::LD3LN_WB_H_register; 643263509Sdim case AArch64::LD3LN_WB_S_fixed: return AArch64::LD3LN_WB_S_register; 644263509Sdim case AArch64::LD3LN_WB_D_fixed: return AArch64::LD3LN_WB_D_register; 645263509Sdim 646263509Sdim case AArch64::LD4LN_WB_B_fixed: return AArch64::LD4LN_WB_B_register; 647263509Sdim case AArch64::LD4LN_WB_H_fixed: return AArch64::LD4LN_WB_H_register; 648263509Sdim case AArch64::LD4LN_WB_S_fixed: return AArch64::LD4LN_WB_S_register; 649263509Sdim case AArch64::LD4LN_WB_D_fixed: return AArch64::LD4LN_WB_D_register; 650263509Sdim 651263509Sdim // Post-index of lane stores 652263509Sdim case AArch64::ST2LN_WB_B_fixed: return AArch64::ST2LN_WB_B_register; 653263509Sdim case AArch64::ST2LN_WB_H_fixed: return AArch64::ST2LN_WB_H_register; 654263509Sdim case AArch64::ST2LN_WB_S_fixed: return AArch64::ST2LN_WB_S_register; 655263509Sdim case AArch64::ST2LN_WB_D_fixed: return AArch64::ST2LN_WB_D_register; 656263509Sdim 657263509Sdim case AArch64::ST3LN_WB_B_fixed: return AArch64::ST3LN_WB_B_register; 658263509Sdim case AArch64::ST3LN_WB_H_fixed: return AArch64::ST3LN_WB_H_register; 659263509Sdim case AArch64::ST3LN_WB_S_fixed: return AArch64::ST3LN_WB_S_register; 660263509Sdim case AArch64::ST3LN_WB_D_fixed: return AArch64::ST3LN_WB_D_register; 661263509Sdim 662263509Sdim case AArch64::ST4LN_WB_B_fixed: return AArch64::ST4LN_WB_B_register; 663263509Sdim case AArch64::ST4LN_WB_H_fixed: return AArch64::ST4LN_WB_H_register; 664263509Sdim case AArch64::ST4LN_WB_S_fixed: return AArch64::ST4LN_WB_S_register; 665263509Sdim case AArch64::ST4LN_WB_D_fixed: return AArch64::ST4LN_WB_D_register; 666263509Sdim } 667263509Sdim return Opc; // If not one we handle, return it unchanged. 668263509Sdim} 669263509Sdim 670263509SdimSDNode *AArch64DAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, 671263509Sdim unsigned NumVecs, 672263509Sdim const uint16_t *Opcodes) { 673263509Sdim assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range"); 674263509Sdim 675263509Sdim EVT VT = N->getValueType(0); 676263509Sdim unsigned OpcodeIndex; 677263509Sdim bool is64BitVector = VT.is64BitVector(); 678263509Sdim switch (VT.getScalarType().getSizeInBits()) { 679263509Sdim case 8: OpcodeIndex = is64BitVector ? 0 : 4; break; 680263509Sdim case 16: OpcodeIndex = is64BitVector ? 1 : 5; break; 681263509Sdim case 32: OpcodeIndex = is64BitVector ? 2 : 6; break; 682263509Sdim case 64: OpcodeIndex = is64BitVector ? 3 : 7; break; 683263509Sdim default: llvm_unreachable("unhandled vector load type"); 684263509Sdim } 685263509Sdim unsigned Opc = Opcodes[OpcodeIndex]; 686263509Sdim 687263509Sdim SmallVector<SDValue, 2> Ops; 688263509Sdim unsigned AddrOpIdx = isUpdating ? 1 : 2; 689263509Sdim Ops.push_back(N->getOperand(AddrOpIdx)); // Push back the Memory Address 690263509Sdim 691263509Sdim if (isUpdating) { 692263509Sdim SDValue Inc = N->getOperand(AddrOpIdx + 1); 693263509Sdim if (!isa<ConstantSDNode>(Inc.getNode())) // Increment in Register 694263509Sdim Opc = getVLDSTRegisterUpdateOpcode(Opc); 695263509Sdim Ops.push_back(Inc); 696263509Sdim } 697263509Sdim 698263509Sdim Ops.push_back(N->getOperand(0)); // Push back the Chain 699263509Sdim 700263509Sdim SmallVector<EVT, 3> ResTys; 701263509Sdim // Push back the type of return super register 702263509Sdim if (NumVecs == 1) 703263509Sdim ResTys.push_back(VT); 704263509Sdim else if (NumVecs == 3) 705263509Sdim ResTys.push_back(MVT::Untyped); 706263509Sdim else { 707263509Sdim EVT ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, 708263509Sdim is64BitVector ? NumVecs : NumVecs * 2); 709263509Sdim ResTys.push_back(ResTy); 710263509Sdim } 711263509Sdim 712263509Sdim if (isUpdating) 713263509Sdim ResTys.push_back(MVT::i64); // Type of the updated register 714263509Sdim ResTys.push_back(MVT::Other); // Type of the Chain 715263509Sdim SDLoc dl(N); 716263509Sdim SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 717263509Sdim 718263509Sdim // Transfer memoperands. 719263509Sdim MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 720263509Sdim MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 721263509Sdim cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1); 722263509Sdim 723263509Sdim if (NumVecs == 1) 724263509Sdim return VLd; 725263509Sdim 726263509Sdim // If NumVecs > 1, the return result is a super register containing 2-4 727263509Sdim // consecutive vector registers. 728263509Sdim SDValue SuperReg = SDValue(VLd, 0); 729263509Sdim 730263509Sdim unsigned Sub0 = is64BitVector ? AArch64::dsub_0 : AArch64::qsub_0; 731263509Sdim for (unsigned Vec = 0; Vec < NumVecs; ++Vec) 732263509Sdim ReplaceUses(SDValue(N, Vec), 733263509Sdim CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg)); 734263509Sdim // Update users of the Chain 735263509Sdim ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1)); 736263509Sdim if (isUpdating) 737263509Sdim ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2)); 738263509Sdim 739263509Sdim return NULL; 740263509Sdim} 741263509Sdim 742263509SdimSDNode *AArch64DAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, 743263509Sdim unsigned NumVecs, 744263509Sdim const uint16_t *Opcodes) { 745263509Sdim assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range"); 746263509Sdim SDLoc dl(N); 747263509Sdim 748263509Sdim MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 749263509Sdim MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 750263509Sdim 751263509Sdim unsigned AddrOpIdx = isUpdating ? 1 : 2; 752263509Sdim unsigned Vec0Idx = 3; 753263509Sdim EVT VT = N->getOperand(Vec0Idx).getValueType(); 754263509Sdim unsigned OpcodeIndex; 755263509Sdim bool is64BitVector = VT.is64BitVector(); 756263509Sdim switch (VT.getScalarType().getSizeInBits()) { 757263509Sdim case 8: OpcodeIndex = is64BitVector ? 0 : 4; break; 758263509Sdim case 16: OpcodeIndex = is64BitVector ? 1 : 5; break; 759263509Sdim case 32: OpcodeIndex = is64BitVector ? 2 : 6; break; 760263509Sdim case 64: OpcodeIndex = is64BitVector ? 3 : 7; break; 761263509Sdim default: llvm_unreachable("unhandled vector store type"); 762263509Sdim } 763263509Sdim unsigned Opc = Opcodes[OpcodeIndex]; 764263509Sdim 765263509Sdim SmallVector<EVT, 2> ResTys; 766263509Sdim if (isUpdating) 767263509Sdim ResTys.push_back(MVT::i64); 768263509Sdim ResTys.push_back(MVT::Other); // Type for the Chain 769263509Sdim 770263509Sdim SmallVector<SDValue, 6> Ops; 771263509Sdim Ops.push_back(N->getOperand(AddrOpIdx)); // Push back the Memory Address 772263509Sdim 773263509Sdim if (isUpdating) { 774263509Sdim SDValue Inc = N->getOperand(AddrOpIdx + 1); 775263509Sdim if (!isa<ConstantSDNode>(Inc.getNode())) // Increment in Register 776263509Sdim Opc = getVLDSTRegisterUpdateOpcode(Opc); 777263509Sdim Ops.push_back(Inc); 778263509Sdim } 779263509Sdim 780263509Sdim SmallVector<SDValue, 4> Regs(N->op_begin() + Vec0Idx, 781263509Sdim N->op_begin() + Vec0Idx + NumVecs); 782263509Sdim SDValue SrcReg = is64BitVector ? createDTuple(Regs) : createQTuple(Regs); 783263509Sdim Ops.push_back(SrcReg); 784263509Sdim 785263509Sdim // Push back the Chain 786263509Sdim Ops.push_back(N->getOperand(0)); 787263509Sdim 788263509Sdim // Transfer memoperands. 789263509Sdim SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 790263509Sdim cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1); 791263509Sdim 792263509Sdim return VSt; 793263509Sdim} 794263509Sdim 795263509SdimSDValue 796263509SdimAArch64DAGToDAGISel::getTargetSubregToReg(int SRIdx, SDLoc DL, EVT VT, EVT VTD, 797263509Sdim SDValue Operand) { 798263509Sdim SDNode *Reg = CurDAG->getMachineNode(TargetOpcode::SUBREG_TO_REG, DL, 799263509Sdim VT, VTD, MVT::Other, 800263509Sdim CurDAG->getTargetConstant(0, MVT::i64), 801263509Sdim Operand, 802263509Sdim CurDAG->getTargetConstant(AArch64::sub_64, MVT::i32)); 803263509Sdim return SDValue(Reg, 0); 804263509Sdim} 805263509Sdim 806263509SdimSDNode *AArch64DAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating, 807263509Sdim unsigned NumVecs, 808263509Sdim const uint16_t *Opcodes) { 809263509Sdim assert(NumVecs >=2 && NumVecs <= 4 && "Load Dup NumVecs out-of-range"); 810263509Sdim SDLoc dl(N); 811263509Sdim 812263509Sdim EVT VT = N->getValueType(0); 813263509Sdim unsigned OpcodeIndex; 814263509Sdim bool is64BitVector = VT.is64BitVector(); 815263509Sdim switch (VT.getScalarType().getSizeInBits()) { 816263509Sdim case 8: OpcodeIndex = is64BitVector ? 0 : 4; break; 817263509Sdim case 16: OpcodeIndex = is64BitVector ? 1 : 5; break; 818263509Sdim case 32: OpcodeIndex = is64BitVector ? 2 : 6; break; 819263509Sdim case 64: OpcodeIndex = is64BitVector ? 3 : 7; break; 820263509Sdim default: llvm_unreachable("unhandled vector duplicate lane load type"); 821263509Sdim } 822263509Sdim unsigned Opc = Opcodes[OpcodeIndex]; 823263509Sdim 824263509Sdim SDValue SuperReg; 825263509Sdim SmallVector<SDValue, 6> Ops; 826263509Sdim Ops.push_back(N->getOperand(1)); // Push back the Memory Address 827263509Sdim if (isUpdating) { 828263509Sdim SDValue Inc = N->getOperand(2); 829263509Sdim if (!isa<ConstantSDNode>(Inc.getNode())) // Increment in Register 830263509Sdim Opc = getVLDSTRegisterUpdateOpcode(Opc); 831263509Sdim Ops.push_back(Inc); 832263509Sdim } 833263509Sdim Ops.push_back(N->getOperand(0)); // Push back the Chain 834263509Sdim 835263509Sdim SmallVector<EVT, 3> ResTys; 836263509Sdim // Push back the type of return super register 837263509Sdim if (NumVecs == 3) 838263509Sdim ResTys.push_back(MVT::Untyped); 839263509Sdim else { 840263509Sdim EVT ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, 841263509Sdim is64BitVector ? NumVecs : NumVecs * 2); 842263509Sdim ResTys.push_back(ResTy); 843263509Sdim } 844263509Sdim if (isUpdating) 845263509Sdim ResTys.push_back(MVT::i64); // Type of the updated register 846263509Sdim ResTys.push_back(MVT::Other); // Type of the Chain 847263509Sdim SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 848263509Sdim 849263509Sdim // Transfer memoperands. 850263509Sdim MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 851263509Sdim MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 852263509Sdim cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1); 853263509Sdim 854263509Sdim SuperReg = SDValue(VLdDup, 0); 855263509Sdim unsigned Sub0 = is64BitVector ? AArch64::dsub_0 : AArch64::qsub_0; 856263509Sdim // Update uses of each registers in super register 857263509Sdim for (unsigned Vec = 0; Vec < NumVecs; ++Vec) 858263509Sdim ReplaceUses(SDValue(N, Vec), 859263509Sdim CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg)); 860263509Sdim // Update uses of the Chain 861263509Sdim ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1)); 862263509Sdim if (isUpdating) 863263509Sdim ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2)); 864263509Sdim return NULL; 865263509Sdim} 866263509Sdim 867263509Sdim// We only have 128-bit vector type of load/store lane instructions. 868263509Sdim// If it is 64-bit vector, we also select it to the 128-bit instructions. 869263509Sdim// Just use SUBREG_TO_REG to adapt the input to 128-bit vector and 870263509Sdim// EXTRACT_SUBREG to get the 64-bit vector from the 128-bit vector output. 871263509SdimSDNode *AArch64DAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, 872263509Sdim bool isUpdating, unsigned NumVecs, 873263509Sdim const uint16_t *Opcodes) { 874263509Sdim assert(NumVecs >= 2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range"); 875263509Sdim SDLoc dl(N); 876263509Sdim unsigned AddrOpIdx = isUpdating ? 1 : 2; 877263509Sdim unsigned Vec0Idx = 3; 878263509Sdim 879263509Sdim SDValue Chain = N->getOperand(0); 880263509Sdim unsigned Lane = 881263509Sdim cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue(); 882263509Sdim EVT VT = N->getOperand(Vec0Idx).getValueType(); 883263509Sdim bool is64BitVector = VT.is64BitVector(); 884263509Sdim EVT VT64; // 64-bit Vector Type 885263509Sdim 886263509Sdim if (is64BitVector) { 887263509Sdim VT64 = VT; 888263509Sdim VT = EVT::getVectorVT(*CurDAG->getContext(), VT.getVectorElementType(), 889263509Sdim VT.getVectorNumElements() * 2); 890263509Sdim } 891263509Sdim 892263509Sdim unsigned OpcodeIndex; 893263509Sdim switch (VT.getScalarType().getSizeInBits()) { 894263509Sdim case 8: OpcodeIndex = 0; break; 895263509Sdim case 16: OpcodeIndex = 1; break; 896263509Sdim case 32: OpcodeIndex = 2; break; 897263509Sdim case 64: OpcodeIndex = 3; break; 898263509Sdim default: llvm_unreachable("unhandled vector lane load/store type"); 899263509Sdim } 900263509Sdim unsigned Opc = Opcodes[OpcodeIndex]; 901263509Sdim 902263509Sdim SmallVector<EVT, 3> ResTys; 903263509Sdim if (IsLoad) { 904263509Sdim // Push back the type of return super register 905263509Sdim if (NumVecs == 3) 906263509Sdim ResTys.push_back(MVT::Untyped); 907263509Sdim else { 908263509Sdim EVT ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, 909263509Sdim is64BitVector ? NumVecs : NumVecs * 2); 910263509Sdim ResTys.push_back(ResTy); 911263509Sdim } 912263509Sdim } 913263509Sdim if (isUpdating) 914263509Sdim ResTys.push_back(MVT::i64); // Type of the updated register 915263509Sdim ResTys.push_back(MVT::Other); // Type of Chain 916263509Sdim SmallVector<SDValue, 5> Ops; 917263509Sdim Ops.push_back(N->getOperand(AddrOpIdx)); // Push back the Memory Address 918263509Sdim if (isUpdating) { 919263509Sdim SDValue Inc = N->getOperand(AddrOpIdx + 1); 920263509Sdim if (!isa<ConstantSDNode>(Inc.getNode())) // Increment in Register 921263509Sdim Opc = getVLDSTRegisterUpdateOpcode(Opc); 922263509Sdim Ops.push_back(Inc); 923263509Sdim } 924263509Sdim 925263509Sdim SmallVector<SDValue, 4> Regs(N->op_begin() + Vec0Idx, 926263509Sdim N->op_begin() + Vec0Idx + NumVecs); 927263509Sdim if (is64BitVector) 928263509Sdim for (unsigned i = 0; i < Regs.size(); i++) 929263509Sdim Regs[i] = getTargetSubregToReg(AArch64::sub_64, dl, VT, VT64, Regs[i]); 930263509Sdim SDValue SuperReg = createQTuple(Regs); 931263509Sdim 932263509Sdim Ops.push_back(SuperReg); // Source Reg 933263509Sdim SDValue LaneValue = CurDAG->getTargetConstant(Lane, MVT::i32); 934263509Sdim Ops.push_back(LaneValue); 935263509Sdim Ops.push_back(Chain); // Push back the Chain 936263509Sdim 937263509Sdim SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 938263509Sdim MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 939263509Sdim MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 940263509Sdim cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1); 941263509Sdim if (!IsLoad) 942263509Sdim return VLdLn; 943263509Sdim 944263509Sdim // Extract the subregisters. 945263509Sdim SuperReg = SDValue(VLdLn, 0); 946263509Sdim unsigned Sub0 = AArch64::qsub_0; 947263509Sdim // Update uses of each registers in super register 948263509Sdim for (unsigned Vec = 0; Vec < NumVecs; ++Vec) { 949263509Sdim SDValue SUB0 = CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg); 950263509Sdim if (is64BitVector) { 951263509Sdim SUB0 = CurDAG->getTargetExtractSubreg(AArch64::sub_64, dl, VT64, SUB0); 952263509Sdim } 953263509Sdim ReplaceUses(SDValue(N, Vec), SUB0); 954263509Sdim } 955263509Sdim ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1)); 956263509Sdim if (isUpdating) 957263509Sdim ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2)); 958263509Sdim return NULL; 959263509Sdim} 960263509Sdim 961263509Sdimunsigned AArch64DAGToDAGISel::getTBLOpc(bool IsExt, bool Is64Bit, 962263509Sdim unsigned NumOfVec) { 963263509Sdim assert(NumOfVec >= 1 && NumOfVec <= 4 && "VST NumVecs out-of-range"); 964263509Sdim 965263509Sdim unsigned Opc = 0; 966263509Sdim switch (NumOfVec) { 967263509Sdim default: 968263509Sdim break; 969263509Sdim case 1: 970263509Sdim if (IsExt) 971263509Sdim Opc = Is64Bit ? AArch64::TBX1_8b : AArch64::TBX1_16b; 972263509Sdim else 973263509Sdim Opc = Is64Bit ? AArch64::TBL1_8b : AArch64::TBL1_16b; 974263509Sdim break; 975263509Sdim case 2: 976263509Sdim if (IsExt) 977263509Sdim Opc = Is64Bit ? AArch64::TBX2_8b : AArch64::TBX2_16b; 978263509Sdim else 979263509Sdim Opc = Is64Bit ? AArch64::TBL2_8b : AArch64::TBL2_16b; 980263509Sdim break; 981263509Sdim case 3: 982263509Sdim if (IsExt) 983263509Sdim Opc = Is64Bit ? AArch64::TBX3_8b : AArch64::TBX3_16b; 984263509Sdim else 985263509Sdim Opc = Is64Bit ? AArch64::TBL3_8b : AArch64::TBL3_16b; 986263509Sdim break; 987263509Sdim case 4: 988263509Sdim if (IsExt) 989263509Sdim Opc = Is64Bit ? AArch64::TBX4_8b : AArch64::TBX4_16b; 990263509Sdim else 991263509Sdim Opc = Is64Bit ? AArch64::TBL4_8b : AArch64::TBL4_16b; 992263509Sdim break; 993263509Sdim } 994263509Sdim 995263509Sdim return Opc; 996263509Sdim} 997263509Sdim 998263509SdimSDNode *AArch64DAGToDAGISel::SelectVTBL(SDNode *N, unsigned NumVecs, 999263509Sdim bool IsExt) { 1000263509Sdim assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range"); 1001263509Sdim SDLoc dl(N); 1002263509Sdim 1003263509Sdim // Check the element of look up table is 64-bit or not 1004263509Sdim unsigned Vec0Idx = IsExt ? 2 : 1; 1005263509Sdim assert(!N->getOperand(Vec0Idx + 0).getValueType().is64BitVector() && 1006263509Sdim "The element of lookup table for vtbl and vtbx must be 128-bit"); 1007263509Sdim 1008263509Sdim // Check the return value type is 64-bit or not 1009263509Sdim EVT ResVT = N->getValueType(0); 1010263509Sdim bool is64BitRes = ResVT.is64BitVector(); 1011263509Sdim 1012263509Sdim // Create new SDValue for vector list 1013263509Sdim SmallVector<SDValue, 4> Regs(N->op_begin() + Vec0Idx, 1014263509Sdim N->op_begin() + Vec0Idx + NumVecs); 1015263509Sdim SDValue TblReg = createQTuple(Regs); 1016263509Sdim unsigned Opc = getTBLOpc(IsExt, is64BitRes, NumVecs); 1017263509Sdim 1018263509Sdim SmallVector<SDValue, 3> Ops; 1019263509Sdim if (IsExt) 1020263509Sdim Ops.push_back(N->getOperand(1)); 1021263509Sdim Ops.push_back(TblReg); 1022263509Sdim Ops.push_back(N->getOperand(Vec0Idx + NumVecs)); 1023263509Sdim return CurDAG->getMachineNode(Opc, dl, ResVT, Ops); 1024263509Sdim} 1025263509Sdim 1026249259SdimSDNode *AArch64DAGToDAGISel::Select(SDNode *Node) { 1027249259Sdim // Dump information about the Node being selected 1028249259Sdim DEBUG(dbgs() << "Selecting: "; Node->dump(CurDAG); dbgs() << "\n"); 1029249259Sdim 1030249259Sdim if (Node->isMachineOpcode()) { 1031249259Sdim DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n"); 1032255946Sdim Node->setNodeId(-1); 1033249259Sdim return NULL; 1034249259Sdim } 1035249259Sdim 1036249259Sdim switch (Node->getOpcode()) { 1037252723Sdim case ISD::ATOMIC_LOAD_ADD: 1038252723Sdim return SelectAtomic(Node, 1039252723Sdim AArch64::ATOMIC_LOAD_ADD_I8, 1040252723Sdim AArch64::ATOMIC_LOAD_ADD_I16, 1041252723Sdim AArch64::ATOMIC_LOAD_ADD_I32, 1042252723Sdim AArch64::ATOMIC_LOAD_ADD_I64); 1043252723Sdim case ISD::ATOMIC_LOAD_SUB: 1044252723Sdim return SelectAtomic(Node, 1045252723Sdim AArch64::ATOMIC_LOAD_SUB_I8, 1046252723Sdim AArch64::ATOMIC_LOAD_SUB_I16, 1047252723Sdim AArch64::ATOMIC_LOAD_SUB_I32, 1048252723Sdim AArch64::ATOMIC_LOAD_SUB_I64); 1049252723Sdim case ISD::ATOMIC_LOAD_AND: 1050252723Sdim return SelectAtomic(Node, 1051252723Sdim AArch64::ATOMIC_LOAD_AND_I8, 1052252723Sdim AArch64::ATOMIC_LOAD_AND_I16, 1053252723Sdim AArch64::ATOMIC_LOAD_AND_I32, 1054252723Sdim AArch64::ATOMIC_LOAD_AND_I64); 1055252723Sdim case ISD::ATOMIC_LOAD_OR: 1056252723Sdim return SelectAtomic(Node, 1057252723Sdim AArch64::ATOMIC_LOAD_OR_I8, 1058252723Sdim AArch64::ATOMIC_LOAD_OR_I16, 1059252723Sdim AArch64::ATOMIC_LOAD_OR_I32, 1060252723Sdim AArch64::ATOMIC_LOAD_OR_I64); 1061252723Sdim case ISD::ATOMIC_LOAD_XOR: 1062252723Sdim return SelectAtomic(Node, 1063252723Sdim AArch64::ATOMIC_LOAD_XOR_I8, 1064252723Sdim AArch64::ATOMIC_LOAD_XOR_I16, 1065252723Sdim AArch64::ATOMIC_LOAD_XOR_I32, 1066252723Sdim AArch64::ATOMIC_LOAD_XOR_I64); 1067252723Sdim case ISD::ATOMIC_LOAD_NAND: 1068252723Sdim return SelectAtomic(Node, 1069252723Sdim AArch64::ATOMIC_LOAD_NAND_I8, 1070252723Sdim AArch64::ATOMIC_LOAD_NAND_I16, 1071252723Sdim AArch64::ATOMIC_LOAD_NAND_I32, 1072252723Sdim AArch64::ATOMIC_LOAD_NAND_I64); 1073252723Sdim case ISD::ATOMIC_LOAD_MIN: 1074252723Sdim return SelectAtomic(Node, 1075252723Sdim AArch64::ATOMIC_LOAD_MIN_I8, 1076252723Sdim AArch64::ATOMIC_LOAD_MIN_I16, 1077252723Sdim AArch64::ATOMIC_LOAD_MIN_I32, 1078252723Sdim AArch64::ATOMIC_LOAD_MIN_I64); 1079252723Sdim case ISD::ATOMIC_LOAD_MAX: 1080252723Sdim return SelectAtomic(Node, 1081252723Sdim AArch64::ATOMIC_LOAD_MAX_I8, 1082252723Sdim AArch64::ATOMIC_LOAD_MAX_I16, 1083252723Sdim AArch64::ATOMIC_LOAD_MAX_I32, 1084252723Sdim AArch64::ATOMIC_LOAD_MAX_I64); 1085252723Sdim case ISD::ATOMIC_LOAD_UMIN: 1086252723Sdim return SelectAtomic(Node, 1087252723Sdim AArch64::ATOMIC_LOAD_UMIN_I8, 1088252723Sdim AArch64::ATOMIC_LOAD_UMIN_I16, 1089252723Sdim AArch64::ATOMIC_LOAD_UMIN_I32, 1090252723Sdim AArch64::ATOMIC_LOAD_UMIN_I64); 1091252723Sdim case ISD::ATOMIC_LOAD_UMAX: 1092252723Sdim return SelectAtomic(Node, 1093252723Sdim AArch64::ATOMIC_LOAD_UMAX_I8, 1094252723Sdim AArch64::ATOMIC_LOAD_UMAX_I16, 1095252723Sdim AArch64::ATOMIC_LOAD_UMAX_I32, 1096252723Sdim AArch64::ATOMIC_LOAD_UMAX_I64); 1097252723Sdim case ISD::ATOMIC_SWAP: 1098252723Sdim return SelectAtomic(Node, 1099252723Sdim AArch64::ATOMIC_SWAP_I8, 1100252723Sdim AArch64::ATOMIC_SWAP_I16, 1101252723Sdim AArch64::ATOMIC_SWAP_I32, 1102252723Sdim AArch64::ATOMIC_SWAP_I64); 1103252723Sdim case ISD::ATOMIC_CMP_SWAP: 1104252723Sdim return SelectAtomic(Node, 1105252723Sdim AArch64::ATOMIC_CMP_SWAP_I8, 1106252723Sdim AArch64::ATOMIC_CMP_SWAP_I16, 1107252723Sdim AArch64::ATOMIC_CMP_SWAP_I32, 1108252723Sdim AArch64::ATOMIC_CMP_SWAP_I64); 1109249259Sdim case ISD::FrameIndex: { 1110249259Sdim int FI = cast<FrameIndexSDNode>(Node)->getIndex(); 1111263509Sdim EVT PtrTy = getTargetLowering()->getPointerTy(); 1112249259Sdim SDValue TFI = CurDAG->getTargetFrameIndex(FI, PtrTy); 1113249259Sdim return CurDAG->SelectNodeTo(Node, AArch64::ADDxxi_lsl0_s, PtrTy, 1114249259Sdim TFI, CurDAG->getTargetConstant(0, PtrTy)); 1115249259Sdim } 1116249259Sdim case ISD::ConstantPool: { 1117249259Sdim // Constant pools are fine, just create a Target entry. 1118249259Sdim ConstantPoolSDNode *CN = cast<ConstantPoolSDNode>(Node); 1119249259Sdim const Constant *C = CN->getConstVal(); 1120249259Sdim SDValue CP = CurDAG->getTargetConstantPool(C, CN->getValueType(0)); 1121249259Sdim 1122249259Sdim ReplaceUses(SDValue(Node, 0), CP); 1123249259Sdim return NULL; 1124249259Sdim } 1125249259Sdim case ISD::Constant: { 1126249259Sdim SDNode *ResNode = 0; 1127249259Sdim if (cast<ConstantSDNode>(Node)->getZExtValue() == 0) { 1128249259Sdim // XZR and WZR are probably even better than an actual move: most of the 1129249259Sdim // time they can be folded into another instruction with *no* cost. 1130249259Sdim 1131249259Sdim EVT Ty = Node->getValueType(0); 1132249259Sdim assert((Ty == MVT::i32 || Ty == MVT::i64) && "unexpected type"); 1133249259Sdim uint16_t Register = Ty == MVT::i32 ? AArch64::WZR : AArch64::XZR; 1134249259Sdim ResNode = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), 1135263509Sdim SDLoc(Node), 1136249259Sdim Register, Ty).getNode(); 1137249259Sdim } 1138249259Sdim 1139249259Sdim // Next best option is a move-immediate, see if we can do that. 1140249259Sdim if (!ResNode) { 1141249259Sdim ResNode = TrySelectToMoveImm(Node); 1142249259Sdim } 1143249259Sdim 1144249259Sdim if (ResNode) 1145249259Sdim return ResNode; 1146249259Sdim 1147249259Sdim // If even that fails we fall back to a lit-pool entry at the moment. Future 1148249259Sdim // tuning may change this to a sequence of MOVZ/MOVN/MOVK instructions. 1149249259Sdim ResNode = SelectToLitPool(Node); 1150249259Sdim assert(ResNode && "We need *some* way to materialise a constant"); 1151249259Sdim 1152249259Sdim // We want to continue selection at this point since the litpool access 1153249259Sdim // generated used generic nodes for simplicity. 1154249259Sdim ReplaceUses(SDValue(Node, 0), SDValue(ResNode, 0)); 1155249259Sdim Node = ResNode; 1156249259Sdim break; 1157249259Sdim } 1158249259Sdim case ISD::ConstantFP: { 1159249259Sdim if (A64Imms::isFPImm(cast<ConstantFPSDNode>(Node)->getValueAPF())) { 1160249259Sdim // FMOV will take care of it from TableGen 1161249259Sdim break; 1162249259Sdim } 1163249259Sdim 1164249259Sdim SDNode *ResNode = LowerToFPLitPool(Node); 1165249259Sdim ReplaceUses(SDValue(Node, 0), SDValue(ResNode, 0)); 1166249259Sdim 1167249259Sdim // We want to continue selection at this point since the litpool access 1168249259Sdim // generated used generic nodes for simplicity. 1169249259Sdim Node = ResNode; 1170249259Sdim break; 1171249259Sdim } 1172263509Sdim case AArch64ISD::NEON_LD1_UPD: { 1173263509Sdim static const uint16_t Opcodes[] = { 1174263509Sdim AArch64::LD1WB_8B_fixed, AArch64::LD1WB_4H_fixed, 1175263509Sdim AArch64::LD1WB_2S_fixed, AArch64::LD1WB_1D_fixed, 1176263509Sdim AArch64::LD1WB_16B_fixed, AArch64::LD1WB_8H_fixed, 1177263509Sdim AArch64::LD1WB_4S_fixed, AArch64::LD1WB_2D_fixed 1178263509Sdim }; 1179263509Sdim return SelectVLD(Node, true, 1, Opcodes); 1180263509Sdim } 1181263509Sdim case AArch64ISD::NEON_LD2_UPD: { 1182263509Sdim static const uint16_t Opcodes[] = { 1183263509Sdim AArch64::LD2WB_8B_fixed, AArch64::LD2WB_4H_fixed, 1184263509Sdim AArch64::LD2WB_2S_fixed, AArch64::LD1x2WB_1D_fixed, 1185263509Sdim AArch64::LD2WB_16B_fixed, AArch64::LD2WB_8H_fixed, 1186263509Sdim AArch64::LD2WB_4S_fixed, AArch64::LD2WB_2D_fixed 1187263509Sdim }; 1188263509Sdim return SelectVLD(Node, true, 2, Opcodes); 1189263509Sdim } 1190263509Sdim case AArch64ISD::NEON_LD3_UPD: { 1191263509Sdim static const uint16_t Opcodes[] = { 1192263509Sdim AArch64::LD3WB_8B_fixed, AArch64::LD3WB_4H_fixed, 1193263509Sdim AArch64::LD3WB_2S_fixed, AArch64::LD1x3WB_1D_fixed, 1194263509Sdim AArch64::LD3WB_16B_fixed, AArch64::LD3WB_8H_fixed, 1195263509Sdim AArch64::LD3WB_4S_fixed, AArch64::LD3WB_2D_fixed 1196263509Sdim }; 1197263509Sdim return SelectVLD(Node, true, 3, Opcodes); 1198263509Sdim } 1199263509Sdim case AArch64ISD::NEON_LD4_UPD: { 1200263509Sdim static const uint16_t Opcodes[] = { 1201263509Sdim AArch64::LD4WB_8B_fixed, AArch64::LD4WB_4H_fixed, 1202263509Sdim AArch64::LD4WB_2S_fixed, AArch64::LD1x4WB_1D_fixed, 1203263509Sdim AArch64::LD4WB_16B_fixed, AArch64::LD4WB_8H_fixed, 1204263509Sdim AArch64::LD4WB_4S_fixed, AArch64::LD4WB_2D_fixed 1205263509Sdim }; 1206263509Sdim return SelectVLD(Node, true, 4, Opcodes); 1207263509Sdim } 1208263509Sdim case AArch64ISD::NEON_LD1x2_UPD: { 1209263509Sdim static const uint16_t Opcodes[] = { 1210263509Sdim AArch64::LD1x2WB_8B_fixed, AArch64::LD1x2WB_4H_fixed, 1211263509Sdim AArch64::LD1x2WB_2S_fixed, AArch64::LD1x2WB_1D_fixed, 1212263509Sdim AArch64::LD1x2WB_16B_fixed, AArch64::LD1x2WB_8H_fixed, 1213263509Sdim AArch64::LD1x2WB_4S_fixed, AArch64::LD1x2WB_2D_fixed 1214263509Sdim }; 1215263509Sdim return SelectVLD(Node, true, 2, Opcodes); 1216263509Sdim } 1217263509Sdim case AArch64ISD::NEON_LD1x3_UPD: { 1218263509Sdim static const uint16_t Opcodes[] = { 1219263509Sdim AArch64::LD1x3WB_8B_fixed, AArch64::LD1x3WB_4H_fixed, 1220263509Sdim AArch64::LD1x3WB_2S_fixed, AArch64::LD1x3WB_1D_fixed, 1221263509Sdim AArch64::LD1x3WB_16B_fixed, AArch64::LD1x3WB_8H_fixed, 1222263509Sdim AArch64::LD1x3WB_4S_fixed, AArch64::LD1x3WB_2D_fixed 1223263509Sdim }; 1224263509Sdim return SelectVLD(Node, true, 3, Opcodes); 1225263509Sdim } 1226263509Sdim case AArch64ISD::NEON_LD1x4_UPD: { 1227263509Sdim static const uint16_t Opcodes[] = { 1228263509Sdim AArch64::LD1x4WB_8B_fixed, AArch64::LD1x4WB_4H_fixed, 1229263509Sdim AArch64::LD1x4WB_2S_fixed, AArch64::LD1x4WB_1D_fixed, 1230263509Sdim AArch64::LD1x4WB_16B_fixed, AArch64::LD1x4WB_8H_fixed, 1231263509Sdim AArch64::LD1x4WB_4S_fixed, AArch64::LD1x4WB_2D_fixed 1232263509Sdim }; 1233263509Sdim return SelectVLD(Node, true, 4, Opcodes); 1234263509Sdim } 1235263509Sdim case AArch64ISD::NEON_ST1_UPD: { 1236263509Sdim static const uint16_t Opcodes[] = { 1237263509Sdim AArch64::ST1WB_8B_fixed, AArch64::ST1WB_4H_fixed, 1238263509Sdim AArch64::ST1WB_2S_fixed, AArch64::ST1WB_1D_fixed, 1239263509Sdim AArch64::ST1WB_16B_fixed, AArch64::ST1WB_8H_fixed, 1240263509Sdim AArch64::ST1WB_4S_fixed, AArch64::ST1WB_2D_fixed 1241263509Sdim }; 1242263509Sdim return SelectVST(Node, true, 1, Opcodes); 1243263509Sdim } 1244263509Sdim case AArch64ISD::NEON_ST2_UPD: { 1245263509Sdim static const uint16_t Opcodes[] = { 1246263509Sdim AArch64::ST2WB_8B_fixed, AArch64::ST2WB_4H_fixed, 1247263509Sdim AArch64::ST2WB_2S_fixed, AArch64::ST1x2WB_1D_fixed, 1248263509Sdim AArch64::ST2WB_16B_fixed, AArch64::ST2WB_8H_fixed, 1249263509Sdim AArch64::ST2WB_4S_fixed, AArch64::ST2WB_2D_fixed 1250263509Sdim }; 1251263509Sdim return SelectVST(Node, true, 2, Opcodes); 1252263509Sdim } 1253263509Sdim case AArch64ISD::NEON_ST3_UPD: { 1254263509Sdim static const uint16_t Opcodes[] = { 1255263509Sdim AArch64::ST3WB_8B_fixed, AArch64::ST3WB_4H_fixed, 1256263509Sdim AArch64::ST3WB_2S_fixed, AArch64::ST1x3WB_1D_fixed, 1257263509Sdim AArch64::ST3WB_16B_fixed, AArch64::ST3WB_8H_fixed, 1258263509Sdim AArch64::ST3WB_4S_fixed, AArch64::ST3WB_2D_fixed 1259263509Sdim }; 1260263509Sdim return SelectVST(Node, true, 3, Opcodes); 1261263509Sdim } 1262263509Sdim case AArch64ISD::NEON_ST4_UPD: { 1263263509Sdim static const uint16_t Opcodes[] = { 1264263509Sdim AArch64::ST4WB_8B_fixed, AArch64::ST4WB_4H_fixed, 1265263509Sdim AArch64::ST4WB_2S_fixed, AArch64::ST1x4WB_1D_fixed, 1266263509Sdim AArch64::ST4WB_16B_fixed, AArch64::ST4WB_8H_fixed, 1267263509Sdim AArch64::ST4WB_4S_fixed, AArch64::ST4WB_2D_fixed 1268263509Sdim }; 1269263509Sdim return SelectVST(Node, true, 4, Opcodes); 1270263509Sdim } 1271263509Sdim case AArch64ISD::NEON_LD2DUP: { 1272263509Sdim static const uint16_t Opcodes[] = { 1273263509Sdim AArch64::LD2R_8B, AArch64::LD2R_4H, AArch64::LD2R_2S, 1274263509Sdim AArch64::LD2R_1D, AArch64::LD2R_16B, AArch64::LD2R_8H, 1275263509Sdim AArch64::LD2R_4S, AArch64::LD2R_2D 1276263509Sdim }; 1277263509Sdim return SelectVLDDup(Node, false, 2, Opcodes); 1278263509Sdim } 1279263509Sdim case AArch64ISD::NEON_LD3DUP: { 1280263509Sdim static const uint16_t Opcodes[] = { 1281263509Sdim AArch64::LD3R_8B, AArch64::LD3R_4H, AArch64::LD3R_2S, 1282263509Sdim AArch64::LD3R_1D, AArch64::LD3R_16B, AArch64::LD3R_8H, 1283263509Sdim AArch64::LD3R_4S, AArch64::LD3R_2D 1284263509Sdim }; 1285263509Sdim return SelectVLDDup(Node, false, 3, Opcodes); 1286263509Sdim } 1287263509Sdim case AArch64ISD::NEON_LD4DUP: { 1288263509Sdim static const uint16_t Opcodes[] = { 1289263509Sdim AArch64::LD4R_8B, AArch64::LD4R_4H, AArch64::LD4R_2S, 1290263509Sdim AArch64::LD4R_1D, AArch64::LD4R_16B, AArch64::LD4R_8H, 1291263509Sdim AArch64::LD4R_4S, AArch64::LD4R_2D 1292263509Sdim }; 1293263509Sdim return SelectVLDDup(Node, false, 4, Opcodes); 1294263509Sdim } 1295263509Sdim case AArch64ISD::NEON_LD2DUP_UPD: { 1296263509Sdim static const uint16_t Opcodes[] = { 1297263509Sdim AArch64::LD2R_WB_8B_fixed, AArch64::LD2R_WB_4H_fixed, 1298263509Sdim AArch64::LD2R_WB_2S_fixed, AArch64::LD2R_WB_1D_fixed, 1299263509Sdim AArch64::LD2R_WB_16B_fixed, AArch64::LD2R_WB_8H_fixed, 1300263509Sdim AArch64::LD2R_WB_4S_fixed, AArch64::LD2R_WB_2D_fixed 1301263509Sdim }; 1302263509Sdim return SelectVLDDup(Node, true, 2, Opcodes); 1303263509Sdim } 1304263509Sdim case AArch64ISD::NEON_LD3DUP_UPD: { 1305263509Sdim static const uint16_t Opcodes[] = { 1306263509Sdim AArch64::LD3R_WB_8B_fixed, AArch64::LD3R_WB_4H_fixed, 1307263509Sdim AArch64::LD3R_WB_2S_fixed, AArch64::LD3R_WB_1D_fixed, 1308263509Sdim AArch64::LD3R_WB_16B_fixed, AArch64::LD3R_WB_8H_fixed, 1309263509Sdim AArch64::LD3R_WB_4S_fixed, AArch64::LD3R_WB_2D_fixed 1310263509Sdim }; 1311263509Sdim return SelectVLDDup(Node, true, 3, Opcodes); 1312263509Sdim } 1313263509Sdim case AArch64ISD::NEON_LD4DUP_UPD: { 1314263509Sdim static const uint16_t Opcodes[] = { 1315263509Sdim AArch64::LD4R_WB_8B_fixed, AArch64::LD4R_WB_4H_fixed, 1316263509Sdim AArch64::LD4R_WB_2S_fixed, AArch64::LD4R_WB_1D_fixed, 1317263509Sdim AArch64::LD4R_WB_16B_fixed, AArch64::LD4R_WB_8H_fixed, 1318263509Sdim AArch64::LD4R_WB_4S_fixed, AArch64::LD4R_WB_2D_fixed 1319263509Sdim }; 1320263509Sdim return SelectVLDDup(Node, true, 4, Opcodes); 1321263509Sdim } 1322263509Sdim case AArch64ISD::NEON_LD2LN_UPD: { 1323263509Sdim static const uint16_t Opcodes[] = { 1324263509Sdim AArch64::LD2LN_WB_B_fixed, AArch64::LD2LN_WB_H_fixed, 1325263509Sdim AArch64::LD2LN_WB_S_fixed, AArch64::LD2LN_WB_D_fixed 1326263509Sdim }; 1327263509Sdim return SelectVLDSTLane(Node, true, true, 2, Opcodes); 1328263509Sdim } 1329263509Sdim case AArch64ISD::NEON_LD3LN_UPD: { 1330263509Sdim static const uint16_t Opcodes[] = { 1331263509Sdim AArch64::LD3LN_WB_B_fixed, AArch64::LD3LN_WB_H_fixed, 1332263509Sdim AArch64::LD3LN_WB_S_fixed, AArch64::LD3LN_WB_D_fixed 1333263509Sdim }; 1334263509Sdim return SelectVLDSTLane(Node, true, true, 3, Opcodes); 1335263509Sdim } 1336263509Sdim case AArch64ISD::NEON_LD4LN_UPD: { 1337263509Sdim static const uint16_t Opcodes[] = { 1338263509Sdim AArch64::LD4LN_WB_B_fixed, AArch64::LD4LN_WB_H_fixed, 1339263509Sdim AArch64::LD4LN_WB_S_fixed, AArch64::LD4LN_WB_D_fixed 1340263509Sdim }; 1341263509Sdim return SelectVLDSTLane(Node, true, true, 4, Opcodes); 1342263509Sdim } 1343263509Sdim case AArch64ISD::NEON_ST2LN_UPD: { 1344263509Sdim static const uint16_t Opcodes[] = { 1345263509Sdim AArch64::ST2LN_WB_B_fixed, AArch64::ST2LN_WB_H_fixed, 1346263509Sdim AArch64::ST2LN_WB_S_fixed, AArch64::ST2LN_WB_D_fixed 1347263509Sdim }; 1348263509Sdim return SelectVLDSTLane(Node, false, true, 2, Opcodes); 1349263509Sdim } 1350263509Sdim case AArch64ISD::NEON_ST3LN_UPD: { 1351263509Sdim static const uint16_t Opcodes[] = { 1352263509Sdim AArch64::ST3LN_WB_B_fixed, AArch64::ST3LN_WB_H_fixed, 1353263509Sdim AArch64::ST3LN_WB_S_fixed, AArch64::ST3LN_WB_D_fixed 1354263509Sdim }; 1355263509Sdim return SelectVLDSTLane(Node, false, true, 3, Opcodes); 1356263509Sdim } 1357263509Sdim case AArch64ISD::NEON_ST4LN_UPD: { 1358263509Sdim static const uint16_t Opcodes[] = { 1359263509Sdim AArch64::ST4LN_WB_B_fixed, AArch64::ST4LN_WB_H_fixed, 1360263509Sdim AArch64::ST4LN_WB_S_fixed, AArch64::ST4LN_WB_D_fixed 1361263509Sdim }; 1362263509Sdim return SelectVLDSTLane(Node, false, true, 4, Opcodes); 1363263509Sdim } 1364263509Sdim case AArch64ISD::NEON_ST1x2_UPD: { 1365263509Sdim static const uint16_t Opcodes[] = { 1366263509Sdim AArch64::ST1x2WB_8B_fixed, AArch64::ST1x2WB_4H_fixed, 1367263509Sdim AArch64::ST1x2WB_2S_fixed, AArch64::ST1x2WB_1D_fixed, 1368263509Sdim AArch64::ST1x2WB_16B_fixed, AArch64::ST1x2WB_8H_fixed, 1369263509Sdim AArch64::ST1x2WB_4S_fixed, AArch64::ST1x2WB_2D_fixed 1370263509Sdim }; 1371263509Sdim return SelectVST(Node, true, 2, Opcodes); 1372263509Sdim } 1373263509Sdim case AArch64ISD::NEON_ST1x3_UPD: { 1374263509Sdim static const uint16_t Opcodes[] = { 1375263509Sdim AArch64::ST1x3WB_8B_fixed, AArch64::ST1x3WB_4H_fixed, 1376263509Sdim AArch64::ST1x3WB_2S_fixed, AArch64::ST1x3WB_1D_fixed, 1377263509Sdim AArch64::ST1x3WB_16B_fixed, AArch64::ST1x3WB_8H_fixed, 1378263509Sdim AArch64::ST1x3WB_4S_fixed, AArch64::ST1x3WB_2D_fixed 1379263509Sdim }; 1380263509Sdim return SelectVST(Node, true, 3, Opcodes); 1381263509Sdim } 1382263509Sdim case AArch64ISD::NEON_ST1x4_UPD: { 1383263509Sdim static const uint16_t Opcodes[] = { 1384263509Sdim AArch64::ST1x4WB_8B_fixed, AArch64::ST1x4WB_4H_fixed, 1385263509Sdim AArch64::ST1x4WB_2S_fixed, AArch64::ST1x4WB_1D_fixed, 1386263509Sdim AArch64::ST1x4WB_16B_fixed, AArch64::ST1x4WB_8H_fixed, 1387263509Sdim AArch64::ST1x4WB_4S_fixed, AArch64::ST1x4WB_2D_fixed 1388263509Sdim }; 1389263509Sdim return SelectVST(Node, true, 4, Opcodes); 1390263509Sdim } 1391263509Sdim case ISD::INTRINSIC_WO_CHAIN: { 1392263509Sdim unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(0))->getZExtValue(); 1393263509Sdim bool IsExt = false; 1394263509Sdim switch (IntNo) { 1395263509Sdim default: 1396263509Sdim break; 1397263509Sdim case Intrinsic::aarch64_neon_vtbx1: 1398263509Sdim IsExt = true; 1399263509Sdim case Intrinsic::aarch64_neon_vtbl1: 1400263509Sdim return SelectVTBL(Node, 1, IsExt); 1401263509Sdim case Intrinsic::aarch64_neon_vtbx2: 1402263509Sdim IsExt = true; 1403263509Sdim case Intrinsic::aarch64_neon_vtbl2: 1404263509Sdim return SelectVTBL(Node, 2, IsExt); 1405263509Sdim case Intrinsic::aarch64_neon_vtbx3: 1406263509Sdim IsExt = true; 1407263509Sdim case Intrinsic::aarch64_neon_vtbl3: 1408263509Sdim return SelectVTBL(Node, 3, IsExt); 1409263509Sdim case Intrinsic::aarch64_neon_vtbx4: 1410263509Sdim IsExt = true; 1411263509Sdim case Intrinsic::aarch64_neon_vtbl4: 1412263509Sdim return SelectVTBL(Node, 4, IsExt); 1413263509Sdim } 1414263509Sdim break; 1415263509Sdim } 1416263509Sdim case ISD::INTRINSIC_VOID: 1417263509Sdim case ISD::INTRINSIC_W_CHAIN: { 1418263509Sdim unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue(); 1419263509Sdim switch (IntNo) { 1420263509Sdim default: 1421263509Sdim break; 1422263509Sdim case Intrinsic::arm_neon_vld1: { 1423263509Sdim static const uint16_t Opcodes[] = { 1424263509Sdim AArch64::LD1_8B, AArch64::LD1_4H, AArch64::LD1_2S, AArch64::LD1_1D, 1425263509Sdim AArch64::LD1_16B, AArch64::LD1_8H, AArch64::LD1_4S, AArch64::LD1_2D 1426263509Sdim }; 1427263509Sdim return SelectVLD(Node, false, 1, Opcodes); 1428263509Sdim } 1429263509Sdim case Intrinsic::arm_neon_vld2: { 1430263509Sdim static const uint16_t Opcodes[] = { 1431263509Sdim AArch64::LD2_8B, AArch64::LD2_4H, AArch64::LD2_2S, AArch64::LD1x2_1D, 1432263509Sdim AArch64::LD2_16B, AArch64::LD2_8H, AArch64::LD2_4S, AArch64::LD2_2D 1433263509Sdim }; 1434263509Sdim return SelectVLD(Node, false, 2, Opcodes); 1435263509Sdim } 1436263509Sdim case Intrinsic::arm_neon_vld3: { 1437263509Sdim static const uint16_t Opcodes[] = { 1438263509Sdim AArch64::LD3_8B, AArch64::LD3_4H, AArch64::LD3_2S, AArch64::LD1x3_1D, 1439263509Sdim AArch64::LD3_16B, AArch64::LD3_8H, AArch64::LD3_4S, AArch64::LD3_2D 1440263509Sdim }; 1441263509Sdim return SelectVLD(Node, false, 3, Opcodes); 1442263509Sdim } 1443263509Sdim case Intrinsic::arm_neon_vld4: { 1444263509Sdim static const uint16_t Opcodes[] = { 1445263509Sdim AArch64::LD4_8B, AArch64::LD4_4H, AArch64::LD4_2S, AArch64::LD1x4_1D, 1446263509Sdim AArch64::LD4_16B, AArch64::LD4_8H, AArch64::LD4_4S, AArch64::LD4_2D 1447263509Sdim }; 1448263509Sdim return SelectVLD(Node, false, 4, Opcodes); 1449263509Sdim } 1450263509Sdim case Intrinsic::aarch64_neon_vld1x2: { 1451263509Sdim static const uint16_t Opcodes[] = { 1452263509Sdim AArch64::LD1x2_8B, AArch64::LD1x2_4H, AArch64::LD1x2_2S, 1453263509Sdim AArch64::LD1x2_1D, AArch64::LD1x2_16B, AArch64::LD1x2_8H, 1454263509Sdim AArch64::LD1x2_4S, AArch64::LD1x2_2D 1455263509Sdim }; 1456263509Sdim return SelectVLD(Node, false, 2, Opcodes); 1457263509Sdim } 1458263509Sdim case Intrinsic::aarch64_neon_vld1x3: { 1459263509Sdim static const uint16_t Opcodes[] = { 1460263509Sdim AArch64::LD1x3_8B, AArch64::LD1x3_4H, AArch64::LD1x3_2S, 1461263509Sdim AArch64::LD1x3_1D, AArch64::LD1x3_16B, AArch64::LD1x3_8H, 1462263509Sdim AArch64::LD1x3_4S, AArch64::LD1x3_2D 1463263509Sdim }; 1464263509Sdim return SelectVLD(Node, false, 3, Opcodes); 1465263509Sdim } 1466263509Sdim case Intrinsic::aarch64_neon_vld1x4: { 1467263509Sdim static const uint16_t Opcodes[] = { 1468263509Sdim AArch64::LD1x4_8B, AArch64::LD1x4_4H, AArch64::LD1x4_2S, 1469263509Sdim AArch64::LD1x4_1D, AArch64::LD1x4_16B, AArch64::LD1x4_8H, 1470263509Sdim AArch64::LD1x4_4S, AArch64::LD1x4_2D 1471263509Sdim }; 1472263509Sdim return SelectVLD(Node, false, 4, Opcodes); 1473263509Sdim } 1474263509Sdim case Intrinsic::arm_neon_vst1: { 1475263509Sdim static const uint16_t Opcodes[] = { 1476263509Sdim AArch64::ST1_8B, AArch64::ST1_4H, AArch64::ST1_2S, AArch64::ST1_1D, 1477263509Sdim AArch64::ST1_16B, AArch64::ST1_8H, AArch64::ST1_4S, AArch64::ST1_2D 1478263509Sdim }; 1479263509Sdim return SelectVST(Node, false, 1, Opcodes); 1480263509Sdim } 1481263509Sdim case Intrinsic::arm_neon_vst2: { 1482263509Sdim static const uint16_t Opcodes[] = { 1483263509Sdim AArch64::ST2_8B, AArch64::ST2_4H, AArch64::ST2_2S, AArch64::ST1x2_1D, 1484263509Sdim AArch64::ST2_16B, AArch64::ST2_8H, AArch64::ST2_4S, AArch64::ST2_2D 1485263509Sdim }; 1486263509Sdim return SelectVST(Node, false, 2, Opcodes); 1487263509Sdim } 1488263509Sdim case Intrinsic::arm_neon_vst3: { 1489263509Sdim static const uint16_t Opcodes[] = { 1490263509Sdim AArch64::ST3_8B, AArch64::ST3_4H, AArch64::ST3_2S, AArch64::ST1x3_1D, 1491263509Sdim AArch64::ST3_16B, AArch64::ST3_8H, AArch64::ST3_4S, AArch64::ST3_2D 1492263509Sdim }; 1493263509Sdim return SelectVST(Node, false, 3, Opcodes); 1494263509Sdim } 1495263509Sdim case Intrinsic::arm_neon_vst4: { 1496263509Sdim static const uint16_t Opcodes[] = { 1497263509Sdim AArch64::ST4_8B, AArch64::ST4_4H, AArch64::ST4_2S, AArch64::ST1x4_1D, 1498263509Sdim AArch64::ST4_16B, AArch64::ST4_8H, AArch64::ST4_4S, AArch64::ST4_2D 1499263509Sdim }; 1500263509Sdim return SelectVST(Node, false, 4, Opcodes); 1501263509Sdim } 1502263509Sdim case Intrinsic::aarch64_neon_vst1x2: { 1503263509Sdim static const uint16_t Opcodes[] = { 1504263509Sdim AArch64::ST1x2_8B, AArch64::ST1x2_4H, AArch64::ST1x2_2S, 1505263509Sdim AArch64::ST1x2_1D, AArch64::ST1x2_16B, AArch64::ST1x2_8H, 1506263509Sdim AArch64::ST1x2_4S, AArch64::ST1x2_2D 1507263509Sdim }; 1508263509Sdim return SelectVST(Node, false, 2, Opcodes); 1509263509Sdim } 1510263509Sdim case Intrinsic::aarch64_neon_vst1x3: { 1511263509Sdim static const uint16_t Opcodes[] = { 1512263509Sdim AArch64::ST1x3_8B, AArch64::ST1x3_4H, AArch64::ST1x3_2S, 1513263509Sdim AArch64::ST1x3_1D, AArch64::ST1x3_16B, AArch64::ST1x3_8H, 1514263509Sdim AArch64::ST1x3_4S, AArch64::ST1x3_2D 1515263509Sdim }; 1516263509Sdim return SelectVST(Node, false, 3, Opcodes); 1517263509Sdim } 1518263509Sdim case Intrinsic::aarch64_neon_vst1x4: { 1519263509Sdim static const uint16_t Opcodes[] = { 1520263509Sdim AArch64::ST1x4_8B, AArch64::ST1x4_4H, AArch64::ST1x4_2S, 1521263509Sdim AArch64::ST1x4_1D, AArch64::ST1x4_16B, AArch64::ST1x4_8H, 1522263509Sdim AArch64::ST1x4_4S, AArch64::ST1x4_2D 1523263509Sdim }; 1524263509Sdim return SelectVST(Node, false, 4, Opcodes); 1525263509Sdim } 1526263509Sdim case Intrinsic::arm_neon_vld2lane: { 1527263509Sdim static const uint16_t Opcodes[] = { 1528263509Sdim AArch64::LD2LN_B, AArch64::LD2LN_H, AArch64::LD2LN_S, AArch64::LD2LN_D 1529263509Sdim }; 1530263509Sdim return SelectVLDSTLane(Node, true, false, 2, Opcodes); 1531263509Sdim } 1532263509Sdim case Intrinsic::arm_neon_vld3lane: { 1533263509Sdim static const uint16_t Opcodes[] = { 1534263509Sdim AArch64::LD3LN_B, AArch64::LD3LN_H, AArch64::LD3LN_S, AArch64::LD3LN_D 1535263509Sdim }; 1536263509Sdim return SelectVLDSTLane(Node, true, false, 3, Opcodes); 1537263509Sdim } 1538263509Sdim case Intrinsic::arm_neon_vld4lane: { 1539263509Sdim static const uint16_t Opcodes[] = { 1540263509Sdim AArch64::LD4LN_B, AArch64::LD4LN_H, AArch64::LD4LN_S, AArch64::LD4LN_D 1541263509Sdim }; 1542263509Sdim return SelectVLDSTLane(Node, true, false, 4, Opcodes); 1543263509Sdim } 1544263509Sdim case Intrinsic::arm_neon_vst2lane: { 1545263509Sdim static const uint16_t Opcodes[] = { 1546263509Sdim AArch64::ST2LN_B, AArch64::ST2LN_H, AArch64::ST2LN_S, AArch64::ST2LN_D 1547263509Sdim }; 1548263509Sdim return SelectVLDSTLane(Node, false, false, 2, Opcodes); 1549263509Sdim } 1550263509Sdim case Intrinsic::arm_neon_vst3lane: { 1551263509Sdim static const uint16_t Opcodes[] = { 1552263509Sdim AArch64::ST3LN_B, AArch64::ST3LN_H, AArch64::ST3LN_S, AArch64::ST3LN_D 1553263509Sdim }; 1554263509Sdim return SelectVLDSTLane(Node, false, false, 3, Opcodes); 1555263509Sdim } 1556263509Sdim case Intrinsic::arm_neon_vst4lane: { 1557263509Sdim static const uint16_t Opcodes[] = { 1558263509Sdim AArch64::ST4LN_B, AArch64::ST4LN_H, AArch64::ST4LN_S, AArch64::ST4LN_D 1559263509Sdim }; 1560263509Sdim return SelectVLDSTLane(Node, false, false, 4, Opcodes); 1561263509Sdim } 1562263509Sdim } // End of switch IntNo 1563263509Sdim break; 1564263509Sdim } // End of case ISD::INTRINSIC_VOID and :ISD::INTRINSIC_W_CHAIN 1565249259Sdim default: 1566249259Sdim break; // Let generic code handle it 1567249259Sdim } 1568249259Sdim 1569249259Sdim SDNode *ResNode = SelectCode(Node); 1570249259Sdim 1571249259Sdim DEBUG(dbgs() << "=> "; 1572249259Sdim if (ResNode == NULL || ResNode == Node) 1573249259Sdim Node->dump(CurDAG); 1574249259Sdim else 1575249259Sdim ResNode->dump(CurDAG); 1576249259Sdim dbgs() << "\n"); 1577249259Sdim 1578249259Sdim return ResNode; 1579249259Sdim} 1580249259Sdim 1581249259Sdim/// This pass converts a legalized DAG into a AArch64-specific DAG, ready for 1582249259Sdim/// instruction scheduling. 1583249259SdimFunctionPass *llvm::createAArch64ISelDAG(AArch64TargetMachine &TM, 1584249259Sdim CodeGenOpt::Level OptLevel) { 1585249259Sdim return new AArch64DAGToDAGISel(TM, OptLevel); 1586249259Sdim} 1587