1249259Sdim//===-- AArch64ISelDAGToDAG.cpp - A dag to dag inst selector for AArch64 --===// 2249259Sdim// 3249259Sdim// The LLVM Compiler Infrastructure 4249259Sdim// 5249259Sdim// This file is distributed under the University of Illinois Open Source 6249259Sdim// License. See LICENSE.TXT for details. 7249259Sdim// 8249259Sdim//===----------------------------------------------------------------------===// 9249259Sdim// 10249259Sdim// This file defines an instruction selector for the AArch64 target. 11249259Sdim// 12249259Sdim//===----------------------------------------------------------------------===// 13249259Sdim 14249259Sdim#define DEBUG_TYPE "aarch64-isel" 15249259Sdim#include "AArch64.h" 16249259Sdim#include "AArch64InstrInfo.h" 17249259Sdim#include "AArch64Subtarget.h" 18249259Sdim#include "AArch64TargetMachine.h" 19249259Sdim#include "Utils/AArch64BaseInfo.h" 20249259Sdim#include "llvm/ADT/APSInt.h" 21249259Sdim#include "llvm/CodeGen/SelectionDAGISel.h" 22249259Sdim#include "llvm/IR/GlobalValue.h" 23249259Sdim#include "llvm/Support/Debug.h" 24249259Sdim#include "llvm/Support/raw_ostream.h" 25249259Sdim 26249259Sdimusing namespace llvm; 27249259Sdim 28249259Sdim//===--------------------------------------------------------------------===// 29249259Sdim/// AArch64 specific code to select AArch64 machine instructions for 30249259Sdim/// SelectionDAG operations. 31249259Sdim/// 32249259Sdimnamespace { 33249259Sdim 34249259Sdimclass AArch64DAGToDAGISel : public SelectionDAGISel { 35249259Sdim AArch64TargetMachine &TM; 36249259Sdim const AArch64InstrInfo *TII; 37249259Sdim 38249259Sdim /// Keep a pointer to the AArch64Subtarget around so that we can 39249259Sdim /// make the right decision when generating code for different targets. 40249259Sdim const AArch64Subtarget *Subtarget; 41249259Sdim 42249259Sdimpublic: 43249259Sdim explicit AArch64DAGToDAGISel(AArch64TargetMachine &tm, 44249259Sdim CodeGenOpt::Level OptLevel) 45249259Sdim : SelectionDAGISel(tm, OptLevel), TM(tm), 46249259Sdim TII(static_cast<const AArch64InstrInfo*>(TM.getInstrInfo())), 47249259Sdim Subtarget(&TM.getSubtarget<AArch64Subtarget>()) { 48249259Sdim } 49249259Sdim 50249259Sdim virtual const char *getPassName() const { 51249259Sdim return "AArch64 Instruction Selection"; 52249259Sdim } 53249259Sdim 54249259Sdim // Include the pieces autogenerated from the target description. 55249259Sdim#include "AArch64GenDAGISel.inc" 56249259Sdim 57249259Sdim template<unsigned MemSize> 58249259Sdim bool SelectOffsetUImm12(SDValue N, SDValue &UImm12) { 59249259Sdim const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N); 60249259Sdim if (!CN || CN->getZExtValue() % MemSize != 0 61249259Sdim || CN->getZExtValue() / MemSize > 0xfff) 62249259Sdim return false; 63249259Sdim 64249259Sdim UImm12 = CurDAG->getTargetConstant(CN->getZExtValue() / MemSize, MVT::i64); 65249259Sdim return true; 66249259Sdim } 67249259Sdim 68249259Sdim template<unsigned RegWidth> 69249259Sdim bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos) { 70249259Sdim return SelectCVTFixedPosOperand(N, FixedPos, RegWidth); 71249259Sdim } 72249259Sdim 73251662Sdim /// Used for pre-lowered address-reference nodes, so we already know 74251662Sdim /// the fields match. This operand's job is simply to add an 75251662Sdim /// appropriate shift operand (i.e. 0) to the MOVZ/MOVK instruction. 76251662Sdim bool SelectMOVWAddressRef(SDValue N, SDValue &Imm, SDValue &Shift) { 77251662Sdim Imm = N; 78251662Sdim Shift = CurDAG->getTargetConstant(0, MVT::i32); 79251662Sdim return true; 80251662Sdim } 81251662Sdim 82249259Sdim bool SelectFPZeroOperand(SDValue N, SDValue &Dummy); 83249259Sdim 84249259Sdim bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, 85249259Sdim unsigned RegWidth); 86249259Sdim 87249259Sdim bool SelectInlineAsmMemoryOperand(const SDValue &Op, 88249259Sdim char ConstraintCode, 89249259Sdim std::vector<SDValue> &OutOps); 90249259Sdim 91249259Sdim bool SelectLogicalImm(SDValue N, SDValue &Imm); 92249259Sdim 93249259Sdim template<unsigned RegWidth> 94249259Sdim bool SelectTSTBOperand(SDValue N, SDValue &FixedPos) { 95249259Sdim return SelectTSTBOperand(N, FixedPos, RegWidth); 96249259Sdim } 97249259Sdim 98249259Sdim bool SelectTSTBOperand(SDValue N, SDValue &FixedPos, unsigned RegWidth); 99249259Sdim 100251662Sdim SDNode *SelectAtomic(SDNode *N, unsigned Op8, unsigned Op16, unsigned Op32, 101251662Sdim unsigned Op64); 102251662Sdim 103251662Sdim /// Put the given constant into a pool and return a DAG which will give its 104251662Sdim /// address. 105251662Sdim SDValue getConstantPoolItemAddress(DebugLoc DL, const Constant *CV); 106251662Sdim 107249259Sdim SDNode *TrySelectToMoveImm(SDNode *N); 108249259Sdim SDNode *LowerToFPLitPool(SDNode *Node); 109249259Sdim SDNode *SelectToLitPool(SDNode *N); 110249259Sdim 111249259Sdim SDNode* Select(SDNode*); 112249259Sdimprivate: 113249259Sdim}; 114249259Sdim} 115249259Sdim 116249259Sdimbool 117249259SdimAArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, 118249259Sdim unsigned RegWidth) { 119249259Sdim const ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N); 120249259Sdim if (!CN) return false; 121249259Sdim 122249259Sdim // An FCVT[SU] instruction performs: convertToInt(Val * 2^fbits) where fbits 123249259Sdim // is between 1 and 32 for a destination w-register, or 1 and 64 for an 124249259Sdim // x-register. 125249259Sdim // 126249259Sdim // By this stage, we've detected (fp_to_[su]int (fmul Val, THIS_NODE)) so we 127249259Sdim // want THIS_NODE to be 2^fbits. This is much easier to deal with using 128249259Sdim // integers. 129249259Sdim bool IsExact; 130249259Sdim 131249259Sdim // fbits is between 1 and 64 in the worst-case, which means the fmul 132249259Sdim // could have 2^64 as an actual operand. Need 65 bits of precision. 133249259Sdim APSInt IntVal(65, true); 134249259Sdim CN->getValueAPF().convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact); 135249259Sdim 136249259Sdim // N.b. isPowerOf2 also checks for > 0. 137249259Sdim if (!IsExact || !IntVal.isPowerOf2()) return false; 138249259Sdim unsigned FBits = IntVal.logBase2(); 139249259Sdim 140249259Sdim // Checks above should have guaranteed that we haven't lost information in 141249259Sdim // finding FBits, but it must still be in range. 142249259Sdim if (FBits == 0 || FBits > RegWidth) return false; 143249259Sdim 144249259Sdim FixedPos = CurDAG->getTargetConstant(64 - FBits, MVT::i32); 145249259Sdim return true; 146249259Sdim} 147249259Sdim 148249259Sdimbool 149249259SdimAArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(const SDValue &Op, 150249259Sdim char ConstraintCode, 151249259Sdim std::vector<SDValue> &OutOps) { 152249259Sdim switch (ConstraintCode) { 153249259Sdim default: llvm_unreachable("Unrecognised AArch64 memory constraint"); 154249259Sdim case 'm': 155249259Sdim // FIXME: more freedom is actually permitted for 'm'. We can go 156249259Sdim // hunting for a base and an offset if we want. Of course, since 157249259Sdim // we don't really know how the operand is going to be used we're 158249259Sdim // probably restricted to the load/store pair's simm7 as an offset 159249259Sdim // range anyway. 160249259Sdim case 'Q': 161249259Sdim OutOps.push_back(Op); 162249259Sdim } 163249259Sdim 164249259Sdim return false; 165249259Sdim} 166249259Sdim 167249259Sdimbool 168249259SdimAArch64DAGToDAGISel::SelectFPZeroOperand(SDValue N, SDValue &Dummy) { 169249259Sdim ConstantFPSDNode *Imm = dyn_cast<ConstantFPSDNode>(N); 170249259Sdim if (!Imm || !Imm->getValueAPF().isPosZero()) 171249259Sdim return false; 172249259Sdim 173249259Sdim // Doesn't actually carry any information, but keeps TableGen quiet. 174249259Sdim Dummy = CurDAG->getTargetConstant(0, MVT::i32); 175249259Sdim return true; 176249259Sdim} 177249259Sdim 178249259Sdimbool AArch64DAGToDAGISel::SelectLogicalImm(SDValue N, SDValue &Imm) { 179249259Sdim uint32_t Bits; 180249259Sdim uint32_t RegWidth = N.getValueType().getSizeInBits(); 181249259Sdim 182249259Sdim ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N); 183249259Sdim if (!CN) return false; 184249259Sdim 185249259Sdim if (!A64Imms::isLogicalImm(RegWidth, CN->getZExtValue(), Bits)) 186249259Sdim return false; 187249259Sdim 188249259Sdim Imm = CurDAG->getTargetConstant(Bits, MVT::i32); 189249259Sdim return true; 190249259Sdim} 191249259Sdim 192249259SdimSDNode *AArch64DAGToDAGISel::TrySelectToMoveImm(SDNode *Node) { 193249259Sdim SDNode *ResNode; 194249259Sdim DebugLoc dl = Node->getDebugLoc(); 195249259Sdim EVT DestType = Node->getValueType(0); 196249259Sdim unsigned DestWidth = DestType.getSizeInBits(); 197249259Sdim 198249259Sdim unsigned MOVOpcode; 199249259Sdim EVT MOVType; 200249259Sdim int UImm16, Shift; 201249259Sdim uint32_t LogicalBits; 202249259Sdim 203249259Sdim uint64_t BitPat = cast<ConstantSDNode>(Node)->getZExtValue(); 204249259Sdim if (A64Imms::isMOVZImm(DestWidth, BitPat, UImm16, Shift)) { 205249259Sdim MOVType = DestType; 206249259Sdim MOVOpcode = DestWidth == 64 ? AArch64::MOVZxii : AArch64::MOVZwii; 207249259Sdim } else if (A64Imms::isMOVNImm(DestWidth, BitPat, UImm16, Shift)) { 208249259Sdim MOVType = DestType; 209249259Sdim MOVOpcode = DestWidth == 64 ? AArch64::MOVNxii : AArch64::MOVNwii; 210249259Sdim } else if (DestWidth == 64 && A64Imms::isMOVNImm(32, BitPat, UImm16, Shift)) { 211249259Sdim // To get something like 0x0000_0000_ffff_1234 into a 64-bit register we can 212249259Sdim // use a 32-bit instruction: "movn w0, 0xedbc". 213249259Sdim MOVType = MVT::i32; 214249259Sdim MOVOpcode = AArch64::MOVNwii; 215249259Sdim } else if (A64Imms::isLogicalImm(DestWidth, BitPat, LogicalBits)) { 216249259Sdim MOVOpcode = DestWidth == 64 ? AArch64::ORRxxi : AArch64::ORRwwi; 217249259Sdim uint16_t ZR = DestWidth == 64 ? AArch64::XZR : AArch64::WZR; 218249259Sdim 219249259Sdim return CurDAG->getMachineNode(MOVOpcode, dl, DestType, 220249259Sdim CurDAG->getRegister(ZR, DestType), 221249259Sdim CurDAG->getTargetConstant(LogicalBits, MVT::i32)); 222249259Sdim } else { 223249259Sdim // Can't handle it in one instruction. There's scope for permitting two (or 224249259Sdim // more) instructions, but that'll need more thought. 225249259Sdim return NULL; 226249259Sdim } 227249259Sdim 228249259Sdim ResNode = CurDAG->getMachineNode(MOVOpcode, dl, MOVType, 229249259Sdim CurDAG->getTargetConstant(UImm16, MVT::i32), 230249259Sdim CurDAG->getTargetConstant(Shift, MVT::i32)); 231249259Sdim 232249259Sdim if (MOVType != DestType) { 233249259Sdim ResNode = CurDAG->getMachineNode(TargetOpcode::SUBREG_TO_REG, dl, 234249259Sdim MVT::i64, MVT::i32, MVT::Other, 235249259Sdim CurDAG->getTargetConstant(0, MVT::i64), 236249259Sdim SDValue(ResNode, 0), 237249259Sdim CurDAG->getTargetConstant(AArch64::sub_32, MVT::i32)); 238249259Sdim } 239249259Sdim 240249259Sdim return ResNode; 241249259Sdim} 242249259Sdim 243251662SdimSDValue 244251662SdimAArch64DAGToDAGISel::getConstantPoolItemAddress(DebugLoc DL, 245251662Sdim const Constant *CV) { 246251662Sdim EVT PtrVT = TLI.getPointerTy(); 247251662Sdim 248251662Sdim switch (TLI.getTargetMachine().getCodeModel()) { 249251662Sdim case CodeModel::Small: { 250251662Sdim unsigned Alignment = 251251662Sdim TLI.getDataLayout()->getABITypeAlignment(CV->getType()); 252251662Sdim return CurDAG->getNode( 253251662Sdim AArch64ISD::WrapperSmall, DL, PtrVT, 254251662Sdim CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_NO_FLAG), 255251662Sdim CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_LO12), 256251662Sdim CurDAG->getConstant(Alignment, MVT::i32)); 257251662Sdim } 258251662Sdim case CodeModel::Large: { 259251662Sdim SDNode *LitAddr; 260251662Sdim LitAddr = CurDAG->getMachineNode( 261251662Sdim AArch64::MOVZxii, DL, PtrVT, 262251662Sdim CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_ABS_G3), 263251662Sdim CurDAG->getTargetConstant(0, MVT::i32)); 264251662Sdim LitAddr = CurDAG->getMachineNode( 265251662Sdim AArch64::MOVKxii, DL, PtrVT, SDValue(LitAddr, 0), 266251662Sdim CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_ABS_G2_NC), 267251662Sdim CurDAG->getTargetConstant(0, MVT::i32)); 268251662Sdim LitAddr = CurDAG->getMachineNode( 269251662Sdim AArch64::MOVKxii, DL, PtrVT, SDValue(LitAddr, 0), 270251662Sdim CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_ABS_G1_NC), 271251662Sdim CurDAG->getTargetConstant(0, MVT::i32)); 272251662Sdim LitAddr = CurDAG->getMachineNode( 273251662Sdim AArch64::MOVKxii, DL, PtrVT, SDValue(LitAddr, 0), 274251662Sdim CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0, AArch64II::MO_ABS_G0_NC), 275251662Sdim CurDAG->getTargetConstant(0, MVT::i32)); 276251662Sdim return SDValue(LitAddr, 0); 277251662Sdim } 278251662Sdim default: 279251662Sdim llvm_unreachable("Only small and large code models supported now"); 280251662Sdim } 281251662Sdim} 282251662Sdim 283249259SdimSDNode *AArch64DAGToDAGISel::SelectToLitPool(SDNode *Node) { 284249259Sdim DebugLoc DL = Node->getDebugLoc(); 285249259Sdim uint64_t UnsignedVal = cast<ConstantSDNode>(Node)->getZExtValue(); 286249259Sdim int64_t SignedVal = cast<ConstantSDNode>(Node)->getSExtValue(); 287249259Sdim EVT DestType = Node->getValueType(0); 288249259Sdim 289249259Sdim // Since we may end up loading a 64-bit constant from a 32-bit entry the 290249259Sdim // constant in the pool may have a different type to the eventual node. 291249259Sdim ISD::LoadExtType Extension; 292249259Sdim EVT MemType; 293249259Sdim 294249259Sdim assert((DestType == MVT::i64 || DestType == MVT::i32) 295249259Sdim && "Only expect integer constants at the moment"); 296249259Sdim 297249259Sdim if (DestType == MVT::i32) { 298249259Sdim Extension = ISD::NON_EXTLOAD; 299249259Sdim MemType = MVT::i32; 300249259Sdim } else if (UnsignedVal <= UINT32_MAX) { 301249259Sdim Extension = ISD::ZEXTLOAD; 302249259Sdim MemType = MVT::i32; 303249259Sdim } else if (SignedVal >= INT32_MIN && SignedVal <= INT32_MAX) { 304249259Sdim Extension = ISD::SEXTLOAD; 305249259Sdim MemType = MVT::i32; 306249259Sdim } else { 307249259Sdim Extension = ISD::NON_EXTLOAD; 308249259Sdim MemType = MVT::i64; 309249259Sdim } 310249259Sdim 311249259Sdim Constant *CV = ConstantInt::get(Type::getIntNTy(*CurDAG->getContext(), 312249259Sdim MemType.getSizeInBits()), 313249259Sdim UnsignedVal); 314251662Sdim SDValue PoolAddr = getConstantPoolItemAddress(DL, CV); 315249259Sdim unsigned Alignment = TLI.getDataLayout()->getABITypeAlignment(CV->getType()); 316249259Sdim 317249259Sdim return CurDAG->getExtLoad(Extension, DL, DestType, CurDAG->getEntryNode(), 318249259Sdim PoolAddr, 319249259Sdim MachinePointerInfo::getConstantPool(), MemType, 320249259Sdim /* isVolatile = */ false, 321249259Sdim /* isNonTemporal = */ false, 322249259Sdim Alignment).getNode(); 323249259Sdim} 324249259Sdim 325249259SdimSDNode *AArch64DAGToDAGISel::LowerToFPLitPool(SDNode *Node) { 326249259Sdim DebugLoc DL = Node->getDebugLoc(); 327249259Sdim const ConstantFP *FV = cast<ConstantFPSDNode>(Node)->getConstantFPValue(); 328249259Sdim EVT DestType = Node->getValueType(0); 329249259Sdim 330249259Sdim unsigned Alignment = TLI.getDataLayout()->getABITypeAlignment(FV->getType()); 331251662Sdim SDValue PoolAddr = getConstantPoolItemAddress(DL, FV); 332249259Sdim 333249259Sdim return CurDAG->getLoad(DestType, DL, CurDAG->getEntryNode(), PoolAddr, 334249259Sdim MachinePointerInfo::getConstantPool(), 335249259Sdim /* isVolatile = */ false, 336249259Sdim /* isNonTemporal = */ false, 337249259Sdim /* isInvariant = */ true, 338249259Sdim Alignment).getNode(); 339249259Sdim} 340249259Sdim 341249259Sdimbool 342249259SdimAArch64DAGToDAGISel::SelectTSTBOperand(SDValue N, SDValue &FixedPos, 343249259Sdim unsigned RegWidth) { 344249259Sdim const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N); 345249259Sdim if (!CN) return false; 346249259Sdim 347249259Sdim uint64_t Val = CN->getZExtValue(); 348249259Sdim 349249259Sdim if (!isPowerOf2_64(Val)) return false; 350249259Sdim 351249259Sdim unsigned TestedBit = Log2_64(Val); 352249259Sdim // Checks above should have guaranteed that we haven't lost information in 353249259Sdim // finding TestedBit, but it must still be in range. 354249259Sdim if (TestedBit >= RegWidth) return false; 355249259Sdim 356249259Sdim FixedPos = CurDAG->getTargetConstant(TestedBit, MVT::i64); 357249259Sdim return true; 358249259Sdim} 359249259Sdim 360251662SdimSDNode *AArch64DAGToDAGISel::SelectAtomic(SDNode *Node, unsigned Op8, 361251662Sdim unsigned Op16,unsigned Op32, 362251662Sdim unsigned Op64) { 363251662Sdim // Mostly direct translation to the given operations, except that we preserve 364251662Sdim // the AtomicOrdering for use later on. 365251662Sdim AtomicSDNode *AN = cast<AtomicSDNode>(Node); 366251662Sdim EVT VT = AN->getMemoryVT(); 367251662Sdim 368251662Sdim unsigned Op; 369251662Sdim if (VT == MVT::i8) 370251662Sdim Op = Op8; 371251662Sdim else if (VT == MVT::i16) 372251662Sdim Op = Op16; 373251662Sdim else if (VT == MVT::i32) 374251662Sdim Op = Op32; 375251662Sdim else if (VT == MVT::i64) 376251662Sdim Op = Op64; 377251662Sdim else 378251662Sdim llvm_unreachable("Unexpected atomic operation"); 379251662Sdim 380251662Sdim SmallVector<SDValue, 4> Ops; 381251662Sdim for (unsigned i = 1; i < AN->getNumOperands(); ++i) 382251662Sdim Ops.push_back(AN->getOperand(i)); 383251662Sdim 384251662Sdim Ops.push_back(CurDAG->getTargetConstant(AN->getOrdering(), MVT::i32)); 385251662Sdim Ops.push_back(AN->getOperand(0)); // Chain moves to the end 386251662Sdim 387251662Sdim return CurDAG->SelectNodeTo(Node, Op, 388251662Sdim AN->getValueType(0), MVT::Other, 389251662Sdim &Ops[0], Ops.size()); 390251662Sdim} 391251662Sdim 392249259SdimSDNode *AArch64DAGToDAGISel::Select(SDNode *Node) { 393249259Sdim // Dump information about the Node being selected 394249259Sdim DEBUG(dbgs() << "Selecting: "; Node->dump(CurDAG); dbgs() << "\n"); 395249259Sdim 396249259Sdim if (Node->isMachineOpcode()) { 397249259Sdim DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n"); 398255804Sdim Node->setNodeId(-1); 399249259Sdim return NULL; 400249259Sdim } 401249259Sdim 402249259Sdim switch (Node->getOpcode()) { 403251662Sdim case ISD::ATOMIC_LOAD_ADD: 404251662Sdim return SelectAtomic(Node, 405251662Sdim AArch64::ATOMIC_LOAD_ADD_I8, 406251662Sdim AArch64::ATOMIC_LOAD_ADD_I16, 407251662Sdim AArch64::ATOMIC_LOAD_ADD_I32, 408251662Sdim AArch64::ATOMIC_LOAD_ADD_I64); 409251662Sdim case ISD::ATOMIC_LOAD_SUB: 410251662Sdim return SelectAtomic(Node, 411251662Sdim AArch64::ATOMIC_LOAD_SUB_I8, 412251662Sdim AArch64::ATOMIC_LOAD_SUB_I16, 413251662Sdim AArch64::ATOMIC_LOAD_SUB_I32, 414251662Sdim AArch64::ATOMIC_LOAD_SUB_I64); 415251662Sdim case ISD::ATOMIC_LOAD_AND: 416251662Sdim return SelectAtomic(Node, 417251662Sdim AArch64::ATOMIC_LOAD_AND_I8, 418251662Sdim AArch64::ATOMIC_LOAD_AND_I16, 419251662Sdim AArch64::ATOMIC_LOAD_AND_I32, 420251662Sdim AArch64::ATOMIC_LOAD_AND_I64); 421251662Sdim case ISD::ATOMIC_LOAD_OR: 422251662Sdim return SelectAtomic(Node, 423251662Sdim AArch64::ATOMIC_LOAD_OR_I8, 424251662Sdim AArch64::ATOMIC_LOAD_OR_I16, 425251662Sdim AArch64::ATOMIC_LOAD_OR_I32, 426251662Sdim AArch64::ATOMIC_LOAD_OR_I64); 427251662Sdim case ISD::ATOMIC_LOAD_XOR: 428251662Sdim return SelectAtomic(Node, 429251662Sdim AArch64::ATOMIC_LOAD_XOR_I8, 430251662Sdim AArch64::ATOMIC_LOAD_XOR_I16, 431251662Sdim AArch64::ATOMIC_LOAD_XOR_I32, 432251662Sdim AArch64::ATOMIC_LOAD_XOR_I64); 433251662Sdim case ISD::ATOMIC_LOAD_NAND: 434251662Sdim return SelectAtomic(Node, 435251662Sdim AArch64::ATOMIC_LOAD_NAND_I8, 436251662Sdim AArch64::ATOMIC_LOAD_NAND_I16, 437251662Sdim AArch64::ATOMIC_LOAD_NAND_I32, 438251662Sdim AArch64::ATOMIC_LOAD_NAND_I64); 439251662Sdim case ISD::ATOMIC_LOAD_MIN: 440251662Sdim return SelectAtomic(Node, 441251662Sdim AArch64::ATOMIC_LOAD_MIN_I8, 442251662Sdim AArch64::ATOMIC_LOAD_MIN_I16, 443251662Sdim AArch64::ATOMIC_LOAD_MIN_I32, 444251662Sdim AArch64::ATOMIC_LOAD_MIN_I64); 445251662Sdim case ISD::ATOMIC_LOAD_MAX: 446251662Sdim return SelectAtomic(Node, 447251662Sdim AArch64::ATOMIC_LOAD_MAX_I8, 448251662Sdim AArch64::ATOMIC_LOAD_MAX_I16, 449251662Sdim AArch64::ATOMIC_LOAD_MAX_I32, 450251662Sdim AArch64::ATOMIC_LOAD_MAX_I64); 451251662Sdim case ISD::ATOMIC_LOAD_UMIN: 452251662Sdim return SelectAtomic(Node, 453251662Sdim AArch64::ATOMIC_LOAD_UMIN_I8, 454251662Sdim AArch64::ATOMIC_LOAD_UMIN_I16, 455251662Sdim AArch64::ATOMIC_LOAD_UMIN_I32, 456251662Sdim AArch64::ATOMIC_LOAD_UMIN_I64); 457251662Sdim case ISD::ATOMIC_LOAD_UMAX: 458251662Sdim return SelectAtomic(Node, 459251662Sdim AArch64::ATOMIC_LOAD_UMAX_I8, 460251662Sdim AArch64::ATOMIC_LOAD_UMAX_I16, 461251662Sdim AArch64::ATOMIC_LOAD_UMAX_I32, 462251662Sdim AArch64::ATOMIC_LOAD_UMAX_I64); 463251662Sdim case ISD::ATOMIC_SWAP: 464251662Sdim return SelectAtomic(Node, 465251662Sdim AArch64::ATOMIC_SWAP_I8, 466251662Sdim AArch64::ATOMIC_SWAP_I16, 467251662Sdim AArch64::ATOMIC_SWAP_I32, 468251662Sdim AArch64::ATOMIC_SWAP_I64); 469251662Sdim case ISD::ATOMIC_CMP_SWAP: 470251662Sdim return SelectAtomic(Node, 471251662Sdim AArch64::ATOMIC_CMP_SWAP_I8, 472251662Sdim AArch64::ATOMIC_CMP_SWAP_I16, 473251662Sdim AArch64::ATOMIC_CMP_SWAP_I32, 474251662Sdim AArch64::ATOMIC_CMP_SWAP_I64); 475249259Sdim case ISD::FrameIndex: { 476249259Sdim int FI = cast<FrameIndexSDNode>(Node)->getIndex(); 477249259Sdim EVT PtrTy = TLI.getPointerTy(); 478249259Sdim SDValue TFI = CurDAG->getTargetFrameIndex(FI, PtrTy); 479249259Sdim return CurDAG->SelectNodeTo(Node, AArch64::ADDxxi_lsl0_s, PtrTy, 480249259Sdim TFI, CurDAG->getTargetConstant(0, PtrTy)); 481249259Sdim } 482249259Sdim case ISD::ConstantPool: { 483249259Sdim // Constant pools are fine, just create a Target entry. 484249259Sdim ConstantPoolSDNode *CN = cast<ConstantPoolSDNode>(Node); 485249259Sdim const Constant *C = CN->getConstVal(); 486249259Sdim SDValue CP = CurDAG->getTargetConstantPool(C, CN->getValueType(0)); 487249259Sdim 488249259Sdim ReplaceUses(SDValue(Node, 0), CP); 489249259Sdim return NULL; 490249259Sdim } 491249259Sdim case ISD::Constant: { 492249259Sdim SDNode *ResNode = 0; 493249259Sdim if (cast<ConstantSDNode>(Node)->getZExtValue() == 0) { 494249259Sdim // XZR and WZR are probably even better than an actual move: most of the 495249259Sdim // time they can be folded into another instruction with *no* cost. 496249259Sdim 497249259Sdim EVT Ty = Node->getValueType(0); 498249259Sdim assert((Ty == MVT::i32 || Ty == MVT::i64) && "unexpected type"); 499249259Sdim uint16_t Register = Ty == MVT::i32 ? AArch64::WZR : AArch64::XZR; 500249259Sdim ResNode = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), 501249259Sdim Node->getDebugLoc(), 502249259Sdim Register, Ty).getNode(); 503249259Sdim } 504249259Sdim 505249259Sdim // Next best option is a move-immediate, see if we can do that. 506249259Sdim if (!ResNode) { 507249259Sdim ResNode = TrySelectToMoveImm(Node); 508249259Sdim } 509249259Sdim 510249259Sdim if (ResNode) 511249259Sdim return ResNode; 512249259Sdim 513249259Sdim // If even that fails we fall back to a lit-pool entry at the moment. Future 514249259Sdim // tuning may change this to a sequence of MOVZ/MOVN/MOVK instructions. 515249259Sdim ResNode = SelectToLitPool(Node); 516249259Sdim assert(ResNode && "We need *some* way to materialise a constant"); 517249259Sdim 518249259Sdim // We want to continue selection at this point since the litpool access 519249259Sdim // generated used generic nodes for simplicity. 520249259Sdim ReplaceUses(SDValue(Node, 0), SDValue(ResNode, 0)); 521249259Sdim Node = ResNode; 522249259Sdim break; 523249259Sdim } 524249259Sdim case ISD::ConstantFP: { 525249259Sdim if (A64Imms::isFPImm(cast<ConstantFPSDNode>(Node)->getValueAPF())) { 526249259Sdim // FMOV will take care of it from TableGen 527249259Sdim break; 528249259Sdim } 529249259Sdim 530249259Sdim SDNode *ResNode = LowerToFPLitPool(Node); 531249259Sdim ReplaceUses(SDValue(Node, 0), SDValue(ResNode, 0)); 532249259Sdim 533249259Sdim // We want to continue selection at this point since the litpool access 534249259Sdim // generated used generic nodes for simplicity. 535249259Sdim Node = ResNode; 536249259Sdim break; 537249259Sdim } 538249259Sdim default: 539249259Sdim break; // Let generic code handle it 540249259Sdim } 541249259Sdim 542249259Sdim SDNode *ResNode = SelectCode(Node); 543249259Sdim 544249259Sdim DEBUG(dbgs() << "=> "; 545249259Sdim if (ResNode == NULL || ResNode == Node) 546249259Sdim Node->dump(CurDAG); 547249259Sdim else 548249259Sdim ResNode->dump(CurDAG); 549249259Sdim dbgs() << "\n"); 550249259Sdim 551249259Sdim return ResNode; 552249259Sdim} 553249259Sdim 554249259Sdim/// This pass converts a legalized DAG into a AArch64-specific DAG, ready for 555249259Sdim/// instruction scheduling. 556249259SdimFunctionPass *llvm::createAArch64ISelDAG(AArch64TargetMachine &TM, 557249259Sdim CodeGenOpt::Level OptLevel) { 558249259Sdim return new AArch64DAGToDAGISel(TM, OptLevel); 559249259Sdim} 560