1193323Sed//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===// 2193323Sed// 3193323Sed// The LLVM Compiler Infrastructure 4193323Sed// 5193323Sed// This file is distributed under the University of Illinois Open Source 6193323Sed// License. See LICENSE.TXT for details. 7193323Sed// 8193323Sed//===----------------------------------------------------------------------===// 9193323Sed// 10193323Sed// This file defines an instruction selector for the ARM target. 11193323Sed// 12193323Sed//===----------------------------------------------------------------------===// 13193323Sed 14193323Sed#include "ARM.h" 15218893Sdim#include "ARMBaseInstrInfo.h" 16193323Sed#include "ARMTargetMachine.h" 17226633Sdim#include "MCTargetDesc/ARMAddressingModes.h" 18288943Sdim#include "llvm/ADT/StringSwitch.h" 19193323Sed#include "llvm/CodeGen/MachineFrameInfo.h" 20193323Sed#include "llvm/CodeGen/MachineFunction.h" 21193323Sed#include "llvm/CodeGen/MachineInstrBuilder.h" 22249423Sdim#include "llvm/CodeGen/MachineRegisterInfo.h" 23193323Sed#include "llvm/CodeGen/SelectionDAG.h" 24193323Sed#include "llvm/CodeGen/SelectionDAGISel.h" 25249423Sdim#include "llvm/IR/CallingConv.h" 26249423Sdim#include "llvm/IR/Constants.h" 27249423Sdim#include "llvm/IR/DerivedTypes.h" 28249423Sdim#include "llvm/IR/Function.h" 29249423Sdim#include "llvm/IR/Intrinsics.h" 30249423Sdim#include "llvm/IR/LLVMContext.h" 31207631Srdivacky#include "llvm/Support/CommandLine.h" 32193323Sed#include "llvm/Support/Compiler.h" 33193323Sed#include "llvm/Support/Debug.h" 34198090Srdivacky#include "llvm/Support/ErrorHandling.h" 35249423Sdim#include "llvm/Target/TargetLowering.h" 36249423Sdim#include "llvm/Target/TargetOptions.h" 37198090Srdivacky 38193323Sedusing namespace llvm; 39193323Sed 40276479Sdim#define DEBUG_TYPE "arm-isel" 41276479Sdim 42212904Sdimstatic cl::opt<bool> 43212904SdimDisableShifterOp("disable-shifter-op", cl::Hidden, 44212904Sdim cl::desc("Disable isel of shifter-op"), 45212904Sdim cl::init(false)); 46212904Sdim 47218893Sdimstatic cl::opt<bool> 48218893SdimCheckVMLxHazard("check-vmlx-hazard", cl::Hidden, 49218893Sdim cl::desc("Check fp vmla / vmls hazard at isel time"), 50221345Sdim cl::init(true)); 51218893Sdim 52193323Sed//===--------------------------------------------------------------------===// 53193323Sed/// ARMDAGToDAGISel - ARM specific code to select ARM machine 54193323Sed/// instructions for SelectionDAG operations. 55193323Sed/// 56193323Sednamespace { 57218893Sdim 58218893Sdimenum AddrMode2Type { 59218893Sdim AM2_BASE, // Simple AM2 (+-imm12) 60218893Sdim AM2_SHOP // Shifter-op AM2 61218893Sdim}; 62218893Sdim 63193323Sedclass ARMDAGToDAGISel : public SelectionDAGISel { 64193323Sed /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can 65193323Sed /// make the right decision when generating code for different targets. 66193323Sed const ARMSubtarget *Subtarget; 67193323Sed 68193323Sedpublic: 69276479Sdim explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm, CodeGenOpt::Level OptLevel) 70276479Sdim : SelectionDAGISel(tm, OptLevel) {} 71276479Sdim 72276479Sdim bool runOnMachineFunction(MachineFunction &MF) override { 73276479Sdim // Reset the subtarget each time through. 74288943Sdim Subtarget = &MF.getSubtarget<ARMSubtarget>(); 75276479Sdim SelectionDAGISel::runOnMachineFunction(MF); 76276479Sdim return true; 77193323Sed } 78193323Sed 79276479Sdim const char *getPassName() const override { 80193323Sed return "ARM Instruction Selection"; 81194612Sed } 82194612Sed 83276479Sdim void PreprocessISelDAG() override; 84249423Sdim 85198090Srdivacky /// getI32Imm - Return a target constant of type i32 with the specified 86198090Srdivacky /// value. 87288943Sdim inline SDValue getI32Imm(unsigned Imm, SDLoc dl) { 88288943Sdim return CurDAG->getTargetConstant(Imm, dl, MVT::i32); 89194612Sed } 90194612Sed 91276479Sdim SDNode *Select(SDNode *N) override; 92203954Srdivacky 93218893Sdim 94218893Sdim bool hasNoVMLxHazardUse(SDNode *N) const; 95218893Sdim bool isShifterOpProfitable(const SDValue &Shift, 96218893Sdim ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt); 97226633Sdim bool SelectRegShifterOperand(SDValue N, SDValue &A, 98221345Sdim SDValue &B, SDValue &C, 99221345Sdim bool CheckProfitability = true); 100226633Sdim bool SelectImmShifterOperand(SDValue N, SDValue &A, 101226633Sdim SDValue &B, bool CheckProfitability = true); 102226633Sdim bool SelectShiftRegShifterOperand(SDValue N, SDValue &A, 103221345Sdim SDValue &B, SDValue &C) { 104221345Sdim // Don't apply the profitability check 105226633Sdim return SelectRegShifterOperand(N, A, B, C, false); 106221345Sdim } 107226633Sdim bool SelectShiftImmShifterOperand(SDValue N, SDValue &A, 108226633Sdim SDValue &B) { 109226633Sdim // Don't apply the profitability check 110226633Sdim return SelectImmShifterOperand(N, A, B, false); 111226633Sdim } 112221345Sdim 113218893Sdim bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm); 114218893Sdim bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc); 115218893Sdim 116218893Sdim AddrMode2Type SelectAddrMode2Worker(SDValue N, SDValue &Base, 117218893Sdim SDValue &Offset, SDValue &Opc); 118218893Sdim bool SelectAddrMode2Base(SDValue N, SDValue &Base, SDValue &Offset, 119218893Sdim SDValue &Opc) { 120218893Sdim return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_BASE; 121218893Sdim } 122218893Sdim 123218893Sdim bool SelectAddrMode2ShOp(SDValue N, SDValue &Base, SDValue &Offset, 124218893Sdim SDValue &Opc) { 125218893Sdim return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_SHOP; 126218893Sdim } 127218893Sdim 128218893Sdim bool SelectAddrMode2(SDValue N, SDValue &Base, SDValue &Offset, 129218893Sdim SDValue &Opc) { 130218893Sdim SelectAddrMode2Worker(N, Base, Offset, Opc); 131218893Sdim// return SelectAddrMode2ShOp(N, Base, Offset, Opc); 132218893Sdim // This always matches one way or another. 133218893Sdim return true; 134218893Sdim } 135218893Sdim 136261991Sdim bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) { 137261991Sdim const ConstantSDNode *CN = cast<ConstantSDNode>(N); 138288943Sdim Pred = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(N), MVT::i32); 139261991Sdim Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32); 140261991Sdim return true; 141261991Sdim } 142261991Sdim 143226633Sdim bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N, 144193323Sed SDValue &Offset, SDValue &Opc); 145226633Sdim bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N, 146226633Sdim SDValue &Offset, SDValue &Opc); 147226633Sdim bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N, 148226633Sdim SDValue &Offset, SDValue &Opc); 149226633Sdim bool SelectAddrOffsetNone(SDValue N, SDValue &Base); 150218893Sdim bool SelectAddrMode3(SDValue N, SDValue &Base, 151193323Sed SDValue &Offset, SDValue &Opc); 152202375Srdivacky bool SelectAddrMode3Offset(SDNode *Op, SDValue N, 153193323Sed SDValue &Offset, SDValue &Opc); 154218893Sdim bool SelectAddrMode5(SDValue N, SDValue &Base, 155193323Sed SDValue &Offset); 156218893Sdim bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align); 157219077Sdim bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset); 158193323Sed 159218893Sdim bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label); 160193323Sed 161218893Sdim // Thumb Addressing Modes: 162218893Sdim bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset); 163218893Sdim bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base, 164218893Sdim SDValue &OffImm); 165218893Sdim bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base, 166218893Sdim SDValue &OffImm); 167218893Sdim bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base, 168218893Sdim SDValue &OffImm); 169218893Sdim bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base, 170218893Sdim SDValue &OffImm); 171218893Sdim bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm); 172193323Sed 173218893Sdim // Thumb 2 Addressing Modes: 174218893Sdim bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm); 175218893Sdim bool SelectT2AddrModeImm8(SDValue N, SDValue &Base, 176195340Sed SDValue &OffImm); 177202375Srdivacky bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N, 178195340Sed SDValue &OffImm); 179218893Sdim bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base, 180195340Sed SDValue &OffReg, SDValue &ShImm); 181261991Sdim bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm); 182195340Sed 183218893Sdim inline bool is_so_imm(unsigned Imm) const { 184218893Sdim return ARM_AM::getSOImmVal(Imm) != -1; 185218893Sdim } 186218893Sdim 187218893Sdim inline bool is_so_imm_not(unsigned Imm) const { 188218893Sdim return ARM_AM::getSOImmVal(~Imm) != -1; 189218893Sdim } 190218893Sdim 191218893Sdim inline bool is_t2_so_imm(unsigned Imm) const { 192218893Sdim return ARM_AM::getT2SOImmVal(Imm) != -1; 193218893Sdim } 194218893Sdim 195218893Sdim inline bool is_t2_so_imm_not(unsigned Imm) const { 196218893Sdim return ARM_AM::getT2SOImmVal(~Imm) != -1; 197218893Sdim } 198218893Sdim 199193323Sed // Include the pieces autogenerated from the target description. 200193323Sed#include "ARMGenDAGISel.inc" 201193323Sed 202193323Sedprivate: 203195340Sed /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for 204195340Sed /// ARM. 205202375Srdivacky SDNode *SelectARMIndexedLoad(SDNode *N); 206202375Srdivacky SDNode *SelectT2IndexedLoad(SDNode *N); 207195340Sed 208206083Srdivacky /// SelectVLD - Select NEON load intrinsics. NumVecs should be 209206083Srdivacky /// 1, 2, 3 or 4. The opcode arrays specify the instructions used for 210198090Srdivacky /// loads of D registers and even subregs and odd subregs of Q registers. 211206083Srdivacky /// For NumVecs <= 2, QOpcodes1 is not used. 212218893Sdim SDNode *SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs, 213239462Sdim const uint16_t *DOpcodes, 214239462Sdim const uint16_t *QOpcodes0, const uint16_t *QOpcodes1); 215198090Srdivacky 216198113Srdivacky /// SelectVST - Select NEON store intrinsics. NumVecs should 217206083Srdivacky /// be 1, 2, 3 or 4. The opcode arrays specify the instructions used for 218198113Srdivacky /// stores of D registers and even subregs and odd subregs of Q registers. 219206083Srdivacky /// For NumVecs <= 2, QOpcodes1 is not used. 220218893Sdim SDNode *SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, 221239462Sdim const uint16_t *DOpcodes, 222239462Sdim const uint16_t *QOpcodes0, const uint16_t *QOpcodes1); 223198113Srdivacky 224198090Srdivacky /// SelectVLDSTLane - Select NEON load/store lane intrinsics. NumVecs should 225198090Srdivacky /// be 2, 3 or 4. The opcode arrays specify the instructions used for 226218893Sdim /// load/store of D registers and Q registers. 227218893Sdim SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, 228218893Sdim bool isUpdating, unsigned NumVecs, 229239462Sdim const uint16_t *DOpcodes, const uint16_t *QOpcodes); 230198090Srdivacky 231218893Sdim /// SelectVLDDup - Select NEON load-duplicate intrinsics. NumVecs 232218893Sdim /// should be 2, 3 or 4. The opcode array specifies the instructions used 233218893Sdim /// for loading D registers. (Q registers are not supported.) 234218893Sdim SDNode *SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs, 235239462Sdim const uint16_t *Opcodes); 236218893Sdim 237210299Sed /// SelectVTBL - Select NEON VTBL and VTBX intrinsics. NumVecs should be 2, 238210299Sed /// 3 or 4. These are custom-selected so that a REG_SEQUENCE can be 239210299Sed /// generated to force the table registers to be consecutive. 240210299Sed SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc); 241210299Sed 242198090Srdivacky /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM. 243207618Srdivacky SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned); 244198090Srdivacky 245226633Sdim // Select special operations if node forms integer ABS pattern 246226633Sdim SDNode *SelectABSOp(SDNode *N); 247226633Sdim 248288943Sdim SDNode *SelectReadRegister(SDNode *N); 249288943Sdim SDNode *SelectWriteRegister(SDNode *N); 250288943Sdim 251249423Sdim SDNode *SelectInlineAsm(SDNode *N); 252249423Sdim 253208599Srdivacky SDNode *SelectConcatVector(SDNode *N); 254208599Srdivacky 255195340Sed /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 256195340Sed /// inline asm expressions. 257288943Sdim bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, 258276479Sdim std::vector<SDValue> &OutOps) override; 259198090Srdivacky 260249423Sdim // Form pairs of consecutive R, S, D, or Q registers. 261249423Sdim SDNode *createGPRPairNode(EVT VT, SDValue V0, SDValue V1); 262249423Sdim SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1); 263249423Sdim SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1); 264249423Sdim SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1); 265208599Srdivacky 266210299Sed // Form sequences of 4 consecutive S, D, or Q registers. 267249423Sdim SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); 268249423Sdim SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); 269249423Sdim SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); 270208599Srdivacky 271218893Sdim // Get the alignment operand for a NEON VLD or VST instruction. 272288943Sdim SDValue GetVLDSTAlign(SDValue Align, SDLoc dl, unsigned NumVecs, 273288943Sdim bool is64BitVector); 274296417Sdim 275296417Sdim /// Returns the number of instructions required to materialize the given 276296417Sdim /// constant in a register, or 3 if a literal pool load is needed. 277296417Sdim unsigned ConstantMaterializationCost(unsigned Val) const; 278296417Sdim 279296417Sdim /// Checks if N is a multiplication by a constant where we can extract out a 280296417Sdim /// power of two from the constant so that it can be used in a shift, but only 281296417Sdim /// if it simplifies the materialization of the constant. Returns true if it 282296417Sdim /// is, and assigns to PowerOfTwo the power of two that should be extracted 283296417Sdim /// out and to NewMulConst the new constant to be multiplied by. 284296417Sdim bool canExtractShiftFromMul(const SDValue &N, unsigned MaxShift, 285296417Sdim unsigned &PowerOfTwo, SDValue &NewMulConst) const; 286296417Sdim 287296417Sdim /// Replace N with M in CurDAG, in a way that also ensures that M gets 288296417Sdim /// selected when N would have been selected. 289296417Sdim void replaceDAGValue(const SDValue &N, SDValue M); 290193323Sed}; 291193323Sed} 292193323Sed 293198090Srdivacky/// isInt32Immediate - This method tests to see if the node is a 32-bit constant 294198090Srdivacky/// operand. If so Imm will receive the 32-bit value. 295198090Srdivackystatic bool isInt32Immediate(SDNode *N, unsigned &Imm) { 296198090Srdivacky if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) { 297198090Srdivacky Imm = cast<ConstantSDNode>(N)->getZExtValue(); 298198090Srdivacky return true; 299198090Srdivacky } 300198090Srdivacky return false; 301198090Srdivacky} 302198090Srdivacky 303198090Srdivacky// isInt32Immediate - This method tests to see if a constant operand. 304198090Srdivacky// If so Imm will receive the 32 bit value. 305198090Srdivackystatic bool isInt32Immediate(SDValue N, unsigned &Imm) { 306198090Srdivacky return isInt32Immediate(N.getNode(), Imm); 307198090Srdivacky} 308198090Srdivacky 309198090Srdivacky// isOpcWithIntImmediate - This method tests to see if the node is a specific 310198090Srdivacky// opcode and that it has a immediate integer right operand. 311198090Srdivacky// If so Imm will receive the 32 bit value. 312198090Srdivackystatic bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) { 313198090Srdivacky return N->getOpcode() == Opc && 314198090Srdivacky isInt32Immediate(N->getOperand(1).getNode(), Imm); 315198090Srdivacky} 316198090Srdivacky 317218893Sdim/// \brief Check whether a particular node is a constant value representable as 318243830Sdim/// (N * Scale) where (N in [\p RangeMin, \p RangeMax). 319218893Sdim/// 320218893Sdim/// \param ScaledConstant [out] - On success, the pre-scaled constant value. 321226633Sdimstatic bool isScaledConstantInRange(SDValue Node, int Scale, 322218893Sdim int RangeMin, int RangeMax, 323218893Sdim int &ScaledConstant) { 324226633Sdim assert(Scale > 0 && "Invalid scale!"); 325198090Srdivacky 326218893Sdim // Check that this is a constant. 327218893Sdim const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node); 328218893Sdim if (!C) 329218893Sdim return false; 330218893Sdim 331218893Sdim ScaledConstant = (int) C->getZExtValue(); 332218893Sdim if ((ScaledConstant % Scale) != 0) 333218893Sdim return false; 334218893Sdim 335218893Sdim ScaledConstant /= Scale; 336218893Sdim return ScaledConstant >= RangeMin && ScaledConstant < RangeMax; 337218893Sdim} 338218893Sdim 339249423Sdimvoid ARMDAGToDAGISel::PreprocessISelDAG() { 340249423Sdim if (!Subtarget->hasV6T2Ops()) 341249423Sdim return; 342249423Sdim 343249423Sdim bool isThumb2 = Subtarget->isThumb(); 344249423Sdim for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 345249423Sdim E = CurDAG->allnodes_end(); I != E; ) { 346296417Sdim SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues. 347249423Sdim 348249423Sdim if (N->getOpcode() != ISD::ADD) 349249423Sdim continue; 350249423Sdim 351249423Sdim // Look for (add X1, (and (srl X2, c1), c2)) where c2 is constant with 352249423Sdim // leading zeros, followed by consecutive set bits, followed by 1 or 2 353249423Sdim // trailing zeros, e.g. 1020. 354249423Sdim // Transform the expression to 355249423Sdim // (add X1, (shl (and (srl X2, c1), (c2>>tz)), tz)) where tz is the number 356249423Sdim // of trailing zeros of c2. The left shift would be folded as an shifter 357249423Sdim // operand of 'add' and the 'and' and 'srl' would become a bits extraction 358249423Sdim // node (UBFX). 359249423Sdim 360249423Sdim SDValue N0 = N->getOperand(0); 361249423Sdim SDValue N1 = N->getOperand(1); 362249423Sdim unsigned And_imm = 0; 363249423Sdim if (!isOpcWithIntImmediate(N1.getNode(), ISD::AND, And_imm)) { 364249423Sdim if (isOpcWithIntImmediate(N0.getNode(), ISD::AND, And_imm)) 365249423Sdim std::swap(N0, N1); 366249423Sdim } 367249423Sdim if (!And_imm) 368249423Sdim continue; 369249423Sdim 370249423Sdim // Check if the AND mask is an immediate of the form: 000.....1111111100 371261991Sdim unsigned TZ = countTrailingZeros(And_imm); 372249423Sdim if (TZ != 1 && TZ != 2) 373249423Sdim // Be conservative here. Shifter operands aren't always free. e.g. On 374249423Sdim // Swift, left shifter operand of 1 / 2 for free but others are not. 375249423Sdim // e.g. 376249423Sdim // ubfx r3, r1, #16, #8 377249423Sdim // ldr.w r3, [r0, r3, lsl #2] 378249423Sdim // vs. 379249423Sdim // mov.w r9, #1020 380249423Sdim // and.w r2, r9, r1, lsr #14 381249423Sdim // ldr r2, [r0, r2] 382249423Sdim continue; 383249423Sdim And_imm >>= TZ; 384249423Sdim if (And_imm & (And_imm + 1)) 385249423Sdim continue; 386249423Sdim 387249423Sdim // Look for (and (srl X, c1), c2). 388249423Sdim SDValue Srl = N1.getOperand(0); 389249423Sdim unsigned Srl_imm = 0; 390249423Sdim if (!isOpcWithIntImmediate(Srl.getNode(), ISD::SRL, Srl_imm) || 391249423Sdim (Srl_imm <= 2)) 392249423Sdim continue; 393249423Sdim 394249423Sdim // Make sure first operand is not a shifter operand which would prevent 395249423Sdim // folding of the left shift. 396249423Sdim SDValue CPTmp0; 397249423Sdim SDValue CPTmp1; 398249423Sdim SDValue CPTmp2; 399249423Sdim if (isThumb2) { 400296417Sdim if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1)) 401249423Sdim continue; 402249423Sdim } else { 403249423Sdim if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) || 404249423Sdim SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2)) 405249423Sdim continue; 406249423Sdim } 407249423Sdim 408249423Sdim // Now make the transformation. 409261991Sdim Srl = CurDAG->getNode(ISD::SRL, SDLoc(Srl), MVT::i32, 410249423Sdim Srl.getOperand(0), 411288943Sdim CurDAG->getConstant(Srl_imm + TZ, SDLoc(Srl), 412288943Sdim MVT::i32)); 413261991Sdim N1 = CurDAG->getNode(ISD::AND, SDLoc(N1), MVT::i32, 414288943Sdim Srl, 415288943Sdim CurDAG->getConstant(And_imm, SDLoc(Srl), MVT::i32)); 416261991Sdim N1 = CurDAG->getNode(ISD::SHL, SDLoc(N1), MVT::i32, 417288943Sdim N1, CurDAG->getConstant(TZ, SDLoc(Srl), MVT::i32)); 418249423Sdim CurDAG->UpdateNodeOperands(N, N0, N1); 419276479Sdim } 420249423Sdim} 421249423Sdim 422218893Sdim/// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS 423218893Sdim/// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at 424218893Sdim/// least on current ARM implementations) which should be avoidded. 425218893Sdimbool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const { 426218893Sdim if (OptLevel == CodeGenOpt::None) 427218893Sdim return true; 428218893Sdim 429218893Sdim if (!CheckVMLxHazard) 430218893Sdim return true; 431218893Sdim 432276479Sdim if (!Subtarget->isCortexA7() && !Subtarget->isCortexA8() && 433276479Sdim !Subtarget->isCortexA9() && !Subtarget->isSwift()) 434218893Sdim return true; 435218893Sdim 436218893Sdim if (!N->hasOneUse()) 437218893Sdim return false; 438218893Sdim 439218893Sdim SDNode *Use = *N->use_begin(); 440218893Sdim if (Use->getOpcode() == ISD::CopyToReg) 441218893Sdim return true; 442218893Sdim if (Use->isMachineOpcode()) { 443276479Sdim const ARMBaseInstrInfo *TII = static_cast<const ARMBaseInstrInfo *>( 444280031Sdim CurDAG->getSubtarget().getInstrInfo()); 445261991Sdim 446224145Sdim const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode()); 447224145Sdim if (MCID.mayStore()) 448218893Sdim return true; 449224145Sdim unsigned Opcode = MCID.getOpcode(); 450218893Sdim if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD) 451218893Sdim return true; 452218893Sdim // vmlx feeding into another vmlx. We actually want to unfold 453218893Sdim // the use later in the MLxExpansion pass. e.g. 454218893Sdim // vmla 455218893Sdim // vmla (stall 8 cycles) 456218893Sdim // 457218893Sdim // vmul (5 cycles) 458218893Sdim // vadd (5 cycles) 459218893Sdim // vmla 460218893Sdim // This adds up to about 18 - 19 cycles. 461218893Sdim // 462218893Sdim // vmla 463218893Sdim // vmul (stall 4 cycles) 464218893Sdim // vadd adds up to about 14 cycles. 465218893Sdim return TII->isFpMLxInstruction(Opcode); 466218893Sdim } 467218893Sdim 468218893Sdim return false; 469218893Sdim} 470218893Sdim 471218893Sdimbool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift, 472218893Sdim ARM_AM::ShiftOpc ShOpcVal, 473218893Sdim unsigned ShAmt) { 474243830Sdim if (!Subtarget->isLikeA9() && !Subtarget->isSwift()) 475218893Sdim return true; 476218893Sdim if (Shift.hasOneUse()) 477218893Sdim return true; 478218893Sdim // R << 2 is free. 479243830Sdim return ShOpcVal == ARM_AM::lsl && 480243830Sdim (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1)); 481218893Sdim} 482218893Sdim 483296417Sdimunsigned ARMDAGToDAGISel::ConstantMaterializationCost(unsigned Val) const { 484296417Sdim if (Subtarget->isThumb()) { 485296417Sdim if (Val <= 255) return 1; // MOV 486296417Sdim if (Subtarget->hasV6T2Ops() && Val <= 0xffff) return 1; // MOVW 487296417Sdim if (~Val <= 255) return 2; // MOV + MVN 488296417Sdim if (ARM_AM::isThumbImmShiftedVal(Val)) return 2; // MOV + LSL 489296417Sdim } else { 490296417Sdim if (ARM_AM::getSOImmVal(Val) != -1) return 1; // MOV 491296417Sdim if (ARM_AM::getSOImmVal(~Val) != -1) return 1; // MVN 492296417Sdim if (Subtarget->hasV6T2Ops() && Val <= 0xffff) return 1; // MOVW 493296417Sdim if (ARM_AM::isSOImmTwoPartVal(Val)) return 2; // two instrs 494296417Sdim } 495296417Sdim if (Subtarget->useMovt(*MF)) return 2; // MOVW + MOVT 496296417Sdim return 3; // Literal pool load 497296417Sdim} 498296417Sdim 499296417Sdimbool ARMDAGToDAGISel::canExtractShiftFromMul(const SDValue &N, 500296417Sdim unsigned MaxShift, 501296417Sdim unsigned &PowerOfTwo, 502296417Sdim SDValue &NewMulConst) const { 503296417Sdim assert(N.getOpcode() == ISD::MUL); 504296417Sdim assert(MaxShift > 0); 505296417Sdim 506296417Sdim // If the multiply is used in more than one place then changing the constant 507296417Sdim // will make other uses incorrect, so don't. 508296417Sdim if (!N.hasOneUse()) return false; 509296417Sdim // Check if the multiply is by a constant 510296417Sdim ConstantSDNode *MulConst = dyn_cast<ConstantSDNode>(N.getOperand(1)); 511296417Sdim if (!MulConst) return false; 512296417Sdim // If the constant is used in more than one place then modifying it will mean 513296417Sdim // we need to materialize two constants instead of one, which is a bad idea. 514296417Sdim if (!MulConst->hasOneUse()) return false; 515296417Sdim unsigned MulConstVal = MulConst->getZExtValue(); 516296417Sdim if (MulConstVal == 0) return false; 517296417Sdim 518296417Sdim // Find the largest power of 2 that MulConstVal is a multiple of 519296417Sdim PowerOfTwo = MaxShift; 520296417Sdim while ((MulConstVal % (1 << PowerOfTwo)) != 0) { 521296417Sdim --PowerOfTwo; 522296417Sdim if (PowerOfTwo == 0) return false; 523296417Sdim } 524296417Sdim 525296417Sdim // Only optimise if the new cost is better 526296417Sdim unsigned NewMulConstVal = MulConstVal / (1 << PowerOfTwo); 527296417Sdim NewMulConst = CurDAG->getConstant(NewMulConstVal, SDLoc(N), MVT::i32); 528296417Sdim unsigned OldCost = ConstantMaterializationCost(MulConstVal); 529296417Sdim unsigned NewCost = ConstantMaterializationCost(NewMulConstVal); 530296417Sdim return NewCost < OldCost; 531296417Sdim} 532296417Sdim 533296417Sdimvoid ARMDAGToDAGISel::replaceDAGValue(const SDValue &N, SDValue M) { 534296417Sdim CurDAG->RepositionNode(N.getNode()->getIterator(), M.getNode()); 535296417Sdim CurDAG->ReplaceAllUsesWith(N, M); 536296417Sdim} 537296417Sdim 538226633Sdimbool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N, 539195340Sed SDValue &BaseReg, 540226633Sdim SDValue &Opc, 541226633Sdim bool CheckProfitability) { 542226633Sdim if (DisableShifterOp) 543226633Sdim return false; 544226633Sdim 545296417Sdim // If N is a multiply-by-constant and it's profitable to extract a shift and 546296417Sdim // use it in a shifted operand do so. 547296417Sdim if (N.getOpcode() == ISD::MUL) { 548296417Sdim unsigned PowerOfTwo = 0; 549296417Sdim SDValue NewMulConst; 550296417Sdim if (canExtractShiftFromMul(N, 31, PowerOfTwo, NewMulConst)) { 551296417Sdim BaseReg = SDValue(Select(CurDAG->getNode(ISD::MUL, SDLoc(N), MVT::i32, 552296417Sdim N.getOperand(0), NewMulConst) 553296417Sdim .getNode()), 554296417Sdim 0); 555296417Sdim replaceDAGValue(N.getOperand(1), NewMulConst); 556296417Sdim Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ARM_AM::lsl, 557296417Sdim PowerOfTwo), 558296417Sdim SDLoc(N), MVT::i32); 559296417Sdim return true; 560296417Sdim } 561296417Sdim } 562296417Sdim 563226633Sdim ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 564226633Sdim 565226633Sdim // Don't match base register only case. That is matched to a separate 566226633Sdim // lower complexity pattern with explicit register operand. 567226633Sdim if (ShOpcVal == ARM_AM::no_shift) return false; 568226633Sdim 569226633Sdim BaseReg = N.getOperand(0); 570226633Sdim unsigned ShImmVal = 0; 571226633Sdim ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); 572226633Sdim if (!RHS) return false; 573226633Sdim ShImmVal = RHS->getZExtValue() & 31; 574226633Sdim Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal), 575288943Sdim SDLoc(N), MVT::i32); 576226633Sdim return true; 577226633Sdim} 578226633Sdim 579226633Sdimbool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N, 580226633Sdim SDValue &BaseReg, 581195340Sed SDValue &ShReg, 582221345Sdim SDValue &Opc, 583221345Sdim bool CheckProfitability) { 584212904Sdim if (DisableShifterOp) 585212904Sdim return false; 586212904Sdim 587226633Sdim ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 588195340Sed 589195340Sed // Don't match base register only case. That is matched to a separate 590195340Sed // lower complexity pattern with explicit register operand. 591195340Sed if (ShOpcVal == ARM_AM::no_shift) return false; 592198090Srdivacky 593195340Sed BaseReg = N.getOperand(0); 594195340Sed unsigned ShImmVal = 0; 595226633Sdim ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); 596226633Sdim if (RHS) return false; 597226633Sdim 598226633Sdim ShReg = N.getOperand(1); 599226633Sdim if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal)) 600226633Sdim return false; 601195340Sed Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal), 602288943Sdim SDLoc(N), MVT::i32); 603195340Sed return true; 604195340Sed} 605195340Sed 606226633Sdim 607218893Sdimbool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N, 608218893Sdim SDValue &Base, 609218893Sdim SDValue &OffImm) { 610218893Sdim // Match simple R + imm12 operands. 611218893Sdim 612218893Sdim // Base only. 613218893Sdim if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 614218893Sdim !CurDAG->isBaseWithConstantOffset(N)) { 615218893Sdim if (N.getOpcode() == ISD::FrameIndex) { 616218893Sdim // Match frame index. 617218893Sdim int FI = cast<FrameIndexSDNode>(N)->getIndex(); 618288943Sdim Base = CurDAG->getTargetFrameIndex( 619288943Sdim FI, TLI->getPointerTy(CurDAG->getDataLayout())); 620288943Sdim OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); 621218893Sdim return true; 622218893Sdim } 623221345Sdim 624218893Sdim if (N.getOpcode() == ARMISD::Wrapper && 625296417Sdim N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress && 626296417Sdim N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) { 627218893Sdim Base = N.getOperand(0); 628218893Sdim } else 629218893Sdim Base = N; 630288943Sdim OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); 631218893Sdim return true; 632218893Sdim } 633218893Sdim 634218893Sdim if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 635280031Sdim int RHSC = (int)RHS->getSExtValue(); 636218893Sdim if (N.getOpcode() == ISD::SUB) 637218893Sdim RHSC = -RHSC; 638218893Sdim 639280031Sdim if (RHSC > -0x1000 && RHSC < 0x1000) { // 12 bits 640218893Sdim Base = N.getOperand(0); 641218893Sdim if (Base.getOpcode() == ISD::FrameIndex) { 642218893Sdim int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 643288943Sdim Base = CurDAG->getTargetFrameIndex( 644288943Sdim FI, TLI->getPointerTy(CurDAG->getDataLayout())); 645218893Sdim } 646288943Sdim OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32); 647218893Sdim return true; 648218893Sdim } 649218893Sdim } 650218893Sdim 651218893Sdim // Base only. 652218893Sdim Base = N; 653288943Sdim OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); 654218893Sdim return true; 655218893Sdim} 656218893Sdim 657218893Sdim 658218893Sdim 659218893Sdimbool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, 660193323Sed SDValue &Opc) { 661218893Sdim if (N.getOpcode() == ISD::MUL && 662243830Sdim ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) { 663193323Sed if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 664193323Sed // X * [3,5,9] -> X + X * [2,4,8] etc. 665193323Sed int RHSC = (int)RHS->getZExtValue(); 666193323Sed if (RHSC & 1) { 667193323Sed RHSC = RHSC & ~1; 668193323Sed ARM_AM::AddrOpc AddSub = ARM_AM::add; 669193323Sed if (RHSC < 0) { 670193323Sed AddSub = ARM_AM::sub; 671193323Sed RHSC = - RHSC; 672193323Sed } 673193323Sed if (isPowerOf2_32(RHSC)) { 674193323Sed unsigned ShAmt = Log2_32(RHSC); 675193323Sed Base = Offset = N.getOperand(0); 676193323Sed Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, 677193323Sed ARM_AM::lsl), 678288943Sdim SDLoc(N), MVT::i32); 679193323Sed return true; 680193323Sed } 681193323Sed } 682193323Sed } 683193323Sed } 684193323Sed 685218893Sdim if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 686218893Sdim // ISD::OR that is equivalent to an ISD::ADD. 687218893Sdim !CurDAG->isBaseWithConstantOffset(N)) 688218893Sdim return false; 689218893Sdim 690218893Sdim // Leave simple R +/- imm12 operands for LDRi12 691218893Sdim if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) { 692218893Sdim int RHSC; 693218893Sdim if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, 694218893Sdim -0x1000+1, 0x1000, RHSC)) // 12 bits. 695218893Sdim return false; 696218893Sdim } 697218893Sdim 698218893Sdim // Otherwise this is R +/- [possibly shifted] R. 699218893Sdim ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add; 700226633Sdim ARM_AM::ShiftOpc ShOpcVal = 701226633Sdim ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode()); 702218893Sdim unsigned ShAmt = 0; 703218893Sdim 704218893Sdim Base = N.getOperand(0); 705218893Sdim Offset = N.getOperand(1); 706218893Sdim 707218893Sdim if (ShOpcVal != ARM_AM::no_shift) { 708218893Sdim // Check to see if the RHS of the shift is a constant, if not, we can't fold 709218893Sdim // it. 710218893Sdim if (ConstantSDNode *Sh = 711218893Sdim dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) { 712218893Sdim ShAmt = Sh->getZExtValue(); 713218893Sdim if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt)) 714218893Sdim Offset = N.getOperand(1).getOperand(0); 715218893Sdim else { 716218893Sdim ShAmt = 0; 717218893Sdim ShOpcVal = ARM_AM::no_shift; 718218893Sdim } 719218893Sdim } else { 720218893Sdim ShOpcVal = ARM_AM::no_shift; 721218893Sdim } 722218893Sdim } 723218893Sdim 724218893Sdim // Try matching (R shl C) + (R). 725218893Sdim if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift && 726243830Sdim !(Subtarget->isLikeA9() || Subtarget->isSwift() || 727243830Sdim N.getOperand(0).hasOneUse())) { 728226633Sdim ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode()); 729218893Sdim if (ShOpcVal != ARM_AM::no_shift) { 730218893Sdim // Check to see if the RHS of the shift is a constant, if not, we can't 731218893Sdim // fold it. 732218893Sdim if (ConstantSDNode *Sh = 733218893Sdim dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) { 734218893Sdim ShAmt = Sh->getZExtValue(); 735226633Sdim if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) { 736218893Sdim Offset = N.getOperand(0).getOperand(0); 737218893Sdim Base = N.getOperand(1); 738218893Sdim } else { 739218893Sdim ShAmt = 0; 740218893Sdim ShOpcVal = ARM_AM::no_shift; 741218893Sdim } 742218893Sdim } else { 743218893Sdim ShOpcVal = ARM_AM::no_shift; 744218893Sdim } 745218893Sdim } 746218893Sdim } 747218893Sdim 748296417Sdim // If Offset is a multiply-by-constant and it's profitable to extract a shift 749296417Sdim // and use it in a shifted operand do so. 750296417Sdim if (Offset.getOpcode() == ISD::MUL && N.hasOneUse()) { 751296417Sdim unsigned PowerOfTwo = 0; 752296417Sdim SDValue NewMulConst; 753296417Sdim if (canExtractShiftFromMul(Offset, 31, PowerOfTwo, NewMulConst)) { 754296417Sdim replaceDAGValue(Offset.getOperand(1), NewMulConst); 755296417Sdim ShAmt = PowerOfTwo; 756296417Sdim ShOpcVal = ARM_AM::lsl; 757296417Sdim } 758296417Sdim } 759296417Sdim 760218893Sdim Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), 761288943Sdim SDLoc(N), MVT::i32); 762218893Sdim return true; 763218893Sdim} 764218893Sdim 765218893Sdim 766218893Sdim//----- 767218893Sdim 768218893SdimAddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N, 769218893Sdim SDValue &Base, 770218893Sdim SDValue &Offset, 771218893Sdim SDValue &Opc) { 772218893Sdim if (N.getOpcode() == ISD::MUL && 773243830Sdim (!(Subtarget->isLikeA9() || Subtarget->isSwift()) || N.hasOneUse())) { 774218893Sdim if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 775218893Sdim // X * [3,5,9] -> X + X * [2,4,8] etc. 776218893Sdim int RHSC = (int)RHS->getZExtValue(); 777218893Sdim if (RHSC & 1) { 778218893Sdim RHSC = RHSC & ~1; 779218893Sdim ARM_AM::AddrOpc AddSub = ARM_AM::add; 780218893Sdim if (RHSC < 0) { 781218893Sdim AddSub = ARM_AM::sub; 782218893Sdim RHSC = - RHSC; 783218893Sdim } 784218893Sdim if (isPowerOf2_32(RHSC)) { 785218893Sdim unsigned ShAmt = Log2_32(RHSC); 786218893Sdim Base = Offset = N.getOperand(0); 787218893Sdim Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, 788218893Sdim ARM_AM::lsl), 789288943Sdim SDLoc(N), MVT::i32); 790218893Sdim return AM2_SHOP; 791218893Sdim } 792218893Sdim } 793218893Sdim } 794218893Sdim } 795218893Sdim 796218893Sdim if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 797218893Sdim // ISD::OR that is equivalent to an ADD. 798218893Sdim !CurDAG->isBaseWithConstantOffset(N)) { 799193323Sed Base = N; 800193323Sed if (N.getOpcode() == ISD::FrameIndex) { 801193323Sed int FI = cast<FrameIndexSDNode>(N)->getIndex(); 802288943Sdim Base = CurDAG->getTargetFrameIndex( 803288943Sdim FI, TLI->getPointerTy(CurDAG->getDataLayout())); 804199989Srdivacky } else if (N.getOpcode() == ARMISD::Wrapper && 805296417Sdim N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress && 806296417Sdim N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) { 807193323Sed Base = N.getOperand(0); 808193323Sed } 809193323Sed Offset = CurDAG->getRegister(0, MVT::i32); 810193323Sed Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0, 811193323Sed ARM_AM::no_shift), 812288943Sdim SDLoc(N), MVT::i32); 813218893Sdim return AM2_BASE; 814193323Sed } 815198090Srdivacky 816193323Sed // Match simple R +/- imm12 operands. 817218893Sdim if (N.getOpcode() != ISD::SUB) { 818218893Sdim int RHSC; 819218893Sdim if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, 820218893Sdim -0x1000+1, 0x1000, RHSC)) { // 12 bits. 821218893Sdim Base = N.getOperand(0); 822218893Sdim if (Base.getOpcode() == ISD::FrameIndex) { 823218893Sdim int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 824288943Sdim Base = CurDAG->getTargetFrameIndex( 825288943Sdim FI, TLI->getPointerTy(CurDAG->getDataLayout())); 826218893Sdim } 827218893Sdim Offset = CurDAG->getRegister(0, MVT::i32); 828193323Sed 829218893Sdim ARM_AM::AddrOpc AddSub = ARM_AM::add; 830218893Sdim if (RHSC < 0) { 831218893Sdim AddSub = ARM_AM::sub; 832218893Sdim RHSC = - RHSC; 833193323Sed } 834218893Sdim Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC, 835218893Sdim ARM_AM::no_shift), 836288943Sdim SDLoc(N), MVT::i32); 837218893Sdim return AM2_BASE; 838193323Sed } 839218893Sdim } 840198090Srdivacky 841243830Sdim if ((Subtarget->isLikeA9() || Subtarget->isSwift()) && !N.hasOneUse()) { 842218893Sdim // Compute R +/- (R << N) and reuse it. 843218893Sdim Base = N; 844218893Sdim Offset = CurDAG->getRegister(0, MVT::i32); 845218893Sdim Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0, 846218893Sdim ARM_AM::no_shift), 847288943Sdim SDLoc(N), MVT::i32); 848218893Sdim return AM2_BASE; 849218893Sdim } 850218893Sdim 851198892Srdivacky // Otherwise this is R +/- [possibly shifted] R. 852218893Sdim ARM_AM::AddrOpc AddSub = N.getOpcode() != ISD::SUB ? ARM_AM::add:ARM_AM::sub; 853226633Sdim ARM_AM::ShiftOpc ShOpcVal = 854226633Sdim ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode()); 855193323Sed unsigned ShAmt = 0; 856198090Srdivacky 857193323Sed Base = N.getOperand(0); 858193323Sed Offset = N.getOperand(1); 859198090Srdivacky 860193323Sed if (ShOpcVal != ARM_AM::no_shift) { 861193323Sed // Check to see if the RHS of the shift is a constant, if not, we can't fold 862193323Sed // it. 863193323Sed if (ConstantSDNode *Sh = 864193323Sed dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) { 865193323Sed ShAmt = Sh->getZExtValue(); 866218893Sdim if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt)) 867218893Sdim Offset = N.getOperand(1).getOperand(0); 868218893Sdim else { 869218893Sdim ShAmt = 0; 870218893Sdim ShOpcVal = ARM_AM::no_shift; 871218893Sdim } 872193323Sed } else { 873193323Sed ShOpcVal = ARM_AM::no_shift; 874193323Sed } 875193323Sed } 876198090Srdivacky 877193323Sed // Try matching (R shl C) + (R). 878218893Sdim if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift && 879243830Sdim !(Subtarget->isLikeA9() || Subtarget->isSwift() || 880243830Sdim N.getOperand(0).hasOneUse())) { 881226633Sdim ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode()); 882193323Sed if (ShOpcVal != ARM_AM::no_shift) { 883193323Sed // Check to see if the RHS of the shift is a constant, if not, we can't 884193323Sed // fold it. 885193323Sed if (ConstantSDNode *Sh = 886193323Sed dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) { 887193323Sed ShAmt = Sh->getZExtValue(); 888226633Sdim if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) { 889218893Sdim Offset = N.getOperand(0).getOperand(0); 890218893Sdim Base = N.getOperand(1); 891218893Sdim } else { 892218893Sdim ShAmt = 0; 893218893Sdim ShOpcVal = ARM_AM::no_shift; 894218893Sdim } 895193323Sed } else { 896193323Sed ShOpcVal = ARM_AM::no_shift; 897193323Sed } 898193323Sed } 899193323Sed } 900198090Srdivacky 901193323Sed Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), 902288943Sdim SDLoc(N), MVT::i32); 903218893Sdim return AM2_SHOP; 904193323Sed} 905193323Sed 906226633Sdimbool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N, 907193323Sed SDValue &Offset, SDValue &Opc) { 908202375Srdivacky unsigned Opcode = Op->getOpcode(); 909193323Sed ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 910193323Sed ? cast<LoadSDNode>(Op)->getAddressingMode() 911193323Sed : cast<StoreSDNode>(Op)->getAddressingMode(); 912193323Sed ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 913193323Sed ? ARM_AM::add : ARM_AM::sub; 914218893Sdim int Val; 915226633Sdim if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) 916226633Sdim return false; 917193323Sed 918193323Sed Offset = N; 919226633Sdim ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 920193323Sed unsigned ShAmt = 0; 921193323Sed if (ShOpcVal != ARM_AM::no_shift) { 922193323Sed // Check to see if the RHS of the shift is a constant, if not, we can't fold 923193323Sed // it. 924193323Sed if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 925193323Sed ShAmt = Sh->getZExtValue(); 926218893Sdim if (isShifterOpProfitable(N, ShOpcVal, ShAmt)) 927218893Sdim Offset = N.getOperand(0); 928218893Sdim else { 929218893Sdim ShAmt = 0; 930218893Sdim ShOpcVal = ARM_AM::no_shift; 931218893Sdim } 932193323Sed } else { 933193323Sed ShOpcVal = ARM_AM::no_shift; 934193323Sed } 935193323Sed } 936193323Sed 937193323Sed Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), 938288943Sdim SDLoc(N), MVT::i32); 939193323Sed return true; 940193323Sed} 941193323Sed 942226633Sdimbool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N, 943226633Sdim SDValue &Offset, SDValue &Opc) { 944226633Sdim unsigned Opcode = Op->getOpcode(); 945226633Sdim ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 946226633Sdim ? cast<LoadSDNode>(Op)->getAddressingMode() 947226633Sdim : cast<StoreSDNode>(Op)->getAddressingMode(); 948226633Sdim ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 949226633Sdim ? ARM_AM::add : ARM_AM::sub; 950226633Sdim int Val; 951226633Sdim if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits. 952226633Sdim if (AddSub == ARM_AM::sub) Val *= -1; 953226633Sdim Offset = CurDAG->getRegister(0, MVT::i32); 954288943Sdim Opc = CurDAG->getTargetConstant(Val, SDLoc(Op), MVT::i32); 955226633Sdim return true; 956226633Sdim } 957193323Sed 958226633Sdim return false; 959226633Sdim} 960226633Sdim 961226633Sdim 962226633Sdimbool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N, 963226633Sdim SDValue &Offset, SDValue &Opc) { 964226633Sdim unsigned Opcode = Op->getOpcode(); 965226633Sdim ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 966226633Sdim ? cast<LoadSDNode>(Op)->getAddressingMode() 967226633Sdim : cast<StoreSDNode>(Op)->getAddressingMode(); 968226633Sdim ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 969226633Sdim ? ARM_AM::add : ARM_AM::sub; 970226633Sdim int Val; 971226633Sdim if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits. 972226633Sdim Offset = CurDAG->getRegister(0, MVT::i32); 973226633Sdim Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val, 974226633Sdim ARM_AM::no_shift), 975288943Sdim SDLoc(Op), MVT::i32); 976226633Sdim return true; 977226633Sdim } 978226633Sdim 979226633Sdim return false; 980226633Sdim} 981226633Sdim 982226633Sdimbool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) { 983226633Sdim Base = N; 984226633Sdim return true; 985226633Sdim} 986226633Sdim 987218893Sdimbool ARMDAGToDAGISel::SelectAddrMode3(SDValue N, 988193323Sed SDValue &Base, SDValue &Offset, 989193323Sed SDValue &Opc) { 990193323Sed if (N.getOpcode() == ISD::SUB) { 991193323Sed // X - C is canonicalize to X + -C, no need to handle it here. 992193323Sed Base = N.getOperand(0); 993193323Sed Offset = N.getOperand(1); 994288943Sdim Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0), SDLoc(N), 995288943Sdim MVT::i32); 996193323Sed return true; 997193323Sed } 998198090Srdivacky 999218893Sdim if (!CurDAG->isBaseWithConstantOffset(N)) { 1000193323Sed Base = N; 1001193323Sed if (N.getOpcode() == ISD::FrameIndex) { 1002193323Sed int FI = cast<FrameIndexSDNode>(N)->getIndex(); 1003288943Sdim Base = CurDAG->getTargetFrameIndex( 1004288943Sdim FI, TLI->getPointerTy(CurDAG->getDataLayout())); 1005193323Sed } 1006193323Sed Offset = CurDAG->getRegister(0, MVT::i32); 1007288943Sdim Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N), 1008288943Sdim MVT::i32); 1009193323Sed return true; 1010193323Sed } 1011198090Srdivacky 1012193323Sed // If the RHS is +/- imm8, fold into addr mode. 1013218893Sdim int RHSC; 1014218893Sdim if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, 1015218893Sdim -256 + 1, 256, RHSC)) { // 8 bits. 1016218893Sdim Base = N.getOperand(0); 1017218893Sdim if (Base.getOpcode() == ISD::FrameIndex) { 1018218893Sdim int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1019288943Sdim Base = CurDAG->getTargetFrameIndex( 1020288943Sdim FI, TLI->getPointerTy(CurDAG->getDataLayout())); 1021218893Sdim } 1022218893Sdim Offset = CurDAG->getRegister(0, MVT::i32); 1023193323Sed 1024218893Sdim ARM_AM::AddrOpc AddSub = ARM_AM::add; 1025218893Sdim if (RHSC < 0) { 1026218893Sdim AddSub = ARM_AM::sub; 1027218893Sdim RHSC = -RHSC; 1028193323Sed } 1029288943Sdim Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC), SDLoc(N), 1030288943Sdim MVT::i32); 1031218893Sdim return true; 1032193323Sed } 1033198090Srdivacky 1034193323Sed Base = N.getOperand(0); 1035193323Sed Offset = N.getOperand(1); 1036288943Sdim Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N), 1037288943Sdim MVT::i32); 1038193323Sed return true; 1039193323Sed} 1040193323Sed 1041202375Srdivackybool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N, 1042193323Sed SDValue &Offset, SDValue &Opc) { 1043202375Srdivacky unsigned Opcode = Op->getOpcode(); 1044193323Sed ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 1045193323Sed ? cast<LoadSDNode>(Op)->getAddressingMode() 1046193323Sed : cast<StoreSDNode>(Op)->getAddressingMode(); 1047193323Sed ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 1048193323Sed ? ARM_AM::add : ARM_AM::sub; 1049218893Sdim int Val; 1050218893Sdim if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits. 1051218893Sdim Offset = CurDAG->getRegister(0, MVT::i32); 1052288943Sdim Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), SDLoc(Op), 1053288943Sdim MVT::i32); 1054218893Sdim return true; 1055193323Sed } 1056193323Sed 1057193323Sed Offset = N; 1058288943Sdim Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), SDLoc(Op), 1059288943Sdim MVT::i32); 1060193323Sed return true; 1061193323Sed} 1062193323Sed 1063218893Sdimbool ARMDAGToDAGISel::SelectAddrMode5(SDValue N, 1064193323Sed SDValue &Base, SDValue &Offset) { 1065218893Sdim if (!CurDAG->isBaseWithConstantOffset(N)) { 1066193323Sed Base = N; 1067193323Sed if (N.getOpcode() == ISD::FrameIndex) { 1068193323Sed int FI = cast<FrameIndexSDNode>(N)->getIndex(); 1069288943Sdim Base = CurDAG->getTargetFrameIndex( 1070288943Sdim FI, TLI->getPointerTy(CurDAG->getDataLayout())); 1071199989Srdivacky } else if (N.getOpcode() == ARMISD::Wrapper && 1072296417Sdim N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress && 1073296417Sdim N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) { 1074193323Sed Base = N.getOperand(0); 1075193323Sed } 1076193323Sed Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0), 1077288943Sdim SDLoc(N), MVT::i32); 1078193323Sed return true; 1079193323Sed } 1080198090Srdivacky 1081193323Sed // If the RHS is +/- imm8, fold into addr mode. 1082218893Sdim int RHSC; 1083218893Sdim if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 1084218893Sdim -256 + 1, 256, RHSC)) { 1085218893Sdim Base = N.getOperand(0); 1086218893Sdim if (Base.getOpcode() == ISD::FrameIndex) { 1087218893Sdim int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1088288943Sdim Base = CurDAG->getTargetFrameIndex( 1089288943Sdim FI, TLI->getPointerTy(CurDAG->getDataLayout())); 1090218893Sdim } 1091193323Sed 1092218893Sdim ARM_AM::AddrOpc AddSub = ARM_AM::add; 1093218893Sdim if (RHSC < 0) { 1094218893Sdim AddSub = ARM_AM::sub; 1095218893Sdim RHSC = -RHSC; 1096193323Sed } 1097218893Sdim Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC), 1098288943Sdim SDLoc(N), MVT::i32); 1099218893Sdim return true; 1100193323Sed } 1101198090Srdivacky 1102193323Sed Base = N; 1103193323Sed Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0), 1104288943Sdim SDLoc(N), MVT::i32); 1105193323Sed return true; 1106193323Sed} 1107193323Sed 1108218893Sdimbool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr, 1109218893Sdim SDValue &Align) { 1110195340Sed Addr = N; 1111218893Sdim 1112218893Sdim unsigned Alignment = 0; 1113288943Sdim 1114288943Sdim MemSDNode *MemN = cast<MemSDNode>(Parent); 1115288943Sdim 1116288943Sdim if (isa<LSBaseSDNode>(MemN) || 1117288943Sdim ((MemN->getOpcode() == ARMISD::VST1_UPD || 1118288943Sdim MemN->getOpcode() == ARMISD::VLD1_UPD) && 1119288943Sdim MemN->getConstantOperandVal(MemN->getNumOperands() - 1) == 1)) { 1120218893Sdim // This case occurs only for VLD1-lane/dup and VST1-lane instructions. 1121218893Sdim // The maximum alignment is equal to the memory size being referenced. 1122288943Sdim unsigned MMOAlign = MemN->getAlignment(); 1123288943Sdim unsigned MemSize = MemN->getMemoryVT().getSizeInBits() / 8; 1124288943Sdim if (MMOAlign >= MemSize && MemSize > 1) 1125218893Sdim Alignment = MemSize; 1126218893Sdim } else { 1127218893Sdim // All other uses of addrmode6 are for intrinsics. For now just record 1128218893Sdim // the raw alignment value; it will be refined later based on the legal 1129218893Sdim // alignment operands for the intrinsic. 1130288943Sdim Alignment = MemN->getAlignment(); 1131218893Sdim } 1132218893Sdim 1133288943Sdim Align = CurDAG->getTargetConstant(Alignment, SDLoc(N), MVT::i32); 1134195340Sed return true; 1135195340Sed} 1136195340Sed 1137219077Sdimbool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N, 1138219077Sdim SDValue &Offset) { 1139219077Sdim LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op); 1140219077Sdim ISD::MemIndexedMode AM = LdSt->getAddressingMode(); 1141219077Sdim if (AM != ISD::POST_INC) 1142219077Sdim return false; 1143219077Sdim Offset = N; 1144219077Sdim if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) { 1145219077Sdim if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits()) 1146219077Sdim Offset = CurDAG->getRegister(0, MVT::i32); 1147219077Sdim } 1148219077Sdim return true; 1149219077Sdim} 1150219077Sdim 1151218893Sdimbool ARMDAGToDAGISel::SelectAddrModePC(SDValue N, 1152198090Srdivacky SDValue &Offset, SDValue &Label) { 1153193323Sed if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) { 1154193323Sed Offset = N.getOperand(0); 1155193323Sed SDValue N1 = N.getOperand(1); 1156218893Sdim Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(), 1157288943Sdim SDLoc(N), MVT::i32); 1158193323Sed return true; 1159193323Sed } 1160218893Sdim 1161193323Sed return false; 1162193323Sed} 1163193323Sed 1164218893Sdim 1165218893Sdim//===----------------------------------------------------------------------===// 1166218893Sdim// Thumb Addressing Modes 1167218893Sdim//===----------------------------------------------------------------------===// 1168218893Sdim 1169218893Sdimbool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N, 1170193323Sed SDValue &Base, SDValue &Offset){ 1171218893Sdim if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) { 1172198090Srdivacky ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N); 1173210299Sed if (!NC || !NC->isNullValue()) 1174198090Srdivacky return false; 1175198090Srdivacky 1176198090Srdivacky Base = Offset = N; 1177193323Sed return true; 1178193323Sed } 1179193323Sed 1180193323Sed Base = N.getOperand(0); 1181193323Sed Offset = N.getOperand(1); 1182193323Sed return true; 1183193323Sed} 1184193323Sed 1185193323Sedbool 1186218893SdimARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, 1187218893Sdim SDValue &Base, SDValue &OffImm) { 1188218893Sdim if (!CurDAG->isBaseWithConstantOffset(N)) { 1189296417Sdim if (N.getOpcode() == ISD::ADD) { 1190296417Sdim return false; // We want to select register offset instead 1191296417Sdim } else if (N.getOpcode() == ARMISD::Wrapper && 1192296417Sdim N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress && 1193296417Sdim N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) { 1194199989Srdivacky Base = N.getOperand(0); 1195218893Sdim } else { 1196199989Srdivacky Base = N; 1197218893Sdim } 1198199989Srdivacky 1199288943Sdim OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); 1200193323Sed return true; 1201193323Sed } 1202193323Sed 1203193323Sed // If the RHS is + imm5 * scale, fold into addr mode. 1204218893Sdim int RHSC; 1205218893Sdim if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) { 1206218893Sdim Base = N.getOperand(0); 1207288943Sdim OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32); 1208218893Sdim return true; 1209193323Sed } 1210193323Sed 1211296417Sdim // Offset is too large, so use register offset instead. 1212296417Sdim return false; 1213193323Sed} 1214193323Sed 1215218893Sdimbool 1216218893SdimARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base, 1217218893Sdim SDValue &OffImm) { 1218218893Sdim return SelectThumbAddrModeImm5S(N, 4, Base, OffImm); 1219193323Sed} 1220193323Sed 1221218893Sdimbool 1222218893SdimARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base, 1223218893Sdim SDValue &OffImm) { 1224218893Sdim return SelectThumbAddrModeImm5S(N, 2, Base, OffImm); 1225193323Sed} 1226193323Sed 1227218893Sdimbool 1228218893SdimARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base, 1229218893Sdim SDValue &OffImm) { 1230218893Sdim return SelectThumbAddrModeImm5S(N, 1, Base, OffImm); 1231193323Sed} 1232193323Sed 1233218893Sdimbool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N, 1234218893Sdim SDValue &Base, SDValue &OffImm) { 1235193323Sed if (N.getOpcode() == ISD::FrameIndex) { 1236193323Sed int FI = cast<FrameIndexSDNode>(N)->getIndex(); 1237288943Sdim // Only multiples of 4 are allowed for the offset, so the frame object 1238288943Sdim // alignment must be at least 4. 1239288943Sdim MachineFrameInfo *MFI = MF->getFrameInfo(); 1240288943Sdim if (MFI->getObjectAlignment(FI) < 4) 1241288943Sdim MFI->setObjectAlignment(FI, 4); 1242288943Sdim Base = CurDAG->getTargetFrameIndex( 1243288943Sdim FI, TLI->getPointerTy(CurDAG->getDataLayout())); 1244288943Sdim OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); 1245193323Sed return true; 1246193323Sed } 1247193323Sed 1248218893Sdim if (!CurDAG->isBaseWithConstantOffset(N)) 1249193323Sed return false; 1250193323Sed 1251193323Sed RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0)); 1252193323Sed if (N.getOperand(0).getOpcode() == ISD::FrameIndex || 1253193323Sed (LHSR && LHSR->getReg() == ARM::SP)) { 1254193323Sed // If the RHS is + imm8 * scale, fold into addr mode. 1255218893Sdim int RHSC; 1256218893Sdim if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) { 1257218893Sdim Base = N.getOperand(0); 1258218893Sdim if (Base.getOpcode() == ISD::FrameIndex) { 1259218893Sdim int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1260288943Sdim // For LHS+RHS to result in an offset that's a multiple of 4 the object 1261288943Sdim // indexed by the LHS must be 4-byte aligned. 1262288943Sdim MachineFrameInfo *MFI = MF->getFrameInfo(); 1263288943Sdim if (MFI->getObjectAlignment(FI) < 4) 1264288943Sdim MFI->setObjectAlignment(FI, 4); 1265288943Sdim Base = CurDAG->getTargetFrameIndex( 1266288943Sdim FI, TLI->getPointerTy(CurDAG->getDataLayout())); 1267193323Sed } 1268288943Sdim OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32); 1269218893Sdim return true; 1270193323Sed } 1271193323Sed } 1272198090Srdivacky 1273193323Sed return false; 1274193323Sed} 1275193323Sed 1276218893Sdim 1277218893Sdim//===----------------------------------------------------------------------===// 1278218893Sdim// Thumb 2 Addressing Modes 1279218893Sdim//===----------------------------------------------------------------------===// 1280218893Sdim 1281218893Sdim 1282218893Sdimbool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N, 1283195340Sed SDValue &Base, SDValue &OffImm) { 1284195340Sed // Match simple R + imm12 operands. 1285195340Sed 1286198090Srdivacky // Base only. 1287218893Sdim if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 1288218893Sdim !CurDAG->isBaseWithConstantOffset(N)) { 1289198090Srdivacky if (N.getOpcode() == ISD::FrameIndex) { 1290218893Sdim // Match frame index. 1291198090Srdivacky int FI = cast<FrameIndexSDNode>(N)->getIndex(); 1292288943Sdim Base = CurDAG->getTargetFrameIndex( 1293288943Sdim FI, TLI->getPointerTy(CurDAG->getDataLayout())); 1294288943Sdim OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); 1295198090Srdivacky return true; 1296218893Sdim } 1297221345Sdim 1298218893Sdim if (N.getOpcode() == ARMISD::Wrapper && 1299296417Sdim N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress && 1300296417Sdim N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) { 1301198090Srdivacky Base = N.getOperand(0); 1302198090Srdivacky if (Base.getOpcode() == ISD::TargetConstantPool) 1303198090Srdivacky return false; // We want to select t2LDRpci instead. 1304198090Srdivacky } else 1305198090Srdivacky Base = N; 1306288943Sdim OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); 1307198090Srdivacky return true; 1308198090Srdivacky } 1309198090Srdivacky 1310195340Sed if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1311218893Sdim if (SelectT2AddrModeImm8(N, Base, OffImm)) 1312198090Srdivacky // Let t2LDRi8 handle (R - imm8). 1313198090Srdivacky return false; 1314198090Srdivacky 1315195340Sed int RHSC = (int)RHS->getZExtValue(); 1316198090Srdivacky if (N.getOpcode() == ISD::SUB) 1317198090Srdivacky RHSC = -RHSC; 1318198090Srdivacky 1319198090Srdivacky if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned) 1320195340Sed Base = N.getOperand(0); 1321198090Srdivacky if (Base.getOpcode() == ISD::FrameIndex) { 1322198090Srdivacky int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1323288943Sdim Base = CurDAG->getTargetFrameIndex( 1324288943Sdim FI, TLI->getPointerTy(CurDAG->getDataLayout())); 1325198090Srdivacky } 1326288943Sdim OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32); 1327195340Sed return true; 1328195340Sed } 1329195340Sed } 1330195340Sed 1331198090Srdivacky // Base only. 1332198090Srdivacky Base = N; 1333288943Sdim OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); 1334198090Srdivacky return true; 1335195340Sed} 1336195340Sed 1337218893Sdimbool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N, 1338195340Sed SDValue &Base, SDValue &OffImm) { 1339198090Srdivacky // Match simple R - imm8 operands. 1340218893Sdim if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 1341218893Sdim !CurDAG->isBaseWithConstantOffset(N)) 1342218893Sdim return false; 1343221345Sdim 1344218893Sdim if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1345218893Sdim int RHSC = (int)RHS->getSExtValue(); 1346218893Sdim if (N.getOpcode() == ISD::SUB) 1347218893Sdim RHSC = -RHSC; 1348198090Srdivacky 1349218893Sdim if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative) 1350218893Sdim Base = N.getOperand(0); 1351218893Sdim if (Base.getOpcode() == ISD::FrameIndex) { 1352218893Sdim int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1353288943Sdim Base = CurDAG->getTargetFrameIndex( 1354288943Sdim FI, TLI->getPointerTy(CurDAG->getDataLayout())); 1355195340Sed } 1356288943Sdim OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32); 1357218893Sdim return true; 1358195340Sed } 1359195340Sed } 1360195340Sed 1361195340Sed return false; 1362195340Sed} 1363195340Sed 1364202375Srdivackybool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N, 1365195340Sed SDValue &OffImm){ 1366202375Srdivacky unsigned Opcode = Op->getOpcode(); 1367195340Sed ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 1368195340Sed ? cast<LoadSDNode>(Op)->getAddressingMode() 1369195340Sed : cast<StoreSDNode>(Op)->getAddressingMode(); 1370218893Sdim int RHSC; 1371218893Sdim if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits. 1372218893Sdim OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC)) 1373288943Sdim ? CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32) 1374288943Sdim : CurDAG->getTargetConstant(-RHSC, SDLoc(N), MVT::i32); 1375218893Sdim return true; 1376195340Sed } 1377195340Sed 1378195340Sed return false; 1379195340Sed} 1380195340Sed 1381218893Sdimbool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N, 1382195340Sed SDValue &Base, 1383195340Sed SDValue &OffReg, SDValue &ShImm) { 1384198090Srdivacky // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12. 1385218893Sdim if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) 1386198090Srdivacky return false; 1387198090Srdivacky 1388198090Srdivacky // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8. 1389198090Srdivacky if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1390198090Srdivacky int RHSC = (int)RHS->getZExtValue(); 1391198090Srdivacky if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned) 1392198090Srdivacky return false; 1393198090Srdivacky else if (RHSC < 0 && RHSC >= -255) // 8 bits 1394198090Srdivacky return false; 1395195340Sed } 1396195340Sed 1397195340Sed // Look for (R + R) or (R + (R << [1,2,3])). 1398195340Sed unsigned ShAmt = 0; 1399195340Sed Base = N.getOperand(0); 1400195340Sed OffReg = N.getOperand(1); 1401195340Sed 1402195340Sed // Swap if it is ((R << c) + R). 1403226633Sdim ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode()); 1404195340Sed if (ShOpcVal != ARM_AM::lsl) { 1405226633Sdim ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode()); 1406195340Sed if (ShOpcVal == ARM_AM::lsl) 1407195340Sed std::swap(Base, OffReg); 1408198090Srdivacky } 1409198090Srdivacky 1410195340Sed if (ShOpcVal == ARM_AM::lsl) { 1411195340Sed // Check to see if the RHS of the shift is a constant, if not, we can't fold 1412195340Sed // it. 1413195340Sed if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) { 1414195340Sed ShAmt = Sh->getZExtValue(); 1415218893Sdim if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt)) 1416218893Sdim OffReg = OffReg.getOperand(0); 1417218893Sdim else { 1418195340Sed ShAmt = 0; 1419218893Sdim } 1420195340Sed } 1421198090Srdivacky } 1422198090Srdivacky 1423296417Sdim // If OffReg is a multiply-by-constant and it's profitable to extract a shift 1424296417Sdim // and use it in a shifted operand do so. 1425296417Sdim if (OffReg.getOpcode() == ISD::MUL && N.hasOneUse()) { 1426296417Sdim unsigned PowerOfTwo = 0; 1427296417Sdim SDValue NewMulConst; 1428296417Sdim if (canExtractShiftFromMul(OffReg, 3, PowerOfTwo, NewMulConst)) { 1429296417Sdim replaceDAGValue(OffReg.getOperand(1), NewMulConst); 1430296417Sdim ShAmt = PowerOfTwo; 1431296417Sdim } 1432296417Sdim } 1433296417Sdim 1434288943Sdim ShImm = CurDAG->getTargetConstant(ShAmt, SDLoc(N), MVT::i32); 1435195340Sed 1436195340Sed return true; 1437195340Sed} 1438195340Sed 1439261991Sdimbool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base, 1440261991Sdim SDValue &OffImm) { 1441276479Sdim // This *must* succeed since it's used for the irreplaceable ldrex and strex 1442261991Sdim // instructions. 1443261991Sdim Base = N; 1444288943Sdim OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); 1445261991Sdim 1446261991Sdim if (N.getOpcode() != ISD::ADD || !CurDAG->isBaseWithConstantOffset(N)) 1447261991Sdim return true; 1448261991Sdim 1449261991Sdim ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); 1450261991Sdim if (!RHS) 1451261991Sdim return true; 1452261991Sdim 1453261991Sdim uint32_t RHSC = (int)RHS->getZExtValue(); 1454261991Sdim if (RHSC > 1020 || RHSC % 4 != 0) 1455261991Sdim return true; 1456261991Sdim 1457261991Sdim Base = N.getOperand(0); 1458261991Sdim if (Base.getOpcode() == ISD::FrameIndex) { 1459261991Sdim int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1460288943Sdim Base = CurDAG->getTargetFrameIndex( 1461288943Sdim FI, TLI->getPointerTy(CurDAG->getDataLayout())); 1462261991Sdim } 1463261991Sdim 1464288943Sdim OffImm = CurDAG->getTargetConstant(RHSC/4, SDLoc(N), MVT::i32); 1465261991Sdim return true; 1466261991Sdim} 1467261991Sdim 1468195340Sed//===--------------------------------------------------------------------===// 1469195340Sed 1470193323Sed/// getAL - Returns a ARMCC::AL immediate node. 1471288943Sdimstatic inline SDValue getAL(SelectionDAG *CurDAG, SDLoc dl) { 1472288943Sdim return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, dl, MVT::i32); 1473193323Sed} 1474193323Sed 1475202375SrdivackySDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) { 1476202375Srdivacky LoadSDNode *LD = cast<LoadSDNode>(N); 1477195340Sed ISD::MemIndexedMode AM = LD->getAddressingMode(); 1478195340Sed if (AM == ISD::UNINDEXED) 1479276479Sdim return nullptr; 1480193323Sed 1481198090Srdivacky EVT LoadedVT = LD->getMemoryVT(); 1482195340Sed SDValue Offset, AMOpc; 1483195340Sed bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC); 1484195340Sed unsigned Opcode = 0; 1485195340Sed bool Match = false; 1486226633Sdim if (LoadedVT == MVT::i32 && isPre && 1487226633Sdim SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) { 1488226633Sdim Opcode = ARM::LDR_PRE_IMM; 1489195340Sed Match = true; 1490226633Sdim } else if (LoadedVT == MVT::i32 && !isPre && 1491226633Sdim SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) { 1492226633Sdim Opcode = ARM::LDR_POST_IMM; 1493226633Sdim Match = true; 1494226633Sdim } else if (LoadedVT == MVT::i32 && 1495226633Sdim SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) { 1496226633Sdim Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG; 1497226633Sdim Match = true; 1498226633Sdim 1499195340Sed } else if (LoadedVT == MVT::i16 && 1500202375Srdivacky SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { 1501195340Sed Match = true; 1502195340Sed Opcode = (LD->getExtensionType() == ISD::SEXTLOAD) 1503195340Sed ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST) 1504195340Sed : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST); 1505195340Sed } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) { 1506195340Sed if (LD->getExtensionType() == ISD::SEXTLOAD) { 1507202375Srdivacky if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { 1508195340Sed Match = true; 1509195340Sed Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST; 1510195340Sed } 1511195340Sed } else { 1512226633Sdim if (isPre && 1513226633Sdim SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) { 1514195340Sed Match = true; 1515226633Sdim Opcode = ARM::LDRB_PRE_IMM; 1516226633Sdim } else if (!isPre && 1517226633Sdim SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) { 1518226633Sdim Match = true; 1519226633Sdim Opcode = ARM::LDRB_POST_IMM; 1520226633Sdim } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) { 1521226633Sdim Match = true; 1522226633Sdim Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG; 1523195340Sed } 1524195340Sed } 1525195340Sed } 1526195340Sed 1527195340Sed if (Match) { 1528226633Sdim if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) { 1529226633Sdim SDValue Chain = LD->getChain(); 1530226633Sdim SDValue Base = LD->getBasePtr(); 1531288943Sdim SDValue Ops[]= { Base, AMOpc, getAL(CurDAG, SDLoc(N)), 1532226633Sdim CurDAG->getRegister(0, MVT::i32), Chain }; 1533261991Sdim return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, 1534251662Sdim MVT::i32, MVT::Other, Ops); 1535226633Sdim } else { 1536226633Sdim SDValue Chain = LD->getChain(); 1537226633Sdim SDValue Base = LD->getBasePtr(); 1538288943Sdim SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG, SDLoc(N)), 1539226633Sdim CurDAG->getRegister(0, MVT::i32), Chain }; 1540261991Sdim return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, 1541251662Sdim MVT::i32, MVT::Other, Ops); 1542226633Sdim } 1543195340Sed } 1544195340Sed 1545276479Sdim return nullptr; 1546195340Sed} 1547195340Sed 1548202375SrdivackySDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) { 1549202375Srdivacky LoadSDNode *LD = cast<LoadSDNode>(N); 1550195340Sed ISD::MemIndexedMode AM = LD->getAddressingMode(); 1551195340Sed if (AM == ISD::UNINDEXED) 1552276479Sdim return nullptr; 1553195340Sed 1554198090Srdivacky EVT LoadedVT = LD->getMemoryVT(); 1555195340Sed bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD; 1556195340Sed SDValue Offset; 1557195340Sed bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC); 1558195340Sed unsigned Opcode = 0; 1559195340Sed bool Match = false; 1560202375Srdivacky if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) { 1561198090Srdivacky switch (LoadedVT.getSimpleVT().SimpleTy) { 1562195340Sed case MVT::i32: 1563195340Sed Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST; 1564195340Sed break; 1565195340Sed case MVT::i16: 1566195340Sed if (isSExtLd) 1567195340Sed Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST; 1568195340Sed else 1569195340Sed Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST; 1570195340Sed break; 1571195340Sed case MVT::i8: 1572195340Sed case MVT::i1: 1573195340Sed if (isSExtLd) 1574195340Sed Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST; 1575195340Sed else 1576195340Sed Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST; 1577195340Sed break; 1578195340Sed default: 1579276479Sdim return nullptr; 1580195340Sed } 1581195340Sed Match = true; 1582195340Sed } 1583195340Sed 1584195340Sed if (Match) { 1585195340Sed SDValue Chain = LD->getChain(); 1586195340Sed SDValue Base = LD->getBasePtr(); 1587288943Sdim SDValue Ops[]= { Base, Offset, getAL(CurDAG, SDLoc(N)), 1588195340Sed CurDAG->getRegister(0, MVT::i32), Chain }; 1589261991Sdim return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32, 1590251662Sdim MVT::Other, Ops); 1591195340Sed } 1592195340Sed 1593276479Sdim return nullptr; 1594195340Sed} 1595195340Sed 1596249423Sdim/// \brief Form a GPRPair pseudo register from a pair of GPR regs. 1597249423SdimSDNode *ARMDAGToDAGISel::createGPRPairNode(EVT VT, SDValue V0, SDValue V1) { 1598261991Sdim SDLoc dl(V0.getNode()); 1599224145Sdim SDValue RegClass = 1600288943Sdim CurDAG->getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32); 1601288943Sdim SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32); 1602288943Sdim SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32); 1603249423Sdim const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1604251662Sdim return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1605249423Sdim} 1606249423Sdim 1607249423Sdim/// \brief Form a D register from a pair of S registers. 1608249423SdimSDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) { 1609261991Sdim SDLoc dl(V0.getNode()); 1610249423Sdim SDValue RegClass = 1611288943Sdim CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, dl, MVT::i32); 1612288943Sdim SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32); 1613288943Sdim SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32); 1614224145Sdim const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1615251662Sdim return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1616210299Sed} 1617210299Sed 1618249423Sdim/// \brief Form a quad register from a pair of D registers. 1619249423SdimSDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) { 1620261991Sdim SDLoc dl(V0.getNode()); 1621288943Sdim SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl, 1622288943Sdim MVT::i32); 1623288943Sdim SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32); 1624288943Sdim SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32); 1625224145Sdim const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1626251662Sdim return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1627198090Srdivacky} 1628198090Srdivacky 1629249423Sdim/// \brief Form 4 consecutive D registers from a pair of Q registers. 1630249423SdimSDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) { 1631261991Sdim SDLoc dl(V0.getNode()); 1632288943Sdim SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl, 1633288943Sdim MVT::i32); 1634288943Sdim SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32); 1635288943Sdim SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32); 1636224145Sdim const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1637251662Sdim return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1638208599Srdivacky} 1639208599Srdivacky 1640249423Sdim/// \brief Form 4 consecutive S registers. 1641249423SdimSDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, 1642210299Sed SDValue V2, SDValue V3) { 1643261991Sdim SDLoc dl(V0.getNode()); 1644224145Sdim SDValue RegClass = 1645288943Sdim CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, dl, MVT::i32); 1646288943Sdim SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32); 1647288943Sdim SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32); 1648288943Sdim SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl, MVT::i32); 1649288943Sdim SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl, MVT::i32); 1650224145Sdim const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, 1651224145Sdim V2, SubReg2, V3, SubReg3 }; 1652251662Sdim return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1653210299Sed} 1654210299Sed 1655249423Sdim/// \brief Form 4 consecutive D registers. 1656249423SdimSDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, 1657208599Srdivacky SDValue V2, SDValue V3) { 1658261991Sdim SDLoc dl(V0.getNode()); 1659288943Sdim SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl, 1660288943Sdim MVT::i32); 1661288943Sdim SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32); 1662288943Sdim SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32); 1663288943Sdim SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl, MVT::i32); 1664288943Sdim SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl, MVT::i32); 1665224145Sdim const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, 1666224145Sdim V2, SubReg2, V3, SubReg3 }; 1667251662Sdim return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1668208599Srdivacky} 1669208599Srdivacky 1670249423Sdim/// \brief Form 4 consecutive Q registers. 1671249423SdimSDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, 1672208599Srdivacky SDValue V2, SDValue V3) { 1673261991Sdim SDLoc dl(V0.getNode()); 1674288943Sdim SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl, 1675288943Sdim MVT::i32); 1676288943Sdim SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32); 1677288943Sdim SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32); 1678288943Sdim SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl, MVT::i32); 1679288943Sdim SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl, MVT::i32); 1680224145Sdim const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, 1681224145Sdim V2, SubReg2, V3, SubReg3 }; 1682251662Sdim return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1683208599Srdivacky} 1684208599Srdivacky 1685218893Sdim/// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand 1686218893Sdim/// of a NEON VLD or VST instruction. The supported values depend on the 1687218893Sdim/// number of registers being loaded. 1688288943SdimSDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, SDLoc dl, 1689288943Sdim unsigned NumVecs, bool is64BitVector) { 1690218893Sdim unsigned NumRegs = NumVecs; 1691218893Sdim if (!is64BitVector && NumVecs < 3) 1692218893Sdim NumRegs *= 2; 1693208599Srdivacky 1694218893Sdim unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); 1695218893Sdim if (Alignment >= 32 && NumRegs == 4) 1696218893Sdim Alignment = 32; 1697218893Sdim else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4)) 1698218893Sdim Alignment = 16; 1699218893Sdim else if (Alignment >= 8) 1700218893Sdim Alignment = 8; 1701218893Sdim else 1702218893Sdim Alignment = 0; 1703218893Sdim 1704288943Sdim return CurDAG->getTargetConstant(Alignment, dl, MVT::i32); 1705198090Srdivacky} 1706198090Srdivacky 1707265925Sdimstatic bool isVLDfixed(unsigned Opc) 1708265925Sdim{ 1709265925Sdim switch (Opc) { 1710265925Sdim default: return false; 1711265925Sdim case ARM::VLD1d8wb_fixed : return true; 1712265925Sdim case ARM::VLD1d16wb_fixed : return true; 1713265925Sdim case ARM::VLD1d64Qwb_fixed : return true; 1714265925Sdim case ARM::VLD1d32wb_fixed : return true; 1715265925Sdim case ARM::VLD1d64wb_fixed : return true; 1716265925Sdim case ARM::VLD1d64TPseudoWB_fixed : return true; 1717265925Sdim case ARM::VLD1d64QPseudoWB_fixed : return true; 1718265925Sdim case ARM::VLD1q8wb_fixed : return true; 1719265925Sdim case ARM::VLD1q16wb_fixed : return true; 1720265925Sdim case ARM::VLD1q32wb_fixed : return true; 1721265925Sdim case ARM::VLD1q64wb_fixed : return true; 1722265925Sdim case ARM::VLD2d8wb_fixed : return true; 1723265925Sdim case ARM::VLD2d16wb_fixed : return true; 1724265925Sdim case ARM::VLD2d32wb_fixed : return true; 1725265925Sdim case ARM::VLD2q8PseudoWB_fixed : return true; 1726265925Sdim case ARM::VLD2q16PseudoWB_fixed : return true; 1727265925Sdim case ARM::VLD2q32PseudoWB_fixed : return true; 1728265925Sdim case ARM::VLD2DUPd8wb_fixed : return true; 1729265925Sdim case ARM::VLD2DUPd16wb_fixed : return true; 1730265925Sdim case ARM::VLD2DUPd32wb_fixed : return true; 1731265925Sdim } 1732265925Sdim} 1733265925Sdim 1734265925Sdimstatic bool isVSTfixed(unsigned Opc) 1735265925Sdim{ 1736265925Sdim switch (Opc) { 1737265925Sdim default: return false; 1738265925Sdim case ARM::VST1d8wb_fixed : return true; 1739265925Sdim case ARM::VST1d16wb_fixed : return true; 1740265925Sdim case ARM::VST1d32wb_fixed : return true; 1741265925Sdim case ARM::VST1d64wb_fixed : return true; 1742276479Sdim case ARM::VST1q8wb_fixed : return true; 1743276479Sdim case ARM::VST1q16wb_fixed : return true; 1744276479Sdim case ARM::VST1q32wb_fixed : return true; 1745276479Sdim case ARM::VST1q64wb_fixed : return true; 1746265925Sdim case ARM::VST1d64TPseudoWB_fixed : return true; 1747265925Sdim case ARM::VST1d64QPseudoWB_fixed : return true; 1748265925Sdim case ARM::VST2d8wb_fixed : return true; 1749265925Sdim case ARM::VST2d16wb_fixed : return true; 1750265925Sdim case ARM::VST2d32wb_fixed : return true; 1751265925Sdim case ARM::VST2q8PseudoWB_fixed : return true; 1752265925Sdim case ARM::VST2q16PseudoWB_fixed : return true; 1753265925Sdim case ARM::VST2q32PseudoWB_fixed : return true; 1754265925Sdim } 1755265925Sdim} 1756265925Sdim 1757234353Sdim// Get the register stride update opcode of a VLD/VST instruction that 1758234353Sdim// is otherwise equivalent to the given fixed stride updating instruction. 1759234353Sdimstatic unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) { 1760265925Sdim assert((isVLDfixed(Opc) || isVSTfixed(Opc)) 1761265925Sdim && "Incorrect fixed stride updating instruction."); 1762234353Sdim switch (Opc) { 1763234353Sdim default: break; 1764234353Sdim case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register; 1765234353Sdim case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register; 1766234353Sdim case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register; 1767234353Sdim case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register; 1768234353Sdim case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register; 1769234353Sdim case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register; 1770234353Sdim case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register; 1771234353Sdim case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register; 1772265925Sdim case ARM::VLD1d64Twb_fixed: return ARM::VLD1d64Twb_register; 1773265925Sdim case ARM::VLD1d64Qwb_fixed: return ARM::VLD1d64Qwb_register; 1774265925Sdim case ARM::VLD1d64TPseudoWB_fixed: return ARM::VLD1d64TPseudoWB_register; 1775265925Sdim case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register; 1776234353Sdim 1777234353Sdim case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register; 1778234353Sdim case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register; 1779234353Sdim case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register; 1780234353Sdim case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register; 1781234353Sdim case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register; 1782234353Sdim case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register; 1783234353Sdim case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register; 1784234353Sdim case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register; 1785234353Sdim case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register; 1786234353Sdim case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register; 1787234353Sdim 1788234353Sdim case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register; 1789234353Sdim case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register; 1790234353Sdim case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register; 1791234353Sdim case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register; 1792234353Sdim case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register; 1793234353Sdim case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register; 1794234353Sdim 1795234353Sdim case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register; 1796234353Sdim case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register; 1797234353Sdim case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register; 1798234353Sdim case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register; 1799234353Sdim case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register; 1800234353Sdim case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register; 1801234353Sdim 1802234353Sdim case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register; 1803234353Sdim case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register; 1804234353Sdim case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register; 1805234353Sdim } 1806234353Sdim return Opc; // If not one we handle, return it unchanged. 1807234353Sdim} 1808234353Sdim 1809218893SdimSDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs, 1810239462Sdim const uint16_t *DOpcodes, 1811239462Sdim const uint16_t *QOpcodes0, 1812239462Sdim const uint16_t *QOpcodes1) { 1813206083Srdivacky assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range"); 1814261991Sdim SDLoc dl(N); 1815198090Srdivacky 1816205407Srdivacky SDValue MemAddr, Align; 1817218893Sdim unsigned AddrOpIdx = isUpdating ? 1 : 2; 1818218893Sdim if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) 1819276479Sdim return nullptr; 1820198090Srdivacky 1821198090Srdivacky SDValue Chain = N->getOperand(0); 1822198090Srdivacky EVT VT = N->getValueType(0); 1823198090Srdivacky bool is64BitVector = VT.is64BitVector(); 1824288943Sdim Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector); 1825198090Srdivacky 1826198090Srdivacky unsigned OpcodeIndex; 1827198090Srdivacky switch (VT.getSimpleVT().SimpleTy) { 1828198090Srdivacky default: llvm_unreachable("unhandled vld type"); 1829198090Srdivacky // Double-register operations: 1830198090Srdivacky case MVT::v8i8: OpcodeIndex = 0; break; 1831198090Srdivacky case MVT::v4i16: OpcodeIndex = 1; break; 1832198090Srdivacky case MVT::v2f32: 1833198090Srdivacky case MVT::v2i32: OpcodeIndex = 2; break; 1834198090Srdivacky case MVT::v1i64: OpcodeIndex = 3; break; 1835198090Srdivacky // Quad-register operations: 1836198090Srdivacky case MVT::v16i8: OpcodeIndex = 0; break; 1837198090Srdivacky case MVT::v8i16: OpcodeIndex = 1; break; 1838198090Srdivacky case MVT::v4f32: 1839198090Srdivacky case MVT::v4i32: OpcodeIndex = 2; break; 1840280031Sdim case MVT::v2f64: 1841206083Srdivacky case MVT::v2i64: OpcodeIndex = 3; 1842206083Srdivacky assert(NumVecs == 1 && "v2i64 type only supported for VLD1"); 1843206083Srdivacky break; 1844198090Srdivacky } 1845198090Srdivacky 1846212904Sdim EVT ResTy; 1847212904Sdim if (NumVecs == 1) 1848212904Sdim ResTy = VT; 1849212904Sdim else { 1850212904Sdim unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; 1851212904Sdim if (!is64BitVector) 1852212904Sdim ResTyElts *= 2; 1853212904Sdim ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts); 1854212904Sdim } 1855218893Sdim std::vector<EVT> ResTys; 1856218893Sdim ResTys.push_back(ResTy); 1857218893Sdim if (isUpdating) 1858218893Sdim ResTys.push_back(MVT::i32); 1859218893Sdim ResTys.push_back(MVT::Other); 1860212904Sdim 1861288943Sdim SDValue Pred = getAL(CurDAG, dl); 1862205407Srdivacky SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 1863218893Sdim SDNode *VLd; 1864218893Sdim SmallVector<SDValue, 7> Ops; 1865208599Srdivacky 1866218893Sdim // Double registers and VLD1/VLD2 quad registers are directly supported. 1867218893Sdim if (is64BitVector || NumVecs <= 2) { 1868218893Sdim unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : 1869218893Sdim QOpcodes0[OpcodeIndex]); 1870218893Sdim Ops.push_back(MemAddr); 1871218893Sdim Ops.push_back(Align); 1872218893Sdim if (isUpdating) { 1873218893Sdim SDValue Inc = N->getOperand(AddrOpIdx + 1); 1874234353Sdim // FIXME: VLD1/VLD2 fixed increment doesn't need Reg0. Remove the reg0 1875234353Sdim // case entirely when the rest are updated to that form, too. 1876265925Sdim if ((NumVecs <= 2) && !isa<ConstantSDNode>(Inc.getNode())) 1877234353Sdim Opc = getVLDSTRegisterUpdateOpcode(Opc); 1878265925Sdim // FIXME: We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so 1879234353Sdim // check for that explicitly too. Horribly hacky, but temporary. 1880265925Sdim if ((NumVecs > 2 && !isVLDfixed(Opc)) || 1881234353Sdim !isa<ConstantSDNode>(Inc.getNode())) 1882234353Sdim Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc); 1883208599Srdivacky } 1884218893Sdim Ops.push_back(Pred); 1885218893Sdim Ops.push_back(Reg0); 1886218893Sdim Ops.push_back(Chain); 1887251662Sdim VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 1888198090Srdivacky 1889198090Srdivacky } else { 1890198090Srdivacky // Otherwise, quad registers are loaded with two separate instructions, 1891198090Srdivacky // where one loads the even registers and the other loads the odd registers. 1892212904Sdim EVT AddrTy = MemAddr.getValueType(); 1893198090Srdivacky 1894218893Sdim // Load the even subregs. This is always an updating load, so that it 1895218893Sdim // provides the address to the second load for the odd subregs. 1896212904Sdim SDValue ImplDef = 1897212904Sdim SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0); 1898212904Sdim const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain }; 1899218893Sdim SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, 1900251662Sdim ResTy, AddrTy, MVT::Other, OpsA); 1901212904Sdim Chain = SDValue(VLdA, 2); 1902198090Srdivacky 1903198113Srdivacky // Load the odd subregs. 1904218893Sdim Ops.push_back(SDValue(VLdA, 1)); 1905218893Sdim Ops.push_back(Align); 1906218893Sdim if (isUpdating) { 1907218893Sdim SDValue Inc = N->getOperand(AddrOpIdx + 1); 1908218893Sdim assert(isa<ConstantSDNode>(Inc.getNode()) && 1909218893Sdim "only constant post-increment update allowed for VLD3/4"); 1910218893Sdim (void)Inc; 1911218893Sdim Ops.push_back(Reg0); 1912218893Sdim } 1913218893Sdim Ops.push_back(SDValue(VLdA, 0)); 1914218893Sdim Ops.push_back(Pred); 1915218893Sdim Ops.push_back(Reg0); 1916218893Sdim Ops.push_back(Chain); 1917251662Sdim VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops); 1918212904Sdim } 1919198090Srdivacky 1920221345Sdim // Transfer memoperands. 1921221345Sdim MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1922221345Sdim MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 1923221345Sdim cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1); 1924221345Sdim 1925218893Sdim if (NumVecs == 1) 1926218893Sdim return VLd; 1927218893Sdim 1928218893Sdim // Extract out the subregisters. 1929218893Sdim SDValue SuperReg = SDValue(VLd, 0); 1930218893Sdim assert(ARM::dsub_7 == ARM::dsub_0+7 && 1931218893Sdim ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering"); 1932218893Sdim unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0); 1933218893Sdim for (unsigned Vec = 0; Vec < NumVecs; ++Vec) 1934218893Sdim ReplaceUses(SDValue(N, Vec), 1935218893Sdim CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg)); 1936218893Sdim ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1)); 1937218893Sdim if (isUpdating) 1938218893Sdim ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2)); 1939276479Sdim return nullptr; 1940198090Srdivacky} 1941198090Srdivacky 1942218893SdimSDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, 1943239462Sdim const uint16_t *DOpcodes, 1944239462Sdim const uint16_t *QOpcodes0, 1945239462Sdim const uint16_t *QOpcodes1) { 1946210299Sed assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range"); 1947261991Sdim SDLoc dl(N); 1948198113Srdivacky 1949205407Srdivacky SDValue MemAddr, Align; 1950218893Sdim unsigned AddrOpIdx = isUpdating ? 1 : 2; 1951218893Sdim unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1) 1952218893Sdim if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) 1953276479Sdim return nullptr; 1954198113Srdivacky 1955221345Sdim MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1956221345Sdim MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 1957221345Sdim 1958198113Srdivacky SDValue Chain = N->getOperand(0); 1959218893Sdim EVT VT = N->getOperand(Vec0Idx).getValueType(); 1960198113Srdivacky bool is64BitVector = VT.is64BitVector(); 1961288943Sdim Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector); 1962198113Srdivacky 1963198113Srdivacky unsigned OpcodeIndex; 1964198113Srdivacky switch (VT.getSimpleVT().SimpleTy) { 1965198113Srdivacky default: llvm_unreachable("unhandled vst type"); 1966198113Srdivacky // Double-register operations: 1967198113Srdivacky case MVT::v8i8: OpcodeIndex = 0; break; 1968198113Srdivacky case MVT::v4i16: OpcodeIndex = 1; break; 1969198113Srdivacky case MVT::v2f32: 1970198113Srdivacky case MVT::v2i32: OpcodeIndex = 2; break; 1971198113Srdivacky case MVT::v1i64: OpcodeIndex = 3; break; 1972198113Srdivacky // Quad-register operations: 1973198113Srdivacky case MVT::v16i8: OpcodeIndex = 0; break; 1974198113Srdivacky case MVT::v8i16: OpcodeIndex = 1; break; 1975198113Srdivacky case MVT::v4f32: 1976198113Srdivacky case MVT::v4i32: OpcodeIndex = 2; break; 1977280031Sdim case MVT::v2f64: 1978206083Srdivacky case MVT::v2i64: OpcodeIndex = 3; 1979206083Srdivacky assert(NumVecs == 1 && "v2i64 type only supported for VST1"); 1980206083Srdivacky break; 1981198113Srdivacky } 1982198113Srdivacky 1983218893Sdim std::vector<EVT> ResTys; 1984218893Sdim if (isUpdating) 1985218893Sdim ResTys.push_back(MVT::i32); 1986218893Sdim ResTys.push_back(MVT::Other); 1987218893Sdim 1988288943Sdim SDValue Pred = getAL(CurDAG, dl); 1989205407Srdivacky SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 1990212904Sdim SmallVector<SDValue, 7> Ops; 1991198113Srdivacky 1992218893Sdim // Double registers and VST1/VST2 quad registers are directly supported. 1993218893Sdim if (is64BitVector || NumVecs <= 2) { 1994218893Sdim SDValue SrcReg; 1995212904Sdim if (NumVecs == 1) { 1996218893Sdim SrcReg = N->getOperand(Vec0Idx); 1997218893Sdim } else if (is64BitVector) { 1998208599Srdivacky // Form a REG_SEQUENCE to force register allocation. 1999218893Sdim SDValue V0 = N->getOperand(Vec0Idx + 0); 2000218893Sdim SDValue V1 = N->getOperand(Vec0Idx + 1); 2001208599Srdivacky if (NumVecs == 2) 2002249423Sdim SrcReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0); 2003208599Srdivacky else { 2004218893Sdim SDValue V2 = N->getOperand(Vec0Idx + 2); 2005218893Sdim // If it's a vst3, form a quad D-register and leave the last part as 2006208599Srdivacky // an undef. 2007208599Srdivacky SDValue V3 = (NumVecs == 3) 2008208599Srdivacky ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0) 2009218893Sdim : N->getOperand(Vec0Idx + 3); 2010249423Sdim SrcReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0); 2011208599Srdivacky } 2012212904Sdim } else { 2013212904Sdim // Form a QQ register. 2014218893Sdim SDValue Q0 = N->getOperand(Vec0Idx); 2015218893Sdim SDValue Q1 = N->getOperand(Vec0Idx + 1); 2016249423Sdim SrcReg = SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0); 2017198113Srdivacky } 2018218893Sdim 2019218893Sdim unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : 2020218893Sdim QOpcodes0[OpcodeIndex]); 2021218893Sdim Ops.push_back(MemAddr); 2022218893Sdim Ops.push_back(Align); 2023218893Sdim if (isUpdating) { 2024218893Sdim SDValue Inc = N->getOperand(AddrOpIdx + 1); 2025234353Sdim // FIXME: VST1/VST2 fixed increment doesn't need Reg0. Remove the reg0 2026234353Sdim // case entirely when the rest are updated to that form, too. 2027234353Sdim if (NumVecs <= 2 && !isa<ConstantSDNode>(Inc.getNode())) 2028234353Sdim Opc = getVLDSTRegisterUpdateOpcode(Opc); 2029265925Sdim // FIXME: We use a VST1 for v1i64 even if the pseudo says vld2/3/4, so 2030234353Sdim // check for that explicitly too. Horribly hacky, but temporary. 2031265925Sdim if (!isa<ConstantSDNode>(Inc.getNode())) 2032265925Sdim Ops.push_back(Inc); 2033265925Sdim else if (NumVecs > 2 && !isVSTfixed(Opc)) 2034265925Sdim Ops.push_back(Reg0); 2035218893Sdim } 2036218893Sdim Ops.push_back(SrcReg); 2037212904Sdim Ops.push_back(Pred); 2038218893Sdim Ops.push_back(Reg0); 2039212904Sdim Ops.push_back(Chain); 2040251662Sdim SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 2041221345Sdim 2042221345Sdim // Transfer memoperands. 2043221345Sdim cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1); 2044221345Sdim 2045221345Sdim return VSt; 2046198113Srdivacky } 2047198113Srdivacky 2048198113Srdivacky // Otherwise, quad registers are stored with two separate instructions, 2049198113Srdivacky // where one stores the even registers and the other stores the odd registers. 2050198113Srdivacky 2051210299Sed // Form the QQQQ REG_SEQUENCE. 2052218893Sdim SDValue V0 = N->getOperand(Vec0Idx + 0); 2053218893Sdim SDValue V1 = N->getOperand(Vec0Idx + 1); 2054218893Sdim SDValue V2 = N->getOperand(Vec0Idx + 2); 2055212904Sdim SDValue V3 = (NumVecs == 3) 2056212904Sdim ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) 2057218893Sdim : N->getOperand(Vec0Idx + 3); 2058249423Sdim SDValue RegSeq = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0); 2059198113Srdivacky 2060218893Sdim // Store the even D registers. This is always an updating store, so that it 2061218893Sdim // provides the address to the second store for the odd subregs. 2062218893Sdim const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain }; 2063218893Sdim SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, 2064218893Sdim MemAddr.getValueType(), 2065251662Sdim MVT::Other, OpsA); 2066221345Sdim cast<MachineSDNode>(VStA)->setMemRefs(MemOp, MemOp + 1); 2067218893Sdim Chain = SDValue(VStA, 1); 2068218893Sdim 2069218893Sdim // Store the odd D registers. 2070218893Sdim Ops.push_back(SDValue(VStA, 0)); 2071218893Sdim Ops.push_back(Align); 2072218893Sdim if (isUpdating) { 2073218893Sdim SDValue Inc = N->getOperand(AddrOpIdx + 1); 2074218893Sdim assert(isa<ConstantSDNode>(Inc.getNode()) && 2075218893Sdim "only constant post-increment update allowed for VST3/4"); 2076218893Sdim (void)Inc; 2077218893Sdim Ops.push_back(Reg0); 2078218893Sdim } 2079212904Sdim Ops.push_back(RegSeq); 2080210299Sed Ops.push_back(Pred); 2081218893Sdim Ops.push_back(Reg0); 2082210299Sed Ops.push_back(Chain); 2083221345Sdim SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, 2084251662Sdim Ops); 2085221345Sdim cast<MachineSDNode>(VStB)->setMemRefs(MemOp, MemOp + 1); 2086221345Sdim return VStB; 2087198113Srdivacky} 2088198113Srdivacky 2089202375SrdivackySDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, 2090218893Sdim bool isUpdating, unsigned NumVecs, 2091239462Sdim const uint16_t *DOpcodes, 2092239462Sdim const uint16_t *QOpcodes) { 2093198090Srdivacky assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range"); 2094261991Sdim SDLoc dl(N); 2095198090Srdivacky 2096205407Srdivacky SDValue MemAddr, Align; 2097218893Sdim unsigned AddrOpIdx = isUpdating ? 1 : 2; 2098218893Sdim unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1) 2099218893Sdim if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) 2100276479Sdim return nullptr; 2101198090Srdivacky 2102221345Sdim MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 2103221345Sdim MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 2104221345Sdim 2105198090Srdivacky SDValue Chain = N->getOperand(0); 2106198090Srdivacky unsigned Lane = 2107218893Sdim cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue(); 2108218893Sdim EVT VT = N->getOperand(Vec0Idx).getValueType(); 2109198090Srdivacky bool is64BitVector = VT.is64BitVector(); 2110198090Srdivacky 2111218893Sdim unsigned Alignment = 0; 2112218893Sdim if (NumVecs != 3) { 2113218893Sdim Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); 2114218893Sdim unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8; 2115218893Sdim if (Alignment > NumBytes) 2116218893Sdim Alignment = NumBytes; 2117218893Sdim if (Alignment < 8 && Alignment < NumBytes) 2118218893Sdim Alignment = 0; 2119218893Sdim // Alignment must be a power of two; make sure of that. 2120218893Sdim Alignment = (Alignment & -Alignment); 2121218893Sdim if (Alignment == 1) 2122218893Sdim Alignment = 0; 2123198090Srdivacky } 2124288943Sdim Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32); 2125198090Srdivacky 2126198090Srdivacky unsigned OpcodeIndex; 2127198090Srdivacky switch (VT.getSimpleVT().SimpleTy) { 2128198090Srdivacky default: llvm_unreachable("unhandled vld/vst lane type"); 2129198090Srdivacky // Double-register operations: 2130198090Srdivacky case MVT::v8i8: OpcodeIndex = 0; break; 2131198090Srdivacky case MVT::v4i16: OpcodeIndex = 1; break; 2132198090Srdivacky case MVT::v2f32: 2133198090Srdivacky case MVT::v2i32: OpcodeIndex = 2; break; 2134198090Srdivacky // Quad-register operations: 2135198090Srdivacky case MVT::v8i16: OpcodeIndex = 0; break; 2136198090Srdivacky case MVT::v4f32: 2137198090Srdivacky case MVT::v4i32: OpcodeIndex = 1; break; 2138198090Srdivacky } 2139198090Srdivacky 2140218893Sdim std::vector<EVT> ResTys; 2141218893Sdim if (IsLoad) { 2142218893Sdim unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; 2143218893Sdim if (!is64BitVector) 2144218893Sdim ResTyElts *= 2; 2145218893Sdim ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), 2146218893Sdim MVT::i64, ResTyElts)); 2147218893Sdim } 2148218893Sdim if (isUpdating) 2149218893Sdim ResTys.push_back(MVT::i32); 2150218893Sdim ResTys.push_back(MVT::Other); 2151218893Sdim 2152288943Sdim SDValue Pred = getAL(CurDAG, dl); 2153205407Srdivacky SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2154199989Srdivacky 2155218893Sdim SmallVector<SDValue, 8> Ops; 2156198090Srdivacky Ops.push_back(MemAddr); 2157199481Srdivacky Ops.push_back(Align); 2158218893Sdim if (isUpdating) { 2159218893Sdim SDValue Inc = N->getOperand(AddrOpIdx + 1); 2160218893Sdim Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc); 2161218893Sdim } 2162198090Srdivacky 2163218893Sdim SDValue SuperReg; 2164218893Sdim SDValue V0 = N->getOperand(Vec0Idx + 0); 2165218893Sdim SDValue V1 = N->getOperand(Vec0Idx + 1); 2166218893Sdim if (NumVecs == 2) { 2167218893Sdim if (is64BitVector) 2168249423Sdim SuperReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0); 2169218893Sdim else 2170249423Sdim SuperReg = SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0); 2171198090Srdivacky } else { 2172218893Sdim SDValue V2 = N->getOperand(Vec0Idx + 2); 2173218893Sdim SDValue V3 = (NumVecs == 3) 2174218893Sdim ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) 2175218893Sdim : N->getOperand(Vec0Idx + 3); 2176218893Sdim if (is64BitVector) 2177249423Sdim SuperReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0); 2178218893Sdim else 2179249423Sdim SuperReg = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0); 2180198090Srdivacky } 2181218893Sdim Ops.push_back(SuperReg); 2182288943Sdim Ops.push_back(getI32Imm(Lane, dl)); 2183199989Srdivacky Ops.push_back(Pred); 2184205407Srdivacky Ops.push_back(Reg0); 2185198090Srdivacky Ops.push_back(Chain); 2186198090Srdivacky 2187218893Sdim unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : 2188218893Sdim QOpcodes[OpcodeIndex]); 2189251662Sdim SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 2190221345Sdim cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1); 2191198090Srdivacky if (!IsLoad) 2192218893Sdim return VLdLn; 2193198090Srdivacky 2194218893Sdim // Extract the subregisters. 2195218893Sdim SuperReg = SDValue(VLdLn, 0); 2196218893Sdim assert(ARM::dsub_7 == ARM::dsub_0+7 && 2197218893Sdim ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering"); 2198218893Sdim unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0; 2199218893Sdim for (unsigned Vec = 0; Vec < NumVecs; ++Vec) 2200218893Sdim ReplaceUses(SDValue(N, Vec), 2201218893Sdim CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg)); 2202218893Sdim ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1)); 2203218893Sdim if (isUpdating) 2204218893Sdim ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2)); 2205276479Sdim return nullptr; 2206218893Sdim} 2207208599Srdivacky 2208218893SdimSDNode *ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating, 2209239462Sdim unsigned NumVecs, 2210239462Sdim const uint16_t *Opcodes) { 2211218893Sdim assert(NumVecs >=2 && NumVecs <= 4 && "VLDDup NumVecs out-of-range"); 2212261991Sdim SDLoc dl(N); 2213208599Srdivacky 2214218893Sdim SDValue MemAddr, Align; 2215218893Sdim if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align)) 2216276479Sdim return nullptr; 2217218893Sdim 2218221345Sdim MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 2219221345Sdim MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 2220221345Sdim 2221218893Sdim SDValue Chain = N->getOperand(0); 2222218893Sdim EVT VT = N->getValueType(0); 2223218893Sdim 2224218893Sdim unsigned Alignment = 0; 2225218893Sdim if (NumVecs != 3) { 2226218893Sdim Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); 2227218893Sdim unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8; 2228218893Sdim if (Alignment > NumBytes) 2229218893Sdim Alignment = NumBytes; 2230218893Sdim if (Alignment < 8 && Alignment < NumBytes) 2231218893Sdim Alignment = 0; 2232218893Sdim // Alignment must be a power of two; make sure of that. 2233218893Sdim Alignment = (Alignment & -Alignment); 2234218893Sdim if (Alignment == 1) 2235218893Sdim Alignment = 0; 2236208599Srdivacky } 2237288943Sdim Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32); 2238208599Srdivacky 2239218893Sdim unsigned OpcodeIndex; 2240218893Sdim switch (VT.getSimpleVT().SimpleTy) { 2241218893Sdim default: llvm_unreachable("unhandled vld-dup type"); 2242218893Sdim case MVT::v8i8: OpcodeIndex = 0; break; 2243218893Sdim case MVT::v4i16: OpcodeIndex = 1; break; 2244218893Sdim case MVT::v2f32: 2245218893Sdim case MVT::v2i32: OpcodeIndex = 2; break; 2246218893Sdim } 2247218893Sdim 2248288943Sdim SDValue Pred = getAL(CurDAG, dl); 2249218893Sdim SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2250218893Sdim SDValue SuperReg; 2251218893Sdim unsigned Opc = Opcodes[OpcodeIndex]; 2252218893Sdim SmallVector<SDValue, 6> Ops; 2253218893Sdim Ops.push_back(MemAddr); 2254218893Sdim Ops.push_back(Align); 2255218893Sdim if (isUpdating) { 2256234353Sdim // fixed-stride update instructions don't have an explicit writeback 2257234353Sdim // operand. It's implicit in the opcode itself. 2258218893Sdim SDValue Inc = N->getOperand(2); 2259234353Sdim if (!isa<ConstantSDNode>(Inc.getNode())) 2260234353Sdim Ops.push_back(Inc); 2261234353Sdim // FIXME: VLD3 and VLD4 haven't been updated to that form yet. 2262234353Sdim else if (NumVecs > 2) 2263234353Sdim Ops.push_back(Reg0); 2264218893Sdim } 2265218893Sdim Ops.push_back(Pred); 2266218893Sdim Ops.push_back(Reg0); 2267218893Sdim Ops.push_back(Chain); 2268218893Sdim 2269218893Sdim unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; 2270218893Sdim std::vector<EVT> ResTys; 2271221345Sdim ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,ResTyElts)); 2272218893Sdim if (isUpdating) 2273218893Sdim ResTys.push_back(MVT::i32); 2274218893Sdim ResTys.push_back(MVT::Other); 2275251662Sdim SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 2276221345Sdim cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1); 2277218893Sdim SuperReg = SDValue(VLdDup, 0); 2278218893Sdim 2279218893Sdim // Extract the subregisters. 2280210299Sed assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering"); 2281218893Sdim unsigned SubIdx = ARM::dsub_0; 2282210299Sed for (unsigned Vec = 0; Vec < NumVecs; ++Vec) 2283210299Sed ReplaceUses(SDValue(N, Vec), 2284218893Sdim CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg)); 2285218893Sdim ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1)); 2286218893Sdim if (isUpdating) 2287218893Sdim ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2)); 2288276479Sdim return nullptr; 2289210299Sed} 2290198090Srdivacky 2291210299SedSDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, 2292210299Sed unsigned Opc) { 2293210299Sed assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range"); 2294261991Sdim SDLoc dl(N); 2295210299Sed EVT VT = N->getValueType(0); 2296210299Sed unsigned FirstTblReg = IsExt ? 2 : 1; 2297210299Sed 2298210299Sed // Form a REG_SEQUENCE to force register allocation. 2299210299Sed SDValue RegSeq; 2300210299Sed SDValue V0 = N->getOperand(FirstTblReg + 0); 2301210299Sed SDValue V1 = N->getOperand(FirstTblReg + 1); 2302210299Sed if (NumVecs == 2) 2303249423Sdim RegSeq = SDValue(createDRegPairNode(MVT::v16i8, V0, V1), 0); 2304210299Sed else { 2305210299Sed SDValue V2 = N->getOperand(FirstTblReg + 2); 2306218893Sdim // If it's a vtbl3, form a quad D-register and leave the last part as 2307210299Sed // an undef. 2308210299Sed SDValue V3 = (NumVecs == 3) 2309210299Sed ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) 2310210299Sed : N->getOperand(FirstTblReg + 3); 2311249423Sdim RegSeq = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0); 2312198090Srdivacky } 2313198090Srdivacky 2314210299Sed SmallVector<SDValue, 6> Ops; 2315210299Sed if (IsExt) 2316210299Sed Ops.push_back(N->getOperand(1)); 2317218893Sdim Ops.push_back(RegSeq); 2318210299Sed Ops.push_back(N->getOperand(FirstTblReg + NumVecs)); 2319288943Sdim Ops.push_back(getAL(CurDAG, dl)); // predicate 2320210299Sed Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register 2321251662Sdim return CurDAG->getMachineNode(Opc, dl, VT, Ops); 2322198090Srdivacky} 2323198090Srdivacky 2324202375SrdivackySDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N, 2325207618Srdivacky bool isSigned) { 2326198090Srdivacky if (!Subtarget->hasV6T2Ops()) 2327276479Sdim return nullptr; 2328198090Srdivacky 2329249423Sdim unsigned Opc = isSigned 2330249423Sdim ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX) 2331207618Srdivacky : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX); 2332288943Sdim SDLoc dl(N); 2333207618Srdivacky 2334207618Srdivacky // For unsigned extracts, check for a shift right and mask 2335207618Srdivacky unsigned And_imm = 0; 2336207618Srdivacky if (N->getOpcode() == ISD::AND) { 2337207618Srdivacky if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) { 2338207618Srdivacky 2339207618Srdivacky // The immediate is a mask of the low bits iff imm & (imm+1) == 0 2340207618Srdivacky if (And_imm & (And_imm + 1)) 2341276479Sdim return nullptr; 2342207618Srdivacky 2343207618Srdivacky unsigned Srl_imm = 0; 2344207618Srdivacky if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, 2345207618Srdivacky Srl_imm)) { 2346207618Srdivacky assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!"); 2347207618Srdivacky 2348226633Sdim // Note: The width operand is encoded as width-1. 2349288943Sdim unsigned Width = countTrailingOnes(And_imm) - 1; 2350207618Srdivacky unsigned LSB = Srl_imm; 2351249423Sdim 2352207618Srdivacky SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2353249423Sdim 2354249423Sdim if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) { 2355249423Sdim // It's cheaper to use a right shift to extract the top bits. 2356249423Sdim if (Subtarget->isThumb()) { 2357249423Sdim Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri; 2358249423Sdim SDValue Ops[] = { N->getOperand(0).getOperand(0), 2359288943Sdim CurDAG->getTargetConstant(LSB, dl, MVT::i32), 2360288943Sdim getAL(CurDAG, dl), Reg0, Reg0 }; 2361276479Sdim return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); 2362249423Sdim } 2363249423Sdim 2364249423Sdim // ARM models shift instructions as MOVsi with shifter operand. 2365249423Sdim ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(ISD::SRL); 2366249423Sdim SDValue ShOpc = 2367288943Sdim CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB), dl, 2368249423Sdim MVT::i32); 2369249423Sdim SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc, 2370288943Sdim getAL(CurDAG, dl), Reg0, Reg0 }; 2371276479Sdim return CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops); 2372249423Sdim } 2373249423Sdim 2374207618Srdivacky SDValue Ops[] = { N->getOperand(0).getOperand(0), 2375288943Sdim CurDAG->getTargetConstant(LSB, dl, MVT::i32), 2376288943Sdim CurDAG->getTargetConstant(Width, dl, MVT::i32), 2377288943Sdim getAL(CurDAG, dl), Reg0 }; 2378276479Sdim return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); 2379207618Srdivacky } 2380207618Srdivacky } 2381276479Sdim return nullptr; 2382207618Srdivacky } 2383207618Srdivacky 2384207618Srdivacky // Otherwise, we're looking for a shift of a shift 2385198090Srdivacky unsigned Shl_imm = 0; 2386202375Srdivacky if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) { 2387198090Srdivacky assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!"); 2388198090Srdivacky unsigned Srl_imm = 0; 2389202375Srdivacky if (isInt32Immediate(N->getOperand(1), Srl_imm)) { 2390198090Srdivacky assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!"); 2391226633Sdim // Note: The width operand is encoded as width-1. 2392226633Sdim unsigned Width = 32 - Srl_imm - 1; 2393198090Srdivacky int LSB = Srl_imm - Shl_imm; 2394198396Srdivacky if (LSB < 0) 2395276479Sdim return nullptr; 2396198090Srdivacky SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2397202375Srdivacky SDValue Ops[] = { N->getOperand(0).getOperand(0), 2398288943Sdim CurDAG->getTargetConstant(LSB, dl, MVT::i32), 2399288943Sdim CurDAG->getTargetConstant(Width, dl, MVT::i32), 2400288943Sdim getAL(CurDAG, dl), Reg0 }; 2401276479Sdim return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); 2402198090Srdivacky } 2403198090Srdivacky } 2404280031Sdim 2405280031Sdim if (N->getOpcode() == ISD::SIGN_EXTEND_INREG) { 2406280031Sdim unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits(); 2407280031Sdim unsigned LSB = 0; 2408280031Sdim if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, LSB) && 2409280031Sdim !isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRA, LSB)) 2410280031Sdim return nullptr; 2411280031Sdim 2412280031Sdim if (LSB + Width > 32) 2413280031Sdim return nullptr; 2414280031Sdim 2415280031Sdim SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2416280031Sdim SDValue Ops[] = { N->getOperand(0).getOperand(0), 2417288943Sdim CurDAG->getTargetConstant(LSB, dl, MVT::i32), 2418288943Sdim CurDAG->getTargetConstant(Width - 1, dl, MVT::i32), 2419288943Sdim getAL(CurDAG, dl), Reg0 }; 2420280031Sdim return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); 2421280031Sdim } 2422280031Sdim 2423276479Sdim return nullptr; 2424198090Srdivacky} 2425198090Srdivacky 2426226633Sdim/// Target-specific DAG combining for ISD::XOR. 2427226633Sdim/// Target-independent combining lowers SELECT_CC nodes of the form 2428226633Sdim/// select_cc setg[ge] X, 0, X, -X 2429226633Sdim/// select_cc setgt X, -1, X, -X 2430226633Sdim/// select_cc setl[te] X, 0, -X, X 2431226633Sdim/// select_cc setlt X, 1, -X, X 2432226633Sdim/// which represent Integer ABS into: 2433226633Sdim/// Y = sra (X, size(X)-1); xor (add (X, Y), Y) 2434226633Sdim/// ARM instruction selection detects the latter and matches it to 2435226633Sdim/// ARM::ABS or ARM::t2ABS machine node. 2436226633SdimSDNode *ARMDAGToDAGISel::SelectABSOp(SDNode *N){ 2437226633Sdim SDValue XORSrc0 = N->getOperand(0); 2438226633Sdim SDValue XORSrc1 = N->getOperand(1); 2439226633Sdim EVT VT = N->getValueType(0); 2440226633Sdim 2441226633Sdim if (Subtarget->isThumb1Only()) 2442276479Sdim return nullptr; 2443226633Sdim 2444239462Sdim if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA) 2445276479Sdim return nullptr; 2446226633Sdim 2447226633Sdim SDValue ADDSrc0 = XORSrc0.getOperand(0); 2448226633Sdim SDValue ADDSrc1 = XORSrc0.getOperand(1); 2449226633Sdim SDValue SRASrc0 = XORSrc1.getOperand(0); 2450226633Sdim SDValue SRASrc1 = XORSrc1.getOperand(1); 2451226633Sdim ConstantSDNode *SRAConstant = dyn_cast<ConstantSDNode>(SRASrc1); 2452226633Sdim EVT XType = SRASrc0.getValueType(); 2453226633Sdim unsigned Size = XType.getSizeInBits() - 1; 2454226633Sdim 2455239462Sdim if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 && 2456276479Sdim XType.isInteger() && SRAConstant != nullptr && 2457226633Sdim Size == SRAConstant->getZExtValue()) { 2458239462Sdim unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS; 2459226633Sdim return CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0); 2460226633Sdim } 2461226633Sdim 2462276479Sdim return nullptr; 2463226633Sdim} 2464226633Sdim 2465208599SrdivackySDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) { 2466208599Srdivacky // The only time a CONCAT_VECTORS operation can have legal types is when 2467208599Srdivacky // two 64-bit vectors are concatenated to a 128-bit vector. 2468208599Srdivacky EVT VT = N->getValueType(0); 2469208599Srdivacky if (!VT.is128BitVector() || N->getNumOperands() != 2) 2470208599Srdivacky llvm_unreachable("unexpected CONCAT_VECTORS"); 2471249423Sdim return createDRegPairNode(VT, N->getOperand(0), N->getOperand(1)); 2472208599Srdivacky} 2473208599Srdivacky 2474202375SrdivackySDNode *ARMDAGToDAGISel::Select(SDNode *N) { 2475261991Sdim SDLoc dl(N); 2476193323Sed 2477255804Sdim if (N->isMachineOpcode()) { 2478255804Sdim N->setNodeId(-1); 2479276479Sdim return nullptr; // Already selected. 2480255804Sdim } 2481193323Sed 2482193323Sed switch (N->getOpcode()) { 2483193323Sed default: break; 2484288943Sdim case ISD::WRITE_REGISTER: { 2485288943Sdim SDNode *ResNode = SelectWriteRegister(N); 2486288943Sdim if (ResNode) 2487288943Sdim return ResNode; 2488288943Sdim break; 2489288943Sdim } 2490288943Sdim case ISD::READ_REGISTER: { 2491288943Sdim SDNode *ResNode = SelectReadRegister(N); 2492288943Sdim if (ResNode) 2493288943Sdim return ResNode; 2494288943Sdim break; 2495288943Sdim } 2496249423Sdim case ISD::INLINEASM: { 2497249423Sdim SDNode *ResNode = SelectInlineAsm(N); 2498249423Sdim if (ResNode) 2499249423Sdim return ResNode; 2500249423Sdim break; 2501249423Sdim } 2502226633Sdim case ISD::XOR: { 2503226633Sdim // Select special operations if XOR node forms integer ABS pattern 2504226633Sdim SDNode *ResNode = SelectABSOp(N); 2505226633Sdim if (ResNode) 2506226633Sdim return ResNode; 2507226633Sdim // Other cases are autogenerated. 2508226633Sdim break; 2509226633Sdim } 2510193323Sed case ISD::Constant: { 2511193323Sed unsigned Val = cast<ConstantSDNode>(N)->getZExtValue(); 2512296417Sdim // If we can't materialize the constant we need to use a literal pool 2513296417Sdim if (ConstantMaterializationCost(Val) > 2) { 2514280031Sdim SDValue CPIdx = CurDAG->getTargetConstantPool( 2515280031Sdim ConstantInt::get(Type::getInt32Ty(*CurDAG->getContext()), Val), 2516288943Sdim TLI->getPointerTy(CurDAG->getDataLayout())); 2517193323Sed 2518193323Sed SDNode *ResNode; 2519276479Sdim if (Subtarget->isThumb()) { 2520288943Sdim SDValue Pred = getAL(CurDAG, dl); 2521198090Srdivacky SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2522198090Srdivacky SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() }; 2523218893Sdim ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other, 2524251662Sdim Ops); 2525198090Srdivacky } else { 2526193323Sed SDValue Ops[] = { 2527198090Srdivacky CPIdx, 2528288943Sdim CurDAG->getTargetConstant(0, dl, MVT::i32), 2529288943Sdim getAL(CurDAG, dl), 2530193323Sed CurDAG->getRegister(0, MVT::i32), 2531193323Sed CurDAG->getEntryNode() 2532193323Sed }; 2533198090Srdivacky ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other, 2534251662Sdim Ops); 2535193323Sed } 2536202375Srdivacky ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0)); 2537276479Sdim return nullptr; 2538193323Sed } 2539198090Srdivacky 2540193323Sed // Other cases are autogenerated. 2541193323Sed break; 2542193323Sed } 2543193323Sed case ISD::FrameIndex: { 2544193323Sed // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm. 2545193323Sed int FI = cast<FrameIndexSDNode>(N)->getIndex(); 2546288943Sdim SDValue TFI = CurDAG->getTargetFrameIndex( 2547288943Sdim FI, TLI->getPointerTy(CurDAG->getDataLayout())); 2548198090Srdivacky if (Subtarget->isThumb1Only()) { 2549288943Sdim // Set the alignment of the frame object to 4, to avoid having to generate 2550288943Sdim // more than one ADD 2551288943Sdim MachineFrameInfo *MFI = MF->getFrameInfo(); 2552288943Sdim if (MFI->getObjectAlignment(FI) < 4) 2553288943Sdim MFI->setObjectAlignment(FI, 4); 2554280031Sdim return CurDAG->SelectNodeTo(N, ARM::tADDframe, MVT::i32, TFI, 2555288943Sdim CurDAG->getTargetConstant(0, dl, MVT::i32)); 2556193323Sed } else { 2557198090Srdivacky unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ? 2558198090Srdivacky ARM::t2ADDri : ARM::ADDri); 2559288943Sdim SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, dl, MVT::i32), 2560288943Sdim getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32), 2561198090Srdivacky CurDAG->getRegister(0, MVT::i32) }; 2562276479Sdim return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); 2563193323Sed } 2564193323Sed } 2565198090Srdivacky case ISD::SRL: 2566207618Srdivacky if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false)) 2567198090Srdivacky return I; 2568193323Sed break; 2569280031Sdim case ISD::SIGN_EXTEND_INREG: 2570198090Srdivacky case ISD::SRA: 2571207618Srdivacky if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true)) 2572198090Srdivacky return I; 2573198090Srdivacky break; 2574193323Sed case ISD::MUL: 2575198090Srdivacky if (Subtarget->isThumb1Only()) 2576193323Sed break; 2577202375Srdivacky if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 2578193323Sed unsigned RHSV = C->getZExtValue(); 2579193323Sed if (!RHSV) break; 2580193323Sed if (isPowerOf2_32(RHSV-1)) { // 2^n+1? 2581198090Srdivacky unsigned ShImm = Log2_32(RHSV-1); 2582198090Srdivacky if (ShImm >= 32) 2583198090Srdivacky break; 2584202375Srdivacky SDValue V = N->getOperand(0); 2585198090Srdivacky ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm); 2586288943Sdim SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32); 2587198090Srdivacky SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2588198090Srdivacky if (Subtarget->isThumb()) { 2589288943Sdim SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 }; 2590276479Sdim return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops); 2591198090Srdivacky } else { 2592288943Sdim SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0, 2593288943Sdim Reg0 }; 2594276479Sdim return CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops); 2595198090Srdivacky } 2596193323Sed } 2597193323Sed if (isPowerOf2_32(RHSV+1)) { // 2^n-1? 2598198090Srdivacky unsigned ShImm = Log2_32(RHSV+1); 2599198090Srdivacky if (ShImm >= 32) 2600198090Srdivacky break; 2601202375Srdivacky SDValue V = N->getOperand(0); 2602198090Srdivacky ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm); 2603288943Sdim SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32); 2604198090Srdivacky SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2605198090Srdivacky if (Subtarget->isThumb()) { 2606288943Sdim SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 }; 2607276479Sdim return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops); 2608198090Srdivacky } else { 2609288943Sdim SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0, 2610288943Sdim Reg0 }; 2611276479Sdim return CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops); 2612198090Srdivacky } 2613193323Sed } 2614193323Sed } 2615193323Sed break; 2616198396Srdivacky case ISD::AND: { 2617207618Srdivacky // Check for unsigned bitfield extract 2618207618Srdivacky if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false)) 2619207618Srdivacky return I; 2620207618Srdivacky 2621198396Srdivacky // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits 2622198396Srdivacky // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits 2623198396Srdivacky // are entirely contributed by c2 and lower 16-bits are entirely contributed 2624198396Srdivacky // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)). 2625198396Srdivacky // Select it to: "movt x, ((c1 & 0xffff) >> 16) 2626202375Srdivacky EVT VT = N->getValueType(0); 2627198396Srdivacky if (VT != MVT::i32) 2628198396Srdivacky break; 2629198396Srdivacky unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2()) 2630198396Srdivacky ? ARM::t2MOVTi16 2631198396Srdivacky : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0); 2632198396Srdivacky if (!Opc) 2633198396Srdivacky break; 2634202375Srdivacky SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); 2635198396Srdivacky ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 2636198396Srdivacky if (!N1C) 2637198396Srdivacky break; 2638198396Srdivacky if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) { 2639198396Srdivacky SDValue N2 = N0.getOperand(1); 2640198396Srdivacky ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2); 2641198396Srdivacky if (!N2C) 2642198396Srdivacky break; 2643198396Srdivacky unsigned N1CVal = N1C->getZExtValue(); 2644198396Srdivacky unsigned N2CVal = N2C->getZExtValue(); 2645198396Srdivacky if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) && 2646198396Srdivacky (N1CVal & 0xffffU) == 0xffffU && 2647198396Srdivacky (N2CVal & 0xffffU) == 0x0U) { 2648198396Srdivacky SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16, 2649288943Sdim dl, MVT::i32); 2650198396Srdivacky SDValue Ops[] = { N0.getOperand(0), Imm16, 2651288943Sdim getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) }; 2652251662Sdim return CurDAG->getMachineNode(Opc, dl, VT, Ops); 2653198396Srdivacky } 2654198396Srdivacky } 2655198396Srdivacky break; 2656198396Srdivacky } 2657199481Srdivacky case ARMISD::VMOVRRD: 2658199481Srdivacky return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32, 2659288943Sdim N->getOperand(0), getAL(CurDAG, dl), 2660198090Srdivacky CurDAG->getRegister(0, MVT::i32)); 2661193323Sed case ISD::UMUL_LOHI: { 2662198090Srdivacky if (Subtarget->isThumb1Only()) 2663198090Srdivacky break; 2664198090Srdivacky if (Subtarget->isThumb()) { 2665202375Srdivacky SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2666288943Sdim getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) }; 2667251662Sdim return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops); 2668198090Srdivacky } else { 2669202375Srdivacky SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2670288943Sdim getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32), 2671198090Srdivacky CurDAG->getRegister(0, MVT::i32) }; 2672218893Sdim return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2673218893Sdim ARM::UMULL : ARM::UMULLv5, 2674251662Sdim dl, MVT::i32, MVT::i32, Ops); 2675198090Srdivacky } 2676193323Sed } 2677193323Sed case ISD::SMUL_LOHI: { 2678198090Srdivacky if (Subtarget->isThumb1Only()) 2679198090Srdivacky break; 2680198090Srdivacky if (Subtarget->isThumb()) { 2681202375Srdivacky SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2682288943Sdim getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) }; 2683251662Sdim return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops); 2684198090Srdivacky } else { 2685202375Srdivacky SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2686288943Sdim getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32), 2687193323Sed CurDAG->getRegister(0, MVT::i32) }; 2688218893Sdim return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2689218893Sdim ARM::SMULL : ARM::SMULLv5, 2690251662Sdim dl, MVT::i32, MVT::i32, Ops); 2691198090Srdivacky } 2692193323Sed } 2693243830Sdim case ARMISD::UMLAL:{ 2694243830Sdim if (Subtarget->isThumb()) { 2695243830Sdim SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 2696288943Sdim N->getOperand(3), getAL(CurDAG, dl), 2697243830Sdim CurDAG->getRegister(0, MVT::i32)}; 2698251662Sdim return CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops); 2699243830Sdim }else{ 2700243830Sdim SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 2701288943Sdim N->getOperand(3), getAL(CurDAG, dl), 2702243830Sdim CurDAG->getRegister(0, MVT::i32), 2703243830Sdim CurDAG->getRegister(0, MVT::i32) }; 2704243830Sdim return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2705243830Sdim ARM::UMLAL : ARM::UMLALv5, 2706251662Sdim dl, MVT::i32, MVT::i32, Ops); 2707243830Sdim } 2708243830Sdim } 2709243830Sdim case ARMISD::SMLAL:{ 2710243830Sdim if (Subtarget->isThumb()) { 2711243830Sdim SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 2712288943Sdim N->getOperand(3), getAL(CurDAG, dl), 2713243830Sdim CurDAG->getRegister(0, MVT::i32)}; 2714251662Sdim return CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops); 2715243830Sdim }else{ 2716243830Sdim SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 2717288943Sdim N->getOperand(3), getAL(CurDAG, dl), 2718243830Sdim CurDAG->getRegister(0, MVT::i32), 2719243830Sdim CurDAG->getRegister(0, MVT::i32) }; 2720243830Sdim return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2721243830Sdim ARM::SMLAL : ARM::SMLALv5, 2722251662Sdim dl, MVT::i32, MVT::i32, Ops); 2723243830Sdim } 2724243830Sdim } 2725193323Sed case ISD::LOAD: { 2726276479Sdim SDNode *ResNode = nullptr; 2727198090Srdivacky if (Subtarget->isThumb() && Subtarget->hasThumb2()) 2728202375Srdivacky ResNode = SelectT2IndexedLoad(N); 2729195340Sed else 2730202375Srdivacky ResNode = SelectARMIndexedLoad(N); 2731195340Sed if (ResNode) 2732195340Sed return ResNode; 2733193323Sed // Other cases are autogenerated. 2734193323Sed break; 2735193323Sed } 2736193323Sed case ARMISD::BRCOND: { 2737193323Sed // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) 2738193323Sed // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc) 2739193323Sed // Pattern complexity = 6 cost = 1 size = 0 2740193323Sed 2741193323Sed // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) 2742193323Sed // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc) 2743193323Sed // Pattern complexity = 6 cost = 1 size = 0 2744193323Sed 2745195340Sed // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) 2746195340Sed // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc) 2747195340Sed // Pattern complexity = 6 cost = 1 size = 0 2748195340Sed 2749198090Srdivacky unsigned Opc = Subtarget->isThumb() ? 2750195340Sed ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc; 2751202375Srdivacky SDValue Chain = N->getOperand(0); 2752202375Srdivacky SDValue N1 = N->getOperand(1); 2753202375Srdivacky SDValue N2 = N->getOperand(2); 2754202375Srdivacky SDValue N3 = N->getOperand(3); 2755202375Srdivacky SDValue InFlag = N->getOperand(4); 2756193323Sed assert(N1.getOpcode() == ISD::BasicBlock); 2757193323Sed assert(N2.getOpcode() == ISD::Constant); 2758193323Sed assert(N3.getOpcode() == ISD::Register); 2759193323Sed 2760193323Sed SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned) 2761288943Sdim cast<ConstantSDNode>(N2)->getZExtValue()), dl, 2762193323Sed MVT::i32); 2763193323Sed SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag }; 2764198090Srdivacky SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, 2765251662Sdim MVT::Glue, Ops); 2766193323Sed Chain = SDValue(ResNode, 0); 2767202375Srdivacky if (N->getNumValues() == 2) { 2768193323Sed InFlag = SDValue(ResNode, 1); 2769202375Srdivacky ReplaceUses(SDValue(N, 1), InFlag); 2770193323Sed } 2771202375Srdivacky ReplaceUses(SDValue(N, 0), 2772199511Srdivacky SDValue(Chain.getNode(), Chain.getResNo())); 2773276479Sdim return nullptr; 2774193323Sed } 2775198090Srdivacky case ARMISD::VZIP: { 2776198090Srdivacky unsigned Opc = 0; 2777198090Srdivacky EVT VT = N->getValueType(0); 2778198090Srdivacky switch (VT.getSimpleVT().SimpleTy) { 2779276479Sdim default: return nullptr; 2780198090Srdivacky case MVT::v8i8: Opc = ARM::VZIPd8; break; 2781198090Srdivacky case MVT::v4i16: Opc = ARM::VZIPd16; break; 2782198090Srdivacky case MVT::v2f32: 2783234353Sdim // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm. 2784234353Sdim case MVT::v2i32: Opc = ARM::VTRNd32; break; 2785198090Srdivacky case MVT::v16i8: Opc = ARM::VZIPq8; break; 2786198090Srdivacky case MVT::v8i16: Opc = ARM::VZIPq16; break; 2787198090Srdivacky case MVT::v4f32: 2788198090Srdivacky case MVT::v4i32: Opc = ARM::VZIPq32; break; 2789193323Sed } 2790288943Sdim SDValue Pred = getAL(CurDAG, dl); 2791199989Srdivacky SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2792199989Srdivacky SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; 2793251662Sdim return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops); 2794198090Srdivacky } 2795198090Srdivacky case ARMISD::VUZP: { 2796198090Srdivacky unsigned Opc = 0; 2797198090Srdivacky EVT VT = N->getValueType(0); 2798198090Srdivacky switch (VT.getSimpleVT().SimpleTy) { 2799276479Sdim default: return nullptr; 2800198090Srdivacky case MVT::v8i8: Opc = ARM::VUZPd8; break; 2801198090Srdivacky case MVT::v4i16: Opc = ARM::VUZPd16; break; 2802198090Srdivacky case MVT::v2f32: 2803234353Sdim // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm. 2804234353Sdim case MVT::v2i32: Opc = ARM::VTRNd32; break; 2805198090Srdivacky case MVT::v16i8: Opc = ARM::VUZPq8; break; 2806198090Srdivacky case MVT::v8i16: Opc = ARM::VUZPq16; break; 2807198090Srdivacky case MVT::v4f32: 2808198090Srdivacky case MVT::v4i32: Opc = ARM::VUZPq32; break; 2809193323Sed } 2810288943Sdim SDValue Pred = getAL(CurDAG, dl); 2811199989Srdivacky SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2812199989Srdivacky SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; 2813251662Sdim return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops); 2814198090Srdivacky } 2815198090Srdivacky case ARMISD::VTRN: { 2816198090Srdivacky unsigned Opc = 0; 2817198090Srdivacky EVT VT = N->getValueType(0); 2818198090Srdivacky switch (VT.getSimpleVT().SimpleTy) { 2819276479Sdim default: return nullptr; 2820198090Srdivacky case MVT::v8i8: Opc = ARM::VTRNd8; break; 2821198090Srdivacky case MVT::v4i16: Opc = ARM::VTRNd16; break; 2822198090Srdivacky case MVT::v2f32: 2823198090Srdivacky case MVT::v2i32: Opc = ARM::VTRNd32; break; 2824198090Srdivacky case MVT::v16i8: Opc = ARM::VTRNq8; break; 2825198090Srdivacky case MVT::v8i16: Opc = ARM::VTRNq16; break; 2826198090Srdivacky case MVT::v4f32: 2827198090Srdivacky case MVT::v4i32: Opc = ARM::VTRNq32; break; 2828193323Sed } 2829288943Sdim SDValue Pred = getAL(CurDAG, dl); 2830199989Srdivacky SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2831199989Srdivacky SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; 2832251662Sdim return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops); 2833193323Sed } 2834210299Sed case ARMISD::BUILD_VECTOR: { 2835210299Sed EVT VecVT = N->getValueType(0); 2836210299Sed EVT EltVT = VecVT.getVectorElementType(); 2837210299Sed unsigned NumElts = VecVT.getVectorNumElements(); 2838218893Sdim if (EltVT == MVT::f64) { 2839210299Sed assert(NumElts == 2 && "unexpected type for BUILD_VECTOR"); 2840249423Sdim return createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)); 2841210299Sed } 2842218893Sdim assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR"); 2843210299Sed if (NumElts == 2) 2844249423Sdim return createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)); 2845210299Sed assert(NumElts == 4 && "unexpected type for BUILD_VECTOR"); 2846249423Sdim return createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1), 2847210299Sed N->getOperand(2), N->getOperand(3)); 2848210299Sed } 2849194710Sed 2850218893Sdim case ARMISD::VLD2DUP: { 2851239462Sdim static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16, 2852239462Sdim ARM::VLD2DUPd32 }; 2853218893Sdim return SelectVLDDup(N, false, 2, Opcodes); 2854218893Sdim } 2855218893Sdim 2856218893Sdim case ARMISD::VLD3DUP: { 2857239462Sdim static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo, 2858239462Sdim ARM::VLD3DUPd16Pseudo, 2859239462Sdim ARM::VLD3DUPd32Pseudo }; 2860218893Sdim return SelectVLDDup(N, false, 3, Opcodes); 2861218893Sdim } 2862218893Sdim 2863218893Sdim case ARMISD::VLD4DUP: { 2864239462Sdim static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo, 2865239462Sdim ARM::VLD4DUPd16Pseudo, 2866239462Sdim ARM::VLD4DUPd32Pseudo }; 2867218893Sdim return SelectVLDDup(N, false, 4, Opcodes); 2868218893Sdim } 2869218893Sdim 2870218893Sdim case ARMISD::VLD2DUP_UPD: { 2871239462Sdim static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed, 2872239462Sdim ARM::VLD2DUPd16wb_fixed, 2873239462Sdim ARM::VLD2DUPd32wb_fixed }; 2874218893Sdim return SelectVLDDup(N, true, 2, Opcodes); 2875218893Sdim } 2876218893Sdim 2877218893Sdim case ARMISD::VLD3DUP_UPD: { 2878239462Sdim static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD, 2879239462Sdim ARM::VLD3DUPd16Pseudo_UPD, 2880239462Sdim ARM::VLD3DUPd32Pseudo_UPD }; 2881218893Sdim return SelectVLDDup(N, true, 3, Opcodes); 2882218893Sdim } 2883218893Sdim 2884218893Sdim case ARMISD::VLD4DUP_UPD: { 2885239462Sdim static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD, 2886239462Sdim ARM::VLD4DUPd16Pseudo_UPD, 2887239462Sdim ARM::VLD4DUPd32Pseudo_UPD }; 2888218893Sdim return SelectVLDDup(N, true, 4, Opcodes); 2889218893Sdim } 2890218893Sdim 2891218893Sdim case ARMISD::VLD1_UPD: { 2892239462Sdim static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed, 2893239462Sdim ARM::VLD1d16wb_fixed, 2894239462Sdim ARM::VLD1d32wb_fixed, 2895239462Sdim ARM::VLD1d64wb_fixed }; 2896239462Sdim static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed, 2897239462Sdim ARM::VLD1q16wb_fixed, 2898239462Sdim ARM::VLD1q32wb_fixed, 2899239462Sdim ARM::VLD1q64wb_fixed }; 2900276479Sdim return SelectVLD(N, true, 1, DOpcodes, QOpcodes, nullptr); 2901218893Sdim } 2902218893Sdim 2903218893Sdim case ARMISD::VLD2_UPD: { 2904239462Sdim static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed, 2905239462Sdim ARM::VLD2d16wb_fixed, 2906239462Sdim ARM::VLD2d32wb_fixed, 2907239462Sdim ARM::VLD1q64wb_fixed}; 2908239462Sdim static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed, 2909239462Sdim ARM::VLD2q16PseudoWB_fixed, 2910239462Sdim ARM::VLD2q32PseudoWB_fixed }; 2911276479Sdim return SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr); 2912218893Sdim } 2913218893Sdim 2914218893Sdim case ARMISD::VLD3_UPD: { 2915239462Sdim static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD, 2916239462Sdim ARM::VLD3d16Pseudo_UPD, 2917239462Sdim ARM::VLD3d32Pseudo_UPD, 2918265925Sdim ARM::VLD1d64TPseudoWB_fixed}; 2919239462Sdim static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD, 2920239462Sdim ARM::VLD3q16Pseudo_UPD, 2921239462Sdim ARM::VLD3q32Pseudo_UPD }; 2922239462Sdim static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD, 2923239462Sdim ARM::VLD3q16oddPseudo_UPD, 2924239462Sdim ARM::VLD3q32oddPseudo_UPD }; 2925218893Sdim return SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); 2926218893Sdim } 2927218893Sdim 2928218893Sdim case ARMISD::VLD4_UPD: { 2929239462Sdim static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD, 2930239462Sdim ARM::VLD4d16Pseudo_UPD, 2931239462Sdim ARM::VLD4d32Pseudo_UPD, 2932265925Sdim ARM::VLD1d64QPseudoWB_fixed}; 2933239462Sdim static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD, 2934239462Sdim ARM::VLD4q16Pseudo_UPD, 2935239462Sdim ARM::VLD4q32Pseudo_UPD }; 2936239462Sdim static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD, 2937239462Sdim ARM::VLD4q16oddPseudo_UPD, 2938239462Sdim ARM::VLD4q32oddPseudo_UPD }; 2939218893Sdim return SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); 2940218893Sdim } 2941218893Sdim 2942218893Sdim case ARMISD::VLD2LN_UPD: { 2943239462Sdim static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD, 2944239462Sdim ARM::VLD2LNd16Pseudo_UPD, 2945239462Sdim ARM::VLD2LNd32Pseudo_UPD }; 2946239462Sdim static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD, 2947239462Sdim ARM::VLD2LNq32Pseudo_UPD }; 2948218893Sdim return SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes); 2949218893Sdim } 2950218893Sdim 2951218893Sdim case ARMISD::VLD3LN_UPD: { 2952239462Sdim static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD, 2953239462Sdim ARM::VLD3LNd16Pseudo_UPD, 2954239462Sdim ARM::VLD3LNd32Pseudo_UPD }; 2955239462Sdim static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD, 2956239462Sdim ARM::VLD3LNq32Pseudo_UPD }; 2957218893Sdim return SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes); 2958218893Sdim } 2959218893Sdim 2960218893Sdim case ARMISD::VLD4LN_UPD: { 2961239462Sdim static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD, 2962239462Sdim ARM::VLD4LNd16Pseudo_UPD, 2963239462Sdim ARM::VLD4LNd32Pseudo_UPD }; 2964239462Sdim static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD, 2965239462Sdim ARM::VLD4LNq32Pseudo_UPD }; 2966218893Sdim return SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes); 2967218893Sdim } 2968218893Sdim 2969218893Sdim case ARMISD::VST1_UPD: { 2970239462Sdim static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed, 2971239462Sdim ARM::VST1d16wb_fixed, 2972239462Sdim ARM::VST1d32wb_fixed, 2973239462Sdim ARM::VST1d64wb_fixed }; 2974239462Sdim static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed, 2975239462Sdim ARM::VST1q16wb_fixed, 2976239462Sdim ARM::VST1q32wb_fixed, 2977239462Sdim ARM::VST1q64wb_fixed }; 2978276479Sdim return SelectVST(N, true, 1, DOpcodes, QOpcodes, nullptr); 2979218893Sdim } 2980218893Sdim 2981218893Sdim case ARMISD::VST2_UPD: { 2982239462Sdim static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed, 2983239462Sdim ARM::VST2d16wb_fixed, 2984239462Sdim ARM::VST2d32wb_fixed, 2985239462Sdim ARM::VST1q64wb_fixed}; 2986239462Sdim static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed, 2987239462Sdim ARM::VST2q16PseudoWB_fixed, 2988239462Sdim ARM::VST2q32PseudoWB_fixed }; 2989276479Sdim return SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr); 2990218893Sdim } 2991218893Sdim 2992218893Sdim case ARMISD::VST3_UPD: { 2993239462Sdim static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD, 2994239462Sdim ARM::VST3d16Pseudo_UPD, 2995239462Sdim ARM::VST3d32Pseudo_UPD, 2996239462Sdim ARM::VST1d64TPseudoWB_fixed}; 2997239462Sdim static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD, 2998239462Sdim ARM::VST3q16Pseudo_UPD, 2999239462Sdim ARM::VST3q32Pseudo_UPD }; 3000239462Sdim static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD, 3001239462Sdim ARM::VST3q16oddPseudo_UPD, 3002239462Sdim ARM::VST3q32oddPseudo_UPD }; 3003218893Sdim return SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); 3004218893Sdim } 3005218893Sdim 3006218893Sdim case ARMISD::VST4_UPD: { 3007239462Sdim static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD, 3008239462Sdim ARM::VST4d16Pseudo_UPD, 3009239462Sdim ARM::VST4d32Pseudo_UPD, 3010239462Sdim ARM::VST1d64QPseudoWB_fixed}; 3011239462Sdim static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD, 3012239462Sdim ARM::VST4q16Pseudo_UPD, 3013239462Sdim ARM::VST4q32Pseudo_UPD }; 3014239462Sdim static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD, 3015239462Sdim ARM::VST4q16oddPseudo_UPD, 3016239462Sdim ARM::VST4q32oddPseudo_UPD }; 3017218893Sdim return SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); 3018218893Sdim } 3019218893Sdim 3020218893Sdim case ARMISD::VST2LN_UPD: { 3021239462Sdim static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD, 3022239462Sdim ARM::VST2LNd16Pseudo_UPD, 3023239462Sdim ARM::VST2LNd32Pseudo_UPD }; 3024239462Sdim static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD, 3025239462Sdim ARM::VST2LNq32Pseudo_UPD }; 3026218893Sdim return SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes); 3027218893Sdim } 3028218893Sdim 3029218893Sdim case ARMISD::VST3LN_UPD: { 3030239462Sdim static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD, 3031239462Sdim ARM::VST3LNd16Pseudo_UPD, 3032239462Sdim ARM::VST3LNd32Pseudo_UPD }; 3033239462Sdim static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD, 3034239462Sdim ARM::VST3LNq32Pseudo_UPD }; 3035218893Sdim return SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes); 3036218893Sdim } 3037218893Sdim 3038218893Sdim case ARMISD::VST4LN_UPD: { 3039239462Sdim static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD, 3040239462Sdim ARM::VST4LNd16Pseudo_UPD, 3041239462Sdim ARM::VST4LNd32Pseudo_UPD }; 3042239462Sdim static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD, 3043239462Sdim ARM::VST4LNq32Pseudo_UPD }; 3044218893Sdim return SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes); 3045218893Sdim } 3046218893Sdim 3047198090Srdivacky case ISD::INTRINSIC_VOID: 3048198090Srdivacky case ISD::INTRINSIC_W_CHAIN: { 3049198090Srdivacky unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 3050198090Srdivacky switch (IntNo) { 3051198090Srdivacky default: 3052198090Srdivacky break; 3053194710Sed 3054276479Sdim case Intrinsic::arm_ldaexd: 3055223017Sdim case Intrinsic::arm_ldrexd: { 3056261991Sdim SDLoc dl(N); 3057223017Sdim SDValue Chain = N->getOperand(0); 3058276479Sdim SDValue MemAddr = N->getOperand(2); 3059249423Sdim bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2(); 3060223017Sdim 3061276479Sdim bool IsAcquire = IntNo == Intrinsic::arm_ldaexd; 3062276479Sdim unsigned NewOpc = isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD) 3063276479Sdim : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD); 3064276479Sdim 3065223017Sdim // arm_ldrexd returns a i64 value in {i32, i32} 3066223017Sdim std::vector<EVT> ResTys; 3067249423Sdim if (isThumb) { 3068249423Sdim ResTys.push_back(MVT::i32); 3069249423Sdim ResTys.push_back(MVT::i32); 3070249423Sdim } else 3071249423Sdim ResTys.push_back(MVT::Untyped); 3072223017Sdim ResTys.push_back(MVT::Other); 3073223017Sdim 3074249423Sdim // Place arguments in the right order. 3075223017Sdim SmallVector<SDValue, 7> Ops; 3076223017Sdim Ops.push_back(MemAddr); 3077288943Sdim Ops.push_back(getAL(CurDAG, dl)); 3078223017Sdim Ops.push_back(CurDAG->getRegister(0, MVT::i32)); 3079223017Sdim Ops.push_back(Chain); 3080251662Sdim SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops); 3081223017Sdim // Transfer memoperands. 3082223017Sdim MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 3083223017Sdim MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 3084223017Sdim cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1); 3085223017Sdim 3086223017Sdim // Remap uses. 3087249423Sdim SDValue OutChain = isThumb ? SDValue(Ld, 2) : SDValue(Ld, 1); 3088223017Sdim if (!SDValue(N, 0).use_empty()) { 3089249423Sdim SDValue Result; 3090249423Sdim if (isThumb) 3091249423Sdim Result = SDValue(Ld, 0); 3092249423Sdim else { 3093288943Sdim SDValue SubRegIdx = 3094288943Sdim CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32); 3095249423Sdim SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 3096249423Sdim dl, MVT::i32, SDValue(Ld, 0), SubRegIdx); 3097249423Sdim Result = SDValue(ResNode,0); 3098249423Sdim } 3099223017Sdim ReplaceUses(SDValue(N, 0), Result); 3100223017Sdim } 3101223017Sdim if (!SDValue(N, 1).use_empty()) { 3102249423Sdim SDValue Result; 3103249423Sdim if (isThumb) 3104249423Sdim Result = SDValue(Ld, 1); 3105249423Sdim else { 3106288943Sdim SDValue SubRegIdx = 3107288943Sdim CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32); 3108249423Sdim SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 3109249423Sdim dl, MVT::i32, SDValue(Ld, 0), SubRegIdx); 3110249423Sdim Result = SDValue(ResNode,0); 3111249423Sdim } 3112223017Sdim ReplaceUses(SDValue(N, 1), Result); 3113223017Sdim } 3114249423Sdim ReplaceUses(SDValue(N, 2), OutChain); 3115276479Sdim return nullptr; 3116223017Sdim } 3117276479Sdim case Intrinsic::arm_stlexd: 3118223017Sdim case Intrinsic::arm_strexd: { 3119261991Sdim SDLoc dl(N); 3120223017Sdim SDValue Chain = N->getOperand(0); 3121223017Sdim SDValue Val0 = N->getOperand(2); 3122223017Sdim SDValue Val1 = N->getOperand(3); 3123223017Sdim SDValue MemAddr = N->getOperand(4); 3124223017Sdim 3125223017Sdim // Store exclusive double return a i32 value which is the return status 3126223017Sdim // of the issued store. 3127288943Sdim const EVT ResTys[] = {MVT::i32, MVT::Other}; 3128223017Sdim 3129249423Sdim bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2(); 3130249423Sdim // Place arguments in the right order. 3131223017Sdim SmallVector<SDValue, 7> Ops; 3132249423Sdim if (isThumb) { 3133249423Sdim Ops.push_back(Val0); 3134249423Sdim Ops.push_back(Val1); 3135249423Sdim } else 3136249423Sdim // arm_strexd uses GPRPair. 3137249423Sdim Ops.push_back(SDValue(createGPRPairNode(MVT::Untyped, Val0, Val1), 0)); 3138223017Sdim Ops.push_back(MemAddr); 3139288943Sdim Ops.push_back(getAL(CurDAG, dl)); 3140223017Sdim Ops.push_back(CurDAG->getRegister(0, MVT::i32)); 3141223017Sdim Ops.push_back(Chain); 3142223017Sdim 3143276479Sdim bool IsRelease = IntNo == Intrinsic::arm_stlexd; 3144276479Sdim unsigned NewOpc = isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD) 3145276479Sdim : (IsRelease ? ARM::STLEXD : ARM::STREXD); 3146223017Sdim 3147251662Sdim SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops); 3148223017Sdim // Transfer memoperands. 3149223017Sdim MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 3150223017Sdim MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 3151223017Sdim cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1); 3152223017Sdim 3153223017Sdim return St; 3154223017Sdim } 3155223017Sdim 3156206083Srdivacky case Intrinsic::arm_neon_vld1: { 3157239462Sdim static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16, 3158239462Sdim ARM::VLD1d32, ARM::VLD1d64 }; 3159239462Sdim static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16, 3160239462Sdim ARM::VLD1q32, ARM::VLD1q64}; 3161276479Sdim return SelectVLD(N, false, 1, DOpcodes, QOpcodes, nullptr); 3162206083Srdivacky } 3163206083Srdivacky 3164198090Srdivacky case Intrinsic::arm_neon_vld2: { 3165239462Sdim static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16, 3166239462Sdim ARM::VLD2d32, ARM::VLD1q64 }; 3167239462Sdim static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo, 3168239462Sdim ARM::VLD2q32Pseudo }; 3169276479Sdim return SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr); 3170198090Srdivacky } 3171194710Sed 3172198090Srdivacky case Intrinsic::arm_neon_vld3: { 3173239462Sdim static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo, 3174239462Sdim ARM::VLD3d16Pseudo, 3175239462Sdim ARM::VLD3d32Pseudo, 3176239462Sdim ARM::VLD1d64TPseudo }; 3177239462Sdim static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD, 3178239462Sdim ARM::VLD3q16Pseudo_UPD, 3179239462Sdim ARM::VLD3q32Pseudo_UPD }; 3180239462Sdim static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo, 3181239462Sdim ARM::VLD3q16oddPseudo, 3182239462Sdim ARM::VLD3q32oddPseudo }; 3183218893Sdim return SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1); 3184198090Srdivacky } 3185198090Srdivacky 3186198090Srdivacky case Intrinsic::arm_neon_vld4: { 3187239462Sdim static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo, 3188239462Sdim ARM::VLD4d16Pseudo, 3189239462Sdim ARM::VLD4d32Pseudo, 3190239462Sdim ARM::VLD1d64QPseudo }; 3191239462Sdim static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD, 3192239462Sdim ARM::VLD4q16Pseudo_UPD, 3193239462Sdim ARM::VLD4q32Pseudo_UPD }; 3194239462Sdim static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo, 3195239462Sdim ARM::VLD4q16oddPseudo, 3196239462Sdim ARM::VLD4q32oddPseudo }; 3197218893Sdim return SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1); 3198198090Srdivacky } 3199198090Srdivacky 3200198090Srdivacky case Intrinsic::arm_neon_vld2lane: { 3201239462Sdim static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo, 3202239462Sdim ARM::VLD2LNd16Pseudo, 3203239462Sdim ARM::VLD2LNd32Pseudo }; 3204239462Sdim static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo, 3205239462Sdim ARM::VLD2LNq32Pseudo }; 3206218893Sdim return SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes); 3207198090Srdivacky } 3208198090Srdivacky 3209198090Srdivacky case Intrinsic::arm_neon_vld3lane: { 3210239462Sdim static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo, 3211239462Sdim ARM::VLD3LNd16Pseudo, 3212239462Sdim ARM::VLD3LNd32Pseudo }; 3213239462Sdim static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo, 3214239462Sdim ARM::VLD3LNq32Pseudo }; 3215218893Sdim return SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes); 3216198090Srdivacky } 3217198090Srdivacky 3218198090Srdivacky case Intrinsic::arm_neon_vld4lane: { 3219239462Sdim static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo, 3220239462Sdim ARM::VLD4LNd16Pseudo, 3221239462Sdim ARM::VLD4LNd32Pseudo }; 3222239462Sdim static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo, 3223239462Sdim ARM::VLD4LNq32Pseudo }; 3224218893Sdim return SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes); 3225198090Srdivacky } 3226198090Srdivacky 3227206083Srdivacky case Intrinsic::arm_neon_vst1: { 3228239462Sdim static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16, 3229239462Sdim ARM::VST1d32, ARM::VST1d64 }; 3230239462Sdim static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16, 3231239462Sdim ARM::VST1q32, ARM::VST1q64 }; 3232276479Sdim return SelectVST(N, false, 1, DOpcodes, QOpcodes, nullptr); 3233206083Srdivacky } 3234206083Srdivacky 3235198090Srdivacky case Intrinsic::arm_neon_vst2: { 3236239462Sdim static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16, 3237239462Sdim ARM::VST2d32, ARM::VST1q64 }; 3238239462Sdim static uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo, 3239239462Sdim ARM::VST2q32Pseudo }; 3240276479Sdim return SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr); 3241198090Srdivacky } 3242194710Sed 3243198090Srdivacky case Intrinsic::arm_neon_vst3: { 3244239462Sdim static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo, 3245239462Sdim ARM::VST3d16Pseudo, 3246239462Sdim ARM::VST3d32Pseudo, 3247239462Sdim ARM::VST1d64TPseudo }; 3248239462Sdim static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD, 3249239462Sdim ARM::VST3q16Pseudo_UPD, 3250239462Sdim ARM::VST3q32Pseudo_UPD }; 3251239462Sdim static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo, 3252239462Sdim ARM::VST3q16oddPseudo, 3253239462Sdim ARM::VST3q32oddPseudo }; 3254218893Sdim return SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1); 3255194710Sed } 3256194710Sed 3257198090Srdivacky case Intrinsic::arm_neon_vst4: { 3258239462Sdim static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo, 3259239462Sdim ARM::VST4d16Pseudo, 3260239462Sdim ARM::VST4d32Pseudo, 3261239462Sdim ARM::VST1d64QPseudo }; 3262239462Sdim static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD, 3263239462Sdim ARM::VST4q16Pseudo_UPD, 3264239462Sdim ARM::VST4q32Pseudo_UPD }; 3265239462Sdim static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo, 3266239462Sdim ARM::VST4q16oddPseudo, 3267239462Sdim ARM::VST4q32oddPseudo }; 3268218893Sdim return SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1); 3269198090Srdivacky } 3270198090Srdivacky 3271198090Srdivacky case Intrinsic::arm_neon_vst2lane: { 3272239462Sdim static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo, 3273239462Sdim ARM::VST2LNd16Pseudo, 3274239462Sdim ARM::VST2LNd32Pseudo }; 3275239462Sdim static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo, 3276239462Sdim ARM::VST2LNq32Pseudo }; 3277218893Sdim return SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes); 3278198090Srdivacky } 3279198090Srdivacky 3280198090Srdivacky case Intrinsic::arm_neon_vst3lane: { 3281239462Sdim static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo, 3282239462Sdim ARM::VST3LNd16Pseudo, 3283239462Sdim ARM::VST3LNd32Pseudo }; 3284239462Sdim static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo, 3285239462Sdim ARM::VST3LNq32Pseudo }; 3286218893Sdim return SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes); 3287198090Srdivacky } 3288198090Srdivacky 3289198090Srdivacky case Intrinsic::arm_neon_vst4lane: { 3290239462Sdim static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo, 3291239462Sdim ARM::VST4LNd16Pseudo, 3292239462Sdim ARM::VST4LNd32Pseudo }; 3293239462Sdim static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo, 3294239462Sdim ARM::VST4LNq32Pseudo }; 3295218893Sdim return SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes); 3296198090Srdivacky } 3297198090Srdivacky } 3298208599Srdivacky break; 3299194710Sed } 3300208599Srdivacky 3301210299Sed case ISD::INTRINSIC_WO_CHAIN: { 3302210299Sed unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 3303210299Sed switch (IntNo) { 3304210299Sed default: 3305210299Sed break; 3306210299Sed 3307210299Sed case Intrinsic::arm_neon_vtbl2: 3308234353Sdim return SelectVTBL(N, false, 2, ARM::VTBL2); 3309210299Sed case Intrinsic::arm_neon_vtbl3: 3310218893Sdim return SelectVTBL(N, false, 3, ARM::VTBL3Pseudo); 3311210299Sed case Intrinsic::arm_neon_vtbl4: 3312218893Sdim return SelectVTBL(N, false, 4, ARM::VTBL4Pseudo); 3313210299Sed 3314210299Sed case Intrinsic::arm_neon_vtbx2: 3315234353Sdim return SelectVTBL(N, true, 2, ARM::VTBX2); 3316210299Sed case Intrinsic::arm_neon_vtbx3: 3317218893Sdim return SelectVTBL(N, true, 3, ARM::VTBX3Pseudo); 3318210299Sed case Intrinsic::arm_neon_vtbx4: 3319218893Sdim return SelectVTBL(N, true, 4, ARM::VTBX4Pseudo); 3320210299Sed } 3321210299Sed break; 3322210299Sed } 3323210299Sed 3324221345Sdim case ARMISD::VTBL1: { 3325261991Sdim SDLoc dl(N); 3326221345Sdim EVT VT = N->getValueType(0); 3327221345Sdim SmallVector<SDValue, 6> Ops; 3328221345Sdim 3329221345Sdim Ops.push_back(N->getOperand(0)); 3330221345Sdim Ops.push_back(N->getOperand(1)); 3331288943Sdim Ops.push_back(getAL(CurDAG, dl)); // Predicate 3332221345Sdim Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register 3333251662Sdim return CurDAG->getMachineNode(ARM::VTBL1, dl, VT, Ops); 3334221345Sdim } 3335221345Sdim case ARMISD::VTBL2: { 3336261991Sdim SDLoc dl(N); 3337221345Sdim EVT VT = N->getValueType(0); 3338221345Sdim 3339221345Sdim // Form a REG_SEQUENCE to force register allocation. 3340221345Sdim SDValue V0 = N->getOperand(0); 3341221345Sdim SDValue V1 = N->getOperand(1); 3342249423Sdim SDValue RegSeq = SDValue(createDRegPairNode(MVT::v16i8, V0, V1), 0); 3343221345Sdim 3344221345Sdim SmallVector<SDValue, 6> Ops; 3345221345Sdim Ops.push_back(RegSeq); 3346221345Sdim Ops.push_back(N->getOperand(2)); 3347288943Sdim Ops.push_back(getAL(CurDAG, dl)); // Predicate 3348221345Sdim Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register 3349251662Sdim return CurDAG->getMachineNode(ARM::VTBL2, dl, VT, Ops); 3350221345Sdim } 3351221345Sdim 3352208599Srdivacky case ISD::CONCAT_VECTORS: 3353208599Srdivacky return SelectConcatVector(N); 3354194710Sed } 3355194710Sed 3356202375Srdivacky return SelectCode(N); 3357193323Sed} 3358193323Sed 3359288943Sdim// Inspect a register string of the form 3360288943Sdim// cp<coprocessor>:<opc1>:c<CRn>:c<CRm>:<opc2> (32bit) or 3361288943Sdim// cp<coprocessor>:<opc1>:c<CRm> (64bit) inspect the fields of the string 3362288943Sdim// and obtain the integer operands from them, adding these operands to the 3363288943Sdim// provided vector. 3364288943Sdimstatic void getIntOperandsFromRegisterString(StringRef RegString, 3365288943Sdim SelectionDAG *CurDAG, SDLoc DL, 3366288943Sdim std::vector<SDValue>& Ops) { 3367288943Sdim SmallVector<StringRef, 5> Fields; 3368296417Sdim RegString.split(Fields, ':'); 3369288943Sdim 3370288943Sdim if (Fields.size() > 1) { 3371288943Sdim bool AllIntFields = true; 3372288943Sdim 3373288943Sdim for (StringRef Field : Fields) { 3374288943Sdim // Need to trim out leading 'cp' characters and get the integer field. 3375288943Sdim unsigned IntField; 3376288943Sdim AllIntFields &= !Field.trim("CPcp").getAsInteger(10, IntField); 3377288943Sdim Ops.push_back(CurDAG->getTargetConstant(IntField, DL, MVT::i32)); 3378288943Sdim } 3379288943Sdim 3380288943Sdim assert(AllIntFields && 3381288943Sdim "Unexpected non-integer value in special register string."); 3382288943Sdim } 3383288943Sdim} 3384288943Sdim 3385288943Sdim// Maps a Banked Register string to its mask value. The mask value returned is 3386288943Sdim// for use in the MRSbanked / MSRbanked instruction nodes as the Banked Register 3387288943Sdim// mask operand, which expresses which register is to be used, e.g. r8, and in 3388288943Sdim// which mode it is to be used, e.g. usr. Returns -1 to signify that the string 3389288943Sdim// was invalid. 3390288943Sdimstatic inline int getBankedRegisterMask(StringRef RegString) { 3391288943Sdim return StringSwitch<int>(RegString.lower()) 3392288943Sdim .Case("r8_usr", 0x00) 3393288943Sdim .Case("r9_usr", 0x01) 3394288943Sdim .Case("r10_usr", 0x02) 3395288943Sdim .Case("r11_usr", 0x03) 3396288943Sdim .Case("r12_usr", 0x04) 3397288943Sdim .Case("sp_usr", 0x05) 3398288943Sdim .Case("lr_usr", 0x06) 3399288943Sdim .Case("r8_fiq", 0x08) 3400288943Sdim .Case("r9_fiq", 0x09) 3401288943Sdim .Case("r10_fiq", 0x0a) 3402288943Sdim .Case("r11_fiq", 0x0b) 3403288943Sdim .Case("r12_fiq", 0x0c) 3404288943Sdim .Case("sp_fiq", 0x0d) 3405288943Sdim .Case("lr_fiq", 0x0e) 3406288943Sdim .Case("lr_irq", 0x10) 3407288943Sdim .Case("sp_irq", 0x11) 3408288943Sdim .Case("lr_svc", 0x12) 3409288943Sdim .Case("sp_svc", 0x13) 3410288943Sdim .Case("lr_abt", 0x14) 3411288943Sdim .Case("sp_abt", 0x15) 3412288943Sdim .Case("lr_und", 0x16) 3413288943Sdim .Case("sp_und", 0x17) 3414288943Sdim .Case("lr_mon", 0x1c) 3415288943Sdim .Case("sp_mon", 0x1d) 3416288943Sdim .Case("elr_hyp", 0x1e) 3417288943Sdim .Case("sp_hyp", 0x1f) 3418288943Sdim .Case("spsr_fiq", 0x2e) 3419288943Sdim .Case("spsr_irq", 0x30) 3420288943Sdim .Case("spsr_svc", 0x32) 3421288943Sdim .Case("spsr_abt", 0x34) 3422288943Sdim .Case("spsr_und", 0x36) 3423288943Sdim .Case("spsr_mon", 0x3c) 3424288943Sdim .Case("spsr_hyp", 0x3e) 3425288943Sdim .Default(-1); 3426288943Sdim} 3427288943Sdim 3428288943Sdim// Maps a MClass special register string to its value for use in the 3429288943Sdim// t2MRS_M / t2MSR_M instruction nodes as the SYSm value operand. 3430288943Sdim// Returns -1 to signify that the string was invalid. 3431288943Sdimstatic inline int getMClassRegisterSYSmValueMask(StringRef RegString) { 3432288943Sdim return StringSwitch<int>(RegString.lower()) 3433288943Sdim .Case("apsr", 0x0) 3434288943Sdim .Case("iapsr", 0x1) 3435288943Sdim .Case("eapsr", 0x2) 3436288943Sdim .Case("xpsr", 0x3) 3437288943Sdim .Case("ipsr", 0x5) 3438288943Sdim .Case("epsr", 0x6) 3439288943Sdim .Case("iepsr", 0x7) 3440288943Sdim .Case("msp", 0x8) 3441288943Sdim .Case("psp", 0x9) 3442288943Sdim .Case("primask", 0x10) 3443288943Sdim .Case("basepri", 0x11) 3444288943Sdim .Case("basepri_max", 0x12) 3445288943Sdim .Case("faultmask", 0x13) 3446288943Sdim .Case("control", 0x14) 3447288943Sdim .Default(-1); 3448288943Sdim} 3449288943Sdim 3450288943Sdim// The flags here are common to those allowed for apsr in the A class cores and 3451288943Sdim// those allowed for the special registers in the M class cores. Returns a 3452288943Sdim// value representing which flags were present, -1 if invalid. 3453296417Sdimstatic inline int getMClassFlagsMask(StringRef Flags, bool hasDSP) { 3454288943Sdim if (Flags.empty()) 3455296417Sdim return 0x2 | (int)hasDSP; 3456288943Sdim 3457288943Sdim return StringSwitch<int>(Flags) 3458288943Sdim .Case("g", 0x1) 3459288943Sdim .Case("nzcvq", 0x2) 3460288943Sdim .Case("nzcvqg", 0x3) 3461288943Sdim .Default(-1); 3462288943Sdim} 3463288943Sdim 3464288943Sdimstatic int getMClassRegisterMask(StringRef Reg, StringRef Flags, bool IsRead, 3465288943Sdim const ARMSubtarget *Subtarget) { 3466288943Sdim // Ensure that the register (without flags) was a valid M Class special 3467288943Sdim // register. 3468288943Sdim int SYSmvalue = getMClassRegisterSYSmValueMask(Reg); 3469288943Sdim if (SYSmvalue == -1) 3470288943Sdim return -1; 3471288943Sdim 3472288943Sdim // basepri, basepri_max and faultmask are only valid for V7m. 3473288943Sdim if (!Subtarget->hasV7Ops() && SYSmvalue >= 0x11 && SYSmvalue <= 0x13) 3474288943Sdim return -1; 3475288943Sdim 3476288943Sdim // If it was a read then we won't be expecting flags and so at this point 3477288943Sdim // we can return the mask. 3478288943Sdim if (IsRead) { 3479288943Sdim assert (Flags.empty() && "Unexpected flags for reading M class register."); 3480288943Sdim return SYSmvalue; 3481288943Sdim } 3482288943Sdim 3483288943Sdim // We know we are now handling a write so need to get the mask for the flags. 3484296417Sdim int Mask = getMClassFlagsMask(Flags, Subtarget->hasDSP()); 3485288943Sdim 3486288943Sdim // Only apsr, iapsr, eapsr, xpsr can have flags. The other register values 3487288943Sdim // shouldn't have flags present. 3488288943Sdim if ((SYSmvalue < 0x4 && Mask == -1) || (SYSmvalue > 0x4 && !Flags.empty())) 3489288943Sdim return -1; 3490288943Sdim 3491288943Sdim // The _g and _nzcvqg versions are only valid if the DSP extension is 3492288943Sdim // available. 3493296417Sdim if (!Subtarget->hasDSP() && (Mask & 0x1)) 3494288943Sdim return -1; 3495288943Sdim 3496288943Sdim // The register was valid so need to put the mask in the correct place 3497288943Sdim // (the flags need to be in bits 11-10) and combine with the SYSmvalue to 3498288943Sdim // construct the operand for the instruction node. 3499288943Sdim if (SYSmvalue < 0x4) 3500288943Sdim return SYSmvalue | Mask << 10; 3501288943Sdim 3502288943Sdim return SYSmvalue; 3503288943Sdim} 3504288943Sdim 3505288943Sdimstatic int getARClassRegisterMask(StringRef Reg, StringRef Flags) { 3506288943Sdim // The mask operand contains the special register (R Bit) in bit 4, whether 3507288943Sdim // the register is spsr (R bit is 1) or one of cpsr/apsr (R bit is 0), and 3508288943Sdim // bits 3-0 contains the fields to be accessed in the special register, set by 3509288943Sdim // the flags provided with the register. 3510288943Sdim int Mask = 0; 3511288943Sdim if (Reg == "apsr") { 3512288943Sdim // The flags permitted for apsr are the same flags that are allowed in 3513288943Sdim // M class registers. We get the flag value and then shift the flags into 3514288943Sdim // the correct place to combine with the mask. 3515296417Sdim Mask = getMClassFlagsMask(Flags, true); 3516288943Sdim if (Mask == -1) 3517288943Sdim return -1; 3518288943Sdim return Mask << 2; 3519288943Sdim } 3520288943Sdim 3521288943Sdim if (Reg != "cpsr" && Reg != "spsr") { 3522288943Sdim return -1; 3523288943Sdim } 3524288943Sdim 3525288943Sdim // This is the same as if the flags were "fc" 3526288943Sdim if (Flags.empty() || Flags == "all") 3527288943Sdim return Mask | 0x9; 3528288943Sdim 3529288943Sdim // Inspect the supplied flags string and set the bits in the mask for 3530288943Sdim // the relevant and valid flags allowed for cpsr and spsr. 3531288943Sdim for (char Flag : Flags) { 3532288943Sdim int FlagVal; 3533288943Sdim switch (Flag) { 3534288943Sdim case 'c': 3535288943Sdim FlagVal = 0x1; 3536288943Sdim break; 3537288943Sdim case 'x': 3538288943Sdim FlagVal = 0x2; 3539288943Sdim break; 3540288943Sdim case 's': 3541288943Sdim FlagVal = 0x4; 3542288943Sdim break; 3543288943Sdim case 'f': 3544288943Sdim FlagVal = 0x8; 3545288943Sdim break; 3546288943Sdim default: 3547288943Sdim FlagVal = 0; 3548288943Sdim } 3549288943Sdim 3550288943Sdim // This avoids allowing strings where the same flag bit appears twice. 3551288943Sdim if (!FlagVal || (Mask & FlagVal)) 3552288943Sdim return -1; 3553288943Sdim Mask |= FlagVal; 3554288943Sdim } 3555288943Sdim 3556288943Sdim // If the register is spsr then we need to set the R bit. 3557288943Sdim if (Reg == "spsr") 3558288943Sdim Mask |= 0x10; 3559288943Sdim 3560288943Sdim return Mask; 3561288943Sdim} 3562288943Sdim 3563288943Sdim// Lower the read_register intrinsic to ARM specific DAG nodes 3564288943Sdim// using the supplied metadata string to select the instruction node to use 3565288943Sdim// and the registers/masks to construct as operands for the node. 3566288943SdimSDNode *ARMDAGToDAGISel::SelectReadRegister(SDNode *N){ 3567288943Sdim const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1)); 3568288943Sdim const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0)); 3569288943Sdim bool IsThumb2 = Subtarget->isThumb2(); 3570288943Sdim SDLoc DL(N); 3571288943Sdim 3572288943Sdim std::vector<SDValue> Ops; 3573288943Sdim getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops); 3574288943Sdim 3575288943Sdim if (!Ops.empty()) { 3576288943Sdim // If the special register string was constructed of fields (as defined 3577288943Sdim // in the ACLE) then need to lower to MRC node (32 bit) or 3578288943Sdim // MRRC node(64 bit), we can make the distinction based on the number of 3579288943Sdim // operands we have. 3580288943Sdim unsigned Opcode; 3581288943Sdim SmallVector<EVT, 3> ResTypes; 3582288943Sdim if (Ops.size() == 5){ 3583288943Sdim Opcode = IsThumb2 ? ARM::t2MRC : ARM::MRC; 3584288943Sdim ResTypes.append({ MVT::i32, MVT::Other }); 3585288943Sdim } else { 3586288943Sdim assert(Ops.size() == 3 && 3587288943Sdim "Invalid number of fields in special register string."); 3588288943Sdim Opcode = IsThumb2 ? ARM::t2MRRC : ARM::MRRC; 3589288943Sdim ResTypes.append({ MVT::i32, MVT::i32, MVT::Other }); 3590288943Sdim } 3591288943Sdim 3592288943Sdim Ops.push_back(getAL(CurDAG, DL)); 3593288943Sdim Ops.push_back(CurDAG->getRegister(0, MVT::i32)); 3594288943Sdim Ops.push_back(N->getOperand(0)); 3595288943Sdim return CurDAG->getMachineNode(Opcode, DL, ResTypes, Ops); 3596288943Sdim } 3597288943Sdim 3598288943Sdim std::string SpecialReg = RegString->getString().lower(); 3599288943Sdim 3600288943Sdim int BankedReg = getBankedRegisterMask(SpecialReg); 3601288943Sdim if (BankedReg != -1) { 3602288943Sdim Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32), 3603288943Sdim getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32), 3604288943Sdim N->getOperand(0) }; 3605288943Sdim return CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSbanked : ARM::MRSbanked, 3606288943Sdim DL, MVT::i32, MVT::Other, Ops); 3607288943Sdim } 3608288943Sdim 3609288943Sdim // The VFP registers are read by creating SelectionDAG nodes with opcodes 3610288943Sdim // corresponding to the register that is being read from. So we switch on the 3611288943Sdim // string to find which opcode we need to use. 3612288943Sdim unsigned Opcode = StringSwitch<unsigned>(SpecialReg) 3613288943Sdim .Case("fpscr", ARM::VMRS) 3614288943Sdim .Case("fpexc", ARM::VMRS_FPEXC) 3615288943Sdim .Case("fpsid", ARM::VMRS_FPSID) 3616288943Sdim .Case("mvfr0", ARM::VMRS_MVFR0) 3617288943Sdim .Case("mvfr1", ARM::VMRS_MVFR1) 3618288943Sdim .Case("mvfr2", ARM::VMRS_MVFR2) 3619288943Sdim .Case("fpinst", ARM::VMRS_FPINST) 3620288943Sdim .Case("fpinst2", ARM::VMRS_FPINST2) 3621288943Sdim .Default(0); 3622288943Sdim 3623288943Sdim // If an opcode was found then we can lower the read to a VFP instruction. 3624288943Sdim if (Opcode) { 3625288943Sdim if (!Subtarget->hasVFP2()) 3626288943Sdim return nullptr; 3627288943Sdim if (Opcode == ARM::VMRS_MVFR2 && !Subtarget->hasFPARMv8()) 3628288943Sdim return nullptr; 3629288943Sdim 3630288943Sdim Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32), 3631288943Sdim N->getOperand(0) }; 3632288943Sdim return CurDAG->getMachineNode(Opcode, DL, MVT::i32, MVT::Other, Ops); 3633288943Sdim } 3634288943Sdim 3635288943Sdim // If the target is M Class then need to validate that the register string 3636288943Sdim // is an acceptable value, so check that a mask can be constructed from the 3637288943Sdim // string. 3638288943Sdim if (Subtarget->isMClass()) { 3639288943Sdim int SYSmValue = getMClassRegisterMask(SpecialReg, "", true, Subtarget); 3640288943Sdim if (SYSmValue == -1) 3641288943Sdim return nullptr; 3642288943Sdim 3643288943Sdim SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32), 3644288943Sdim getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32), 3645288943Sdim N->getOperand(0) }; 3646288943Sdim return CurDAG->getMachineNode(ARM::t2MRS_M, DL, MVT::i32, MVT::Other, Ops); 3647288943Sdim } 3648288943Sdim 3649288943Sdim // Here we know the target is not M Class so we need to check if it is one 3650288943Sdim // of the remaining possible values which are apsr, cpsr or spsr. 3651288943Sdim if (SpecialReg == "apsr" || SpecialReg == "cpsr") { 3652288943Sdim Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32), 3653288943Sdim N->getOperand(0) }; 3654288943Sdim return CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRS_AR : ARM::MRS, DL, 3655288943Sdim MVT::i32, MVT::Other, Ops); 3656288943Sdim } 3657288943Sdim 3658288943Sdim if (SpecialReg == "spsr") { 3659288943Sdim Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32), 3660288943Sdim N->getOperand(0) }; 3661288943Sdim return CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSsys_AR : ARM::MRSsys, 3662288943Sdim DL, MVT::i32, MVT::Other, Ops); 3663288943Sdim } 3664288943Sdim 3665288943Sdim return nullptr; 3666288943Sdim} 3667288943Sdim 3668288943Sdim// Lower the write_register intrinsic to ARM specific DAG nodes 3669288943Sdim// using the supplied metadata string to select the instruction node to use 3670288943Sdim// and the registers/masks to use in the nodes 3671288943SdimSDNode *ARMDAGToDAGISel::SelectWriteRegister(SDNode *N){ 3672288943Sdim const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1)); 3673288943Sdim const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0)); 3674288943Sdim bool IsThumb2 = Subtarget->isThumb2(); 3675288943Sdim SDLoc DL(N); 3676288943Sdim 3677288943Sdim std::vector<SDValue> Ops; 3678288943Sdim getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops); 3679288943Sdim 3680288943Sdim if (!Ops.empty()) { 3681288943Sdim // If the special register string was constructed of fields (as defined 3682288943Sdim // in the ACLE) then need to lower to MCR node (32 bit) or 3683288943Sdim // MCRR node(64 bit), we can make the distinction based on the number of 3684288943Sdim // operands we have. 3685288943Sdim unsigned Opcode; 3686288943Sdim if (Ops.size() == 5) { 3687288943Sdim Opcode = IsThumb2 ? ARM::t2MCR : ARM::MCR; 3688288943Sdim Ops.insert(Ops.begin()+2, N->getOperand(2)); 3689288943Sdim } else { 3690288943Sdim assert(Ops.size() == 3 && 3691288943Sdim "Invalid number of fields in special register string."); 3692288943Sdim Opcode = IsThumb2 ? ARM::t2MCRR : ARM::MCRR; 3693288943Sdim SDValue WriteValue[] = { N->getOperand(2), N->getOperand(3) }; 3694288943Sdim Ops.insert(Ops.begin()+2, WriteValue, WriteValue+2); 3695288943Sdim } 3696288943Sdim 3697288943Sdim Ops.push_back(getAL(CurDAG, DL)); 3698288943Sdim Ops.push_back(CurDAG->getRegister(0, MVT::i32)); 3699288943Sdim Ops.push_back(N->getOperand(0)); 3700288943Sdim 3701288943Sdim return CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops); 3702288943Sdim } 3703288943Sdim 3704288943Sdim std::string SpecialReg = RegString->getString().lower(); 3705288943Sdim int BankedReg = getBankedRegisterMask(SpecialReg); 3706288943Sdim if (BankedReg != -1) { 3707288943Sdim Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32), N->getOperand(2), 3708288943Sdim getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32), 3709288943Sdim N->getOperand(0) }; 3710288943Sdim return CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSRbanked : ARM::MSRbanked, 3711288943Sdim DL, MVT::Other, Ops); 3712288943Sdim } 3713288943Sdim 3714288943Sdim // The VFP registers are written to by creating SelectionDAG nodes with 3715288943Sdim // opcodes corresponding to the register that is being written. So we switch 3716288943Sdim // on the string to find which opcode we need to use. 3717288943Sdim unsigned Opcode = StringSwitch<unsigned>(SpecialReg) 3718288943Sdim .Case("fpscr", ARM::VMSR) 3719288943Sdim .Case("fpexc", ARM::VMSR_FPEXC) 3720288943Sdim .Case("fpsid", ARM::VMSR_FPSID) 3721288943Sdim .Case("fpinst", ARM::VMSR_FPINST) 3722288943Sdim .Case("fpinst2", ARM::VMSR_FPINST2) 3723288943Sdim .Default(0); 3724288943Sdim 3725288943Sdim if (Opcode) { 3726288943Sdim if (!Subtarget->hasVFP2()) 3727288943Sdim return nullptr; 3728288943Sdim Ops = { N->getOperand(2), getAL(CurDAG, DL), 3729288943Sdim CurDAG->getRegister(0, MVT::i32), N->getOperand(0) }; 3730288943Sdim return CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops); 3731288943Sdim } 3732288943Sdim 3733288943Sdim SmallVector<StringRef, 5> Fields; 3734296417Sdim StringRef(SpecialReg).split(Fields, '_', 1, false); 3735288943Sdim std::string Reg = Fields[0].str(); 3736288943Sdim StringRef Flags = Fields.size() == 2 ? Fields[1] : ""; 3737288943Sdim 3738288943Sdim // If the target was M Class then need to validate the special register value 3739288943Sdim // and retrieve the mask for use in the instruction node. 3740288943Sdim if (Subtarget->isMClass()) { 3741288943Sdim // basepri_max gets split so need to correct Reg and Flags. 3742288943Sdim if (SpecialReg == "basepri_max") { 3743288943Sdim Reg = SpecialReg; 3744288943Sdim Flags = ""; 3745288943Sdim } 3746288943Sdim int SYSmValue = getMClassRegisterMask(Reg, Flags, false, Subtarget); 3747288943Sdim if (SYSmValue == -1) 3748288943Sdim return nullptr; 3749288943Sdim 3750288943Sdim SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32), 3751288943Sdim N->getOperand(2), getAL(CurDAG, DL), 3752288943Sdim CurDAG->getRegister(0, MVT::i32), N->getOperand(0) }; 3753288943Sdim return CurDAG->getMachineNode(ARM::t2MSR_M, DL, MVT::Other, Ops); 3754288943Sdim } 3755288943Sdim 3756288943Sdim // We then check to see if a valid mask can be constructed for one of the 3757288943Sdim // register string values permitted for the A and R class cores. These values 3758288943Sdim // are apsr, spsr and cpsr; these are also valid on older cores. 3759288943Sdim int Mask = getARClassRegisterMask(Reg, Flags); 3760288943Sdim if (Mask != -1) { 3761288943Sdim Ops = { CurDAG->getTargetConstant(Mask, DL, MVT::i32), N->getOperand(2), 3762288943Sdim getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32), 3763288943Sdim N->getOperand(0) }; 3764288943Sdim return CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSR_AR : ARM::MSR, 3765288943Sdim DL, MVT::Other, Ops); 3766288943Sdim } 3767288943Sdim 3768288943Sdim return nullptr; 3769288943Sdim} 3770288943Sdim 3771249423SdimSDNode *ARMDAGToDAGISel::SelectInlineAsm(SDNode *N){ 3772249423Sdim std::vector<SDValue> AsmNodeOperands; 3773249423Sdim unsigned Flag, Kind; 3774249423Sdim bool Changed = false; 3775249423Sdim unsigned NumOps = N->getNumOperands(); 3776249423Sdim 3777249423Sdim // Normally, i64 data is bounded to two arbitrary GRPs for "%r" constraint. 3778249423Sdim // However, some instrstions (e.g. ldrexd/strexd in ARM mode) require 3779249423Sdim // (even/even+1) GPRs and use %n and %Hn to refer to the individual regs 3780249423Sdim // respectively. Since there is no constraint to explicitly specify a 3781261991Sdim // reg pair, we use GPRPair reg class for "%r" for 64-bit data. For Thumb, 3782261991Sdim // the 64-bit data may be referred by H, Q, R modifiers, so we still pack 3783261991Sdim // them into a GPRPair. 3784249423Sdim 3785261991Sdim SDLoc dl(N); 3786276479Sdim SDValue Glue = N->getGluedNode() ? N->getOperand(NumOps-1) 3787276479Sdim : SDValue(nullptr,0); 3788249423Sdim 3789261991Sdim SmallVector<bool, 8> OpChanged; 3790249423Sdim // Glue node will be appended late. 3791261991Sdim for(unsigned i = 0, e = N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) { 3792249423Sdim SDValue op = N->getOperand(i); 3793249423Sdim AsmNodeOperands.push_back(op); 3794249423Sdim 3795249423Sdim if (i < InlineAsm::Op_FirstOperand) 3796249423Sdim continue; 3797249423Sdim 3798249423Sdim if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(i))) { 3799249423Sdim Flag = C->getZExtValue(); 3800249423Sdim Kind = InlineAsm::getKind(Flag); 3801249423Sdim } 3802249423Sdim else 3803249423Sdim continue; 3804249423Sdim 3805261991Sdim // Immediate operands to inline asm in the SelectionDAG are modeled with 3806261991Sdim // two operands. The first is a constant of value InlineAsm::Kind_Imm, and 3807261991Sdim // the second is a constant with the value of the immediate. If we get here 3808261991Sdim // and we have a Kind_Imm, skip the next operand, and continue. 3809261991Sdim if (Kind == InlineAsm::Kind_Imm) { 3810261991Sdim SDValue op = N->getOperand(++i); 3811261991Sdim AsmNodeOperands.push_back(op); 3812261991Sdim continue; 3813261991Sdim } 3814261991Sdim 3815261991Sdim unsigned NumRegs = InlineAsm::getNumOperandRegisters(Flag); 3816261991Sdim if (NumRegs) 3817261991Sdim OpChanged.push_back(false); 3818261991Sdim 3819261991Sdim unsigned DefIdx = 0; 3820261991Sdim bool IsTiedToChangedOp = false; 3821261991Sdim // If it's a use that is tied with a previous def, it has no 3822261991Sdim // reg class constraint. 3823261991Sdim if (Changed && InlineAsm::isUseOperandTiedToDef(Flag, DefIdx)) 3824261991Sdim IsTiedToChangedOp = OpChanged[DefIdx]; 3825261991Sdim 3826249423Sdim if (Kind != InlineAsm::Kind_RegUse && Kind != InlineAsm::Kind_RegDef 3827249423Sdim && Kind != InlineAsm::Kind_RegDefEarlyClobber) 3828249423Sdim continue; 3829249423Sdim 3830249423Sdim unsigned RC; 3831249423Sdim bool HasRC = InlineAsm::hasRegClassConstraint(Flag, RC); 3832261991Sdim if ((!IsTiedToChangedOp && (!HasRC || RC != ARM::GPRRegClassID)) 3833261991Sdim || NumRegs != 2) 3834249423Sdim continue; 3835249423Sdim 3836261991Sdim assert((i+2 < NumOps) && "Invalid number of operands in inline asm"); 3837249423Sdim SDValue V0 = N->getOperand(i+1); 3838249423Sdim SDValue V1 = N->getOperand(i+2); 3839249423Sdim unsigned Reg0 = cast<RegisterSDNode>(V0)->getReg(); 3840249423Sdim unsigned Reg1 = cast<RegisterSDNode>(V1)->getReg(); 3841249423Sdim SDValue PairedReg; 3842249423Sdim MachineRegisterInfo &MRI = MF->getRegInfo(); 3843249423Sdim 3844249423Sdim if (Kind == InlineAsm::Kind_RegDef || 3845249423Sdim Kind == InlineAsm::Kind_RegDefEarlyClobber) { 3846249423Sdim // Replace the two GPRs with 1 GPRPair and copy values from GPRPair to 3847249423Sdim // the original GPRs. 3848249423Sdim 3849249423Sdim unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass); 3850249423Sdim PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped); 3851249423Sdim SDValue Chain = SDValue(N,0); 3852249423Sdim 3853249423Sdim SDNode *GU = N->getGluedUser(); 3854249423Sdim SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped, 3855249423Sdim Chain.getValue(1)); 3856249423Sdim 3857249423Sdim // Extract values from a GPRPair reg and copy to the original GPR reg. 3858249423Sdim SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32, 3859249423Sdim RegCopy); 3860249423Sdim SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32, 3861249423Sdim RegCopy); 3862249423Sdim SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0, 3863249423Sdim RegCopy.getValue(1)); 3864249423Sdim SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1)); 3865249423Sdim 3866249423Sdim // Update the original glue user. 3867249423Sdim std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1); 3868249423Sdim Ops.push_back(T1.getValue(1)); 3869276479Sdim CurDAG->UpdateNodeOperands(GU, Ops); 3870249423Sdim } 3871249423Sdim else { 3872249423Sdim // For Kind == InlineAsm::Kind_RegUse, we first copy two GPRs into a 3873249423Sdim // GPRPair and then pass the GPRPair to the inline asm. 3874249423Sdim SDValue Chain = AsmNodeOperands[InlineAsm::Op_InputChain]; 3875249423Sdim 3876249423Sdim // As REG_SEQ doesn't take RegisterSDNode, we copy them first. 3877249423Sdim SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32, 3878249423Sdim Chain.getValue(1)); 3879249423Sdim SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32, 3880249423Sdim T0.getValue(1)); 3881249423Sdim SDValue Pair = SDValue(createGPRPairNode(MVT::Untyped, T0, T1), 0); 3882249423Sdim 3883249423Sdim // Copy REG_SEQ into a GPRPair-typed VR and replace the original two 3884249423Sdim // i32 VRs of inline asm with it. 3885249423Sdim unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass); 3886249423Sdim PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped); 3887249423Sdim Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair, T1.getValue(1)); 3888249423Sdim 3889249423Sdim AsmNodeOperands[InlineAsm::Op_InputChain] = Chain; 3890249423Sdim Glue = Chain.getValue(1); 3891249423Sdim } 3892249423Sdim 3893249423Sdim Changed = true; 3894249423Sdim 3895249423Sdim if(PairedReg.getNode()) { 3896261991Sdim OpChanged[OpChanged.size() -1 ] = true; 3897249423Sdim Flag = InlineAsm::getFlagWord(Kind, 1 /* RegNum*/); 3898261991Sdim if (IsTiedToChangedOp) 3899261991Sdim Flag = InlineAsm::getFlagWordForMatchingOp(Flag, DefIdx); 3900261991Sdim else 3901261991Sdim Flag = InlineAsm::getFlagWordForRegClass(Flag, ARM::GPRPairRegClassID); 3902249423Sdim // Replace the current flag. 3903249423Sdim AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant( 3904288943Sdim Flag, dl, MVT::i32); 3905249423Sdim // Add the new register node and skip the original two GPRs. 3906249423Sdim AsmNodeOperands.push_back(PairedReg); 3907249423Sdim // Skip the next two GPRs. 3908249423Sdim i += 2; 3909249423Sdim } 3910249423Sdim } 3911249423Sdim 3912261991Sdim if (Glue.getNode()) 3913261991Sdim AsmNodeOperands.push_back(Glue); 3914249423Sdim if (!Changed) 3915276479Sdim return nullptr; 3916249423Sdim 3917261991Sdim SDValue New = CurDAG->getNode(ISD::INLINEASM, SDLoc(N), 3918276479Sdim CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands); 3919249423Sdim New->setNodeId(-1); 3920249423Sdim return New.getNode(); 3921249423Sdim} 3922249423Sdim 3923249423Sdim 3924193323Sedbool ARMDAGToDAGISel:: 3925288943SdimSelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, 3926193323Sed std::vector<SDValue> &OutOps) { 3927288943Sdim switch(ConstraintID) { 3928288943Sdim default: 3929288943Sdim llvm_unreachable("Unexpected asm memory constraint"); 3930288943Sdim case InlineAsm::Constraint_i: 3931288943Sdim // FIXME: It seems strange that 'i' is needed here since it's supposed to 3932288943Sdim // be an immediate and not a memory constraint. 3933288943Sdim // Fallthrough. 3934288943Sdim case InlineAsm::Constraint_m: 3935296417Sdim case InlineAsm::Constraint_o: 3936288943Sdim case InlineAsm::Constraint_Q: 3937288943Sdim case InlineAsm::Constraint_Um: 3938288943Sdim case InlineAsm::Constraint_Un: 3939288943Sdim case InlineAsm::Constraint_Uq: 3940288943Sdim case InlineAsm::Constraint_Us: 3941288943Sdim case InlineAsm::Constraint_Ut: 3942288943Sdim case InlineAsm::Constraint_Uv: 3943288943Sdim case InlineAsm::Constraint_Uy: 3944288943Sdim // Require the address to be in a register. That is safe for all ARM 3945288943Sdim // variants and it is hard to do anything much smarter without knowing 3946288943Sdim // how the operand is used. 3947288943Sdim OutOps.push_back(Op); 3948288943Sdim return false; 3949288943Sdim } 3950288943Sdim return true; 3951193323Sed} 3952193323Sed 3953193323Sed/// createARMISelDag - This pass converts a legalized DAG into a 3954193323Sed/// ARM-specific DAG, ready for instruction scheduling. 3955193323Sed/// 3956198090SrdivackyFunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM, 3957198090Srdivacky CodeGenOpt::Level OptLevel) { 3958198090Srdivacky return new ARMDAGToDAGISel(TM, OptLevel); 3959193323Sed} 3960