ARMISelDAGToDAG.cpp revision 280031 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARM.h"
15#include "ARMBaseInstrInfo.h"
16#include "ARMTargetMachine.h"
17#include "MCTargetDesc/ARMAddressingModes.h"
18#include "llvm/CodeGen/MachineFrameInfo.h"
19#include "llvm/CodeGen/MachineFunction.h"
20#include "llvm/CodeGen/MachineInstrBuilder.h"
21#include "llvm/CodeGen/MachineRegisterInfo.h"
22#include "llvm/CodeGen/SelectionDAG.h"
23#include "llvm/CodeGen/SelectionDAGISel.h"
24#include "llvm/IR/CallingConv.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/DerivedTypes.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/Intrinsics.h"
29#include "llvm/IR/LLVMContext.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/Compiler.h"
32#include "llvm/Support/Debug.h"
33#include "llvm/Support/ErrorHandling.h"
34#include "llvm/Target/TargetLowering.h"
35#include "llvm/Target/TargetOptions.h"
36
37using namespace llvm;
38
39#define DEBUG_TYPE "arm-isel"
40
41static cl::opt<bool>
42DisableShifterOp("disable-shifter-op", cl::Hidden,
43  cl::desc("Disable isel of shifter-op"),
44  cl::init(false));
45
46static cl::opt<bool>
47CheckVMLxHazard("check-vmlx-hazard", cl::Hidden,
48  cl::desc("Check fp vmla / vmls hazard at isel time"),
49  cl::init(true));
50
51//===--------------------------------------------------------------------===//
52/// ARMDAGToDAGISel - ARM specific code to select ARM machine
53/// instructions for SelectionDAG operations.
54///
55namespace {
56
57enum AddrMode2Type {
58  AM2_BASE, // Simple AM2 (+-imm12)
59  AM2_SHOP  // Shifter-op AM2
60};
61
62class ARMDAGToDAGISel : public SelectionDAGISel {
63  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
64  /// make the right decision when generating code for different targets.
65  const ARMSubtarget *Subtarget;
66
67public:
68  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm, CodeGenOpt::Level OptLevel)
69      : SelectionDAGISel(tm, OptLevel) {}
70
71  bool runOnMachineFunction(MachineFunction &MF) override {
72    // Reset the subtarget each time through.
73    Subtarget = &MF.getTarget().getSubtarget<ARMSubtarget>();
74    SelectionDAGISel::runOnMachineFunction(MF);
75    return true;
76  }
77
78  const char *getPassName() const override {
79    return "ARM Instruction Selection";
80  }
81
82  void PreprocessISelDAG() override;
83
84  /// getI32Imm - Return a target constant of type i32 with the specified
85  /// value.
86  inline SDValue getI32Imm(unsigned Imm) {
87    return CurDAG->getTargetConstant(Imm, MVT::i32);
88  }
89
90  SDNode *Select(SDNode *N) override;
91
92
93  bool hasNoVMLxHazardUse(SDNode *N) const;
94  bool isShifterOpProfitable(const SDValue &Shift,
95                             ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt);
96  bool SelectRegShifterOperand(SDValue N, SDValue &A,
97                               SDValue &B, SDValue &C,
98                               bool CheckProfitability = true);
99  bool SelectImmShifterOperand(SDValue N, SDValue &A,
100                               SDValue &B, bool CheckProfitability = true);
101  bool SelectShiftRegShifterOperand(SDValue N, SDValue &A,
102                                    SDValue &B, SDValue &C) {
103    // Don't apply the profitability check
104    return SelectRegShifterOperand(N, A, B, C, false);
105  }
106  bool SelectShiftImmShifterOperand(SDValue N, SDValue &A,
107                                    SDValue &B) {
108    // Don't apply the profitability check
109    return SelectImmShifterOperand(N, A, B, false);
110  }
111
112  bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
113  bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
114
115  AddrMode2Type SelectAddrMode2Worker(SDValue N, SDValue &Base,
116                                      SDValue &Offset, SDValue &Opc);
117  bool SelectAddrMode2Base(SDValue N, SDValue &Base, SDValue &Offset,
118                           SDValue &Opc) {
119    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_BASE;
120  }
121
122  bool SelectAddrMode2ShOp(SDValue N, SDValue &Base, SDValue &Offset,
123                           SDValue &Opc) {
124    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_SHOP;
125  }
126
127  bool SelectAddrMode2(SDValue N, SDValue &Base, SDValue &Offset,
128                       SDValue &Opc) {
129    SelectAddrMode2Worker(N, Base, Offset, Opc);
130//    return SelectAddrMode2ShOp(N, Base, Offset, Opc);
131    // This always matches one way or another.
132    return true;
133  }
134
135  bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
136    const ConstantSDNode *CN = cast<ConstantSDNode>(N);
137    Pred = CurDAG->getTargetConstant(CN->getZExtValue(), MVT::i32);
138    Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32);
139    return true;
140  }
141
142  bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
143                             SDValue &Offset, SDValue &Opc);
144  bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
145                             SDValue &Offset, SDValue &Opc);
146  bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
147                             SDValue &Offset, SDValue &Opc);
148  bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
149  bool SelectAddrMode3(SDValue N, SDValue &Base,
150                       SDValue &Offset, SDValue &Opc);
151  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
152                             SDValue &Offset, SDValue &Opc);
153  bool SelectAddrMode5(SDValue N, SDValue &Base,
154                       SDValue &Offset);
155  bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
156  bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
157
158  bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
159
160  // Thumb Addressing Modes:
161  bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
162  bool SelectThumbAddrModeRI(SDValue N, SDValue &Base, SDValue &Offset,
163                             unsigned Scale);
164  bool SelectThumbAddrModeRI5S1(SDValue N, SDValue &Base, SDValue &Offset);
165  bool SelectThumbAddrModeRI5S2(SDValue N, SDValue &Base, SDValue &Offset);
166  bool SelectThumbAddrModeRI5S4(SDValue N, SDValue &Base, SDValue &Offset);
167  bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
168                                SDValue &OffImm);
169  bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
170                                 SDValue &OffImm);
171  bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
172                                 SDValue &OffImm);
173  bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
174                                 SDValue &OffImm);
175  bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
176
177  // Thumb 2 Addressing Modes:
178  bool SelectT2ShifterOperandReg(SDValue N,
179                                 SDValue &BaseReg, SDValue &Opc);
180  bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
181  bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
182                            SDValue &OffImm);
183  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
184                                 SDValue &OffImm);
185  bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
186                             SDValue &OffReg, SDValue &ShImm);
187  bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm);
188
189  inline bool is_so_imm(unsigned Imm) const {
190    return ARM_AM::getSOImmVal(Imm) != -1;
191  }
192
193  inline bool is_so_imm_not(unsigned Imm) const {
194    return ARM_AM::getSOImmVal(~Imm) != -1;
195  }
196
197  inline bool is_t2_so_imm(unsigned Imm) const {
198    return ARM_AM::getT2SOImmVal(Imm) != -1;
199  }
200
201  inline bool is_t2_so_imm_not(unsigned Imm) const {
202    return ARM_AM::getT2SOImmVal(~Imm) != -1;
203  }
204
205  // Include the pieces autogenerated from the target description.
206#include "ARMGenDAGISel.inc"
207
208private:
209  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
210  /// ARM.
211  SDNode *SelectARMIndexedLoad(SDNode *N);
212  SDNode *SelectT2IndexedLoad(SDNode *N);
213
214  /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
215  /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
216  /// loads of D registers and even subregs and odd subregs of Q registers.
217  /// For NumVecs <= 2, QOpcodes1 is not used.
218  SDNode *SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
219                    const uint16_t *DOpcodes,
220                    const uint16_t *QOpcodes0, const uint16_t *QOpcodes1);
221
222  /// SelectVST - Select NEON store intrinsics.  NumVecs should
223  /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
224  /// stores of D registers and even subregs and odd subregs of Q registers.
225  /// For NumVecs <= 2, QOpcodes1 is not used.
226  SDNode *SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
227                    const uint16_t *DOpcodes,
228                    const uint16_t *QOpcodes0, const uint16_t *QOpcodes1);
229
230  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
231  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
232  /// load/store of D registers and Q registers.
233  SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad,
234                          bool isUpdating, unsigned NumVecs,
235                          const uint16_t *DOpcodes, const uint16_t *QOpcodes);
236
237  /// SelectVLDDup - Select NEON load-duplicate intrinsics.  NumVecs
238  /// should be 2, 3 or 4.  The opcode array specifies the instructions used
239  /// for loading D registers.  (Q registers are not supported.)
240  SDNode *SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs,
241                       const uint16_t *Opcodes);
242
243  /// SelectVTBL - Select NEON VTBL and VTBX intrinsics.  NumVecs should be 2,
244  /// 3 or 4.  These are custom-selected so that a REG_SEQUENCE can be
245  /// generated to force the table registers to be consecutive.
246  SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc);
247
248  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
249  SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
250
251  // Select special operations if node forms integer ABS pattern
252  SDNode *SelectABSOp(SDNode *N);
253
254  SDNode *SelectInlineAsm(SDNode *N);
255
256  SDNode *SelectConcatVector(SDNode *N);
257
258  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
259  /// inline asm expressions.
260  bool SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
261                                    std::vector<SDValue> &OutOps) override;
262
263  // Form pairs of consecutive R, S, D, or Q registers.
264  SDNode *createGPRPairNode(EVT VT, SDValue V0, SDValue V1);
265  SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1);
266  SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1);
267  SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1);
268
269  // Form sequences of 4 consecutive S, D, or Q registers.
270  SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
271  SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
272  SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
273
274  // Get the alignment operand for a NEON VLD or VST instruction.
275  SDValue GetVLDSTAlign(SDValue Align, unsigned NumVecs, bool is64BitVector);
276};
277}
278
279/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
280/// operand. If so Imm will receive the 32-bit value.
281static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
282  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
283    Imm = cast<ConstantSDNode>(N)->getZExtValue();
284    return true;
285  }
286  return false;
287}
288
289// isInt32Immediate - This method tests to see if a constant operand.
290// If so Imm will receive the 32 bit value.
291static bool isInt32Immediate(SDValue N, unsigned &Imm) {
292  return isInt32Immediate(N.getNode(), Imm);
293}
294
295// isOpcWithIntImmediate - This method tests to see if the node is a specific
296// opcode and that it has a immediate integer right operand.
297// If so Imm will receive the 32 bit value.
298static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
299  return N->getOpcode() == Opc &&
300         isInt32Immediate(N->getOperand(1).getNode(), Imm);
301}
302
303/// \brief Check whether a particular node is a constant value representable as
304/// (N * Scale) where (N in [\p RangeMin, \p RangeMax).
305///
306/// \param ScaledConstant [out] - On success, the pre-scaled constant value.
307static bool isScaledConstantInRange(SDValue Node, int Scale,
308                                    int RangeMin, int RangeMax,
309                                    int &ScaledConstant) {
310  assert(Scale > 0 && "Invalid scale!");
311
312  // Check that this is a constant.
313  const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node);
314  if (!C)
315    return false;
316
317  ScaledConstant = (int) C->getZExtValue();
318  if ((ScaledConstant % Scale) != 0)
319    return false;
320
321  ScaledConstant /= Scale;
322  return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
323}
324
325void ARMDAGToDAGISel::PreprocessISelDAG() {
326  if (!Subtarget->hasV6T2Ops())
327    return;
328
329  bool isThumb2 = Subtarget->isThumb();
330  for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
331       E = CurDAG->allnodes_end(); I != E; ) {
332    SDNode *N = I++;  // Preincrement iterator to avoid invalidation issues.
333
334    if (N->getOpcode() != ISD::ADD)
335      continue;
336
337    // Look for (add X1, (and (srl X2, c1), c2)) where c2 is constant with
338    // leading zeros, followed by consecutive set bits, followed by 1 or 2
339    // trailing zeros, e.g. 1020.
340    // Transform the expression to
341    // (add X1, (shl (and (srl X2, c1), (c2>>tz)), tz)) where tz is the number
342    // of trailing zeros of c2. The left shift would be folded as an shifter
343    // operand of 'add' and the 'and' and 'srl' would become a bits extraction
344    // node (UBFX).
345
346    SDValue N0 = N->getOperand(0);
347    SDValue N1 = N->getOperand(1);
348    unsigned And_imm = 0;
349    if (!isOpcWithIntImmediate(N1.getNode(), ISD::AND, And_imm)) {
350      if (isOpcWithIntImmediate(N0.getNode(), ISD::AND, And_imm))
351        std::swap(N0, N1);
352    }
353    if (!And_imm)
354      continue;
355
356    // Check if the AND mask is an immediate of the form: 000.....1111111100
357    unsigned TZ = countTrailingZeros(And_imm);
358    if (TZ != 1 && TZ != 2)
359      // Be conservative here. Shifter operands aren't always free. e.g. On
360      // Swift, left shifter operand of 1 / 2 for free but others are not.
361      // e.g.
362      //  ubfx   r3, r1, #16, #8
363      //  ldr.w  r3, [r0, r3, lsl #2]
364      // vs.
365      //  mov.w  r9, #1020
366      //  and.w  r2, r9, r1, lsr #14
367      //  ldr    r2, [r0, r2]
368      continue;
369    And_imm >>= TZ;
370    if (And_imm & (And_imm + 1))
371      continue;
372
373    // Look for (and (srl X, c1), c2).
374    SDValue Srl = N1.getOperand(0);
375    unsigned Srl_imm = 0;
376    if (!isOpcWithIntImmediate(Srl.getNode(), ISD::SRL, Srl_imm) ||
377        (Srl_imm <= 2))
378      continue;
379
380    // Make sure first operand is not a shifter operand which would prevent
381    // folding of the left shift.
382    SDValue CPTmp0;
383    SDValue CPTmp1;
384    SDValue CPTmp2;
385    if (isThumb2) {
386      if (SelectT2ShifterOperandReg(N0, CPTmp0, CPTmp1))
387        continue;
388    } else {
389      if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) ||
390          SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2))
391        continue;
392    }
393
394    // Now make the transformation.
395    Srl = CurDAG->getNode(ISD::SRL, SDLoc(Srl), MVT::i32,
396                          Srl.getOperand(0),
397                          CurDAG->getConstant(Srl_imm+TZ, MVT::i32));
398    N1 = CurDAG->getNode(ISD::AND, SDLoc(N1), MVT::i32,
399                         Srl, CurDAG->getConstant(And_imm, MVT::i32));
400    N1 = CurDAG->getNode(ISD::SHL, SDLoc(N1), MVT::i32,
401                         N1, CurDAG->getConstant(TZ, MVT::i32));
402    CurDAG->UpdateNodeOperands(N, N0, N1);
403  }
404}
405
406/// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS
407/// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at
408/// least on current ARM implementations) which should be avoidded.
409bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
410  if (OptLevel == CodeGenOpt::None)
411    return true;
412
413  if (!CheckVMLxHazard)
414    return true;
415
416  if (!Subtarget->isCortexA7() && !Subtarget->isCortexA8() &&
417      !Subtarget->isCortexA9() && !Subtarget->isSwift())
418    return true;
419
420  if (!N->hasOneUse())
421    return false;
422
423  SDNode *Use = *N->use_begin();
424  if (Use->getOpcode() == ISD::CopyToReg)
425    return true;
426  if (Use->isMachineOpcode()) {
427    const ARMBaseInstrInfo *TII = static_cast<const ARMBaseInstrInfo *>(
428        CurDAG->getSubtarget().getInstrInfo());
429
430    const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode());
431    if (MCID.mayStore())
432      return true;
433    unsigned Opcode = MCID.getOpcode();
434    if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
435      return true;
436    // vmlx feeding into another vmlx. We actually want to unfold
437    // the use later in the MLxExpansion pass. e.g.
438    // vmla
439    // vmla (stall 8 cycles)
440    //
441    // vmul (5 cycles)
442    // vadd (5 cycles)
443    // vmla
444    // This adds up to about 18 - 19 cycles.
445    //
446    // vmla
447    // vmul (stall 4 cycles)
448    // vadd adds up to about 14 cycles.
449    return TII->isFpMLxInstruction(Opcode);
450  }
451
452  return false;
453}
454
455bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
456                                            ARM_AM::ShiftOpc ShOpcVal,
457                                            unsigned ShAmt) {
458  if (!Subtarget->isLikeA9() && !Subtarget->isSwift())
459    return true;
460  if (Shift.hasOneUse())
461    return true;
462  // R << 2 is free.
463  return ShOpcVal == ARM_AM::lsl &&
464         (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1));
465}
466
467bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
468                                              SDValue &BaseReg,
469                                              SDValue &Opc,
470                                              bool CheckProfitability) {
471  if (DisableShifterOp)
472    return false;
473
474  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
475
476  // Don't match base register only case. That is matched to a separate
477  // lower complexity pattern with explicit register operand.
478  if (ShOpcVal == ARM_AM::no_shift) return false;
479
480  BaseReg = N.getOperand(0);
481  unsigned ShImmVal = 0;
482  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
483  if (!RHS) return false;
484  ShImmVal = RHS->getZExtValue() & 31;
485  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
486                                  MVT::i32);
487  return true;
488}
489
490bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
491                                              SDValue &BaseReg,
492                                              SDValue &ShReg,
493                                              SDValue &Opc,
494                                              bool CheckProfitability) {
495  if (DisableShifterOp)
496    return false;
497
498  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
499
500  // Don't match base register only case. That is matched to a separate
501  // lower complexity pattern with explicit register operand.
502  if (ShOpcVal == ARM_AM::no_shift) return false;
503
504  BaseReg = N.getOperand(0);
505  unsigned ShImmVal = 0;
506  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
507  if (RHS) return false;
508
509  ShReg = N.getOperand(1);
510  if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
511    return false;
512  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
513                                  MVT::i32);
514  return true;
515}
516
517
518bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
519                                          SDValue &Base,
520                                          SDValue &OffImm) {
521  // Match simple R + imm12 operands.
522
523  // Base only.
524  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
525      !CurDAG->isBaseWithConstantOffset(N)) {
526    if (N.getOpcode() == ISD::FrameIndex) {
527      // Match frame index.
528      int FI = cast<FrameIndexSDNode>(N)->getIndex();
529      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
530      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
531      return true;
532    }
533
534    if (N.getOpcode() == ARMISD::Wrapper &&
535        N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
536      Base = N.getOperand(0);
537    } else
538      Base = N;
539    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
540    return true;
541  }
542
543  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
544    int RHSC = (int)RHS->getSExtValue();
545    if (N.getOpcode() == ISD::SUB)
546      RHSC = -RHSC;
547
548    if (RHSC > -0x1000 && RHSC < 0x1000) { // 12 bits
549      Base   = N.getOperand(0);
550      if (Base.getOpcode() == ISD::FrameIndex) {
551        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
552        Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
553      }
554      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
555      return true;
556    }
557  }
558
559  // Base only.
560  Base = N;
561  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
562  return true;
563}
564
565
566
567bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
568                                      SDValue &Opc) {
569  if (N.getOpcode() == ISD::MUL &&
570      ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) {
571    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
572      // X * [3,5,9] -> X + X * [2,4,8] etc.
573      int RHSC = (int)RHS->getZExtValue();
574      if (RHSC & 1) {
575        RHSC = RHSC & ~1;
576        ARM_AM::AddrOpc AddSub = ARM_AM::add;
577        if (RHSC < 0) {
578          AddSub = ARM_AM::sub;
579          RHSC = - RHSC;
580        }
581        if (isPowerOf2_32(RHSC)) {
582          unsigned ShAmt = Log2_32(RHSC);
583          Base = Offset = N.getOperand(0);
584          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
585                                                            ARM_AM::lsl),
586                                          MVT::i32);
587          return true;
588        }
589      }
590    }
591  }
592
593  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
594      // ISD::OR that is equivalent to an ISD::ADD.
595      !CurDAG->isBaseWithConstantOffset(N))
596    return false;
597
598  // Leave simple R +/- imm12 operands for LDRi12
599  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) {
600    int RHSC;
601    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
602                                -0x1000+1, 0x1000, RHSC)) // 12 bits.
603      return false;
604  }
605
606  // Otherwise this is R +/- [possibly shifted] R.
607  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add;
608  ARM_AM::ShiftOpc ShOpcVal =
609    ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
610  unsigned ShAmt = 0;
611
612  Base   = N.getOperand(0);
613  Offset = N.getOperand(1);
614
615  if (ShOpcVal != ARM_AM::no_shift) {
616    // Check to see if the RHS of the shift is a constant, if not, we can't fold
617    // it.
618    if (ConstantSDNode *Sh =
619           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
620      ShAmt = Sh->getZExtValue();
621      if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
622        Offset = N.getOperand(1).getOperand(0);
623      else {
624        ShAmt = 0;
625        ShOpcVal = ARM_AM::no_shift;
626      }
627    } else {
628      ShOpcVal = ARM_AM::no_shift;
629    }
630  }
631
632  // Try matching (R shl C) + (R).
633  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
634      !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
635        N.getOperand(0).hasOneUse())) {
636    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
637    if (ShOpcVal != ARM_AM::no_shift) {
638      // Check to see if the RHS of the shift is a constant, if not, we can't
639      // fold it.
640      if (ConstantSDNode *Sh =
641          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
642        ShAmt = Sh->getZExtValue();
643        if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
644          Offset = N.getOperand(0).getOperand(0);
645          Base = N.getOperand(1);
646        } else {
647          ShAmt = 0;
648          ShOpcVal = ARM_AM::no_shift;
649        }
650      } else {
651        ShOpcVal = ARM_AM::no_shift;
652      }
653    }
654  }
655
656  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
657                                  MVT::i32);
658  return true;
659}
660
661
662//-----
663
664AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N,
665                                                     SDValue &Base,
666                                                     SDValue &Offset,
667                                                     SDValue &Opc) {
668  if (N.getOpcode() == ISD::MUL &&
669      (!(Subtarget->isLikeA9() || Subtarget->isSwift()) || N.hasOneUse())) {
670    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
671      // X * [3,5,9] -> X + X * [2,4,8] etc.
672      int RHSC = (int)RHS->getZExtValue();
673      if (RHSC & 1) {
674        RHSC = RHSC & ~1;
675        ARM_AM::AddrOpc AddSub = ARM_AM::add;
676        if (RHSC < 0) {
677          AddSub = ARM_AM::sub;
678          RHSC = - RHSC;
679        }
680        if (isPowerOf2_32(RHSC)) {
681          unsigned ShAmt = Log2_32(RHSC);
682          Base = Offset = N.getOperand(0);
683          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
684                                                            ARM_AM::lsl),
685                                          MVT::i32);
686          return AM2_SHOP;
687        }
688      }
689    }
690  }
691
692  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
693      // ISD::OR that is equivalent to an ADD.
694      !CurDAG->isBaseWithConstantOffset(N)) {
695    Base = N;
696    if (N.getOpcode() == ISD::FrameIndex) {
697      int FI = cast<FrameIndexSDNode>(N)->getIndex();
698      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
699    } else if (N.getOpcode() == ARMISD::Wrapper &&
700               N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
701      Base = N.getOperand(0);
702    }
703    Offset = CurDAG->getRegister(0, MVT::i32);
704    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
705                                                      ARM_AM::no_shift),
706                                    MVT::i32);
707    return AM2_BASE;
708  }
709
710  // Match simple R +/- imm12 operands.
711  if (N.getOpcode() != ISD::SUB) {
712    int RHSC;
713    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
714                                -0x1000+1, 0x1000, RHSC)) { // 12 bits.
715      Base = N.getOperand(0);
716      if (Base.getOpcode() == ISD::FrameIndex) {
717        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
718        Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
719      }
720      Offset = CurDAG->getRegister(0, MVT::i32);
721
722      ARM_AM::AddrOpc AddSub = ARM_AM::add;
723      if (RHSC < 0) {
724        AddSub = ARM_AM::sub;
725        RHSC = - RHSC;
726      }
727      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
728                                                        ARM_AM::no_shift),
729                                      MVT::i32);
730      return AM2_BASE;
731    }
732  }
733
734  if ((Subtarget->isLikeA9() || Subtarget->isSwift()) && !N.hasOneUse()) {
735    // Compute R +/- (R << N) and reuse it.
736    Base = N;
737    Offset = CurDAG->getRegister(0, MVT::i32);
738    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
739                                                      ARM_AM::no_shift),
740                                    MVT::i32);
741    return AM2_BASE;
742  }
743
744  // Otherwise this is R +/- [possibly shifted] R.
745  ARM_AM::AddrOpc AddSub = N.getOpcode() != ISD::SUB ? ARM_AM::add:ARM_AM::sub;
746  ARM_AM::ShiftOpc ShOpcVal =
747    ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
748  unsigned ShAmt = 0;
749
750  Base   = N.getOperand(0);
751  Offset = N.getOperand(1);
752
753  if (ShOpcVal != ARM_AM::no_shift) {
754    // Check to see if the RHS of the shift is a constant, if not, we can't fold
755    // it.
756    if (ConstantSDNode *Sh =
757           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
758      ShAmt = Sh->getZExtValue();
759      if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
760        Offset = N.getOperand(1).getOperand(0);
761      else {
762        ShAmt = 0;
763        ShOpcVal = ARM_AM::no_shift;
764      }
765    } else {
766      ShOpcVal = ARM_AM::no_shift;
767    }
768  }
769
770  // Try matching (R shl C) + (R).
771  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
772      !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
773        N.getOperand(0).hasOneUse())) {
774    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
775    if (ShOpcVal != ARM_AM::no_shift) {
776      // Check to see if the RHS of the shift is a constant, if not, we can't
777      // fold it.
778      if (ConstantSDNode *Sh =
779          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
780        ShAmt = Sh->getZExtValue();
781        if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
782          Offset = N.getOperand(0).getOperand(0);
783          Base = N.getOperand(1);
784        } else {
785          ShAmt = 0;
786          ShOpcVal = ARM_AM::no_shift;
787        }
788      } else {
789        ShOpcVal = ARM_AM::no_shift;
790      }
791    }
792  }
793
794  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
795                                  MVT::i32);
796  return AM2_SHOP;
797}
798
799bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
800                                            SDValue &Offset, SDValue &Opc) {
801  unsigned Opcode = Op->getOpcode();
802  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
803    ? cast<LoadSDNode>(Op)->getAddressingMode()
804    : cast<StoreSDNode>(Op)->getAddressingMode();
805  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
806    ? ARM_AM::add : ARM_AM::sub;
807  int Val;
808  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val))
809    return false;
810
811  Offset = N;
812  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
813  unsigned ShAmt = 0;
814  if (ShOpcVal != ARM_AM::no_shift) {
815    // Check to see if the RHS of the shift is a constant, if not, we can't fold
816    // it.
817    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
818      ShAmt = Sh->getZExtValue();
819      if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
820        Offset = N.getOperand(0);
821      else {
822        ShAmt = 0;
823        ShOpcVal = ARM_AM::no_shift;
824      }
825    } else {
826      ShOpcVal = ARM_AM::no_shift;
827    }
828  }
829
830  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
831                                  MVT::i32);
832  return true;
833}
834
835bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
836                                            SDValue &Offset, SDValue &Opc) {
837  unsigned Opcode = Op->getOpcode();
838  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
839    ? cast<LoadSDNode>(Op)->getAddressingMode()
840    : cast<StoreSDNode>(Op)->getAddressingMode();
841  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
842    ? ARM_AM::add : ARM_AM::sub;
843  int Val;
844  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
845    if (AddSub == ARM_AM::sub) Val *= -1;
846    Offset = CurDAG->getRegister(0, MVT::i32);
847    Opc = CurDAG->getTargetConstant(Val, MVT::i32);
848    return true;
849  }
850
851  return false;
852}
853
854
855bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
856                                            SDValue &Offset, SDValue &Opc) {
857  unsigned Opcode = Op->getOpcode();
858  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
859    ? cast<LoadSDNode>(Op)->getAddressingMode()
860    : cast<StoreSDNode>(Op)->getAddressingMode();
861  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
862    ? ARM_AM::add : ARM_AM::sub;
863  int Val;
864  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
865    Offset = CurDAG->getRegister(0, MVT::i32);
866    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
867                                                      ARM_AM::no_shift),
868                                    MVT::i32);
869    return true;
870  }
871
872  return false;
873}
874
875bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
876  Base = N;
877  return true;
878}
879
880bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
881                                      SDValue &Base, SDValue &Offset,
882                                      SDValue &Opc) {
883  if (N.getOpcode() == ISD::SUB) {
884    // X - C  is canonicalize to X + -C, no need to handle it here.
885    Base = N.getOperand(0);
886    Offset = N.getOperand(1);
887    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
888    return true;
889  }
890
891  if (!CurDAG->isBaseWithConstantOffset(N)) {
892    Base = N;
893    if (N.getOpcode() == ISD::FrameIndex) {
894      int FI = cast<FrameIndexSDNode>(N)->getIndex();
895      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
896    }
897    Offset = CurDAG->getRegister(0, MVT::i32);
898    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
899    return true;
900  }
901
902  // If the RHS is +/- imm8, fold into addr mode.
903  int RHSC;
904  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
905                              -256 + 1, 256, RHSC)) { // 8 bits.
906    Base = N.getOperand(0);
907    if (Base.getOpcode() == ISD::FrameIndex) {
908      int FI = cast<FrameIndexSDNode>(Base)->getIndex();
909      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
910    }
911    Offset = CurDAG->getRegister(0, MVT::i32);
912
913    ARM_AM::AddrOpc AddSub = ARM_AM::add;
914    if (RHSC < 0) {
915      AddSub = ARM_AM::sub;
916      RHSC = -RHSC;
917    }
918    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
919    return true;
920  }
921
922  Base = N.getOperand(0);
923  Offset = N.getOperand(1);
924  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
925  return true;
926}
927
928bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
929                                            SDValue &Offset, SDValue &Opc) {
930  unsigned Opcode = Op->getOpcode();
931  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
932    ? cast<LoadSDNode>(Op)->getAddressingMode()
933    : cast<StoreSDNode>(Op)->getAddressingMode();
934  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
935    ? ARM_AM::add : ARM_AM::sub;
936  int Val;
937  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits.
938    Offset = CurDAG->getRegister(0, MVT::i32);
939    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
940    return true;
941  }
942
943  Offset = N;
944  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
945  return true;
946}
947
948bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
949                                      SDValue &Base, SDValue &Offset) {
950  if (!CurDAG->isBaseWithConstantOffset(N)) {
951    Base = N;
952    if (N.getOpcode() == ISD::FrameIndex) {
953      int FI = cast<FrameIndexSDNode>(N)->getIndex();
954      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
955    } else if (N.getOpcode() == ARMISD::Wrapper &&
956               N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
957      Base = N.getOperand(0);
958    }
959    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
960                                       MVT::i32);
961    return true;
962  }
963
964  // If the RHS is +/- imm8, fold into addr mode.
965  int RHSC;
966  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4,
967                              -256 + 1, 256, RHSC)) {
968    Base = N.getOperand(0);
969    if (Base.getOpcode() == ISD::FrameIndex) {
970      int FI = cast<FrameIndexSDNode>(Base)->getIndex();
971      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
972    }
973
974    ARM_AM::AddrOpc AddSub = ARM_AM::add;
975    if (RHSC < 0) {
976      AddSub = ARM_AM::sub;
977      RHSC = -RHSC;
978    }
979    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
980                                       MVT::i32);
981    return true;
982  }
983
984  Base = N;
985  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
986                                     MVT::i32);
987  return true;
988}
989
990bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
991                                      SDValue &Align) {
992  Addr = N;
993
994  unsigned Alignment = 0;
995  if (LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(Parent)) {
996    // This case occurs only for VLD1-lane/dup and VST1-lane instructions.
997    // The maximum alignment is equal to the memory size being referenced.
998    unsigned LSNAlign = LSN->getAlignment();
999    unsigned MemSize = LSN->getMemoryVT().getSizeInBits() / 8;
1000    if (LSNAlign >= MemSize && MemSize > 1)
1001      Alignment = MemSize;
1002  } else {
1003    // All other uses of addrmode6 are for intrinsics.  For now just record
1004    // the raw alignment value; it will be refined later based on the legal
1005    // alignment operands for the intrinsic.
1006    Alignment = cast<MemIntrinsicSDNode>(Parent)->getAlignment();
1007  }
1008
1009  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
1010  return true;
1011}
1012
1013bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N,
1014                                            SDValue &Offset) {
1015  LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op);
1016  ISD::MemIndexedMode AM = LdSt->getAddressingMode();
1017  if (AM != ISD::POST_INC)
1018    return false;
1019  Offset = N;
1020  if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) {
1021    if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits())
1022      Offset = CurDAG->getRegister(0, MVT::i32);
1023  }
1024  return true;
1025}
1026
1027bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
1028                                       SDValue &Offset, SDValue &Label) {
1029  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
1030    Offset = N.getOperand(0);
1031    SDValue N1 = N.getOperand(1);
1032    Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
1033                                      MVT::i32);
1034    return true;
1035  }
1036
1037  return false;
1038}
1039
1040
1041//===----------------------------------------------------------------------===//
1042//                         Thumb Addressing Modes
1043//===----------------------------------------------------------------------===//
1044
1045bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N,
1046                                            SDValue &Base, SDValue &Offset){
1047  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) {
1048    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
1049    if (!NC || !NC->isNullValue())
1050      return false;
1051
1052    Base = Offset = N;
1053    return true;
1054  }
1055
1056  Base = N.getOperand(0);
1057  Offset = N.getOperand(1);
1058  return true;
1059}
1060
1061bool
1062ARMDAGToDAGISel::SelectThumbAddrModeRI(SDValue N, SDValue &Base,
1063                                       SDValue &Offset, unsigned Scale) {
1064  if (Scale == 4) {
1065    SDValue TmpBase, TmpOffImm;
1066    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
1067      return false;  // We want to select tLDRspi / tSTRspi instead.
1068
1069    if (N.getOpcode() == ARMISD::Wrapper &&
1070        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
1071      return false;  // We want to select tLDRpci instead.
1072  }
1073
1074  if (!CurDAG->isBaseWithConstantOffset(N))
1075    return false;
1076
1077  // Thumb does not have [sp, r] address mode.
1078  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1079  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
1080  if ((LHSR && LHSR->getReg() == ARM::SP) ||
1081      (RHSR && RHSR->getReg() == ARM::SP))
1082    return false;
1083
1084  // FIXME: Why do we explicitly check for a match here and then return false?
1085  // Presumably to allow something else to match, but shouldn't this be
1086  // documented?
1087  int RHSC;
1088  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC))
1089    return false;
1090
1091  Base = N.getOperand(0);
1092  Offset = N.getOperand(1);
1093  return true;
1094}
1095
1096bool
1097ARMDAGToDAGISel::SelectThumbAddrModeRI5S1(SDValue N,
1098                                          SDValue &Base,
1099                                          SDValue &Offset) {
1100  return SelectThumbAddrModeRI(N, Base, Offset, 1);
1101}
1102
1103bool
1104ARMDAGToDAGISel::SelectThumbAddrModeRI5S2(SDValue N,
1105                                          SDValue &Base,
1106                                          SDValue &Offset) {
1107  return SelectThumbAddrModeRI(N, Base, Offset, 2);
1108}
1109
1110bool
1111ARMDAGToDAGISel::SelectThumbAddrModeRI5S4(SDValue N,
1112                                          SDValue &Base,
1113                                          SDValue &Offset) {
1114  return SelectThumbAddrModeRI(N, Base, Offset, 4);
1115}
1116
1117bool
1118ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
1119                                          SDValue &Base, SDValue &OffImm) {
1120  if (Scale == 4) {
1121    SDValue TmpBase, TmpOffImm;
1122    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
1123      return false;  // We want to select tLDRspi / tSTRspi instead.
1124
1125    if (N.getOpcode() == ARMISD::Wrapper &&
1126        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
1127      return false;  // We want to select tLDRpci instead.
1128  }
1129
1130  if (!CurDAG->isBaseWithConstantOffset(N)) {
1131    if (N.getOpcode() == ARMISD::Wrapper &&
1132        N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
1133      Base = N.getOperand(0);
1134    } else {
1135      Base = N;
1136    }
1137
1138    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1139    return true;
1140  }
1141
1142  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1143  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
1144  if ((LHSR && LHSR->getReg() == ARM::SP) ||
1145      (RHSR && RHSR->getReg() == ARM::SP)) {
1146    ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(N.getOperand(0));
1147    ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
1148    unsigned LHSC = LHS ? LHS->getZExtValue() : 0;
1149    unsigned RHSC = RHS ? RHS->getZExtValue() : 0;
1150
1151    // Thumb does not have [sp, #imm5] address mode for non-zero imm5.
1152    if (LHSC != 0 || RHSC != 0) return false;
1153
1154    Base = N;
1155    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1156    return true;
1157  }
1158
1159  // If the RHS is + imm5 * scale, fold into addr mode.
1160  int RHSC;
1161  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
1162    Base = N.getOperand(0);
1163    OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1164    return true;
1165  }
1166
1167  Base = N.getOperand(0);
1168  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1169  return true;
1170}
1171
1172bool
1173ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
1174                                           SDValue &OffImm) {
1175  return SelectThumbAddrModeImm5S(N, 4, Base, OffImm);
1176}
1177
1178bool
1179ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
1180                                           SDValue &OffImm) {
1181  return SelectThumbAddrModeImm5S(N, 2, Base, OffImm);
1182}
1183
1184bool
1185ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
1186                                           SDValue &OffImm) {
1187  return SelectThumbAddrModeImm5S(N, 1, Base, OffImm);
1188}
1189
1190bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
1191                                            SDValue &Base, SDValue &OffImm) {
1192  if (N.getOpcode() == ISD::FrameIndex) {
1193    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1194    Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
1195    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1196    return true;
1197  }
1198
1199  if (!CurDAG->isBaseWithConstantOffset(N))
1200    return false;
1201
1202  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1203  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
1204      (LHSR && LHSR->getReg() == ARM::SP)) {
1205    // If the RHS is + imm8 * scale, fold into addr mode.
1206    int RHSC;
1207    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) {
1208      Base = N.getOperand(0);
1209      if (Base.getOpcode() == ISD::FrameIndex) {
1210        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1211        Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
1212      }
1213      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1214      return true;
1215    }
1216  }
1217
1218  return false;
1219}
1220
1221
1222//===----------------------------------------------------------------------===//
1223//                        Thumb 2 Addressing Modes
1224//===----------------------------------------------------------------------===//
1225
1226
1227bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg,
1228                                                SDValue &Opc) {
1229  if (DisableShifterOp)
1230    return false;
1231
1232  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
1233
1234  // Don't match base register only case. That is matched to a separate
1235  // lower complexity pattern with explicit register operand.
1236  if (ShOpcVal == ARM_AM::no_shift) return false;
1237
1238  BaseReg = N.getOperand(0);
1239  unsigned ShImmVal = 0;
1240  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1241    ShImmVal = RHS->getZExtValue() & 31;
1242    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
1243    return true;
1244  }
1245
1246  return false;
1247}
1248
1249bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
1250                                            SDValue &Base, SDValue &OffImm) {
1251  // Match simple R + imm12 operands.
1252
1253  // Base only.
1254  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1255      !CurDAG->isBaseWithConstantOffset(N)) {
1256    if (N.getOpcode() == ISD::FrameIndex) {
1257      // Match frame index.
1258      int FI = cast<FrameIndexSDNode>(N)->getIndex();
1259      Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
1260      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1261      return true;
1262    }
1263
1264    if (N.getOpcode() == ARMISD::Wrapper &&
1265        N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) {
1266      Base = N.getOperand(0);
1267      if (Base.getOpcode() == ISD::TargetConstantPool)
1268        return false;  // We want to select t2LDRpci instead.
1269    } else
1270      Base = N;
1271    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1272    return true;
1273  }
1274
1275  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1276    if (SelectT2AddrModeImm8(N, Base, OffImm))
1277      // Let t2LDRi8 handle (R - imm8).
1278      return false;
1279
1280    int RHSC = (int)RHS->getZExtValue();
1281    if (N.getOpcode() == ISD::SUB)
1282      RHSC = -RHSC;
1283
1284    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
1285      Base   = N.getOperand(0);
1286      if (Base.getOpcode() == ISD::FrameIndex) {
1287        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1288        Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
1289      }
1290      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1291      return true;
1292    }
1293  }
1294
1295  // Base only.
1296  Base = N;
1297  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1298  return true;
1299}
1300
1301bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
1302                                           SDValue &Base, SDValue &OffImm) {
1303  // Match simple R - imm8 operands.
1304  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1305      !CurDAG->isBaseWithConstantOffset(N))
1306    return false;
1307
1308  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1309    int RHSC = (int)RHS->getSExtValue();
1310    if (N.getOpcode() == ISD::SUB)
1311      RHSC = -RHSC;
1312
1313    if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
1314      Base = N.getOperand(0);
1315      if (Base.getOpcode() == ISD::FrameIndex) {
1316        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1317        Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
1318      }
1319      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1320      return true;
1321    }
1322  }
1323
1324  return false;
1325}
1326
1327bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
1328                                                 SDValue &OffImm){
1329  unsigned Opcode = Op->getOpcode();
1330  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
1331    ? cast<LoadSDNode>(Op)->getAddressingMode()
1332    : cast<StoreSDNode>(Op)->getAddressingMode();
1333  int RHSC;
1334  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
1335    OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
1336      ? CurDAG->getTargetConstant(RHSC, MVT::i32)
1337      : CurDAG->getTargetConstant(-RHSC, MVT::i32);
1338    return true;
1339  }
1340
1341  return false;
1342}
1343
1344bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
1345                                            SDValue &Base,
1346                                            SDValue &OffReg, SDValue &ShImm) {
1347  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
1348  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N))
1349    return false;
1350
1351  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
1352  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1353    int RHSC = (int)RHS->getZExtValue();
1354    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
1355      return false;
1356    else if (RHSC < 0 && RHSC >= -255) // 8 bits
1357      return false;
1358  }
1359
1360  // Look for (R + R) or (R + (R << [1,2,3])).
1361  unsigned ShAmt = 0;
1362  Base   = N.getOperand(0);
1363  OffReg = N.getOperand(1);
1364
1365  // Swap if it is ((R << c) + R).
1366  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode());
1367  if (ShOpcVal != ARM_AM::lsl) {
1368    ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode());
1369    if (ShOpcVal == ARM_AM::lsl)
1370      std::swap(Base, OffReg);
1371  }
1372
1373  if (ShOpcVal == ARM_AM::lsl) {
1374    // Check to see if the RHS of the shift is a constant, if not, we can't fold
1375    // it.
1376    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
1377      ShAmt = Sh->getZExtValue();
1378      if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1379        OffReg = OffReg.getOperand(0);
1380      else {
1381        ShAmt = 0;
1382      }
1383    }
1384  }
1385
1386  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
1387
1388  return true;
1389}
1390
1391bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base,
1392                                                SDValue &OffImm) {
1393  // This *must* succeed since it's used for the irreplaceable ldrex and strex
1394  // instructions.
1395  Base = N;
1396  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1397
1398  if (N.getOpcode() != ISD::ADD || !CurDAG->isBaseWithConstantOffset(N))
1399    return true;
1400
1401  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
1402  if (!RHS)
1403    return true;
1404
1405  uint32_t RHSC = (int)RHS->getZExtValue();
1406  if (RHSC > 1020 || RHSC % 4 != 0)
1407    return true;
1408
1409  Base = N.getOperand(0);
1410  if (Base.getOpcode() == ISD::FrameIndex) {
1411    int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1412    Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
1413  }
1414
1415  OffImm = CurDAG->getTargetConstant(RHSC / 4, MVT::i32);
1416  return true;
1417}
1418
1419//===--------------------------------------------------------------------===//
1420
1421/// getAL - Returns a ARMCC::AL immediate node.
1422static inline SDValue getAL(SelectionDAG *CurDAG) {
1423  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
1424}
1425
1426SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
1427  LoadSDNode *LD = cast<LoadSDNode>(N);
1428  ISD::MemIndexedMode AM = LD->getAddressingMode();
1429  if (AM == ISD::UNINDEXED)
1430    return nullptr;
1431
1432  EVT LoadedVT = LD->getMemoryVT();
1433  SDValue Offset, AMOpc;
1434  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1435  unsigned Opcode = 0;
1436  bool Match = false;
1437  if (LoadedVT == MVT::i32 && isPre &&
1438      SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1439    Opcode = ARM::LDR_PRE_IMM;
1440    Match = true;
1441  } else if (LoadedVT == MVT::i32 && !isPre &&
1442      SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1443    Opcode = ARM::LDR_POST_IMM;
1444    Match = true;
1445  } else if (LoadedVT == MVT::i32 &&
1446      SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1447    Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1448    Match = true;
1449
1450  } else if (LoadedVT == MVT::i16 &&
1451             SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1452    Match = true;
1453    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
1454      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1455      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1456  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1457    if (LD->getExtensionType() == ISD::SEXTLOAD) {
1458      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1459        Match = true;
1460        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1461      }
1462    } else {
1463      if (isPre &&
1464          SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1465        Match = true;
1466        Opcode = ARM::LDRB_PRE_IMM;
1467      } else if (!isPre &&
1468                  SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1469        Match = true;
1470        Opcode = ARM::LDRB_POST_IMM;
1471      } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1472        Match = true;
1473        Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1474      }
1475    }
1476  }
1477
1478  if (Match) {
1479    if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1480      SDValue Chain = LD->getChain();
1481      SDValue Base = LD->getBasePtr();
1482      SDValue Ops[]= { Base, AMOpc, getAL(CurDAG),
1483                       CurDAG->getRegister(0, MVT::i32), Chain };
1484      return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32,
1485                                    MVT::i32, MVT::Other, Ops);
1486    } else {
1487      SDValue Chain = LD->getChain();
1488      SDValue Base = LD->getBasePtr();
1489      SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
1490                       CurDAG->getRegister(0, MVT::i32), Chain };
1491      return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32,
1492                                    MVT::i32, MVT::Other, Ops);
1493    }
1494  }
1495
1496  return nullptr;
1497}
1498
1499SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
1500  LoadSDNode *LD = cast<LoadSDNode>(N);
1501  ISD::MemIndexedMode AM = LD->getAddressingMode();
1502  if (AM == ISD::UNINDEXED)
1503    return nullptr;
1504
1505  EVT LoadedVT = LD->getMemoryVT();
1506  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1507  SDValue Offset;
1508  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1509  unsigned Opcode = 0;
1510  bool Match = false;
1511  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
1512    switch (LoadedVT.getSimpleVT().SimpleTy) {
1513    case MVT::i32:
1514      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1515      break;
1516    case MVT::i16:
1517      if (isSExtLd)
1518        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1519      else
1520        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1521      break;
1522    case MVT::i8:
1523    case MVT::i1:
1524      if (isSExtLd)
1525        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1526      else
1527        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1528      break;
1529    default:
1530      return nullptr;
1531    }
1532    Match = true;
1533  }
1534
1535  if (Match) {
1536    SDValue Chain = LD->getChain();
1537    SDValue Base = LD->getBasePtr();
1538    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
1539                     CurDAG->getRegister(0, MVT::i32), Chain };
1540    return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1541                                  MVT::Other, Ops);
1542  }
1543
1544  return nullptr;
1545}
1546
1547/// \brief Form a GPRPair pseudo register from a pair of GPR regs.
1548SDNode *ARMDAGToDAGISel::createGPRPairNode(EVT VT, SDValue V0, SDValue V1) {
1549  SDLoc dl(V0.getNode());
1550  SDValue RegClass =
1551    CurDAG->getTargetConstant(ARM::GPRPairRegClassID, MVT::i32);
1552  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, MVT::i32);
1553  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, MVT::i32);
1554  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1555  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1556}
1557
1558/// \brief Form a D register from a pair of S registers.
1559SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1560  SDLoc dl(V0.getNode());
1561  SDValue RegClass =
1562    CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, MVT::i32);
1563  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1564  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1565  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1566  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1567}
1568
1569/// \brief Form a quad register from a pair of D registers.
1570SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1571  SDLoc dl(V0.getNode());
1572  SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, MVT::i32);
1573  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1574  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1575  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1576  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1577}
1578
1579/// \brief Form 4 consecutive D registers from a pair of Q registers.
1580SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1581  SDLoc dl(V0.getNode());
1582  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
1583  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1584  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1585  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1586  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1587}
1588
1589/// \brief Form 4 consecutive S registers.
1590SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1,
1591                                   SDValue V2, SDValue V3) {
1592  SDLoc dl(V0.getNode());
1593  SDValue RegClass =
1594    CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, MVT::i32);
1595  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1596  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1597  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32);
1598  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32);
1599  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1600                                    V2, SubReg2, V3, SubReg3 };
1601  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1602}
1603
1604/// \brief Form 4 consecutive D registers.
1605SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1,
1606                                   SDValue V2, SDValue V3) {
1607  SDLoc dl(V0.getNode());
1608  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
1609  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1610  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1611  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32);
1612  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32);
1613  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1614                                    V2, SubReg2, V3, SubReg3 };
1615  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1616}
1617
1618/// \brief Form 4 consecutive Q registers.
1619SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1,
1620                                   SDValue V2, SDValue V3) {
1621  SDLoc dl(V0.getNode());
1622  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, MVT::i32);
1623  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1624  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1625  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32);
1626  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32);
1627  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1628                                    V2, SubReg2, V3, SubReg3 };
1629  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1630}
1631
1632/// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
1633/// of a NEON VLD or VST instruction.  The supported values depend on the
1634/// number of registers being loaded.
1635SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, unsigned NumVecs,
1636                                       bool is64BitVector) {
1637  unsigned NumRegs = NumVecs;
1638  if (!is64BitVector && NumVecs < 3)
1639    NumRegs *= 2;
1640
1641  unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1642  if (Alignment >= 32 && NumRegs == 4)
1643    Alignment = 32;
1644  else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1645    Alignment = 16;
1646  else if (Alignment >= 8)
1647    Alignment = 8;
1648  else
1649    Alignment = 0;
1650
1651  return CurDAG->getTargetConstant(Alignment, MVT::i32);
1652}
1653
1654static bool isVLDfixed(unsigned Opc)
1655{
1656  switch (Opc) {
1657  default: return false;
1658  case ARM::VLD1d8wb_fixed : return true;
1659  case ARM::VLD1d16wb_fixed : return true;
1660  case ARM::VLD1d64Qwb_fixed : return true;
1661  case ARM::VLD1d32wb_fixed : return true;
1662  case ARM::VLD1d64wb_fixed : return true;
1663  case ARM::VLD1d64TPseudoWB_fixed : return true;
1664  case ARM::VLD1d64QPseudoWB_fixed : return true;
1665  case ARM::VLD1q8wb_fixed : return true;
1666  case ARM::VLD1q16wb_fixed : return true;
1667  case ARM::VLD1q32wb_fixed : return true;
1668  case ARM::VLD1q64wb_fixed : return true;
1669  case ARM::VLD2d8wb_fixed : return true;
1670  case ARM::VLD2d16wb_fixed : return true;
1671  case ARM::VLD2d32wb_fixed : return true;
1672  case ARM::VLD2q8PseudoWB_fixed : return true;
1673  case ARM::VLD2q16PseudoWB_fixed : return true;
1674  case ARM::VLD2q32PseudoWB_fixed : return true;
1675  case ARM::VLD2DUPd8wb_fixed : return true;
1676  case ARM::VLD2DUPd16wb_fixed : return true;
1677  case ARM::VLD2DUPd32wb_fixed : return true;
1678  }
1679}
1680
1681static bool isVSTfixed(unsigned Opc)
1682{
1683  switch (Opc) {
1684  default: return false;
1685  case ARM::VST1d8wb_fixed : return true;
1686  case ARM::VST1d16wb_fixed : return true;
1687  case ARM::VST1d32wb_fixed : return true;
1688  case ARM::VST1d64wb_fixed : return true;
1689  case ARM::VST1q8wb_fixed : return true;
1690  case ARM::VST1q16wb_fixed : return true;
1691  case ARM::VST1q32wb_fixed : return true;
1692  case ARM::VST1q64wb_fixed : return true;
1693  case ARM::VST1d64TPseudoWB_fixed : return true;
1694  case ARM::VST1d64QPseudoWB_fixed : return true;
1695  case ARM::VST2d8wb_fixed : return true;
1696  case ARM::VST2d16wb_fixed : return true;
1697  case ARM::VST2d32wb_fixed : return true;
1698  case ARM::VST2q8PseudoWB_fixed : return true;
1699  case ARM::VST2q16PseudoWB_fixed : return true;
1700  case ARM::VST2q32PseudoWB_fixed : return true;
1701  }
1702}
1703
1704// Get the register stride update opcode of a VLD/VST instruction that
1705// is otherwise equivalent to the given fixed stride updating instruction.
1706static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
1707  assert((isVLDfixed(Opc) || isVSTfixed(Opc))
1708    && "Incorrect fixed stride updating instruction.");
1709  switch (Opc) {
1710  default: break;
1711  case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register;
1712  case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register;
1713  case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register;
1714  case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register;
1715  case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register;
1716  case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register;
1717  case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register;
1718  case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register;
1719  case ARM::VLD1d64Twb_fixed: return ARM::VLD1d64Twb_register;
1720  case ARM::VLD1d64Qwb_fixed: return ARM::VLD1d64Qwb_register;
1721  case ARM::VLD1d64TPseudoWB_fixed: return ARM::VLD1d64TPseudoWB_register;
1722  case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register;
1723
1724  case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register;
1725  case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register;
1726  case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register;
1727  case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register;
1728  case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register;
1729  case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register;
1730  case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register;
1731  case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register;
1732  case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register;
1733  case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register;
1734
1735  case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register;
1736  case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register;
1737  case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register;
1738  case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register;
1739  case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register;
1740  case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register;
1741
1742  case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register;
1743  case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register;
1744  case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register;
1745  case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register;
1746  case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register;
1747  case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register;
1748
1749  case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register;
1750  case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register;
1751  case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register;
1752  }
1753  return Opc; // If not one we handle, return it unchanged.
1754}
1755
1756SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
1757                                   const uint16_t *DOpcodes,
1758                                   const uint16_t *QOpcodes0,
1759                                   const uint16_t *QOpcodes1) {
1760  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1761  SDLoc dl(N);
1762
1763  SDValue MemAddr, Align;
1764  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1765  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1766    return nullptr;
1767
1768  SDValue Chain = N->getOperand(0);
1769  EVT VT = N->getValueType(0);
1770  bool is64BitVector = VT.is64BitVector();
1771  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1772
1773  unsigned OpcodeIndex;
1774  switch (VT.getSimpleVT().SimpleTy) {
1775  default: llvm_unreachable("unhandled vld type");
1776    // Double-register operations:
1777  case MVT::v8i8:  OpcodeIndex = 0; break;
1778  case MVT::v4i16: OpcodeIndex = 1; break;
1779  case MVT::v2f32:
1780  case MVT::v2i32: OpcodeIndex = 2; break;
1781  case MVT::v1i64: OpcodeIndex = 3; break;
1782    // Quad-register operations:
1783  case MVT::v16i8: OpcodeIndex = 0; break;
1784  case MVT::v8i16: OpcodeIndex = 1; break;
1785  case MVT::v4f32:
1786  case MVT::v4i32: OpcodeIndex = 2; break;
1787  case MVT::v2f64:
1788  case MVT::v2i64: OpcodeIndex = 3;
1789    assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1790    break;
1791  }
1792
1793  EVT ResTy;
1794  if (NumVecs == 1)
1795    ResTy = VT;
1796  else {
1797    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1798    if (!is64BitVector)
1799      ResTyElts *= 2;
1800    ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1801  }
1802  std::vector<EVT> ResTys;
1803  ResTys.push_back(ResTy);
1804  if (isUpdating)
1805    ResTys.push_back(MVT::i32);
1806  ResTys.push_back(MVT::Other);
1807
1808  SDValue Pred = getAL(CurDAG);
1809  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1810  SDNode *VLd;
1811  SmallVector<SDValue, 7> Ops;
1812
1813  // Double registers and VLD1/VLD2 quad registers are directly supported.
1814  if (is64BitVector || NumVecs <= 2) {
1815    unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1816                    QOpcodes0[OpcodeIndex]);
1817    Ops.push_back(MemAddr);
1818    Ops.push_back(Align);
1819    if (isUpdating) {
1820      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1821      // FIXME: VLD1/VLD2 fixed increment doesn't need Reg0. Remove the reg0
1822      // case entirely when the rest are updated to that form, too.
1823      if ((NumVecs <= 2) && !isa<ConstantSDNode>(Inc.getNode()))
1824        Opc = getVLDSTRegisterUpdateOpcode(Opc);
1825      // FIXME: We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so
1826      // check for that explicitly too. Horribly hacky, but temporary.
1827      if ((NumVecs > 2 && !isVLDfixed(Opc)) ||
1828          !isa<ConstantSDNode>(Inc.getNode()))
1829        Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1830    }
1831    Ops.push_back(Pred);
1832    Ops.push_back(Reg0);
1833    Ops.push_back(Chain);
1834    VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1835
1836  } else {
1837    // Otherwise, quad registers are loaded with two separate instructions,
1838    // where one loads the even registers and the other loads the odd registers.
1839    EVT AddrTy = MemAddr.getValueType();
1840
1841    // Load the even subregs.  This is always an updating load, so that it
1842    // provides the address to the second load for the odd subregs.
1843    SDValue ImplDef =
1844      SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
1845    const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
1846    SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1847                                          ResTy, AddrTy, MVT::Other, OpsA);
1848    Chain = SDValue(VLdA, 2);
1849
1850    // Load the odd subregs.
1851    Ops.push_back(SDValue(VLdA, 1));
1852    Ops.push_back(Align);
1853    if (isUpdating) {
1854      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1855      assert(isa<ConstantSDNode>(Inc.getNode()) &&
1856             "only constant post-increment update allowed for VLD3/4");
1857      (void)Inc;
1858      Ops.push_back(Reg0);
1859    }
1860    Ops.push_back(SDValue(VLdA, 0));
1861    Ops.push_back(Pred);
1862    Ops.push_back(Reg0);
1863    Ops.push_back(Chain);
1864    VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops);
1865  }
1866
1867  // Transfer memoperands.
1868  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1869  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1870  cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1);
1871
1872  if (NumVecs == 1)
1873    return VLd;
1874
1875  // Extract out the subregisters.
1876  SDValue SuperReg = SDValue(VLd, 0);
1877  assert(ARM::dsub_7 == ARM::dsub_0+7 &&
1878         ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1879  unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
1880  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1881    ReplaceUses(SDValue(N, Vec),
1882                CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1883  ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
1884  if (isUpdating)
1885    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2));
1886  return nullptr;
1887}
1888
1889SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
1890                                   const uint16_t *DOpcodes,
1891                                   const uint16_t *QOpcodes0,
1892                                   const uint16_t *QOpcodes1) {
1893  assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1894  SDLoc dl(N);
1895
1896  SDValue MemAddr, Align;
1897  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1898  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1899  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1900    return nullptr;
1901
1902  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1903  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1904
1905  SDValue Chain = N->getOperand(0);
1906  EVT VT = N->getOperand(Vec0Idx).getValueType();
1907  bool is64BitVector = VT.is64BitVector();
1908  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1909
1910  unsigned OpcodeIndex;
1911  switch (VT.getSimpleVT().SimpleTy) {
1912  default: llvm_unreachable("unhandled vst type");
1913    // Double-register operations:
1914  case MVT::v8i8:  OpcodeIndex = 0; break;
1915  case MVT::v4i16: OpcodeIndex = 1; break;
1916  case MVT::v2f32:
1917  case MVT::v2i32: OpcodeIndex = 2; break;
1918  case MVT::v1i64: OpcodeIndex = 3; break;
1919    // Quad-register operations:
1920  case MVT::v16i8: OpcodeIndex = 0; break;
1921  case MVT::v8i16: OpcodeIndex = 1; break;
1922  case MVT::v4f32:
1923  case MVT::v4i32: OpcodeIndex = 2; break;
1924  case MVT::v2f64:
1925  case MVT::v2i64: OpcodeIndex = 3;
1926    assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1927    break;
1928  }
1929
1930  std::vector<EVT> ResTys;
1931  if (isUpdating)
1932    ResTys.push_back(MVT::i32);
1933  ResTys.push_back(MVT::Other);
1934
1935  SDValue Pred = getAL(CurDAG);
1936  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1937  SmallVector<SDValue, 7> Ops;
1938
1939  // Double registers and VST1/VST2 quad registers are directly supported.
1940  if (is64BitVector || NumVecs <= 2) {
1941    SDValue SrcReg;
1942    if (NumVecs == 1) {
1943      SrcReg = N->getOperand(Vec0Idx);
1944    } else if (is64BitVector) {
1945      // Form a REG_SEQUENCE to force register allocation.
1946      SDValue V0 = N->getOperand(Vec0Idx + 0);
1947      SDValue V1 = N->getOperand(Vec0Idx + 1);
1948      if (NumVecs == 2)
1949        SrcReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
1950      else {
1951        SDValue V2 = N->getOperand(Vec0Idx + 2);
1952        // If it's a vst3, form a quad D-register and leave the last part as
1953        // an undef.
1954        SDValue V3 = (NumVecs == 3)
1955          ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1956          : N->getOperand(Vec0Idx + 3);
1957        SrcReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
1958      }
1959    } else {
1960      // Form a QQ register.
1961      SDValue Q0 = N->getOperand(Vec0Idx);
1962      SDValue Q1 = N->getOperand(Vec0Idx + 1);
1963      SrcReg = SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0);
1964    }
1965
1966    unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1967                    QOpcodes0[OpcodeIndex]);
1968    Ops.push_back(MemAddr);
1969    Ops.push_back(Align);
1970    if (isUpdating) {
1971      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1972      // FIXME: VST1/VST2 fixed increment doesn't need Reg0. Remove the reg0
1973      // case entirely when the rest are updated to that form, too.
1974      if (NumVecs <= 2 && !isa<ConstantSDNode>(Inc.getNode()))
1975        Opc = getVLDSTRegisterUpdateOpcode(Opc);
1976      // FIXME: We use a VST1 for v1i64 even if the pseudo says vld2/3/4, so
1977      // check for that explicitly too. Horribly hacky, but temporary.
1978      if  (!isa<ConstantSDNode>(Inc.getNode()))
1979        Ops.push_back(Inc);
1980      else if (NumVecs > 2 && !isVSTfixed(Opc))
1981        Ops.push_back(Reg0);
1982    }
1983    Ops.push_back(SrcReg);
1984    Ops.push_back(Pred);
1985    Ops.push_back(Reg0);
1986    Ops.push_back(Chain);
1987    SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1988
1989    // Transfer memoperands.
1990    cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1);
1991
1992    return VSt;
1993  }
1994
1995  // Otherwise, quad registers are stored with two separate instructions,
1996  // where one stores the even registers and the other stores the odd registers.
1997
1998  // Form the QQQQ REG_SEQUENCE.
1999  SDValue V0 = N->getOperand(Vec0Idx + 0);
2000  SDValue V1 = N->getOperand(Vec0Idx + 1);
2001  SDValue V2 = N->getOperand(Vec0Idx + 2);
2002  SDValue V3 = (NumVecs == 3)
2003    ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2004    : N->getOperand(Vec0Idx + 3);
2005  SDValue RegSeq = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2006
2007  // Store the even D registers.  This is always an updating store, so that it
2008  // provides the address to the second store for the odd subregs.
2009  const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain };
2010  SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
2011                                        MemAddr.getValueType(),
2012                                        MVT::Other, OpsA);
2013  cast<MachineSDNode>(VStA)->setMemRefs(MemOp, MemOp + 1);
2014  Chain = SDValue(VStA, 1);
2015
2016  // Store the odd D registers.
2017  Ops.push_back(SDValue(VStA, 0));
2018  Ops.push_back(Align);
2019  if (isUpdating) {
2020    SDValue Inc = N->getOperand(AddrOpIdx + 1);
2021    assert(isa<ConstantSDNode>(Inc.getNode()) &&
2022           "only constant post-increment update allowed for VST3/4");
2023    (void)Inc;
2024    Ops.push_back(Reg0);
2025  }
2026  Ops.push_back(RegSeq);
2027  Ops.push_back(Pred);
2028  Ops.push_back(Reg0);
2029  Ops.push_back(Chain);
2030  SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
2031                                        Ops);
2032  cast<MachineSDNode>(VStB)->setMemRefs(MemOp, MemOp + 1);
2033  return VStB;
2034}
2035
2036SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
2037                                         bool isUpdating, unsigned NumVecs,
2038                                         const uint16_t *DOpcodes,
2039                                         const uint16_t *QOpcodes) {
2040  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
2041  SDLoc dl(N);
2042
2043  SDValue MemAddr, Align;
2044  unsigned AddrOpIdx = isUpdating ? 1 : 2;
2045  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
2046  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2047    return nullptr;
2048
2049  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2050  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2051
2052  SDValue Chain = N->getOperand(0);
2053  unsigned Lane =
2054    cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue();
2055  EVT VT = N->getOperand(Vec0Idx).getValueType();
2056  bool is64BitVector = VT.is64BitVector();
2057
2058  unsigned Alignment = 0;
2059  if (NumVecs != 3) {
2060    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2061    unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
2062    if (Alignment > NumBytes)
2063      Alignment = NumBytes;
2064    if (Alignment < 8 && Alignment < NumBytes)
2065      Alignment = 0;
2066    // Alignment must be a power of two; make sure of that.
2067    Alignment = (Alignment & -Alignment);
2068    if (Alignment == 1)
2069      Alignment = 0;
2070  }
2071  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
2072
2073  unsigned OpcodeIndex;
2074  switch (VT.getSimpleVT().SimpleTy) {
2075  default: llvm_unreachable("unhandled vld/vst lane type");
2076    // Double-register operations:
2077  case MVT::v8i8:  OpcodeIndex = 0; break;
2078  case MVT::v4i16: OpcodeIndex = 1; break;
2079  case MVT::v2f32:
2080  case MVT::v2i32: OpcodeIndex = 2; break;
2081    // Quad-register operations:
2082  case MVT::v8i16: OpcodeIndex = 0; break;
2083  case MVT::v4f32:
2084  case MVT::v4i32: OpcodeIndex = 1; break;
2085  }
2086
2087  std::vector<EVT> ResTys;
2088  if (IsLoad) {
2089    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2090    if (!is64BitVector)
2091      ResTyElts *= 2;
2092    ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(),
2093                                      MVT::i64, ResTyElts));
2094  }
2095  if (isUpdating)
2096    ResTys.push_back(MVT::i32);
2097  ResTys.push_back(MVT::Other);
2098
2099  SDValue Pred = getAL(CurDAG);
2100  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2101
2102  SmallVector<SDValue, 8> Ops;
2103  Ops.push_back(MemAddr);
2104  Ops.push_back(Align);
2105  if (isUpdating) {
2106    SDValue Inc = N->getOperand(AddrOpIdx + 1);
2107    Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
2108  }
2109
2110  SDValue SuperReg;
2111  SDValue V0 = N->getOperand(Vec0Idx + 0);
2112  SDValue V1 = N->getOperand(Vec0Idx + 1);
2113  if (NumVecs == 2) {
2114    if (is64BitVector)
2115      SuperReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2116    else
2117      SuperReg = SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0);
2118  } else {
2119    SDValue V2 = N->getOperand(Vec0Idx + 2);
2120    SDValue V3 = (NumVecs == 3)
2121      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2122      : N->getOperand(Vec0Idx + 3);
2123    if (is64BitVector)
2124      SuperReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2125    else
2126      SuperReg = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2127  }
2128  Ops.push_back(SuperReg);
2129  Ops.push_back(getI32Imm(Lane));
2130  Ops.push_back(Pred);
2131  Ops.push_back(Reg0);
2132  Ops.push_back(Chain);
2133
2134  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
2135                                  QOpcodes[OpcodeIndex]);
2136  SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2137  cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1);
2138  if (!IsLoad)
2139    return VLdLn;
2140
2141  // Extract the subregisters.
2142  SuperReg = SDValue(VLdLn, 0);
2143  assert(ARM::dsub_7 == ARM::dsub_0+7 &&
2144         ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
2145  unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2146  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2147    ReplaceUses(SDValue(N, Vec),
2148                CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2149  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1));
2150  if (isUpdating)
2151    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2));
2152  return nullptr;
2153}
2154
2155SDNode *ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating,
2156                                      unsigned NumVecs,
2157                                      const uint16_t *Opcodes) {
2158  assert(NumVecs >=2 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
2159  SDLoc dl(N);
2160
2161  SDValue MemAddr, Align;
2162  if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align))
2163    return nullptr;
2164
2165  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2166  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2167
2168  SDValue Chain = N->getOperand(0);
2169  EVT VT = N->getValueType(0);
2170
2171  unsigned Alignment = 0;
2172  if (NumVecs != 3) {
2173    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2174    unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
2175    if (Alignment > NumBytes)
2176      Alignment = NumBytes;
2177    if (Alignment < 8 && Alignment < NumBytes)
2178      Alignment = 0;
2179    // Alignment must be a power of two; make sure of that.
2180    Alignment = (Alignment & -Alignment);
2181    if (Alignment == 1)
2182      Alignment = 0;
2183  }
2184  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
2185
2186  unsigned OpcodeIndex;
2187  switch (VT.getSimpleVT().SimpleTy) {
2188  default: llvm_unreachable("unhandled vld-dup type");
2189  case MVT::v8i8:  OpcodeIndex = 0; break;
2190  case MVT::v4i16: OpcodeIndex = 1; break;
2191  case MVT::v2f32:
2192  case MVT::v2i32: OpcodeIndex = 2; break;
2193  }
2194
2195  SDValue Pred = getAL(CurDAG);
2196  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2197  SDValue SuperReg;
2198  unsigned Opc = Opcodes[OpcodeIndex];
2199  SmallVector<SDValue, 6> Ops;
2200  Ops.push_back(MemAddr);
2201  Ops.push_back(Align);
2202  if (isUpdating) {
2203    // fixed-stride update instructions don't have an explicit writeback
2204    // operand. It's implicit in the opcode itself.
2205    SDValue Inc = N->getOperand(2);
2206    if (!isa<ConstantSDNode>(Inc.getNode()))
2207      Ops.push_back(Inc);
2208    // FIXME: VLD3 and VLD4 haven't been updated to that form yet.
2209    else if (NumVecs > 2)
2210      Ops.push_back(Reg0);
2211  }
2212  Ops.push_back(Pred);
2213  Ops.push_back(Reg0);
2214  Ops.push_back(Chain);
2215
2216  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2217  std::vector<EVT> ResTys;
2218  ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,ResTyElts));
2219  if (isUpdating)
2220    ResTys.push_back(MVT::i32);
2221  ResTys.push_back(MVT::Other);
2222  SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2223  cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1);
2224  SuperReg = SDValue(VLdDup, 0);
2225
2226  // Extract the subregisters.
2227  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
2228  unsigned SubIdx = ARM::dsub_0;
2229  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2230    ReplaceUses(SDValue(N, Vec),
2231                CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
2232  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1));
2233  if (isUpdating)
2234    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2));
2235  return nullptr;
2236}
2237
2238SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs,
2239                                    unsigned Opc) {
2240  assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range");
2241  SDLoc dl(N);
2242  EVT VT = N->getValueType(0);
2243  unsigned FirstTblReg = IsExt ? 2 : 1;
2244
2245  // Form a REG_SEQUENCE to force register allocation.
2246  SDValue RegSeq;
2247  SDValue V0 = N->getOperand(FirstTblReg + 0);
2248  SDValue V1 = N->getOperand(FirstTblReg + 1);
2249  if (NumVecs == 2)
2250    RegSeq = SDValue(createDRegPairNode(MVT::v16i8, V0, V1), 0);
2251  else {
2252    SDValue V2 = N->getOperand(FirstTblReg + 2);
2253    // If it's a vtbl3, form a quad D-register and leave the last part as
2254    // an undef.
2255    SDValue V3 = (NumVecs == 3)
2256      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2257      : N->getOperand(FirstTblReg + 3);
2258    RegSeq = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2259  }
2260
2261  SmallVector<SDValue, 6> Ops;
2262  if (IsExt)
2263    Ops.push_back(N->getOperand(1));
2264  Ops.push_back(RegSeq);
2265  Ops.push_back(N->getOperand(FirstTblReg + NumVecs));
2266  Ops.push_back(getAL(CurDAG)); // predicate
2267  Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register
2268  return CurDAG->getMachineNode(Opc, dl, VT, Ops);
2269}
2270
2271SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
2272                                                     bool isSigned) {
2273  if (!Subtarget->hasV6T2Ops())
2274    return nullptr;
2275
2276  unsigned Opc = isSigned
2277    ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
2278    : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
2279
2280  // For unsigned extracts, check for a shift right and mask
2281  unsigned And_imm = 0;
2282  if (N->getOpcode() == ISD::AND) {
2283    if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
2284
2285      // The immediate is a mask of the low bits iff imm & (imm+1) == 0
2286      if (And_imm & (And_imm + 1))
2287        return nullptr;
2288
2289      unsigned Srl_imm = 0;
2290      if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
2291                                Srl_imm)) {
2292        assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2293
2294        // Note: The width operand is encoded as width-1.
2295        unsigned Width = CountTrailingOnes_32(And_imm) - 1;
2296        unsigned LSB = Srl_imm;
2297
2298        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2299
2300        if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) {
2301          // It's cheaper to use a right shift to extract the top bits.
2302          if (Subtarget->isThumb()) {
2303            Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri;
2304            SDValue Ops[] = { N->getOperand(0).getOperand(0),
2305                              CurDAG->getTargetConstant(LSB, MVT::i32),
2306                              getAL(CurDAG), Reg0, Reg0 };
2307            return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2308          }
2309
2310          // ARM models shift instructions as MOVsi with shifter operand.
2311          ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(ISD::SRL);
2312          SDValue ShOpc =
2313            CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB),
2314                                      MVT::i32);
2315          SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc,
2316                            getAL(CurDAG), Reg0, Reg0 };
2317          return CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops);
2318        }
2319
2320        SDValue Ops[] = { N->getOperand(0).getOperand(0),
2321                          CurDAG->getTargetConstant(LSB, MVT::i32),
2322                          CurDAG->getTargetConstant(Width, MVT::i32),
2323                          getAL(CurDAG), Reg0 };
2324        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2325      }
2326    }
2327    return nullptr;
2328  }
2329
2330  // Otherwise, we're looking for a shift of a shift
2331  unsigned Shl_imm = 0;
2332  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
2333    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
2334    unsigned Srl_imm = 0;
2335    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
2336      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2337      // Note: The width operand is encoded as width-1.
2338      unsigned Width = 32 - Srl_imm - 1;
2339      int LSB = Srl_imm - Shl_imm;
2340      if (LSB < 0)
2341        return nullptr;
2342      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2343      SDValue Ops[] = { N->getOperand(0).getOperand(0),
2344                        CurDAG->getTargetConstant(LSB, MVT::i32),
2345                        CurDAG->getTargetConstant(Width, MVT::i32),
2346                        getAL(CurDAG), Reg0 };
2347      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2348    }
2349  }
2350
2351  if (N->getOpcode() == ISD::SIGN_EXTEND_INREG) {
2352    unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
2353    unsigned LSB = 0;
2354    if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, LSB) &&
2355        !isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRA, LSB))
2356      return nullptr;
2357
2358    if (LSB + Width > 32)
2359      return nullptr;
2360
2361    SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2362    SDValue Ops[] = { N->getOperand(0).getOperand(0),
2363                      CurDAG->getTargetConstant(LSB, MVT::i32),
2364                      CurDAG->getTargetConstant(Width - 1, MVT::i32),
2365                      getAL(CurDAG), Reg0 };
2366    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2367  }
2368
2369  return nullptr;
2370}
2371
2372/// Target-specific DAG combining for ISD::XOR.
2373/// Target-independent combining lowers SELECT_CC nodes of the form
2374/// select_cc setg[ge] X,  0,  X, -X
2375/// select_cc setgt    X, -1,  X, -X
2376/// select_cc setl[te] X,  0, -X,  X
2377/// select_cc setlt    X,  1, -X,  X
2378/// which represent Integer ABS into:
2379/// Y = sra (X, size(X)-1); xor (add (X, Y), Y)
2380/// ARM instruction selection detects the latter and matches it to
2381/// ARM::ABS or ARM::t2ABS machine node.
2382SDNode *ARMDAGToDAGISel::SelectABSOp(SDNode *N){
2383  SDValue XORSrc0 = N->getOperand(0);
2384  SDValue XORSrc1 = N->getOperand(1);
2385  EVT VT = N->getValueType(0);
2386
2387  if (Subtarget->isThumb1Only())
2388    return nullptr;
2389
2390  if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA)
2391    return nullptr;
2392
2393  SDValue ADDSrc0 = XORSrc0.getOperand(0);
2394  SDValue ADDSrc1 = XORSrc0.getOperand(1);
2395  SDValue SRASrc0 = XORSrc1.getOperand(0);
2396  SDValue SRASrc1 = XORSrc1.getOperand(1);
2397  ConstantSDNode *SRAConstant =  dyn_cast<ConstantSDNode>(SRASrc1);
2398  EVT XType = SRASrc0.getValueType();
2399  unsigned Size = XType.getSizeInBits() - 1;
2400
2401  if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 &&
2402      XType.isInteger() && SRAConstant != nullptr &&
2403      Size == SRAConstant->getZExtValue()) {
2404    unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS;
2405    return CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0);
2406  }
2407
2408  return nullptr;
2409}
2410
2411SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
2412  // The only time a CONCAT_VECTORS operation can have legal types is when
2413  // two 64-bit vectors are concatenated to a 128-bit vector.
2414  EVT VT = N->getValueType(0);
2415  if (!VT.is128BitVector() || N->getNumOperands() != 2)
2416    llvm_unreachable("unexpected CONCAT_VECTORS");
2417  return createDRegPairNode(VT, N->getOperand(0), N->getOperand(1));
2418}
2419
2420SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
2421  SDLoc dl(N);
2422
2423  if (N->isMachineOpcode()) {
2424    N->setNodeId(-1);
2425    return nullptr;   // Already selected.
2426  }
2427
2428  switch (N->getOpcode()) {
2429  default: break;
2430  case ISD::INLINEASM: {
2431    SDNode *ResNode = SelectInlineAsm(N);
2432    if (ResNode)
2433      return ResNode;
2434    break;
2435  }
2436  case ISD::XOR: {
2437    // Select special operations if XOR node forms integer ABS pattern
2438    SDNode *ResNode = SelectABSOp(N);
2439    if (ResNode)
2440      return ResNode;
2441    // Other cases are autogenerated.
2442    break;
2443  }
2444  case ISD::Constant: {
2445    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
2446    bool UseCP = true;
2447    if (Subtarget->useMovt(*MF))
2448      // Thumb2-aware targets have the MOVT instruction, so all immediates can
2449      // be done with MOV + MOVT, at worst.
2450      UseCP = false;
2451    else {
2452      if (Subtarget->isThumb()) {
2453        UseCP = (Val > 255 &&                                  // MOV
2454                 ~Val > 255 &&                                 // MOV + MVN
2455                 !ARM_AM::isThumbImmShiftedVal(Val) &&         // MOV + LSL
2456                 !(Subtarget->hasV6T2Ops() && Val <= 0xffff)); // MOVW
2457      } else
2458        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&             // MOV
2459                 ARM_AM::getSOImmVal(~Val) == -1 &&            // MVN
2460                 !ARM_AM::isSOImmTwoPartVal(Val) &&            // two instrs.
2461                 !(Subtarget->hasV6T2Ops() && Val <= 0xffff)); // MOVW
2462    }
2463
2464    if (UseCP) {
2465      SDValue CPIdx = CurDAG->getTargetConstantPool(
2466          ConstantInt::get(Type::getInt32Ty(*CurDAG->getContext()), Val),
2467          TLI->getPointerTy());
2468
2469      SDNode *ResNode;
2470      if (Subtarget->isThumb()) {
2471        SDValue Pred = getAL(CurDAG);
2472        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2473        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
2474        ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
2475                                         Ops);
2476      } else {
2477        SDValue Ops[] = {
2478          CPIdx,
2479          CurDAG->getTargetConstant(0, MVT::i32),
2480          getAL(CurDAG),
2481          CurDAG->getRegister(0, MVT::i32),
2482          CurDAG->getEntryNode()
2483        };
2484        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
2485                                       Ops);
2486      }
2487      ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
2488      return nullptr;
2489    }
2490
2491    // Other cases are autogenerated.
2492    break;
2493  }
2494  case ISD::FrameIndex: {
2495    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
2496    int FI = cast<FrameIndexSDNode>(N)->getIndex();
2497    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy());
2498    if (Subtarget->isThumb1Only()) {
2499      return CurDAG->SelectNodeTo(N, ARM::tADDframe, MVT::i32, TFI,
2500                                  CurDAG->getTargetConstant(0, MVT::i32));
2501    } else {
2502      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
2503                      ARM::t2ADDri : ARM::ADDri);
2504      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
2505                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2506                        CurDAG->getRegister(0, MVT::i32) };
2507      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2508    }
2509  }
2510  case ISD::SRL:
2511    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2512      return I;
2513    break;
2514  case ISD::SIGN_EXTEND_INREG:
2515  case ISD::SRA:
2516    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true))
2517      return I;
2518    break;
2519  case ISD::MUL:
2520    if (Subtarget->isThumb1Only())
2521      break;
2522    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
2523      unsigned RHSV = C->getZExtValue();
2524      if (!RHSV) break;
2525      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
2526        unsigned ShImm = Log2_32(RHSV-1);
2527        if (ShImm >= 32)
2528          break;
2529        SDValue V = N->getOperand(0);
2530        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2531        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2532        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2533        if (Subtarget->isThumb()) {
2534          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2535          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops);
2536        } else {
2537          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2538          return CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops);
2539        }
2540      }
2541      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
2542        unsigned ShImm = Log2_32(RHSV+1);
2543        if (ShImm >= 32)
2544          break;
2545        SDValue V = N->getOperand(0);
2546        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2547        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2548        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2549        if (Subtarget->isThumb()) {
2550          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2551          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops);
2552        } else {
2553          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2554          return CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops);
2555        }
2556      }
2557    }
2558    break;
2559  case ISD::AND: {
2560    // Check for unsigned bitfield extract
2561    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2562      return I;
2563
2564    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
2565    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
2566    // are entirely contributed by c2 and lower 16-bits are entirely contributed
2567    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
2568    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
2569    EVT VT = N->getValueType(0);
2570    if (VT != MVT::i32)
2571      break;
2572    unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
2573      ? ARM::t2MOVTi16
2574      : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
2575    if (!Opc)
2576      break;
2577    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
2578    ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2579    if (!N1C)
2580      break;
2581    if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
2582      SDValue N2 = N0.getOperand(1);
2583      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
2584      if (!N2C)
2585        break;
2586      unsigned N1CVal = N1C->getZExtValue();
2587      unsigned N2CVal = N2C->getZExtValue();
2588      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
2589          (N1CVal & 0xffffU) == 0xffffU &&
2590          (N2CVal & 0xffffU) == 0x0U) {
2591        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
2592                                                  MVT::i32);
2593        SDValue Ops[] = { N0.getOperand(0), Imm16,
2594                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2595        return CurDAG->getMachineNode(Opc, dl, VT, Ops);
2596      }
2597    }
2598    break;
2599  }
2600  case ARMISD::VMOVRRD:
2601    return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
2602                                  N->getOperand(0), getAL(CurDAG),
2603                                  CurDAG->getRegister(0, MVT::i32));
2604  case ISD::UMUL_LOHI: {
2605    if (Subtarget->isThumb1Only())
2606      break;
2607    if (Subtarget->isThumb()) {
2608      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2609                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2610      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops);
2611    } else {
2612      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2613                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2614                        CurDAG->getRegister(0, MVT::i32) };
2615      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2616                                    ARM::UMULL : ARM::UMULLv5,
2617                                    dl, MVT::i32, MVT::i32, Ops);
2618    }
2619  }
2620  case ISD::SMUL_LOHI: {
2621    if (Subtarget->isThumb1Only())
2622      break;
2623    if (Subtarget->isThumb()) {
2624      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2625                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2626      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops);
2627    } else {
2628      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2629                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2630                        CurDAG->getRegister(0, MVT::i32) };
2631      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2632                                    ARM::SMULL : ARM::SMULLv5,
2633                                    dl, MVT::i32, MVT::i32, Ops);
2634    }
2635  }
2636  case ARMISD::UMLAL:{
2637    if (Subtarget->isThumb()) {
2638      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2639                        N->getOperand(3), getAL(CurDAG),
2640                        CurDAG->getRegister(0, MVT::i32)};
2641      return CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops);
2642    }else{
2643      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2644                        N->getOperand(3), getAL(CurDAG),
2645                        CurDAG->getRegister(0, MVT::i32),
2646                        CurDAG->getRegister(0, MVT::i32) };
2647      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2648                                      ARM::UMLAL : ARM::UMLALv5,
2649                                      dl, MVT::i32, MVT::i32, Ops);
2650    }
2651  }
2652  case ARMISD::SMLAL:{
2653    if (Subtarget->isThumb()) {
2654      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2655                        N->getOperand(3), getAL(CurDAG),
2656                        CurDAG->getRegister(0, MVT::i32)};
2657      return CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops);
2658    }else{
2659      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2660                        N->getOperand(3), getAL(CurDAG),
2661                        CurDAG->getRegister(0, MVT::i32),
2662                        CurDAG->getRegister(0, MVT::i32) };
2663      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2664                                      ARM::SMLAL : ARM::SMLALv5,
2665                                      dl, MVT::i32, MVT::i32, Ops);
2666    }
2667  }
2668  case ISD::LOAD: {
2669    SDNode *ResNode = nullptr;
2670    if (Subtarget->isThumb() && Subtarget->hasThumb2())
2671      ResNode = SelectT2IndexedLoad(N);
2672    else
2673      ResNode = SelectARMIndexedLoad(N);
2674    if (ResNode)
2675      return ResNode;
2676    // Other cases are autogenerated.
2677    break;
2678  }
2679  case ARMISD::BRCOND: {
2680    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2681    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2682    // Pattern complexity = 6  cost = 1  size = 0
2683
2684    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2685    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
2686    // Pattern complexity = 6  cost = 1  size = 0
2687
2688    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2689    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2690    // Pattern complexity = 6  cost = 1  size = 0
2691
2692    unsigned Opc = Subtarget->isThumb() ?
2693      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
2694    SDValue Chain = N->getOperand(0);
2695    SDValue N1 = N->getOperand(1);
2696    SDValue N2 = N->getOperand(2);
2697    SDValue N3 = N->getOperand(3);
2698    SDValue InFlag = N->getOperand(4);
2699    assert(N1.getOpcode() == ISD::BasicBlock);
2700    assert(N2.getOpcode() == ISD::Constant);
2701    assert(N3.getOpcode() == ISD::Register);
2702
2703    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
2704                               cast<ConstantSDNode>(N2)->getZExtValue()),
2705                               MVT::i32);
2706    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
2707    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
2708                                             MVT::Glue, Ops);
2709    Chain = SDValue(ResNode, 0);
2710    if (N->getNumValues() == 2) {
2711      InFlag = SDValue(ResNode, 1);
2712      ReplaceUses(SDValue(N, 1), InFlag);
2713    }
2714    ReplaceUses(SDValue(N, 0),
2715                SDValue(Chain.getNode(), Chain.getResNo()));
2716    return nullptr;
2717  }
2718  case ARMISD::VZIP: {
2719    unsigned Opc = 0;
2720    EVT VT = N->getValueType(0);
2721    switch (VT.getSimpleVT().SimpleTy) {
2722    default: return nullptr;
2723    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
2724    case MVT::v4i16: Opc = ARM::VZIPd16; break;
2725    case MVT::v2f32:
2726    // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
2727    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2728    case MVT::v16i8: Opc = ARM::VZIPq8; break;
2729    case MVT::v8i16: Opc = ARM::VZIPq16; break;
2730    case MVT::v4f32:
2731    case MVT::v4i32: Opc = ARM::VZIPq32; break;
2732    }
2733    SDValue Pred = getAL(CurDAG);
2734    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2735    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2736    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops);
2737  }
2738  case ARMISD::VUZP: {
2739    unsigned Opc = 0;
2740    EVT VT = N->getValueType(0);
2741    switch (VT.getSimpleVT().SimpleTy) {
2742    default: return nullptr;
2743    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
2744    case MVT::v4i16: Opc = ARM::VUZPd16; break;
2745    case MVT::v2f32:
2746    // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
2747    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2748    case MVT::v16i8: Opc = ARM::VUZPq8; break;
2749    case MVT::v8i16: Opc = ARM::VUZPq16; break;
2750    case MVT::v4f32:
2751    case MVT::v4i32: Opc = ARM::VUZPq32; break;
2752    }
2753    SDValue Pred = getAL(CurDAG);
2754    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2755    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2756    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops);
2757  }
2758  case ARMISD::VTRN: {
2759    unsigned Opc = 0;
2760    EVT VT = N->getValueType(0);
2761    switch (VT.getSimpleVT().SimpleTy) {
2762    default: return nullptr;
2763    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
2764    case MVT::v4i16: Opc = ARM::VTRNd16; break;
2765    case MVT::v2f32:
2766    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2767    case MVT::v16i8: Opc = ARM::VTRNq8; break;
2768    case MVT::v8i16: Opc = ARM::VTRNq16; break;
2769    case MVT::v4f32:
2770    case MVT::v4i32: Opc = ARM::VTRNq32; break;
2771    }
2772    SDValue Pred = getAL(CurDAG);
2773    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2774    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2775    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops);
2776  }
2777  case ARMISD::BUILD_VECTOR: {
2778    EVT VecVT = N->getValueType(0);
2779    EVT EltVT = VecVT.getVectorElementType();
2780    unsigned NumElts = VecVT.getVectorNumElements();
2781    if (EltVT == MVT::f64) {
2782      assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
2783      return createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1));
2784    }
2785    assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR");
2786    if (NumElts == 2)
2787      return createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1));
2788    assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
2789    return createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1),
2790                     N->getOperand(2), N->getOperand(3));
2791  }
2792
2793  case ARMISD::VLD2DUP: {
2794    static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
2795                                        ARM::VLD2DUPd32 };
2796    return SelectVLDDup(N, false, 2, Opcodes);
2797  }
2798
2799  case ARMISD::VLD3DUP: {
2800    static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
2801                                        ARM::VLD3DUPd16Pseudo,
2802                                        ARM::VLD3DUPd32Pseudo };
2803    return SelectVLDDup(N, false, 3, Opcodes);
2804  }
2805
2806  case ARMISD::VLD4DUP: {
2807    static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
2808                                        ARM::VLD4DUPd16Pseudo,
2809                                        ARM::VLD4DUPd32Pseudo };
2810    return SelectVLDDup(N, false, 4, Opcodes);
2811  }
2812
2813  case ARMISD::VLD2DUP_UPD: {
2814    static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed,
2815                                        ARM::VLD2DUPd16wb_fixed,
2816                                        ARM::VLD2DUPd32wb_fixed };
2817    return SelectVLDDup(N, true, 2, Opcodes);
2818  }
2819
2820  case ARMISD::VLD3DUP_UPD: {
2821    static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
2822                                        ARM::VLD3DUPd16Pseudo_UPD,
2823                                        ARM::VLD3DUPd32Pseudo_UPD };
2824    return SelectVLDDup(N, true, 3, Opcodes);
2825  }
2826
2827  case ARMISD::VLD4DUP_UPD: {
2828    static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
2829                                        ARM::VLD4DUPd16Pseudo_UPD,
2830                                        ARM::VLD4DUPd32Pseudo_UPD };
2831    return SelectVLDDup(N, true, 4, Opcodes);
2832  }
2833
2834  case ARMISD::VLD1_UPD: {
2835    static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
2836                                         ARM::VLD1d16wb_fixed,
2837                                         ARM::VLD1d32wb_fixed,
2838                                         ARM::VLD1d64wb_fixed };
2839    static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
2840                                         ARM::VLD1q16wb_fixed,
2841                                         ARM::VLD1q32wb_fixed,
2842                                         ARM::VLD1q64wb_fixed };
2843    return SelectVLD(N, true, 1, DOpcodes, QOpcodes, nullptr);
2844  }
2845
2846  case ARMISD::VLD2_UPD: {
2847    static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed,
2848                                         ARM::VLD2d16wb_fixed,
2849                                         ARM::VLD2d32wb_fixed,
2850                                         ARM::VLD1q64wb_fixed};
2851    static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed,
2852                                         ARM::VLD2q16PseudoWB_fixed,
2853                                         ARM::VLD2q32PseudoWB_fixed };
2854    return SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr);
2855  }
2856
2857  case ARMISD::VLD3_UPD: {
2858    static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
2859                                         ARM::VLD3d16Pseudo_UPD,
2860                                         ARM::VLD3d32Pseudo_UPD,
2861                                         ARM::VLD1d64TPseudoWB_fixed};
2862    static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
2863                                          ARM::VLD3q16Pseudo_UPD,
2864                                          ARM::VLD3q32Pseudo_UPD };
2865    static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
2866                                          ARM::VLD3q16oddPseudo_UPD,
2867                                          ARM::VLD3q32oddPseudo_UPD };
2868    return SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
2869  }
2870
2871  case ARMISD::VLD4_UPD: {
2872    static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD,
2873                                         ARM::VLD4d16Pseudo_UPD,
2874                                         ARM::VLD4d32Pseudo_UPD,
2875                                         ARM::VLD1d64QPseudoWB_fixed};
2876    static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
2877                                          ARM::VLD4q16Pseudo_UPD,
2878                                          ARM::VLD4q32Pseudo_UPD };
2879    static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
2880                                          ARM::VLD4q16oddPseudo_UPD,
2881                                          ARM::VLD4q32oddPseudo_UPD };
2882    return SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
2883  }
2884
2885  case ARMISD::VLD2LN_UPD: {
2886    static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
2887                                         ARM::VLD2LNd16Pseudo_UPD,
2888                                         ARM::VLD2LNd32Pseudo_UPD };
2889    static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
2890                                         ARM::VLD2LNq32Pseudo_UPD };
2891    return SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes);
2892  }
2893
2894  case ARMISD::VLD3LN_UPD: {
2895    static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
2896                                         ARM::VLD3LNd16Pseudo_UPD,
2897                                         ARM::VLD3LNd32Pseudo_UPD };
2898    static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
2899                                         ARM::VLD3LNq32Pseudo_UPD };
2900    return SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes);
2901  }
2902
2903  case ARMISD::VLD4LN_UPD: {
2904    static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
2905                                         ARM::VLD4LNd16Pseudo_UPD,
2906                                         ARM::VLD4LNd32Pseudo_UPD };
2907    static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
2908                                         ARM::VLD4LNq32Pseudo_UPD };
2909    return SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes);
2910  }
2911
2912  case ARMISD::VST1_UPD: {
2913    static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
2914                                         ARM::VST1d16wb_fixed,
2915                                         ARM::VST1d32wb_fixed,
2916                                         ARM::VST1d64wb_fixed };
2917    static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
2918                                         ARM::VST1q16wb_fixed,
2919                                         ARM::VST1q32wb_fixed,
2920                                         ARM::VST1q64wb_fixed };
2921    return SelectVST(N, true, 1, DOpcodes, QOpcodes, nullptr);
2922  }
2923
2924  case ARMISD::VST2_UPD: {
2925    static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed,
2926                                         ARM::VST2d16wb_fixed,
2927                                         ARM::VST2d32wb_fixed,
2928                                         ARM::VST1q64wb_fixed};
2929    static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed,
2930                                         ARM::VST2q16PseudoWB_fixed,
2931                                         ARM::VST2q32PseudoWB_fixed };
2932    return SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr);
2933  }
2934
2935  case ARMISD::VST3_UPD: {
2936    static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
2937                                         ARM::VST3d16Pseudo_UPD,
2938                                         ARM::VST3d32Pseudo_UPD,
2939                                         ARM::VST1d64TPseudoWB_fixed};
2940    static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
2941                                          ARM::VST3q16Pseudo_UPD,
2942                                          ARM::VST3q32Pseudo_UPD };
2943    static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
2944                                          ARM::VST3q16oddPseudo_UPD,
2945                                          ARM::VST3q32oddPseudo_UPD };
2946    return SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
2947  }
2948
2949  case ARMISD::VST4_UPD: {
2950    static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD,
2951                                         ARM::VST4d16Pseudo_UPD,
2952                                         ARM::VST4d32Pseudo_UPD,
2953                                         ARM::VST1d64QPseudoWB_fixed};
2954    static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
2955                                          ARM::VST4q16Pseudo_UPD,
2956                                          ARM::VST4q32Pseudo_UPD };
2957    static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
2958                                          ARM::VST4q16oddPseudo_UPD,
2959                                          ARM::VST4q32oddPseudo_UPD };
2960    return SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
2961  }
2962
2963  case ARMISD::VST2LN_UPD: {
2964    static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
2965                                         ARM::VST2LNd16Pseudo_UPD,
2966                                         ARM::VST2LNd32Pseudo_UPD };
2967    static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
2968                                         ARM::VST2LNq32Pseudo_UPD };
2969    return SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes);
2970  }
2971
2972  case ARMISD::VST3LN_UPD: {
2973    static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
2974                                         ARM::VST3LNd16Pseudo_UPD,
2975                                         ARM::VST3LNd32Pseudo_UPD };
2976    static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
2977                                         ARM::VST3LNq32Pseudo_UPD };
2978    return SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes);
2979  }
2980
2981  case ARMISD::VST4LN_UPD: {
2982    static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
2983                                         ARM::VST4LNd16Pseudo_UPD,
2984                                         ARM::VST4LNd32Pseudo_UPD };
2985    static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
2986                                         ARM::VST4LNq32Pseudo_UPD };
2987    return SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes);
2988  }
2989
2990  case ISD::INTRINSIC_VOID:
2991  case ISD::INTRINSIC_W_CHAIN: {
2992    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2993    switch (IntNo) {
2994    default:
2995      break;
2996
2997    case Intrinsic::arm_ldaexd:
2998    case Intrinsic::arm_ldrexd: {
2999      SDLoc dl(N);
3000      SDValue Chain = N->getOperand(0);
3001      SDValue MemAddr = N->getOperand(2);
3002      bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2();
3003
3004      bool IsAcquire = IntNo == Intrinsic::arm_ldaexd;
3005      unsigned NewOpc = isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD)
3006                                : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD);
3007
3008      // arm_ldrexd returns a i64 value in {i32, i32}
3009      std::vector<EVT> ResTys;
3010      if (isThumb) {
3011        ResTys.push_back(MVT::i32);
3012        ResTys.push_back(MVT::i32);
3013      } else
3014        ResTys.push_back(MVT::Untyped);
3015      ResTys.push_back(MVT::Other);
3016
3017      // Place arguments in the right order.
3018      SmallVector<SDValue, 7> Ops;
3019      Ops.push_back(MemAddr);
3020      Ops.push_back(getAL(CurDAG));
3021      Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3022      Ops.push_back(Chain);
3023      SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
3024      // Transfer memoperands.
3025      MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
3026      MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3027      cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1);
3028
3029      // Remap uses.
3030      SDValue OutChain = isThumb ? SDValue(Ld, 2) : SDValue(Ld, 1);
3031      if (!SDValue(N, 0).use_empty()) {
3032        SDValue Result;
3033        if (isThumb)
3034          Result = SDValue(Ld, 0);
3035        else {
3036          SDValue SubRegIdx = CurDAG->getTargetConstant(ARM::gsub_0, MVT::i32);
3037          SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
3038              dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
3039          Result = SDValue(ResNode,0);
3040        }
3041        ReplaceUses(SDValue(N, 0), Result);
3042      }
3043      if (!SDValue(N, 1).use_empty()) {
3044        SDValue Result;
3045        if (isThumb)
3046          Result = SDValue(Ld, 1);
3047        else {
3048          SDValue SubRegIdx = CurDAG->getTargetConstant(ARM::gsub_1, MVT::i32);
3049          SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
3050              dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
3051          Result = SDValue(ResNode,0);
3052        }
3053        ReplaceUses(SDValue(N, 1), Result);
3054      }
3055      ReplaceUses(SDValue(N, 2), OutChain);
3056      return nullptr;
3057    }
3058    case Intrinsic::arm_stlexd:
3059    case Intrinsic::arm_strexd: {
3060      SDLoc dl(N);
3061      SDValue Chain = N->getOperand(0);
3062      SDValue Val0 = N->getOperand(2);
3063      SDValue Val1 = N->getOperand(3);
3064      SDValue MemAddr = N->getOperand(4);
3065
3066      // Store exclusive double return a i32 value which is the return status
3067      // of the issued store.
3068      EVT ResTys[] = { MVT::i32, MVT::Other };
3069
3070      bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2();
3071      // Place arguments in the right order.
3072      SmallVector<SDValue, 7> Ops;
3073      if (isThumb) {
3074        Ops.push_back(Val0);
3075        Ops.push_back(Val1);
3076      } else
3077        // arm_strexd uses GPRPair.
3078        Ops.push_back(SDValue(createGPRPairNode(MVT::Untyped, Val0, Val1), 0));
3079      Ops.push_back(MemAddr);
3080      Ops.push_back(getAL(CurDAG));
3081      Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3082      Ops.push_back(Chain);
3083
3084      bool IsRelease = IntNo == Intrinsic::arm_stlexd;
3085      unsigned NewOpc = isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD)
3086                                : (IsRelease ? ARM::STLEXD : ARM::STREXD);
3087
3088      SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
3089      // Transfer memoperands.
3090      MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
3091      MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3092      cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1);
3093
3094      return St;
3095    }
3096
3097    case Intrinsic::arm_neon_vld1: {
3098      static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
3099                                           ARM::VLD1d32, ARM::VLD1d64 };
3100      static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
3101                                           ARM::VLD1q32, ARM::VLD1q64};
3102      return SelectVLD(N, false, 1, DOpcodes, QOpcodes, nullptr);
3103    }
3104
3105    case Intrinsic::arm_neon_vld2: {
3106      static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
3107                                           ARM::VLD2d32, ARM::VLD1q64 };
3108      static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
3109                                           ARM::VLD2q32Pseudo };
3110      return SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr);
3111    }
3112
3113    case Intrinsic::arm_neon_vld3: {
3114      static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo,
3115                                           ARM::VLD3d16Pseudo,
3116                                           ARM::VLD3d32Pseudo,
3117                                           ARM::VLD1d64TPseudo };
3118      static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
3119                                            ARM::VLD3q16Pseudo_UPD,
3120                                            ARM::VLD3q32Pseudo_UPD };
3121      static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo,
3122                                            ARM::VLD3q16oddPseudo,
3123                                            ARM::VLD3q32oddPseudo };
3124      return SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3125    }
3126
3127    case Intrinsic::arm_neon_vld4: {
3128      static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo,
3129                                           ARM::VLD4d16Pseudo,
3130                                           ARM::VLD4d32Pseudo,
3131                                           ARM::VLD1d64QPseudo };
3132      static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
3133                                            ARM::VLD4q16Pseudo_UPD,
3134                                            ARM::VLD4q32Pseudo_UPD };
3135      static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo,
3136                                            ARM::VLD4q16oddPseudo,
3137                                            ARM::VLD4q32oddPseudo };
3138      return SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3139    }
3140
3141    case Intrinsic::arm_neon_vld2lane: {
3142      static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo,
3143                                           ARM::VLD2LNd16Pseudo,
3144                                           ARM::VLD2LNd32Pseudo };
3145      static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo,
3146                                           ARM::VLD2LNq32Pseudo };
3147      return SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes);
3148    }
3149
3150    case Intrinsic::arm_neon_vld3lane: {
3151      static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo,
3152                                           ARM::VLD3LNd16Pseudo,
3153                                           ARM::VLD3LNd32Pseudo };
3154      static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo,
3155                                           ARM::VLD3LNq32Pseudo };
3156      return SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes);
3157    }
3158
3159    case Intrinsic::arm_neon_vld4lane: {
3160      static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo,
3161                                           ARM::VLD4LNd16Pseudo,
3162                                           ARM::VLD4LNd32Pseudo };
3163      static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo,
3164                                           ARM::VLD4LNq32Pseudo };
3165      return SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes);
3166    }
3167
3168    case Intrinsic::arm_neon_vst1: {
3169      static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
3170                                           ARM::VST1d32, ARM::VST1d64 };
3171      static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
3172                                           ARM::VST1q32, ARM::VST1q64 };
3173      return SelectVST(N, false, 1, DOpcodes, QOpcodes, nullptr);
3174    }
3175
3176    case Intrinsic::arm_neon_vst2: {
3177      static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
3178                                           ARM::VST2d32, ARM::VST1q64 };
3179      static uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
3180                                     ARM::VST2q32Pseudo };
3181      return SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr);
3182    }
3183
3184    case Intrinsic::arm_neon_vst3: {
3185      static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo,
3186                                           ARM::VST3d16Pseudo,
3187                                           ARM::VST3d32Pseudo,
3188                                           ARM::VST1d64TPseudo };
3189      static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3190                                            ARM::VST3q16Pseudo_UPD,
3191                                            ARM::VST3q32Pseudo_UPD };
3192      static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo,
3193                                            ARM::VST3q16oddPseudo,
3194                                            ARM::VST3q32oddPseudo };
3195      return SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3196    }
3197
3198    case Intrinsic::arm_neon_vst4: {
3199      static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo,
3200                                           ARM::VST4d16Pseudo,
3201                                           ARM::VST4d32Pseudo,
3202                                           ARM::VST1d64QPseudo };
3203      static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3204                                            ARM::VST4q16Pseudo_UPD,
3205                                            ARM::VST4q32Pseudo_UPD };
3206      static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo,
3207                                            ARM::VST4q16oddPseudo,
3208                                            ARM::VST4q32oddPseudo };
3209      return SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3210    }
3211
3212    case Intrinsic::arm_neon_vst2lane: {
3213      static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo,
3214                                           ARM::VST2LNd16Pseudo,
3215                                           ARM::VST2LNd32Pseudo };
3216      static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo,
3217                                           ARM::VST2LNq32Pseudo };
3218      return SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes);
3219    }
3220
3221    case Intrinsic::arm_neon_vst3lane: {
3222      static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo,
3223                                           ARM::VST3LNd16Pseudo,
3224                                           ARM::VST3LNd32Pseudo };
3225      static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo,
3226                                           ARM::VST3LNq32Pseudo };
3227      return SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes);
3228    }
3229
3230    case Intrinsic::arm_neon_vst4lane: {
3231      static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo,
3232                                           ARM::VST4LNd16Pseudo,
3233                                           ARM::VST4LNd32Pseudo };
3234      static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo,
3235                                           ARM::VST4LNq32Pseudo };
3236      return SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes);
3237    }
3238    }
3239    break;
3240  }
3241
3242  case ISD::INTRINSIC_WO_CHAIN: {
3243    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
3244    switch (IntNo) {
3245    default:
3246      break;
3247
3248    case Intrinsic::arm_neon_vtbl2:
3249      return SelectVTBL(N, false, 2, ARM::VTBL2);
3250    case Intrinsic::arm_neon_vtbl3:
3251      return SelectVTBL(N, false, 3, ARM::VTBL3Pseudo);
3252    case Intrinsic::arm_neon_vtbl4:
3253      return SelectVTBL(N, false, 4, ARM::VTBL4Pseudo);
3254
3255    case Intrinsic::arm_neon_vtbx2:
3256      return SelectVTBL(N, true, 2, ARM::VTBX2);
3257    case Intrinsic::arm_neon_vtbx3:
3258      return SelectVTBL(N, true, 3, ARM::VTBX3Pseudo);
3259    case Intrinsic::arm_neon_vtbx4:
3260      return SelectVTBL(N, true, 4, ARM::VTBX4Pseudo);
3261    }
3262    break;
3263  }
3264
3265  case ARMISD::VTBL1: {
3266    SDLoc dl(N);
3267    EVT VT = N->getValueType(0);
3268    SmallVector<SDValue, 6> Ops;
3269
3270    Ops.push_back(N->getOperand(0));
3271    Ops.push_back(N->getOperand(1));
3272    Ops.push_back(getAL(CurDAG));                    // Predicate
3273    Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
3274    return CurDAG->getMachineNode(ARM::VTBL1, dl, VT, Ops);
3275  }
3276  case ARMISD::VTBL2: {
3277    SDLoc dl(N);
3278    EVT VT = N->getValueType(0);
3279
3280    // Form a REG_SEQUENCE to force register allocation.
3281    SDValue V0 = N->getOperand(0);
3282    SDValue V1 = N->getOperand(1);
3283    SDValue RegSeq = SDValue(createDRegPairNode(MVT::v16i8, V0, V1), 0);
3284
3285    SmallVector<SDValue, 6> Ops;
3286    Ops.push_back(RegSeq);
3287    Ops.push_back(N->getOperand(2));
3288    Ops.push_back(getAL(CurDAG));                    // Predicate
3289    Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
3290    return CurDAG->getMachineNode(ARM::VTBL2, dl, VT, Ops);
3291  }
3292
3293  case ISD::CONCAT_VECTORS:
3294    return SelectConcatVector(N);
3295  }
3296
3297  return SelectCode(N);
3298}
3299
3300SDNode *ARMDAGToDAGISel::SelectInlineAsm(SDNode *N){
3301  std::vector<SDValue> AsmNodeOperands;
3302  unsigned Flag, Kind;
3303  bool Changed = false;
3304  unsigned NumOps = N->getNumOperands();
3305
3306  // Normally, i64 data is bounded to two arbitrary GRPs for "%r" constraint.
3307  // However, some instrstions (e.g. ldrexd/strexd in ARM mode) require
3308  // (even/even+1) GPRs and use %n and %Hn to refer to the individual regs
3309  // respectively. Since there is no constraint to explicitly specify a
3310  // reg pair, we use GPRPair reg class for "%r" for 64-bit data. For Thumb,
3311  // the 64-bit data may be referred by H, Q, R modifiers, so we still pack
3312  // them into a GPRPair.
3313
3314  SDLoc dl(N);
3315  SDValue Glue = N->getGluedNode() ? N->getOperand(NumOps-1)
3316                                   : SDValue(nullptr,0);
3317
3318  SmallVector<bool, 8> OpChanged;
3319  // Glue node will be appended late.
3320  for(unsigned i = 0, e = N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) {
3321    SDValue op = N->getOperand(i);
3322    AsmNodeOperands.push_back(op);
3323
3324    if (i < InlineAsm::Op_FirstOperand)
3325      continue;
3326
3327    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(i))) {
3328      Flag = C->getZExtValue();
3329      Kind = InlineAsm::getKind(Flag);
3330    }
3331    else
3332      continue;
3333
3334    // Immediate operands to inline asm in the SelectionDAG are modeled with
3335    // two operands. The first is a constant of value InlineAsm::Kind_Imm, and
3336    // the second is a constant with the value of the immediate. If we get here
3337    // and we have a Kind_Imm, skip the next operand, and continue.
3338    if (Kind == InlineAsm::Kind_Imm) {
3339      SDValue op = N->getOperand(++i);
3340      AsmNodeOperands.push_back(op);
3341      continue;
3342    }
3343
3344    unsigned NumRegs = InlineAsm::getNumOperandRegisters(Flag);
3345    if (NumRegs)
3346      OpChanged.push_back(false);
3347
3348    unsigned DefIdx = 0;
3349    bool IsTiedToChangedOp = false;
3350    // If it's a use that is tied with a previous def, it has no
3351    // reg class constraint.
3352    if (Changed && InlineAsm::isUseOperandTiedToDef(Flag, DefIdx))
3353      IsTiedToChangedOp = OpChanged[DefIdx];
3354
3355    if (Kind != InlineAsm::Kind_RegUse && Kind != InlineAsm::Kind_RegDef
3356        && Kind != InlineAsm::Kind_RegDefEarlyClobber)
3357      continue;
3358
3359    unsigned RC;
3360    bool HasRC = InlineAsm::hasRegClassConstraint(Flag, RC);
3361    if ((!IsTiedToChangedOp && (!HasRC || RC != ARM::GPRRegClassID))
3362        || NumRegs != 2)
3363      continue;
3364
3365    assert((i+2 < NumOps) && "Invalid number of operands in inline asm");
3366    SDValue V0 = N->getOperand(i+1);
3367    SDValue V1 = N->getOperand(i+2);
3368    unsigned Reg0 = cast<RegisterSDNode>(V0)->getReg();
3369    unsigned Reg1 = cast<RegisterSDNode>(V1)->getReg();
3370    SDValue PairedReg;
3371    MachineRegisterInfo &MRI = MF->getRegInfo();
3372
3373    if (Kind == InlineAsm::Kind_RegDef ||
3374        Kind == InlineAsm::Kind_RegDefEarlyClobber) {
3375      // Replace the two GPRs with 1 GPRPair and copy values from GPRPair to
3376      // the original GPRs.
3377
3378      unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
3379      PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
3380      SDValue Chain = SDValue(N,0);
3381
3382      SDNode *GU = N->getGluedUser();
3383      SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped,
3384                                               Chain.getValue(1));
3385
3386      // Extract values from a GPRPair reg and copy to the original GPR reg.
3387      SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
3388                                                    RegCopy);
3389      SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
3390                                                    RegCopy);
3391      SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0,
3392                                        RegCopy.getValue(1));
3393      SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1));
3394
3395      // Update the original glue user.
3396      std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1);
3397      Ops.push_back(T1.getValue(1));
3398      CurDAG->UpdateNodeOperands(GU, Ops);
3399    }
3400    else {
3401      // For Kind  == InlineAsm::Kind_RegUse, we first copy two GPRs into a
3402      // GPRPair and then pass the GPRPair to the inline asm.
3403      SDValue Chain = AsmNodeOperands[InlineAsm::Op_InputChain];
3404
3405      // As REG_SEQ doesn't take RegisterSDNode, we copy them first.
3406      SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32,
3407                                          Chain.getValue(1));
3408      SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32,
3409                                          T0.getValue(1));
3410      SDValue Pair = SDValue(createGPRPairNode(MVT::Untyped, T0, T1), 0);
3411
3412      // Copy REG_SEQ into a GPRPair-typed VR and replace the original two
3413      // i32 VRs of inline asm with it.
3414      unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
3415      PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
3416      Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair, T1.getValue(1));
3417
3418      AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
3419      Glue = Chain.getValue(1);
3420    }
3421
3422    Changed = true;
3423
3424    if(PairedReg.getNode()) {
3425      OpChanged[OpChanged.size() -1 ] = true;
3426      Flag = InlineAsm::getFlagWord(Kind, 1 /* RegNum*/);
3427      if (IsTiedToChangedOp)
3428        Flag = InlineAsm::getFlagWordForMatchingOp(Flag, DefIdx);
3429      else
3430        Flag = InlineAsm::getFlagWordForRegClass(Flag, ARM::GPRPairRegClassID);
3431      // Replace the current flag.
3432      AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant(
3433          Flag, MVT::i32);
3434      // Add the new register node and skip the original two GPRs.
3435      AsmNodeOperands.push_back(PairedReg);
3436      // Skip the next two GPRs.
3437      i += 2;
3438    }
3439  }
3440
3441  if (Glue.getNode())
3442    AsmNodeOperands.push_back(Glue);
3443  if (!Changed)
3444    return nullptr;
3445
3446  SDValue New = CurDAG->getNode(ISD::INLINEASM, SDLoc(N),
3447      CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
3448  New->setNodeId(-1);
3449  return New.getNode();
3450}
3451
3452
3453bool ARMDAGToDAGISel::
3454SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
3455                             std::vector<SDValue> &OutOps) {
3456  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
3457  // Require the address to be in a register.  That is safe for all ARM
3458  // variants and it is hard to do anything much smarter without knowing
3459  // how the operand is used.
3460  OutOps.push_back(Op);
3461  return false;
3462}
3463
3464/// createARMISelDag - This pass converts a legalized DAG into a
3465/// ARM-specific DAG, ready for instruction scheduling.
3466///
3467FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
3468                                     CodeGenOpt::Level OptLevel) {
3469  return new ARMDAGToDAGISel(TM, OptLevel);
3470}
3471