1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#define DEBUG_TYPE "arm-isel"
15#include "ARM.h"
16#include "ARMBaseInstrInfo.h"
17#include "ARMTargetMachine.h"
18#include "MCTargetDesc/ARMAddressingModes.h"
19#include "llvm/CallingConv.h"
20#include "llvm/Constants.h"
21#include "llvm/DerivedTypes.h"
22#include "llvm/Function.h"
23#include "llvm/Intrinsics.h"
24#include "llvm/LLVMContext.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/SelectionDAG.h"
29#include "llvm/CodeGen/SelectionDAGISel.h"
30#include "llvm/Target/TargetLowering.h"
31#include "llvm/Target/TargetOptions.h"
32#include "llvm/Support/CommandLine.h"
33#include "llvm/Support/Compiler.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/raw_ostream.h"
37
38using namespace llvm;
39
40static cl::opt<bool>
41DisableShifterOp("disable-shifter-op", cl::Hidden,
42  cl::desc("Disable isel of shifter-op"),
43  cl::init(false));
44
45static cl::opt<bool>
46CheckVMLxHazard("check-vmlx-hazard", cl::Hidden,
47  cl::desc("Check fp vmla / vmls hazard at isel time"),
48  cl::init(true));
49
50//===--------------------------------------------------------------------===//
51/// ARMDAGToDAGISel - ARM specific code to select ARM machine
52/// instructions for SelectionDAG operations.
53///
54namespace {
55
56enum AddrMode2Type {
57  AM2_BASE, // Simple AM2 (+-imm12)
58  AM2_SHOP  // Shifter-op AM2
59};
60
61class ARMDAGToDAGISel : public SelectionDAGISel {
62  ARMBaseTargetMachine &TM;
63  const ARMBaseInstrInfo *TII;
64
65  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
66  /// make the right decision when generating code for different targets.
67  const ARMSubtarget *Subtarget;
68
69public:
70  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
71                           CodeGenOpt::Level OptLevel)
72    : SelectionDAGISel(tm, OptLevel), TM(tm),
73      TII(static_cast<const ARMBaseInstrInfo*>(TM.getInstrInfo())),
74      Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
75  }
76
77  virtual const char *getPassName() const {
78    return "ARM Instruction Selection";
79  }
80
81  /// getI32Imm - Return a target constant of type i32 with the specified
82  /// value.
83  inline SDValue getI32Imm(unsigned Imm) {
84    return CurDAG->getTargetConstant(Imm, MVT::i32);
85  }
86
87  SDNode *Select(SDNode *N);
88
89
90  bool hasNoVMLxHazardUse(SDNode *N) const;
91  bool isShifterOpProfitable(const SDValue &Shift,
92                             ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt);
93  bool SelectRegShifterOperand(SDValue N, SDValue &A,
94                               SDValue &B, SDValue &C,
95                               bool CheckProfitability = true);
96  bool SelectImmShifterOperand(SDValue N, SDValue &A,
97                               SDValue &B, bool CheckProfitability = true);
98  bool SelectShiftRegShifterOperand(SDValue N, SDValue &A,
99                                    SDValue &B, SDValue &C) {
100    // Don't apply the profitability check
101    return SelectRegShifterOperand(N, A, B, C, false);
102  }
103  bool SelectShiftImmShifterOperand(SDValue N, SDValue &A,
104                                    SDValue &B) {
105    // Don't apply the profitability check
106    return SelectImmShifterOperand(N, A, B, false);
107  }
108
109  bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
110  bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
111
112  AddrMode2Type SelectAddrMode2Worker(SDValue N, SDValue &Base,
113                                      SDValue &Offset, SDValue &Opc);
114  bool SelectAddrMode2Base(SDValue N, SDValue &Base, SDValue &Offset,
115                           SDValue &Opc) {
116    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_BASE;
117  }
118
119  bool SelectAddrMode2ShOp(SDValue N, SDValue &Base, SDValue &Offset,
120                           SDValue &Opc) {
121    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_SHOP;
122  }
123
124  bool SelectAddrMode2(SDValue N, SDValue &Base, SDValue &Offset,
125                       SDValue &Opc) {
126    SelectAddrMode2Worker(N, Base, Offset, Opc);
127//    return SelectAddrMode2ShOp(N, Base, Offset, Opc);
128    // This always matches one way or another.
129    return true;
130  }
131
132  bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
133                             SDValue &Offset, SDValue &Opc);
134  bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
135                             SDValue &Offset, SDValue &Opc);
136  bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
137                             SDValue &Offset, SDValue &Opc);
138  bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
139  bool SelectAddrMode3(SDValue N, SDValue &Base,
140                       SDValue &Offset, SDValue &Opc);
141  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
142                             SDValue &Offset, SDValue &Opc);
143  bool SelectAddrMode5(SDValue N, SDValue &Base,
144                       SDValue &Offset);
145  bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
146  bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
147
148  bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
149
150  // Thumb Addressing Modes:
151  bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
152  bool SelectThumbAddrModeRI(SDValue N, SDValue &Base, SDValue &Offset,
153                             unsigned Scale);
154  bool SelectThumbAddrModeRI5S1(SDValue N, SDValue &Base, SDValue &Offset);
155  bool SelectThumbAddrModeRI5S2(SDValue N, SDValue &Base, SDValue &Offset);
156  bool SelectThumbAddrModeRI5S4(SDValue N, SDValue &Base, SDValue &Offset);
157  bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
158                                SDValue &OffImm);
159  bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
160                                 SDValue &OffImm);
161  bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
162                                 SDValue &OffImm);
163  bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
164                                 SDValue &OffImm);
165  bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
166
167  // Thumb 2 Addressing Modes:
168  bool SelectT2ShifterOperandReg(SDValue N,
169                                 SDValue &BaseReg, SDValue &Opc);
170  bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
171  bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
172                            SDValue &OffImm);
173  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
174                                 SDValue &OffImm);
175  bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
176                             SDValue &OffReg, SDValue &ShImm);
177
178  inline bool is_so_imm(unsigned Imm) const {
179    return ARM_AM::getSOImmVal(Imm) != -1;
180  }
181
182  inline bool is_so_imm_not(unsigned Imm) const {
183    return ARM_AM::getSOImmVal(~Imm) != -1;
184  }
185
186  inline bool is_t2_so_imm(unsigned Imm) const {
187    return ARM_AM::getT2SOImmVal(Imm) != -1;
188  }
189
190  inline bool is_t2_so_imm_not(unsigned Imm) const {
191    return ARM_AM::getT2SOImmVal(~Imm) != -1;
192  }
193
194  // Include the pieces autogenerated from the target description.
195#include "ARMGenDAGISel.inc"
196
197private:
198  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
199  /// ARM.
200  SDNode *SelectARMIndexedLoad(SDNode *N);
201  SDNode *SelectT2IndexedLoad(SDNode *N);
202
203  /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
204  /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
205  /// loads of D registers and even subregs and odd subregs of Q registers.
206  /// For NumVecs <= 2, QOpcodes1 is not used.
207  SDNode *SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
208                    const uint16_t *DOpcodes,
209                    const uint16_t *QOpcodes0, const uint16_t *QOpcodes1);
210
211  /// SelectVST - Select NEON store intrinsics.  NumVecs should
212  /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
213  /// stores of D registers and even subregs and odd subregs of Q registers.
214  /// For NumVecs <= 2, QOpcodes1 is not used.
215  SDNode *SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
216                    const uint16_t *DOpcodes,
217                    const uint16_t *QOpcodes0, const uint16_t *QOpcodes1);
218
219  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
220  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
221  /// load/store of D registers and Q registers.
222  SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad,
223                          bool isUpdating, unsigned NumVecs,
224                          const uint16_t *DOpcodes, const uint16_t *QOpcodes);
225
226  /// SelectVLDDup - Select NEON load-duplicate intrinsics.  NumVecs
227  /// should be 2, 3 or 4.  The opcode array specifies the instructions used
228  /// for loading D registers.  (Q registers are not supported.)
229  SDNode *SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs,
230                       const uint16_t *Opcodes);
231
232  /// SelectVTBL - Select NEON VTBL and VTBX intrinsics.  NumVecs should be 2,
233  /// 3 or 4.  These are custom-selected so that a REG_SEQUENCE can be
234  /// generated to force the table registers to be consecutive.
235  SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc);
236
237  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
238  SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
239
240  /// SelectCMOVOp - Select CMOV instructions for ARM.
241  SDNode *SelectCMOVOp(SDNode *N);
242  SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
243                              ARMCC::CondCodes CCVal, SDValue CCR,
244                              SDValue InFlag);
245  SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
246                               ARMCC::CondCodes CCVal, SDValue CCR,
247                               SDValue InFlag);
248  SDNode *SelectT2CMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
249                              ARMCC::CondCodes CCVal, SDValue CCR,
250                              SDValue InFlag);
251  SDNode *SelectARMCMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
252                               ARMCC::CondCodes CCVal, SDValue CCR,
253                               SDValue InFlag);
254
255  // Select special operations if node forms integer ABS pattern
256  SDNode *SelectABSOp(SDNode *N);
257
258  SDNode *SelectConcatVector(SDNode *N);
259
260  SDNode *SelectAtomic64(SDNode *Node, unsigned Opc);
261
262  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
263  /// inline asm expressions.
264  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
265                                            char ConstraintCode,
266                                            std::vector<SDValue> &OutOps);
267
268  // Form pairs of consecutive S, D, or Q registers.
269  SDNode *PairSRegs(EVT VT, SDValue V0, SDValue V1);
270  SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
271  SDNode *PairQRegs(EVT VT, SDValue V0, SDValue V1);
272
273  // Form sequences of 4 consecutive S, D, or Q registers.
274  SDNode *QuadSRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
275  SDNode *QuadDRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
276  SDNode *QuadQRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
277
278  // Get the alignment operand for a NEON VLD or VST instruction.
279  SDValue GetVLDSTAlign(SDValue Align, unsigned NumVecs, bool is64BitVector);
280};
281}
282
283/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
284/// operand. If so Imm will receive the 32-bit value.
285static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
286  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
287    Imm = cast<ConstantSDNode>(N)->getZExtValue();
288    return true;
289  }
290  return false;
291}
292
293// isInt32Immediate - This method tests to see if a constant operand.
294// If so Imm will receive the 32 bit value.
295static bool isInt32Immediate(SDValue N, unsigned &Imm) {
296  return isInt32Immediate(N.getNode(), Imm);
297}
298
299// isOpcWithIntImmediate - This method tests to see if the node is a specific
300// opcode and that it has a immediate integer right operand.
301// If so Imm will receive the 32 bit value.
302static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
303  return N->getOpcode() == Opc &&
304         isInt32Immediate(N->getOperand(1).getNode(), Imm);
305}
306
307/// \brief Check whether a particular node is a constant value representable as
308/// (N * Scale) where (N in [\p RangeMin, \p RangeMax).
309///
310/// \param ScaledConstant [out] - On success, the pre-scaled constant value.
311static bool isScaledConstantInRange(SDValue Node, int Scale,
312                                    int RangeMin, int RangeMax,
313                                    int &ScaledConstant) {
314  assert(Scale > 0 && "Invalid scale!");
315
316  // Check that this is a constant.
317  const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node);
318  if (!C)
319    return false;
320
321  ScaledConstant = (int) C->getZExtValue();
322  if ((ScaledConstant % Scale) != 0)
323    return false;
324
325  ScaledConstant /= Scale;
326  return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
327}
328
329/// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS
330/// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at
331/// least on current ARM implementations) which should be avoidded.
332bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
333  if (OptLevel == CodeGenOpt::None)
334    return true;
335
336  if (!CheckVMLxHazard)
337    return true;
338
339  if (!Subtarget->isCortexA7() && !Subtarget->isCortexA8() &&
340      !Subtarget->isLikeA9() && !Subtarget->isSwift())
341    return true;
342
343  if (!N->hasOneUse())
344    return false;
345
346  SDNode *Use = *N->use_begin();
347  if (Use->getOpcode() == ISD::CopyToReg)
348    return true;
349  if (Use->isMachineOpcode()) {
350    const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode());
351    if (MCID.mayStore())
352      return true;
353    unsigned Opcode = MCID.getOpcode();
354    if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
355      return true;
356    // vmlx feeding into another vmlx. We actually want to unfold
357    // the use later in the MLxExpansion pass. e.g.
358    // vmla
359    // vmla (stall 8 cycles)
360    //
361    // vmul (5 cycles)
362    // vadd (5 cycles)
363    // vmla
364    // This adds up to about 18 - 19 cycles.
365    //
366    // vmla
367    // vmul (stall 4 cycles)
368    // vadd adds up to about 14 cycles.
369    return TII->isFpMLxInstruction(Opcode);
370  }
371
372  return false;
373}
374
375bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
376                                            ARM_AM::ShiftOpc ShOpcVal,
377                                            unsigned ShAmt) {
378  if (!Subtarget->isLikeA9() && !Subtarget->isSwift())
379    return true;
380  if (Shift.hasOneUse())
381    return true;
382  // R << 2 is free.
383  return ShOpcVal == ARM_AM::lsl &&
384         (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1));
385}
386
387bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
388                                              SDValue &BaseReg,
389                                              SDValue &Opc,
390                                              bool CheckProfitability) {
391  if (DisableShifterOp)
392    return false;
393
394  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
395
396  // Don't match base register only case. That is matched to a separate
397  // lower complexity pattern with explicit register operand.
398  if (ShOpcVal == ARM_AM::no_shift) return false;
399
400  BaseReg = N.getOperand(0);
401  unsigned ShImmVal = 0;
402  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
403  if (!RHS) return false;
404  ShImmVal = RHS->getZExtValue() & 31;
405  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
406                                  MVT::i32);
407  return true;
408}
409
410bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
411                                              SDValue &BaseReg,
412                                              SDValue &ShReg,
413                                              SDValue &Opc,
414                                              bool CheckProfitability) {
415  if (DisableShifterOp)
416    return false;
417
418  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
419
420  // Don't match base register only case. That is matched to a separate
421  // lower complexity pattern with explicit register operand.
422  if (ShOpcVal == ARM_AM::no_shift) return false;
423
424  BaseReg = N.getOperand(0);
425  unsigned ShImmVal = 0;
426  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
427  if (RHS) return false;
428
429  ShReg = N.getOperand(1);
430  if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
431    return false;
432  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
433                                  MVT::i32);
434  return true;
435}
436
437
438bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
439                                          SDValue &Base,
440                                          SDValue &OffImm) {
441  // Match simple R + imm12 operands.
442
443  // Base only.
444  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
445      !CurDAG->isBaseWithConstantOffset(N)) {
446    if (N.getOpcode() == ISD::FrameIndex) {
447      // Match frame index.
448      int FI = cast<FrameIndexSDNode>(N)->getIndex();
449      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
450      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
451      return true;
452    }
453
454    if (N.getOpcode() == ARMISD::Wrapper &&
455        !(Subtarget->useMovt() &&
456                     N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
457      Base = N.getOperand(0);
458    } else
459      Base = N;
460    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
461    return true;
462  }
463
464  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
465    int RHSC = (int)RHS->getZExtValue();
466    if (N.getOpcode() == ISD::SUB)
467      RHSC = -RHSC;
468
469    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
470      Base   = N.getOperand(0);
471      if (Base.getOpcode() == ISD::FrameIndex) {
472        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
473        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
474      }
475      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
476      return true;
477    }
478  }
479
480  // Base only.
481  Base = N;
482  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
483  return true;
484}
485
486
487
488bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
489                                      SDValue &Opc) {
490  if (N.getOpcode() == ISD::MUL &&
491      ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) {
492    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
493      // X * [3,5,9] -> X + X * [2,4,8] etc.
494      int RHSC = (int)RHS->getZExtValue();
495      if (RHSC & 1) {
496        RHSC = RHSC & ~1;
497        ARM_AM::AddrOpc AddSub = ARM_AM::add;
498        if (RHSC < 0) {
499          AddSub = ARM_AM::sub;
500          RHSC = - RHSC;
501        }
502        if (isPowerOf2_32(RHSC)) {
503          unsigned ShAmt = Log2_32(RHSC);
504          Base = Offset = N.getOperand(0);
505          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
506                                                            ARM_AM::lsl),
507                                          MVT::i32);
508          return true;
509        }
510      }
511    }
512  }
513
514  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
515      // ISD::OR that is equivalent to an ISD::ADD.
516      !CurDAG->isBaseWithConstantOffset(N))
517    return false;
518
519  // Leave simple R +/- imm12 operands for LDRi12
520  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) {
521    int RHSC;
522    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
523                                -0x1000+1, 0x1000, RHSC)) // 12 bits.
524      return false;
525  }
526
527  // Otherwise this is R +/- [possibly shifted] R.
528  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add;
529  ARM_AM::ShiftOpc ShOpcVal =
530    ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
531  unsigned ShAmt = 0;
532
533  Base   = N.getOperand(0);
534  Offset = N.getOperand(1);
535
536  if (ShOpcVal != ARM_AM::no_shift) {
537    // Check to see if the RHS of the shift is a constant, if not, we can't fold
538    // it.
539    if (ConstantSDNode *Sh =
540           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
541      ShAmt = Sh->getZExtValue();
542      if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
543        Offset = N.getOperand(1).getOperand(0);
544      else {
545        ShAmt = 0;
546        ShOpcVal = ARM_AM::no_shift;
547      }
548    } else {
549      ShOpcVal = ARM_AM::no_shift;
550    }
551  }
552
553  // Try matching (R shl C) + (R).
554  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
555      !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
556        N.getOperand(0).hasOneUse())) {
557    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
558    if (ShOpcVal != ARM_AM::no_shift) {
559      // Check to see if the RHS of the shift is a constant, if not, we can't
560      // fold it.
561      if (ConstantSDNode *Sh =
562          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
563        ShAmt = Sh->getZExtValue();
564        if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
565          Offset = N.getOperand(0).getOperand(0);
566          Base = N.getOperand(1);
567        } else {
568          ShAmt = 0;
569          ShOpcVal = ARM_AM::no_shift;
570        }
571      } else {
572        ShOpcVal = ARM_AM::no_shift;
573      }
574    }
575  }
576
577  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
578                                  MVT::i32);
579  return true;
580}
581
582
583//-----
584
585AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N,
586                                                     SDValue &Base,
587                                                     SDValue &Offset,
588                                                     SDValue &Opc) {
589  if (N.getOpcode() == ISD::MUL &&
590      (!(Subtarget->isLikeA9() || Subtarget->isSwift()) || N.hasOneUse())) {
591    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
592      // X * [3,5,9] -> X + X * [2,4,8] etc.
593      int RHSC = (int)RHS->getZExtValue();
594      if (RHSC & 1) {
595        RHSC = RHSC & ~1;
596        ARM_AM::AddrOpc AddSub = ARM_AM::add;
597        if (RHSC < 0) {
598          AddSub = ARM_AM::sub;
599          RHSC = - RHSC;
600        }
601        if (isPowerOf2_32(RHSC)) {
602          unsigned ShAmt = Log2_32(RHSC);
603          Base = Offset = N.getOperand(0);
604          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
605                                                            ARM_AM::lsl),
606                                          MVT::i32);
607          return AM2_SHOP;
608        }
609      }
610    }
611  }
612
613  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
614      // ISD::OR that is equivalent to an ADD.
615      !CurDAG->isBaseWithConstantOffset(N)) {
616    Base = N;
617    if (N.getOpcode() == ISD::FrameIndex) {
618      int FI = cast<FrameIndexSDNode>(N)->getIndex();
619      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
620    } else if (N.getOpcode() == ARMISD::Wrapper &&
621               !(Subtarget->useMovt() &&
622                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
623      Base = N.getOperand(0);
624    }
625    Offset = CurDAG->getRegister(0, MVT::i32);
626    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
627                                                      ARM_AM::no_shift),
628                                    MVT::i32);
629    return AM2_BASE;
630  }
631
632  // Match simple R +/- imm12 operands.
633  if (N.getOpcode() != ISD::SUB) {
634    int RHSC;
635    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
636                                -0x1000+1, 0x1000, RHSC)) { // 12 bits.
637      Base = N.getOperand(0);
638      if (Base.getOpcode() == ISD::FrameIndex) {
639        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
640        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
641      }
642      Offset = CurDAG->getRegister(0, MVT::i32);
643
644      ARM_AM::AddrOpc AddSub = ARM_AM::add;
645      if (RHSC < 0) {
646        AddSub = ARM_AM::sub;
647        RHSC = - RHSC;
648      }
649      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
650                                                        ARM_AM::no_shift),
651                                      MVT::i32);
652      return AM2_BASE;
653    }
654  }
655
656  if ((Subtarget->isLikeA9() || Subtarget->isSwift()) && !N.hasOneUse()) {
657    // Compute R +/- (R << N) and reuse it.
658    Base = N;
659    Offset = CurDAG->getRegister(0, MVT::i32);
660    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
661                                                      ARM_AM::no_shift),
662                                    MVT::i32);
663    return AM2_BASE;
664  }
665
666  // Otherwise this is R +/- [possibly shifted] R.
667  ARM_AM::AddrOpc AddSub = N.getOpcode() != ISD::SUB ? ARM_AM::add:ARM_AM::sub;
668  ARM_AM::ShiftOpc ShOpcVal =
669    ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
670  unsigned ShAmt = 0;
671
672  Base   = N.getOperand(0);
673  Offset = N.getOperand(1);
674
675  if (ShOpcVal != ARM_AM::no_shift) {
676    // Check to see if the RHS of the shift is a constant, if not, we can't fold
677    // it.
678    if (ConstantSDNode *Sh =
679           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
680      ShAmt = Sh->getZExtValue();
681      if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
682        Offset = N.getOperand(1).getOperand(0);
683      else {
684        ShAmt = 0;
685        ShOpcVal = ARM_AM::no_shift;
686      }
687    } else {
688      ShOpcVal = ARM_AM::no_shift;
689    }
690  }
691
692  // Try matching (R shl C) + (R).
693  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
694      !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
695        N.getOperand(0).hasOneUse())) {
696    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
697    if (ShOpcVal != ARM_AM::no_shift) {
698      // Check to see if the RHS of the shift is a constant, if not, we can't
699      // fold it.
700      if (ConstantSDNode *Sh =
701          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
702        ShAmt = Sh->getZExtValue();
703        if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
704          Offset = N.getOperand(0).getOperand(0);
705          Base = N.getOperand(1);
706        } else {
707          ShAmt = 0;
708          ShOpcVal = ARM_AM::no_shift;
709        }
710      } else {
711        ShOpcVal = ARM_AM::no_shift;
712      }
713    }
714  }
715
716  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
717                                  MVT::i32);
718  return AM2_SHOP;
719}
720
721bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
722                                            SDValue &Offset, SDValue &Opc) {
723  unsigned Opcode = Op->getOpcode();
724  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
725    ? cast<LoadSDNode>(Op)->getAddressingMode()
726    : cast<StoreSDNode>(Op)->getAddressingMode();
727  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
728    ? ARM_AM::add : ARM_AM::sub;
729  int Val;
730  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val))
731    return false;
732
733  Offset = N;
734  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
735  unsigned ShAmt = 0;
736  if (ShOpcVal != ARM_AM::no_shift) {
737    // Check to see if the RHS of the shift is a constant, if not, we can't fold
738    // it.
739    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
740      ShAmt = Sh->getZExtValue();
741      if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
742        Offset = N.getOperand(0);
743      else {
744        ShAmt = 0;
745        ShOpcVal = ARM_AM::no_shift;
746      }
747    } else {
748      ShOpcVal = ARM_AM::no_shift;
749    }
750  }
751
752  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
753                                  MVT::i32);
754  return true;
755}
756
757bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
758                                            SDValue &Offset, SDValue &Opc) {
759  unsigned Opcode = Op->getOpcode();
760  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
761    ? cast<LoadSDNode>(Op)->getAddressingMode()
762    : cast<StoreSDNode>(Op)->getAddressingMode();
763  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
764    ? ARM_AM::add : ARM_AM::sub;
765  int Val;
766  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
767    if (AddSub == ARM_AM::sub) Val *= -1;
768    Offset = CurDAG->getRegister(0, MVT::i32);
769    Opc = CurDAG->getTargetConstant(Val, MVT::i32);
770    return true;
771  }
772
773  return false;
774}
775
776
777bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
778                                            SDValue &Offset, SDValue &Opc) {
779  unsigned Opcode = Op->getOpcode();
780  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
781    ? cast<LoadSDNode>(Op)->getAddressingMode()
782    : cast<StoreSDNode>(Op)->getAddressingMode();
783  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
784    ? ARM_AM::add : ARM_AM::sub;
785  int Val;
786  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
787    Offset = CurDAG->getRegister(0, MVT::i32);
788    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
789                                                      ARM_AM::no_shift),
790                                    MVT::i32);
791    return true;
792  }
793
794  return false;
795}
796
797bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
798  Base = N;
799  return true;
800}
801
802bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
803                                      SDValue &Base, SDValue &Offset,
804                                      SDValue &Opc) {
805  if (N.getOpcode() == ISD::SUB) {
806    // X - C  is canonicalize to X + -C, no need to handle it here.
807    Base = N.getOperand(0);
808    Offset = N.getOperand(1);
809    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
810    return true;
811  }
812
813  if (!CurDAG->isBaseWithConstantOffset(N)) {
814    Base = N;
815    if (N.getOpcode() == ISD::FrameIndex) {
816      int FI = cast<FrameIndexSDNode>(N)->getIndex();
817      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
818    }
819    Offset = CurDAG->getRegister(0, MVT::i32);
820    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
821    return true;
822  }
823
824  // If the RHS is +/- imm8, fold into addr mode.
825  int RHSC;
826  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
827                              -256 + 1, 256, RHSC)) { // 8 bits.
828    Base = N.getOperand(0);
829    if (Base.getOpcode() == ISD::FrameIndex) {
830      int FI = cast<FrameIndexSDNode>(Base)->getIndex();
831      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
832    }
833    Offset = CurDAG->getRegister(0, MVT::i32);
834
835    ARM_AM::AddrOpc AddSub = ARM_AM::add;
836    if (RHSC < 0) {
837      AddSub = ARM_AM::sub;
838      RHSC = -RHSC;
839    }
840    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
841    return true;
842  }
843
844  Base = N.getOperand(0);
845  Offset = N.getOperand(1);
846  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
847  return true;
848}
849
850bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
851                                            SDValue &Offset, SDValue &Opc) {
852  unsigned Opcode = Op->getOpcode();
853  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
854    ? cast<LoadSDNode>(Op)->getAddressingMode()
855    : cast<StoreSDNode>(Op)->getAddressingMode();
856  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
857    ? ARM_AM::add : ARM_AM::sub;
858  int Val;
859  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits.
860    Offset = CurDAG->getRegister(0, MVT::i32);
861    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
862    return true;
863  }
864
865  Offset = N;
866  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
867  return true;
868}
869
870bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
871                                      SDValue &Base, SDValue &Offset) {
872  if (!CurDAG->isBaseWithConstantOffset(N)) {
873    Base = N;
874    if (N.getOpcode() == ISD::FrameIndex) {
875      int FI = cast<FrameIndexSDNode>(N)->getIndex();
876      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
877    } else if (N.getOpcode() == ARMISD::Wrapper &&
878               !(Subtarget->useMovt() &&
879                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
880      Base = N.getOperand(0);
881    }
882    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
883                                       MVT::i32);
884    return true;
885  }
886
887  // If the RHS is +/- imm8, fold into addr mode.
888  int RHSC;
889  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4,
890                              -256 + 1, 256, RHSC)) {
891    Base = N.getOperand(0);
892    if (Base.getOpcode() == ISD::FrameIndex) {
893      int FI = cast<FrameIndexSDNode>(Base)->getIndex();
894      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
895    }
896
897    ARM_AM::AddrOpc AddSub = ARM_AM::add;
898    if (RHSC < 0) {
899      AddSub = ARM_AM::sub;
900      RHSC = -RHSC;
901    }
902    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
903                                       MVT::i32);
904    return true;
905  }
906
907  Base = N;
908  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
909                                     MVT::i32);
910  return true;
911}
912
913bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
914                                      SDValue &Align) {
915  Addr = N;
916
917  unsigned Alignment = 0;
918  if (LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(Parent)) {
919    // This case occurs only for VLD1-lane/dup and VST1-lane instructions.
920    // The maximum alignment is equal to the memory size being referenced.
921    unsigned LSNAlign = LSN->getAlignment();
922    unsigned MemSize = LSN->getMemoryVT().getSizeInBits() / 8;
923    if (LSNAlign >= MemSize && MemSize > 1)
924      Alignment = MemSize;
925  } else {
926    // All other uses of addrmode6 are for intrinsics.  For now just record
927    // the raw alignment value; it will be refined later based on the legal
928    // alignment operands for the intrinsic.
929    Alignment = cast<MemIntrinsicSDNode>(Parent)->getAlignment();
930  }
931
932  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
933  return true;
934}
935
936bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N,
937                                            SDValue &Offset) {
938  LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op);
939  ISD::MemIndexedMode AM = LdSt->getAddressingMode();
940  if (AM != ISD::POST_INC)
941    return false;
942  Offset = N;
943  if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) {
944    if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits())
945      Offset = CurDAG->getRegister(0, MVT::i32);
946  }
947  return true;
948}
949
950bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
951                                       SDValue &Offset, SDValue &Label) {
952  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
953    Offset = N.getOperand(0);
954    SDValue N1 = N.getOperand(1);
955    Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
956                                      MVT::i32);
957    return true;
958  }
959
960  return false;
961}
962
963
964//===----------------------------------------------------------------------===//
965//                         Thumb Addressing Modes
966//===----------------------------------------------------------------------===//
967
968bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N,
969                                            SDValue &Base, SDValue &Offset){
970  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) {
971    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
972    if (!NC || !NC->isNullValue())
973      return false;
974
975    Base = Offset = N;
976    return true;
977  }
978
979  Base = N.getOperand(0);
980  Offset = N.getOperand(1);
981  return true;
982}
983
984bool
985ARMDAGToDAGISel::SelectThumbAddrModeRI(SDValue N, SDValue &Base,
986                                       SDValue &Offset, unsigned Scale) {
987  if (Scale == 4) {
988    SDValue TmpBase, TmpOffImm;
989    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
990      return false;  // We want to select tLDRspi / tSTRspi instead.
991
992    if (N.getOpcode() == ARMISD::Wrapper &&
993        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
994      return false;  // We want to select tLDRpci instead.
995  }
996
997  if (!CurDAG->isBaseWithConstantOffset(N))
998    return false;
999
1000  // Thumb does not have [sp, r] address mode.
1001  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1002  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
1003  if ((LHSR && LHSR->getReg() == ARM::SP) ||
1004      (RHSR && RHSR->getReg() == ARM::SP))
1005    return false;
1006
1007  // FIXME: Why do we explicitly check for a match here and then return false?
1008  // Presumably to allow something else to match, but shouldn't this be
1009  // documented?
1010  int RHSC;
1011  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC))
1012    return false;
1013
1014  Base = N.getOperand(0);
1015  Offset = N.getOperand(1);
1016  return true;
1017}
1018
1019bool
1020ARMDAGToDAGISel::SelectThumbAddrModeRI5S1(SDValue N,
1021                                          SDValue &Base,
1022                                          SDValue &Offset) {
1023  return SelectThumbAddrModeRI(N, Base, Offset, 1);
1024}
1025
1026bool
1027ARMDAGToDAGISel::SelectThumbAddrModeRI5S2(SDValue N,
1028                                          SDValue &Base,
1029                                          SDValue &Offset) {
1030  return SelectThumbAddrModeRI(N, Base, Offset, 2);
1031}
1032
1033bool
1034ARMDAGToDAGISel::SelectThumbAddrModeRI5S4(SDValue N,
1035                                          SDValue &Base,
1036                                          SDValue &Offset) {
1037  return SelectThumbAddrModeRI(N, Base, Offset, 4);
1038}
1039
1040bool
1041ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
1042                                          SDValue &Base, SDValue &OffImm) {
1043  if (Scale == 4) {
1044    SDValue TmpBase, TmpOffImm;
1045    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
1046      return false;  // We want to select tLDRspi / tSTRspi instead.
1047
1048    if (N.getOpcode() == ARMISD::Wrapper &&
1049        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
1050      return false;  // We want to select tLDRpci instead.
1051  }
1052
1053  if (!CurDAG->isBaseWithConstantOffset(N)) {
1054    if (N.getOpcode() == ARMISD::Wrapper &&
1055        !(Subtarget->useMovt() &&
1056          N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
1057      Base = N.getOperand(0);
1058    } else {
1059      Base = N;
1060    }
1061
1062    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1063    return true;
1064  }
1065
1066  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1067  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
1068  if ((LHSR && LHSR->getReg() == ARM::SP) ||
1069      (RHSR && RHSR->getReg() == ARM::SP)) {
1070    ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(N.getOperand(0));
1071    ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
1072    unsigned LHSC = LHS ? LHS->getZExtValue() : 0;
1073    unsigned RHSC = RHS ? RHS->getZExtValue() : 0;
1074
1075    // Thumb does not have [sp, #imm5] address mode for non-zero imm5.
1076    if (LHSC != 0 || RHSC != 0) return false;
1077
1078    Base = N;
1079    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1080    return true;
1081  }
1082
1083  // If the RHS is + imm5 * scale, fold into addr mode.
1084  int RHSC;
1085  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
1086    Base = N.getOperand(0);
1087    OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1088    return true;
1089  }
1090
1091  Base = N.getOperand(0);
1092  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1093  return true;
1094}
1095
1096bool
1097ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
1098                                           SDValue &OffImm) {
1099  return SelectThumbAddrModeImm5S(N, 4, Base, OffImm);
1100}
1101
1102bool
1103ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
1104                                           SDValue &OffImm) {
1105  return SelectThumbAddrModeImm5S(N, 2, Base, OffImm);
1106}
1107
1108bool
1109ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
1110                                           SDValue &OffImm) {
1111  return SelectThumbAddrModeImm5S(N, 1, Base, OffImm);
1112}
1113
1114bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
1115                                            SDValue &Base, SDValue &OffImm) {
1116  if (N.getOpcode() == ISD::FrameIndex) {
1117    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1118    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1119    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1120    return true;
1121  }
1122
1123  if (!CurDAG->isBaseWithConstantOffset(N))
1124    return false;
1125
1126  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1127  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
1128      (LHSR && LHSR->getReg() == ARM::SP)) {
1129    // If the RHS is + imm8 * scale, fold into addr mode.
1130    int RHSC;
1131    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) {
1132      Base = N.getOperand(0);
1133      if (Base.getOpcode() == ISD::FrameIndex) {
1134        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1135        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1136      }
1137      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1138      return true;
1139    }
1140  }
1141
1142  return false;
1143}
1144
1145
1146//===----------------------------------------------------------------------===//
1147//                        Thumb 2 Addressing Modes
1148//===----------------------------------------------------------------------===//
1149
1150
1151bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg,
1152                                                SDValue &Opc) {
1153  if (DisableShifterOp)
1154    return false;
1155
1156  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
1157
1158  // Don't match base register only case. That is matched to a separate
1159  // lower complexity pattern with explicit register operand.
1160  if (ShOpcVal == ARM_AM::no_shift) return false;
1161
1162  BaseReg = N.getOperand(0);
1163  unsigned ShImmVal = 0;
1164  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1165    ShImmVal = RHS->getZExtValue() & 31;
1166    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
1167    return true;
1168  }
1169
1170  return false;
1171}
1172
1173bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
1174                                            SDValue &Base, SDValue &OffImm) {
1175  // Match simple R + imm12 operands.
1176
1177  // Base only.
1178  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1179      !CurDAG->isBaseWithConstantOffset(N)) {
1180    if (N.getOpcode() == ISD::FrameIndex) {
1181      // Match frame index.
1182      int FI = cast<FrameIndexSDNode>(N)->getIndex();
1183      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1184      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1185      return true;
1186    }
1187
1188    if (N.getOpcode() == ARMISD::Wrapper &&
1189               !(Subtarget->useMovt() &&
1190                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
1191      Base = N.getOperand(0);
1192      if (Base.getOpcode() == ISD::TargetConstantPool)
1193        return false;  // We want to select t2LDRpci instead.
1194    } else
1195      Base = N;
1196    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1197    return true;
1198  }
1199
1200  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1201    if (SelectT2AddrModeImm8(N, Base, OffImm))
1202      // Let t2LDRi8 handle (R - imm8).
1203      return false;
1204
1205    int RHSC = (int)RHS->getZExtValue();
1206    if (N.getOpcode() == ISD::SUB)
1207      RHSC = -RHSC;
1208
1209    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
1210      Base   = N.getOperand(0);
1211      if (Base.getOpcode() == ISD::FrameIndex) {
1212        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1213        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1214      }
1215      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1216      return true;
1217    }
1218  }
1219
1220  // Base only.
1221  Base = N;
1222  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1223  return true;
1224}
1225
1226bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
1227                                           SDValue &Base, SDValue &OffImm) {
1228  // Match simple R - imm8 operands.
1229  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1230      !CurDAG->isBaseWithConstantOffset(N))
1231    return false;
1232
1233  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1234    int RHSC = (int)RHS->getSExtValue();
1235    if (N.getOpcode() == ISD::SUB)
1236      RHSC = -RHSC;
1237
1238    if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
1239      Base = N.getOperand(0);
1240      if (Base.getOpcode() == ISD::FrameIndex) {
1241        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1242        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1243      }
1244      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1245      return true;
1246    }
1247  }
1248
1249  return false;
1250}
1251
1252bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
1253                                                 SDValue &OffImm){
1254  unsigned Opcode = Op->getOpcode();
1255  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
1256    ? cast<LoadSDNode>(Op)->getAddressingMode()
1257    : cast<StoreSDNode>(Op)->getAddressingMode();
1258  int RHSC;
1259  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
1260    OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
1261      ? CurDAG->getTargetConstant(RHSC, MVT::i32)
1262      : CurDAG->getTargetConstant(-RHSC, MVT::i32);
1263    return true;
1264  }
1265
1266  return false;
1267}
1268
1269bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
1270                                            SDValue &Base,
1271                                            SDValue &OffReg, SDValue &ShImm) {
1272  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
1273  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N))
1274    return false;
1275
1276  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
1277  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1278    int RHSC = (int)RHS->getZExtValue();
1279    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
1280      return false;
1281    else if (RHSC < 0 && RHSC >= -255) // 8 bits
1282      return false;
1283  }
1284
1285  // Look for (R + R) or (R + (R << [1,2,3])).
1286  unsigned ShAmt = 0;
1287  Base   = N.getOperand(0);
1288  OffReg = N.getOperand(1);
1289
1290  // Swap if it is ((R << c) + R).
1291  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode());
1292  if (ShOpcVal != ARM_AM::lsl) {
1293    ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode());
1294    if (ShOpcVal == ARM_AM::lsl)
1295      std::swap(Base, OffReg);
1296  }
1297
1298  if (ShOpcVal == ARM_AM::lsl) {
1299    // Check to see if the RHS of the shift is a constant, if not, we can't fold
1300    // it.
1301    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
1302      ShAmt = Sh->getZExtValue();
1303      if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1304        OffReg = OffReg.getOperand(0);
1305      else {
1306        ShAmt = 0;
1307        ShOpcVal = ARM_AM::no_shift;
1308      }
1309    } else {
1310      ShOpcVal = ARM_AM::no_shift;
1311    }
1312  }
1313
1314  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
1315
1316  return true;
1317}
1318
1319//===--------------------------------------------------------------------===//
1320
1321/// getAL - Returns a ARMCC::AL immediate node.
1322static inline SDValue getAL(SelectionDAG *CurDAG) {
1323  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
1324}
1325
1326SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
1327  LoadSDNode *LD = cast<LoadSDNode>(N);
1328  ISD::MemIndexedMode AM = LD->getAddressingMode();
1329  if (AM == ISD::UNINDEXED)
1330    return NULL;
1331
1332  EVT LoadedVT = LD->getMemoryVT();
1333  SDValue Offset, AMOpc;
1334  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1335  unsigned Opcode = 0;
1336  bool Match = false;
1337  if (LoadedVT == MVT::i32 && isPre &&
1338      SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1339    Opcode = ARM::LDR_PRE_IMM;
1340    Match = true;
1341  } else if (LoadedVT == MVT::i32 && !isPre &&
1342      SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1343    Opcode = ARM::LDR_POST_IMM;
1344    Match = true;
1345  } else if (LoadedVT == MVT::i32 &&
1346      SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1347    Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1348    Match = true;
1349
1350  } else if (LoadedVT == MVT::i16 &&
1351             SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1352    Match = true;
1353    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
1354      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1355      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1356  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1357    if (LD->getExtensionType() == ISD::SEXTLOAD) {
1358      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1359        Match = true;
1360        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1361      }
1362    } else {
1363      if (isPre &&
1364          SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1365        Match = true;
1366        Opcode = ARM::LDRB_PRE_IMM;
1367      } else if (!isPre &&
1368                  SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1369        Match = true;
1370        Opcode = ARM::LDRB_POST_IMM;
1371      } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1372        Match = true;
1373        Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1374      }
1375    }
1376  }
1377
1378  if (Match) {
1379    if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1380      SDValue Chain = LD->getChain();
1381      SDValue Base = LD->getBasePtr();
1382      SDValue Ops[]= { Base, AMOpc, getAL(CurDAG),
1383                       CurDAG->getRegister(0, MVT::i32), Chain };
1384      return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32,
1385                                    MVT::i32, MVT::Other, Ops, 5);
1386    } else {
1387      SDValue Chain = LD->getChain();
1388      SDValue Base = LD->getBasePtr();
1389      SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
1390                       CurDAG->getRegister(0, MVT::i32), Chain };
1391      return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32,
1392                                    MVT::i32, MVT::Other, Ops, 6);
1393    }
1394  }
1395
1396  return NULL;
1397}
1398
1399SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
1400  LoadSDNode *LD = cast<LoadSDNode>(N);
1401  ISD::MemIndexedMode AM = LD->getAddressingMode();
1402  if (AM == ISD::UNINDEXED)
1403    return NULL;
1404
1405  EVT LoadedVT = LD->getMemoryVT();
1406  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1407  SDValue Offset;
1408  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1409  unsigned Opcode = 0;
1410  bool Match = false;
1411  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
1412    switch (LoadedVT.getSimpleVT().SimpleTy) {
1413    case MVT::i32:
1414      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1415      break;
1416    case MVT::i16:
1417      if (isSExtLd)
1418        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1419      else
1420        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1421      break;
1422    case MVT::i8:
1423    case MVT::i1:
1424      if (isSExtLd)
1425        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1426      else
1427        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1428      break;
1429    default:
1430      return NULL;
1431    }
1432    Match = true;
1433  }
1434
1435  if (Match) {
1436    SDValue Chain = LD->getChain();
1437    SDValue Base = LD->getBasePtr();
1438    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
1439                     CurDAG->getRegister(0, MVT::i32), Chain };
1440    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
1441                                  MVT::Other, Ops, 5);
1442  }
1443
1444  return NULL;
1445}
1446
1447/// PairSRegs - Form a D register from a pair of S registers.
1448///
1449SDNode *ARMDAGToDAGISel::PairSRegs(EVT VT, SDValue V0, SDValue V1) {
1450  DebugLoc dl = V0.getNode()->getDebugLoc();
1451  SDValue RegClass =
1452    CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, MVT::i32);
1453  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1454  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1455  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1456  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5);
1457}
1458
1459/// PairDRegs - Form a quad register from a pair of D registers.
1460///
1461SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
1462  DebugLoc dl = V0.getNode()->getDebugLoc();
1463  SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, MVT::i32);
1464  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1465  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1466  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1467  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5);
1468}
1469
1470/// PairQRegs - Form 4 consecutive D registers from a pair of Q registers.
1471///
1472SDNode *ARMDAGToDAGISel::PairQRegs(EVT VT, SDValue V0, SDValue V1) {
1473  DebugLoc dl = V0.getNode()->getDebugLoc();
1474  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
1475  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1476  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1477  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1478  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5);
1479}
1480
1481/// QuadSRegs - Form 4 consecutive S registers.
1482///
1483SDNode *ARMDAGToDAGISel::QuadSRegs(EVT VT, SDValue V0, SDValue V1,
1484                                   SDValue V2, SDValue V3) {
1485  DebugLoc dl = V0.getNode()->getDebugLoc();
1486  SDValue RegClass =
1487    CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, MVT::i32);
1488  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1489  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1490  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32);
1491  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32);
1492  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1493                                    V2, SubReg2, V3, SubReg3 };
1494  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9);
1495}
1496
1497/// QuadDRegs - Form 4 consecutive D registers.
1498///
1499SDNode *ARMDAGToDAGISel::QuadDRegs(EVT VT, SDValue V0, SDValue V1,
1500                                   SDValue V2, SDValue V3) {
1501  DebugLoc dl = V0.getNode()->getDebugLoc();
1502  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
1503  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1504  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1505  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32);
1506  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32);
1507  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1508                                    V2, SubReg2, V3, SubReg3 };
1509  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9);
1510}
1511
1512/// QuadQRegs - Form 4 consecutive Q registers.
1513///
1514SDNode *ARMDAGToDAGISel::QuadQRegs(EVT VT, SDValue V0, SDValue V1,
1515                                   SDValue V2, SDValue V3) {
1516  DebugLoc dl = V0.getNode()->getDebugLoc();
1517  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, MVT::i32);
1518  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1519  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1520  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32);
1521  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32);
1522  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1523                                    V2, SubReg2, V3, SubReg3 };
1524  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9);
1525}
1526
1527/// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
1528/// of a NEON VLD or VST instruction.  The supported values depend on the
1529/// number of registers being loaded.
1530SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, unsigned NumVecs,
1531                                       bool is64BitVector) {
1532  unsigned NumRegs = NumVecs;
1533  if (!is64BitVector && NumVecs < 3)
1534    NumRegs *= 2;
1535
1536  unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1537  if (Alignment >= 32 && NumRegs == 4)
1538    Alignment = 32;
1539  else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1540    Alignment = 16;
1541  else if (Alignment >= 8)
1542    Alignment = 8;
1543  else
1544    Alignment = 0;
1545
1546  return CurDAG->getTargetConstant(Alignment, MVT::i32);
1547}
1548
1549// Get the register stride update opcode of a VLD/VST instruction that
1550// is otherwise equivalent to the given fixed stride updating instruction.
1551static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
1552  switch (Opc) {
1553  default: break;
1554  case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register;
1555  case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register;
1556  case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register;
1557  case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register;
1558  case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register;
1559  case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register;
1560  case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register;
1561  case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register;
1562
1563  case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register;
1564  case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register;
1565  case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register;
1566  case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register;
1567  case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register;
1568  case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register;
1569  case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register;
1570  case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register;
1571  case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register;
1572  case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register;
1573
1574  case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register;
1575  case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register;
1576  case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register;
1577  case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register;
1578  case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register;
1579  case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register;
1580
1581  case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register;
1582  case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register;
1583  case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register;
1584  case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register;
1585  case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register;
1586  case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register;
1587
1588  case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register;
1589  case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register;
1590  case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register;
1591  }
1592  return Opc; // If not one we handle, return it unchanged.
1593}
1594
1595SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
1596                                   const uint16_t *DOpcodes,
1597                                   const uint16_t *QOpcodes0,
1598                                   const uint16_t *QOpcodes1) {
1599  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1600  DebugLoc dl = N->getDebugLoc();
1601
1602  SDValue MemAddr, Align;
1603  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1604  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1605    return NULL;
1606
1607  SDValue Chain = N->getOperand(0);
1608  EVT VT = N->getValueType(0);
1609  bool is64BitVector = VT.is64BitVector();
1610  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1611
1612  unsigned OpcodeIndex;
1613  switch (VT.getSimpleVT().SimpleTy) {
1614  default: llvm_unreachable("unhandled vld type");
1615    // Double-register operations:
1616  case MVT::v8i8:  OpcodeIndex = 0; break;
1617  case MVT::v4i16: OpcodeIndex = 1; break;
1618  case MVT::v2f32:
1619  case MVT::v2i32: OpcodeIndex = 2; break;
1620  case MVT::v1i64: OpcodeIndex = 3; break;
1621    // Quad-register operations:
1622  case MVT::v16i8: OpcodeIndex = 0; break;
1623  case MVT::v8i16: OpcodeIndex = 1; break;
1624  case MVT::v4f32:
1625  case MVT::v4i32: OpcodeIndex = 2; break;
1626  case MVT::v2i64: OpcodeIndex = 3;
1627    assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1628    break;
1629  }
1630
1631  EVT ResTy;
1632  if (NumVecs == 1)
1633    ResTy = VT;
1634  else {
1635    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1636    if (!is64BitVector)
1637      ResTyElts *= 2;
1638    ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1639  }
1640  std::vector<EVT> ResTys;
1641  ResTys.push_back(ResTy);
1642  if (isUpdating)
1643    ResTys.push_back(MVT::i32);
1644  ResTys.push_back(MVT::Other);
1645
1646  SDValue Pred = getAL(CurDAG);
1647  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1648  SDNode *VLd;
1649  SmallVector<SDValue, 7> Ops;
1650
1651  // Double registers and VLD1/VLD2 quad registers are directly supported.
1652  if (is64BitVector || NumVecs <= 2) {
1653    unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1654                    QOpcodes0[OpcodeIndex]);
1655    Ops.push_back(MemAddr);
1656    Ops.push_back(Align);
1657    if (isUpdating) {
1658      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1659      // FIXME: VLD1/VLD2 fixed increment doesn't need Reg0. Remove the reg0
1660      // case entirely when the rest are updated to that form, too.
1661      if ((NumVecs == 1 || NumVecs == 2) && !isa<ConstantSDNode>(Inc.getNode()))
1662        Opc = getVLDSTRegisterUpdateOpcode(Opc);
1663      // We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so
1664      // check for that explicitly too. Horribly hacky, but temporary.
1665      if ((NumVecs != 1 && NumVecs != 2 && Opc != ARM::VLD1q64wb_fixed) ||
1666          !isa<ConstantSDNode>(Inc.getNode()))
1667        Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1668    }
1669    Ops.push_back(Pred);
1670    Ops.push_back(Reg0);
1671    Ops.push_back(Chain);
1672    VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size());
1673
1674  } else {
1675    // Otherwise, quad registers are loaded with two separate instructions,
1676    // where one loads the even registers and the other loads the odd registers.
1677    EVT AddrTy = MemAddr.getValueType();
1678
1679    // Load the even subregs.  This is always an updating load, so that it
1680    // provides the address to the second load for the odd subregs.
1681    SDValue ImplDef =
1682      SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
1683    const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
1684    SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1685                                          ResTy, AddrTy, MVT::Other, OpsA, 7);
1686    Chain = SDValue(VLdA, 2);
1687
1688    // Load the odd subregs.
1689    Ops.push_back(SDValue(VLdA, 1));
1690    Ops.push_back(Align);
1691    if (isUpdating) {
1692      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1693      assert(isa<ConstantSDNode>(Inc.getNode()) &&
1694             "only constant post-increment update allowed for VLD3/4");
1695      (void)Inc;
1696      Ops.push_back(Reg0);
1697    }
1698    Ops.push_back(SDValue(VLdA, 0));
1699    Ops.push_back(Pred);
1700    Ops.push_back(Reg0);
1701    Ops.push_back(Chain);
1702    VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
1703                                 Ops.data(), Ops.size());
1704  }
1705
1706  // Transfer memoperands.
1707  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1708  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1709  cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1);
1710
1711  if (NumVecs == 1)
1712    return VLd;
1713
1714  // Extract out the subregisters.
1715  SDValue SuperReg = SDValue(VLd, 0);
1716  assert(ARM::dsub_7 == ARM::dsub_0+7 &&
1717         ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1718  unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
1719  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1720    ReplaceUses(SDValue(N, Vec),
1721                CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1722  ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
1723  if (isUpdating)
1724    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2));
1725  return NULL;
1726}
1727
1728SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
1729                                   const uint16_t *DOpcodes,
1730                                   const uint16_t *QOpcodes0,
1731                                   const uint16_t *QOpcodes1) {
1732  assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1733  DebugLoc dl = N->getDebugLoc();
1734
1735  SDValue MemAddr, Align;
1736  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1737  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1738  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1739    return NULL;
1740
1741  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1742  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1743
1744  SDValue Chain = N->getOperand(0);
1745  EVT VT = N->getOperand(Vec0Idx).getValueType();
1746  bool is64BitVector = VT.is64BitVector();
1747  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1748
1749  unsigned OpcodeIndex;
1750  switch (VT.getSimpleVT().SimpleTy) {
1751  default: llvm_unreachable("unhandled vst type");
1752    // Double-register operations:
1753  case MVT::v8i8:  OpcodeIndex = 0; break;
1754  case MVT::v4i16: OpcodeIndex = 1; break;
1755  case MVT::v2f32:
1756  case MVT::v2i32: OpcodeIndex = 2; break;
1757  case MVT::v1i64: OpcodeIndex = 3; break;
1758    // Quad-register operations:
1759  case MVT::v16i8: OpcodeIndex = 0; break;
1760  case MVT::v8i16: OpcodeIndex = 1; break;
1761  case MVT::v4f32:
1762  case MVT::v4i32: OpcodeIndex = 2; break;
1763  case MVT::v2i64: OpcodeIndex = 3;
1764    assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1765    break;
1766  }
1767
1768  std::vector<EVT> ResTys;
1769  if (isUpdating)
1770    ResTys.push_back(MVT::i32);
1771  ResTys.push_back(MVT::Other);
1772
1773  SDValue Pred = getAL(CurDAG);
1774  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1775  SmallVector<SDValue, 7> Ops;
1776
1777  // Double registers and VST1/VST2 quad registers are directly supported.
1778  if (is64BitVector || NumVecs <= 2) {
1779    SDValue SrcReg;
1780    if (NumVecs == 1) {
1781      SrcReg = N->getOperand(Vec0Idx);
1782    } else if (is64BitVector) {
1783      // Form a REG_SEQUENCE to force register allocation.
1784      SDValue V0 = N->getOperand(Vec0Idx + 0);
1785      SDValue V1 = N->getOperand(Vec0Idx + 1);
1786      if (NumVecs == 2)
1787        SrcReg = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1788      else {
1789        SDValue V2 = N->getOperand(Vec0Idx + 2);
1790        // If it's a vst3, form a quad D-register and leave the last part as
1791        // an undef.
1792        SDValue V3 = (NumVecs == 3)
1793          ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1794          : N->getOperand(Vec0Idx + 3);
1795        SrcReg = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1796      }
1797    } else {
1798      // Form a QQ register.
1799      SDValue Q0 = N->getOperand(Vec0Idx);
1800      SDValue Q1 = N->getOperand(Vec0Idx + 1);
1801      SrcReg = SDValue(PairQRegs(MVT::v4i64, Q0, Q1), 0);
1802    }
1803
1804    unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1805                    QOpcodes0[OpcodeIndex]);
1806    Ops.push_back(MemAddr);
1807    Ops.push_back(Align);
1808    if (isUpdating) {
1809      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1810      // FIXME: VST1/VST2 fixed increment doesn't need Reg0. Remove the reg0
1811      // case entirely when the rest are updated to that form, too.
1812      if (NumVecs <= 2 && !isa<ConstantSDNode>(Inc.getNode()))
1813        Opc = getVLDSTRegisterUpdateOpcode(Opc);
1814      // We use a VST1 for v1i64 even if the pseudo says vld2/3/4, so
1815      // check for that explicitly too. Horribly hacky, but temporary.
1816      if ((NumVecs > 2 && Opc != ARM::VST1q64wb_fixed) ||
1817          !isa<ConstantSDNode>(Inc.getNode()))
1818        Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1819    }
1820    Ops.push_back(SrcReg);
1821    Ops.push_back(Pred);
1822    Ops.push_back(Reg0);
1823    Ops.push_back(Chain);
1824    SDNode *VSt =
1825      CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size());
1826
1827    // Transfer memoperands.
1828    cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1);
1829
1830    return VSt;
1831  }
1832
1833  // Otherwise, quad registers are stored with two separate instructions,
1834  // where one stores the even registers and the other stores the odd registers.
1835
1836  // Form the QQQQ REG_SEQUENCE.
1837  SDValue V0 = N->getOperand(Vec0Idx + 0);
1838  SDValue V1 = N->getOperand(Vec0Idx + 1);
1839  SDValue V2 = N->getOperand(Vec0Idx + 2);
1840  SDValue V3 = (NumVecs == 3)
1841    ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1842    : N->getOperand(Vec0Idx + 3);
1843  SDValue RegSeq = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1844
1845  // Store the even D registers.  This is always an updating store, so that it
1846  // provides the address to the second store for the odd subregs.
1847  const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain };
1848  SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1849                                        MemAddr.getValueType(),
1850                                        MVT::Other, OpsA, 7);
1851  cast<MachineSDNode>(VStA)->setMemRefs(MemOp, MemOp + 1);
1852  Chain = SDValue(VStA, 1);
1853
1854  // Store the odd D registers.
1855  Ops.push_back(SDValue(VStA, 0));
1856  Ops.push_back(Align);
1857  if (isUpdating) {
1858    SDValue Inc = N->getOperand(AddrOpIdx + 1);
1859    assert(isa<ConstantSDNode>(Inc.getNode()) &&
1860           "only constant post-increment update allowed for VST3/4");
1861    (void)Inc;
1862    Ops.push_back(Reg0);
1863  }
1864  Ops.push_back(RegSeq);
1865  Ops.push_back(Pred);
1866  Ops.push_back(Reg0);
1867  Ops.push_back(Chain);
1868  SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
1869                                        Ops.data(), Ops.size());
1870  cast<MachineSDNode>(VStB)->setMemRefs(MemOp, MemOp + 1);
1871  return VStB;
1872}
1873
1874SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
1875                                         bool isUpdating, unsigned NumVecs,
1876                                         const uint16_t *DOpcodes,
1877                                         const uint16_t *QOpcodes) {
1878  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
1879  DebugLoc dl = N->getDebugLoc();
1880
1881  SDValue MemAddr, Align;
1882  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1883  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1884  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1885    return NULL;
1886
1887  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1888  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1889
1890  SDValue Chain = N->getOperand(0);
1891  unsigned Lane =
1892    cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue();
1893  EVT VT = N->getOperand(Vec0Idx).getValueType();
1894  bool is64BitVector = VT.is64BitVector();
1895
1896  unsigned Alignment = 0;
1897  if (NumVecs != 3) {
1898    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1899    unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
1900    if (Alignment > NumBytes)
1901      Alignment = NumBytes;
1902    if (Alignment < 8 && Alignment < NumBytes)
1903      Alignment = 0;
1904    // Alignment must be a power of two; make sure of that.
1905    Alignment = (Alignment & -Alignment);
1906    if (Alignment == 1)
1907      Alignment = 0;
1908  }
1909  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
1910
1911  unsigned OpcodeIndex;
1912  switch (VT.getSimpleVT().SimpleTy) {
1913  default: llvm_unreachable("unhandled vld/vst lane type");
1914    // Double-register operations:
1915  case MVT::v8i8:  OpcodeIndex = 0; break;
1916  case MVT::v4i16: OpcodeIndex = 1; break;
1917  case MVT::v2f32:
1918  case MVT::v2i32: OpcodeIndex = 2; break;
1919    // Quad-register operations:
1920  case MVT::v8i16: OpcodeIndex = 0; break;
1921  case MVT::v4f32:
1922  case MVT::v4i32: OpcodeIndex = 1; break;
1923  }
1924
1925  std::vector<EVT> ResTys;
1926  if (IsLoad) {
1927    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1928    if (!is64BitVector)
1929      ResTyElts *= 2;
1930    ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(),
1931                                      MVT::i64, ResTyElts));
1932  }
1933  if (isUpdating)
1934    ResTys.push_back(MVT::i32);
1935  ResTys.push_back(MVT::Other);
1936
1937  SDValue Pred = getAL(CurDAG);
1938  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1939
1940  SmallVector<SDValue, 8> Ops;
1941  Ops.push_back(MemAddr);
1942  Ops.push_back(Align);
1943  if (isUpdating) {
1944    SDValue Inc = N->getOperand(AddrOpIdx + 1);
1945    Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1946  }
1947
1948  SDValue SuperReg;
1949  SDValue V0 = N->getOperand(Vec0Idx + 0);
1950  SDValue V1 = N->getOperand(Vec0Idx + 1);
1951  if (NumVecs == 2) {
1952    if (is64BitVector)
1953      SuperReg = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1954    else
1955      SuperReg = SDValue(PairQRegs(MVT::v4i64, V0, V1), 0);
1956  } else {
1957    SDValue V2 = N->getOperand(Vec0Idx + 2);
1958    SDValue V3 = (NumVecs == 3)
1959      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1960      : N->getOperand(Vec0Idx + 3);
1961    if (is64BitVector)
1962      SuperReg = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1963    else
1964      SuperReg = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1965  }
1966  Ops.push_back(SuperReg);
1967  Ops.push_back(getI32Imm(Lane));
1968  Ops.push_back(Pred);
1969  Ops.push_back(Reg0);
1970  Ops.push_back(Chain);
1971
1972  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1973                                  QOpcodes[OpcodeIndex]);
1974  SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys,
1975                                         Ops.data(), Ops.size());
1976  cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1);
1977  if (!IsLoad)
1978    return VLdLn;
1979
1980  // Extract the subregisters.
1981  SuperReg = SDValue(VLdLn, 0);
1982  assert(ARM::dsub_7 == ARM::dsub_0+7 &&
1983         ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1984  unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
1985  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1986    ReplaceUses(SDValue(N, Vec),
1987                CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1988  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1));
1989  if (isUpdating)
1990    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2));
1991  return NULL;
1992}
1993
1994SDNode *ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating,
1995                                      unsigned NumVecs,
1996                                      const uint16_t *Opcodes) {
1997  assert(NumVecs >=2 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
1998  DebugLoc dl = N->getDebugLoc();
1999
2000  SDValue MemAddr, Align;
2001  if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align))
2002    return NULL;
2003
2004  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2005  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2006
2007  SDValue Chain = N->getOperand(0);
2008  EVT VT = N->getValueType(0);
2009
2010  unsigned Alignment = 0;
2011  if (NumVecs != 3) {
2012    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2013    unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
2014    if (Alignment > NumBytes)
2015      Alignment = NumBytes;
2016    if (Alignment < 8 && Alignment < NumBytes)
2017      Alignment = 0;
2018    // Alignment must be a power of two; make sure of that.
2019    Alignment = (Alignment & -Alignment);
2020    if (Alignment == 1)
2021      Alignment = 0;
2022  }
2023  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
2024
2025  unsigned OpcodeIndex;
2026  switch (VT.getSimpleVT().SimpleTy) {
2027  default: llvm_unreachable("unhandled vld-dup type");
2028  case MVT::v8i8:  OpcodeIndex = 0; break;
2029  case MVT::v4i16: OpcodeIndex = 1; break;
2030  case MVT::v2f32:
2031  case MVT::v2i32: OpcodeIndex = 2; break;
2032  }
2033
2034  SDValue Pred = getAL(CurDAG);
2035  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2036  SDValue SuperReg;
2037  unsigned Opc = Opcodes[OpcodeIndex];
2038  SmallVector<SDValue, 6> Ops;
2039  Ops.push_back(MemAddr);
2040  Ops.push_back(Align);
2041  if (isUpdating) {
2042    // fixed-stride update instructions don't have an explicit writeback
2043    // operand. It's implicit in the opcode itself.
2044    SDValue Inc = N->getOperand(2);
2045    if (!isa<ConstantSDNode>(Inc.getNode()))
2046      Ops.push_back(Inc);
2047    // FIXME: VLD3 and VLD4 haven't been updated to that form yet.
2048    else if (NumVecs > 2)
2049      Ops.push_back(Reg0);
2050  }
2051  Ops.push_back(Pred);
2052  Ops.push_back(Reg0);
2053  Ops.push_back(Chain);
2054
2055  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2056  std::vector<EVT> ResTys;
2057  ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,ResTyElts));
2058  if (isUpdating)
2059    ResTys.push_back(MVT::i32);
2060  ResTys.push_back(MVT::Other);
2061  SDNode *VLdDup =
2062    CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size());
2063  cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1);
2064  SuperReg = SDValue(VLdDup, 0);
2065
2066  // Extract the subregisters.
2067  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
2068  unsigned SubIdx = ARM::dsub_0;
2069  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2070    ReplaceUses(SDValue(N, Vec),
2071                CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
2072  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1));
2073  if (isUpdating)
2074    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2));
2075  return NULL;
2076}
2077
2078SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs,
2079                                    unsigned Opc) {
2080  assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range");
2081  DebugLoc dl = N->getDebugLoc();
2082  EVT VT = N->getValueType(0);
2083  unsigned FirstTblReg = IsExt ? 2 : 1;
2084
2085  // Form a REG_SEQUENCE to force register allocation.
2086  SDValue RegSeq;
2087  SDValue V0 = N->getOperand(FirstTblReg + 0);
2088  SDValue V1 = N->getOperand(FirstTblReg + 1);
2089  if (NumVecs == 2)
2090    RegSeq = SDValue(PairDRegs(MVT::v16i8, V0, V1), 0);
2091  else {
2092    SDValue V2 = N->getOperand(FirstTblReg + 2);
2093    // If it's a vtbl3, form a quad D-register and leave the last part as
2094    // an undef.
2095    SDValue V3 = (NumVecs == 3)
2096      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2097      : N->getOperand(FirstTblReg + 3);
2098    RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
2099  }
2100
2101  SmallVector<SDValue, 6> Ops;
2102  if (IsExt)
2103    Ops.push_back(N->getOperand(1));
2104  Ops.push_back(RegSeq);
2105  Ops.push_back(N->getOperand(FirstTblReg + NumVecs));
2106  Ops.push_back(getAL(CurDAG)); // predicate
2107  Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register
2108  return CurDAG->getMachineNode(Opc, dl, VT, Ops.data(), Ops.size());
2109}
2110
2111SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
2112                                                     bool isSigned) {
2113  if (!Subtarget->hasV6T2Ops())
2114    return NULL;
2115
2116  unsigned Opc = isSigned ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
2117    : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
2118
2119
2120  // For unsigned extracts, check for a shift right and mask
2121  unsigned And_imm = 0;
2122  if (N->getOpcode() == ISD::AND) {
2123    if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
2124
2125      // The immediate is a mask of the low bits iff imm & (imm+1) == 0
2126      if (And_imm & (And_imm + 1))
2127        return NULL;
2128
2129      unsigned Srl_imm = 0;
2130      if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
2131                                Srl_imm)) {
2132        assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2133
2134        // Note: The width operand is encoded as width-1.
2135        unsigned Width = CountTrailingOnes_32(And_imm) - 1;
2136        unsigned LSB = Srl_imm;
2137        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2138        SDValue Ops[] = { N->getOperand(0).getOperand(0),
2139                          CurDAG->getTargetConstant(LSB, MVT::i32),
2140                          CurDAG->getTargetConstant(Width, MVT::i32),
2141          getAL(CurDAG), Reg0 };
2142        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2143      }
2144    }
2145    return NULL;
2146  }
2147
2148  // Otherwise, we're looking for a shift of a shift
2149  unsigned Shl_imm = 0;
2150  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
2151    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
2152    unsigned Srl_imm = 0;
2153    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
2154      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2155      // Note: The width operand is encoded as width-1.
2156      unsigned Width = 32 - Srl_imm - 1;
2157      int LSB = Srl_imm - Shl_imm;
2158      if (LSB < 0)
2159        return NULL;
2160      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2161      SDValue Ops[] = { N->getOperand(0).getOperand(0),
2162                        CurDAG->getTargetConstant(LSB, MVT::i32),
2163                        CurDAG->getTargetConstant(Width, MVT::i32),
2164                        getAL(CurDAG), Reg0 };
2165      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2166    }
2167  }
2168  return NULL;
2169}
2170
2171SDNode *ARMDAGToDAGISel::
2172SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2173                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2174  SDValue CPTmp0;
2175  SDValue CPTmp1;
2176  if (SelectT2ShifterOperandReg(TrueVal, CPTmp0, CPTmp1)) {
2177    unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
2178    unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
2179    unsigned Opc = 0;
2180    switch (SOShOp) {
2181    case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
2182    case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
2183    case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
2184    case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
2185    default:
2186      llvm_unreachable("Unknown so_reg opcode!");
2187    }
2188    SDValue SOShImm =
2189      CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
2190    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2191    SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag };
2192    return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6);
2193  }
2194  return 0;
2195}
2196
2197SDNode *ARMDAGToDAGISel::
2198SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2199                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2200  SDValue CPTmp0;
2201  SDValue CPTmp1;
2202  SDValue CPTmp2;
2203  if (SelectImmShifterOperand(TrueVal, CPTmp0, CPTmp2)) {
2204    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2205    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp2, CC, CCR, InFlag };
2206    return CurDAG->SelectNodeTo(N, ARM::MOVCCsi, MVT::i32, Ops, 6);
2207  }
2208
2209  if (SelectRegShifterOperand(TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
2210    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2211    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag };
2212    return CurDAG->SelectNodeTo(N, ARM::MOVCCsr, MVT::i32, Ops, 7);
2213  }
2214  return 0;
2215}
2216
2217SDNode *ARMDAGToDAGISel::
2218SelectT2CMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2219                  ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2220  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
2221  if (!T)
2222    return 0;
2223
2224  unsigned Opc = 0;
2225  unsigned TrueImm = T->getZExtValue();
2226  if (is_t2_so_imm(TrueImm)) {
2227    Opc = ARM::t2MOVCCi;
2228  } else if (TrueImm <= 0xffff) {
2229    Opc = ARM::t2MOVCCi16;
2230  } else if (is_t2_so_imm_not(TrueImm)) {
2231    TrueImm = ~TrueImm;
2232    Opc = ARM::t2MVNCCi;
2233  } else if (TrueVal.getNode()->hasOneUse() && Subtarget->hasV6T2Ops()) {
2234    // Large immediate.
2235    Opc = ARM::t2MOVCCi32imm;
2236  }
2237
2238  if (Opc) {
2239    SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
2240    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2241    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
2242    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2243  }
2244
2245  return 0;
2246}
2247
2248SDNode *ARMDAGToDAGISel::
2249SelectARMCMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2250                   ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2251  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
2252  if (!T)
2253    return 0;
2254
2255  unsigned Opc = 0;
2256  unsigned TrueImm = T->getZExtValue();
2257  bool isSoImm = is_so_imm(TrueImm);
2258  if (isSoImm) {
2259    Opc = ARM::MOVCCi;
2260  } else if (Subtarget->hasV6T2Ops() && TrueImm <= 0xffff) {
2261    Opc = ARM::MOVCCi16;
2262  } else if (is_so_imm_not(TrueImm)) {
2263    TrueImm = ~TrueImm;
2264    Opc = ARM::MVNCCi;
2265  } else if (TrueVal.getNode()->hasOneUse() &&
2266             (Subtarget->hasV6T2Ops() || ARM_AM::isSOImmTwoPartVal(TrueImm))) {
2267    // Large immediate.
2268    Opc = ARM::MOVCCi32imm;
2269  }
2270
2271  if (Opc) {
2272    SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
2273    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2274    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
2275    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2276  }
2277
2278  return 0;
2279}
2280
2281SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) {
2282  EVT VT = N->getValueType(0);
2283  SDValue FalseVal = N->getOperand(0);
2284  SDValue TrueVal  = N->getOperand(1);
2285  SDValue CC = N->getOperand(2);
2286  SDValue CCR = N->getOperand(3);
2287  SDValue InFlag = N->getOperand(4);
2288  assert(CC.getOpcode() == ISD::Constant);
2289  assert(CCR.getOpcode() == ISD::Register);
2290  ARMCC::CondCodes CCVal =
2291    (ARMCC::CondCodes)cast<ConstantSDNode>(CC)->getZExtValue();
2292
2293  if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
2294    // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
2295    // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
2296    // Pattern complexity = 18  cost = 1  size = 0
2297    if (Subtarget->isThumb()) {
2298      SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal,
2299                                        CCVal, CCR, InFlag);
2300      if (!Res)
2301        Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal,
2302                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2303      if (Res)
2304        return Res;
2305    } else {
2306      SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal,
2307                                         CCVal, CCR, InFlag);
2308      if (!Res)
2309        Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal,
2310                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2311      if (Res)
2312        return Res;
2313    }
2314
2315    // Pattern: (ARMcmov:i32 GPR:i32:$false,
2316    //             (imm:i32)<<P:Pred_so_imm>>:$true,
2317    //             (imm:i32):$cc)
2318    // Emits: (MOVCCi:i32 GPR:i32:$false,
2319    //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
2320    // Pattern complexity = 10  cost = 1  size = 0
2321    if (Subtarget->isThumb()) {
2322      SDNode *Res = SelectT2CMOVImmOp(N, FalseVal, TrueVal,
2323                                        CCVal, CCR, InFlag);
2324      if (!Res)
2325        Res = SelectT2CMOVImmOp(N, TrueVal, FalseVal,
2326                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2327      if (Res)
2328        return Res;
2329    } else {
2330      SDNode *Res = SelectARMCMOVImmOp(N, FalseVal, TrueVal,
2331                                         CCVal, CCR, InFlag);
2332      if (!Res)
2333        Res = SelectARMCMOVImmOp(N, TrueVal, FalseVal,
2334                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2335      if (Res)
2336        return Res;
2337    }
2338  }
2339
2340  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2341  // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2342  // Pattern complexity = 6  cost = 1  size = 0
2343  //
2344  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2345  // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2346  // Pattern complexity = 6  cost = 11  size = 0
2347  //
2348  // Also VMOVScc and VMOVDcc.
2349  SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32);
2350  SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag };
2351  unsigned Opc = 0;
2352  switch (VT.getSimpleVT().SimpleTy) {
2353  default: llvm_unreachable("Illegal conditional move type!");
2354  case MVT::i32:
2355    Opc = Subtarget->isThumb()
2356      ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
2357      : ARM::MOVCCr;
2358    break;
2359  case MVT::f32:
2360    Opc = ARM::VMOVScc;
2361    break;
2362  case MVT::f64:
2363    Opc = ARM::VMOVDcc;
2364    break;
2365  }
2366  return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
2367}
2368
2369/// Target-specific DAG combining for ISD::XOR.
2370/// Target-independent combining lowers SELECT_CC nodes of the form
2371/// select_cc setg[ge] X,  0,  X, -X
2372/// select_cc setgt    X, -1,  X, -X
2373/// select_cc setl[te] X,  0, -X,  X
2374/// select_cc setlt    X,  1, -X,  X
2375/// which represent Integer ABS into:
2376/// Y = sra (X, size(X)-1); xor (add (X, Y), Y)
2377/// ARM instruction selection detects the latter and matches it to
2378/// ARM::ABS or ARM::t2ABS machine node.
2379SDNode *ARMDAGToDAGISel::SelectABSOp(SDNode *N){
2380  SDValue XORSrc0 = N->getOperand(0);
2381  SDValue XORSrc1 = N->getOperand(1);
2382  EVT VT = N->getValueType(0);
2383
2384  if (Subtarget->isThumb1Only())
2385    return NULL;
2386
2387  if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA)
2388    return NULL;
2389
2390  SDValue ADDSrc0 = XORSrc0.getOperand(0);
2391  SDValue ADDSrc1 = XORSrc0.getOperand(1);
2392  SDValue SRASrc0 = XORSrc1.getOperand(0);
2393  SDValue SRASrc1 = XORSrc1.getOperand(1);
2394  ConstantSDNode *SRAConstant =  dyn_cast<ConstantSDNode>(SRASrc1);
2395  EVT XType = SRASrc0.getValueType();
2396  unsigned Size = XType.getSizeInBits() - 1;
2397
2398  if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 &&
2399      XType.isInteger() && SRAConstant != NULL &&
2400      Size == SRAConstant->getZExtValue()) {
2401    unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS;
2402    return CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0);
2403  }
2404
2405  return NULL;
2406}
2407
2408SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
2409  // The only time a CONCAT_VECTORS operation can have legal types is when
2410  // two 64-bit vectors are concatenated to a 128-bit vector.
2411  EVT VT = N->getValueType(0);
2412  if (!VT.is128BitVector() || N->getNumOperands() != 2)
2413    llvm_unreachable("unexpected CONCAT_VECTORS");
2414  return PairDRegs(VT, N->getOperand(0), N->getOperand(1));
2415}
2416
2417SDNode *ARMDAGToDAGISel::SelectAtomic64(SDNode *Node, unsigned Opc) {
2418  SmallVector<SDValue, 6> Ops;
2419  Ops.push_back(Node->getOperand(1)); // Ptr
2420  Ops.push_back(Node->getOperand(2)); // Low part of Val1
2421  Ops.push_back(Node->getOperand(3)); // High part of Val1
2422  if (Opc == ARM::ATOMCMPXCHG6432) {
2423    Ops.push_back(Node->getOperand(4)); // Low part of Val2
2424    Ops.push_back(Node->getOperand(5)); // High part of Val2
2425  }
2426  Ops.push_back(Node->getOperand(0)); // Chain
2427  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2428  MemOp[0] = cast<MemSDNode>(Node)->getMemOperand();
2429  SDNode *ResNode = CurDAG->getMachineNode(Opc, Node->getDebugLoc(),
2430                                           MVT::i32, MVT::i32, MVT::Other,
2431                                           Ops.data() ,Ops.size());
2432  cast<MachineSDNode>(ResNode)->setMemRefs(MemOp, MemOp + 1);
2433  return ResNode;
2434}
2435
2436SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
2437  DebugLoc dl = N->getDebugLoc();
2438
2439  if (N->isMachineOpcode())
2440    return NULL;   // Already selected.
2441
2442  switch (N->getOpcode()) {
2443  default: break;
2444  case ISD::XOR: {
2445    // Select special operations if XOR node forms integer ABS pattern
2446    SDNode *ResNode = SelectABSOp(N);
2447    if (ResNode)
2448      return ResNode;
2449    // Other cases are autogenerated.
2450    break;
2451  }
2452  case ISD::Constant: {
2453    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
2454    bool UseCP = true;
2455    if (Subtarget->hasThumb2())
2456      // Thumb2-aware targets have the MOVT instruction, so all immediates can
2457      // be done with MOV + MOVT, at worst.
2458      UseCP = 0;
2459    else {
2460      if (Subtarget->isThumb()) {
2461        UseCP = (Val > 255 &&                          // MOV
2462                 ~Val > 255 &&                         // MOV + MVN
2463                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
2464      } else
2465        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
2466                 ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
2467                 !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
2468    }
2469
2470    if (UseCP) {
2471      SDValue CPIdx =
2472        CurDAG->getTargetConstantPool(ConstantInt::get(
2473                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
2474                                      TLI.getPointerTy());
2475
2476      SDNode *ResNode;
2477      if (Subtarget->isThumb1Only()) {
2478        SDValue Pred = getAL(CurDAG);
2479        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2480        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
2481        ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
2482                                         Ops, 4);
2483      } else {
2484        SDValue Ops[] = {
2485          CPIdx,
2486          CurDAG->getTargetConstant(0, MVT::i32),
2487          getAL(CurDAG),
2488          CurDAG->getRegister(0, MVT::i32),
2489          CurDAG->getEntryNode()
2490        };
2491        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
2492                                       Ops, 5);
2493      }
2494      ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
2495      return NULL;
2496    }
2497
2498    // Other cases are autogenerated.
2499    break;
2500  }
2501  case ISD::FrameIndex: {
2502    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
2503    int FI = cast<FrameIndexSDNode>(N)->getIndex();
2504    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
2505    if (Subtarget->isThumb1Only()) {
2506      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
2507                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2508      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, Ops, 4);
2509    } else {
2510      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
2511                      ARM::t2ADDri : ARM::ADDri);
2512      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
2513                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2514                        CurDAG->getRegister(0, MVT::i32) };
2515      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2516    }
2517  }
2518  case ISD::SRL:
2519    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2520      return I;
2521    break;
2522  case ISD::SRA:
2523    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true))
2524      return I;
2525    break;
2526  case ISD::MUL:
2527    if (Subtarget->isThumb1Only())
2528      break;
2529    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
2530      unsigned RHSV = C->getZExtValue();
2531      if (!RHSV) break;
2532      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
2533        unsigned ShImm = Log2_32(RHSV-1);
2534        if (ShImm >= 32)
2535          break;
2536        SDValue V = N->getOperand(0);
2537        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2538        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2539        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2540        if (Subtarget->isThumb()) {
2541          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2542          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
2543        } else {
2544          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2545          return CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops, 7);
2546        }
2547      }
2548      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
2549        unsigned ShImm = Log2_32(RHSV+1);
2550        if (ShImm >= 32)
2551          break;
2552        SDValue V = N->getOperand(0);
2553        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2554        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2555        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2556        if (Subtarget->isThumb()) {
2557          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2558          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 6);
2559        } else {
2560          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2561          return CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops, 7);
2562        }
2563      }
2564    }
2565    break;
2566  case ISD::AND: {
2567    // Check for unsigned bitfield extract
2568    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2569      return I;
2570
2571    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
2572    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
2573    // are entirely contributed by c2 and lower 16-bits are entirely contributed
2574    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
2575    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
2576    EVT VT = N->getValueType(0);
2577    if (VT != MVT::i32)
2578      break;
2579    unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
2580      ? ARM::t2MOVTi16
2581      : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
2582    if (!Opc)
2583      break;
2584    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
2585    ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2586    if (!N1C)
2587      break;
2588    if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
2589      SDValue N2 = N0.getOperand(1);
2590      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
2591      if (!N2C)
2592        break;
2593      unsigned N1CVal = N1C->getZExtValue();
2594      unsigned N2CVal = N2C->getZExtValue();
2595      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
2596          (N1CVal & 0xffffU) == 0xffffU &&
2597          (N2CVal & 0xffffU) == 0x0U) {
2598        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
2599                                                  MVT::i32);
2600        SDValue Ops[] = { N0.getOperand(0), Imm16,
2601                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2602        return CurDAG->getMachineNode(Opc, dl, VT, Ops, 4);
2603      }
2604    }
2605    break;
2606  }
2607  case ARMISD::VMOVRRD:
2608    return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
2609                                  N->getOperand(0), getAL(CurDAG),
2610                                  CurDAG->getRegister(0, MVT::i32));
2611  case ISD::UMUL_LOHI: {
2612    if (Subtarget->isThumb1Only())
2613      break;
2614    if (Subtarget->isThumb()) {
2615      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2616                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2617                        CurDAG->getRegister(0, MVT::i32) };
2618      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32,Ops,4);
2619    } else {
2620      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2621                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2622                        CurDAG->getRegister(0, MVT::i32) };
2623      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2624                                    ARM::UMULL : ARM::UMULLv5,
2625                                    dl, MVT::i32, MVT::i32, Ops, 5);
2626    }
2627  }
2628  case ISD::SMUL_LOHI: {
2629    if (Subtarget->isThumb1Only())
2630      break;
2631    if (Subtarget->isThumb()) {
2632      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2633                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2634      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32,Ops,4);
2635    } else {
2636      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2637                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2638                        CurDAG->getRegister(0, MVT::i32) };
2639      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2640                                    ARM::SMULL : ARM::SMULLv5,
2641                                    dl, MVT::i32, MVT::i32, Ops, 5);
2642    }
2643  }
2644  case ARMISD::UMLAL:{
2645    if (Subtarget->isThumb()) {
2646      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2647                        N->getOperand(3), getAL(CurDAG),
2648                        CurDAG->getRegister(0, MVT::i32)};
2649      return CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops, 6);
2650    }else{
2651      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2652                        N->getOperand(3), getAL(CurDAG),
2653                        CurDAG->getRegister(0, MVT::i32),
2654                        CurDAG->getRegister(0, MVT::i32) };
2655      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2656                                      ARM::UMLAL : ARM::UMLALv5,
2657                                      dl, MVT::i32, MVT::i32, Ops, 7);
2658    }
2659  }
2660  case ARMISD::SMLAL:{
2661    if (Subtarget->isThumb()) {
2662      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2663                        N->getOperand(3), getAL(CurDAG),
2664                        CurDAG->getRegister(0, MVT::i32)};
2665      return CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops, 6);
2666    }else{
2667      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2668                        N->getOperand(3), getAL(CurDAG),
2669                        CurDAG->getRegister(0, MVT::i32),
2670                        CurDAG->getRegister(0, MVT::i32) };
2671      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2672                                      ARM::SMLAL : ARM::SMLALv5,
2673                                      dl, MVT::i32, MVT::i32, Ops, 7);
2674    }
2675  }
2676  case ISD::LOAD: {
2677    SDNode *ResNode = 0;
2678    if (Subtarget->isThumb() && Subtarget->hasThumb2())
2679      ResNode = SelectT2IndexedLoad(N);
2680    else
2681      ResNode = SelectARMIndexedLoad(N);
2682    if (ResNode)
2683      return ResNode;
2684    // Other cases are autogenerated.
2685    break;
2686  }
2687  case ARMISD::BRCOND: {
2688    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2689    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2690    // Pattern complexity = 6  cost = 1  size = 0
2691
2692    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2693    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
2694    // Pattern complexity = 6  cost = 1  size = 0
2695
2696    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2697    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2698    // Pattern complexity = 6  cost = 1  size = 0
2699
2700    unsigned Opc = Subtarget->isThumb() ?
2701      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
2702    SDValue Chain = N->getOperand(0);
2703    SDValue N1 = N->getOperand(1);
2704    SDValue N2 = N->getOperand(2);
2705    SDValue N3 = N->getOperand(3);
2706    SDValue InFlag = N->getOperand(4);
2707    assert(N1.getOpcode() == ISD::BasicBlock);
2708    assert(N2.getOpcode() == ISD::Constant);
2709    assert(N3.getOpcode() == ISD::Register);
2710
2711    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
2712                               cast<ConstantSDNode>(N2)->getZExtValue()),
2713                               MVT::i32);
2714    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
2715    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
2716                                             MVT::Glue, Ops, 5);
2717    Chain = SDValue(ResNode, 0);
2718    if (N->getNumValues() == 2) {
2719      InFlag = SDValue(ResNode, 1);
2720      ReplaceUses(SDValue(N, 1), InFlag);
2721    }
2722    ReplaceUses(SDValue(N, 0),
2723                SDValue(Chain.getNode(), Chain.getResNo()));
2724    return NULL;
2725  }
2726  case ARMISD::CMOV:
2727    return SelectCMOVOp(N);
2728  case ARMISD::VZIP: {
2729    unsigned Opc = 0;
2730    EVT VT = N->getValueType(0);
2731    switch (VT.getSimpleVT().SimpleTy) {
2732    default: return NULL;
2733    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
2734    case MVT::v4i16: Opc = ARM::VZIPd16; break;
2735    case MVT::v2f32:
2736    // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
2737    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2738    case MVT::v16i8: Opc = ARM::VZIPq8; break;
2739    case MVT::v8i16: Opc = ARM::VZIPq16; break;
2740    case MVT::v4f32:
2741    case MVT::v4i32: Opc = ARM::VZIPq32; break;
2742    }
2743    SDValue Pred = getAL(CurDAG);
2744    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2745    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2746    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2747  }
2748  case ARMISD::VUZP: {
2749    unsigned Opc = 0;
2750    EVT VT = N->getValueType(0);
2751    switch (VT.getSimpleVT().SimpleTy) {
2752    default: return NULL;
2753    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
2754    case MVT::v4i16: Opc = ARM::VUZPd16; break;
2755    case MVT::v2f32:
2756    // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
2757    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2758    case MVT::v16i8: Opc = ARM::VUZPq8; break;
2759    case MVT::v8i16: Opc = ARM::VUZPq16; break;
2760    case MVT::v4f32:
2761    case MVT::v4i32: Opc = ARM::VUZPq32; break;
2762    }
2763    SDValue Pred = getAL(CurDAG);
2764    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2765    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2766    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2767  }
2768  case ARMISD::VTRN: {
2769    unsigned Opc = 0;
2770    EVT VT = N->getValueType(0);
2771    switch (VT.getSimpleVT().SimpleTy) {
2772    default: return NULL;
2773    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
2774    case MVT::v4i16: Opc = ARM::VTRNd16; break;
2775    case MVT::v2f32:
2776    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2777    case MVT::v16i8: Opc = ARM::VTRNq8; break;
2778    case MVT::v8i16: Opc = ARM::VTRNq16; break;
2779    case MVT::v4f32:
2780    case MVT::v4i32: Opc = ARM::VTRNq32; break;
2781    }
2782    SDValue Pred = getAL(CurDAG);
2783    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2784    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2785    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2786  }
2787  case ARMISD::BUILD_VECTOR: {
2788    EVT VecVT = N->getValueType(0);
2789    EVT EltVT = VecVT.getVectorElementType();
2790    unsigned NumElts = VecVT.getVectorNumElements();
2791    if (EltVT == MVT::f64) {
2792      assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
2793      return PairDRegs(VecVT, N->getOperand(0), N->getOperand(1));
2794    }
2795    assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR");
2796    if (NumElts == 2)
2797      return PairSRegs(VecVT, N->getOperand(0), N->getOperand(1));
2798    assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
2799    return QuadSRegs(VecVT, N->getOperand(0), N->getOperand(1),
2800                     N->getOperand(2), N->getOperand(3));
2801  }
2802
2803  case ARMISD::VLD2DUP: {
2804    static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
2805                                        ARM::VLD2DUPd32 };
2806    return SelectVLDDup(N, false, 2, Opcodes);
2807  }
2808
2809  case ARMISD::VLD3DUP: {
2810    static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
2811                                        ARM::VLD3DUPd16Pseudo,
2812                                        ARM::VLD3DUPd32Pseudo };
2813    return SelectVLDDup(N, false, 3, Opcodes);
2814  }
2815
2816  case ARMISD::VLD4DUP: {
2817    static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
2818                                        ARM::VLD4DUPd16Pseudo,
2819                                        ARM::VLD4DUPd32Pseudo };
2820    return SelectVLDDup(N, false, 4, Opcodes);
2821  }
2822
2823  case ARMISD::VLD2DUP_UPD: {
2824    static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed,
2825                                        ARM::VLD2DUPd16wb_fixed,
2826                                        ARM::VLD2DUPd32wb_fixed };
2827    return SelectVLDDup(N, true, 2, Opcodes);
2828  }
2829
2830  case ARMISD::VLD3DUP_UPD: {
2831    static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
2832                                        ARM::VLD3DUPd16Pseudo_UPD,
2833                                        ARM::VLD3DUPd32Pseudo_UPD };
2834    return SelectVLDDup(N, true, 3, Opcodes);
2835  }
2836
2837  case ARMISD::VLD4DUP_UPD: {
2838    static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
2839                                        ARM::VLD4DUPd16Pseudo_UPD,
2840                                        ARM::VLD4DUPd32Pseudo_UPD };
2841    return SelectVLDDup(N, true, 4, Opcodes);
2842  }
2843
2844  case ARMISD::VLD1_UPD: {
2845    static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
2846                                         ARM::VLD1d16wb_fixed,
2847                                         ARM::VLD1d32wb_fixed,
2848                                         ARM::VLD1d64wb_fixed };
2849    static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
2850                                         ARM::VLD1q16wb_fixed,
2851                                         ARM::VLD1q32wb_fixed,
2852                                         ARM::VLD1q64wb_fixed };
2853    return SelectVLD(N, true, 1, DOpcodes, QOpcodes, 0);
2854  }
2855
2856  case ARMISD::VLD2_UPD: {
2857    static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed,
2858                                         ARM::VLD2d16wb_fixed,
2859                                         ARM::VLD2d32wb_fixed,
2860                                         ARM::VLD1q64wb_fixed};
2861    static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed,
2862                                         ARM::VLD2q16PseudoWB_fixed,
2863                                         ARM::VLD2q32PseudoWB_fixed };
2864    return SelectVLD(N, true, 2, DOpcodes, QOpcodes, 0);
2865  }
2866
2867  case ARMISD::VLD3_UPD: {
2868    static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
2869                                         ARM::VLD3d16Pseudo_UPD,
2870                                         ARM::VLD3d32Pseudo_UPD,
2871                                         ARM::VLD1q64wb_fixed};
2872    static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
2873                                          ARM::VLD3q16Pseudo_UPD,
2874                                          ARM::VLD3q32Pseudo_UPD };
2875    static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
2876                                          ARM::VLD3q16oddPseudo_UPD,
2877                                          ARM::VLD3q32oddPseudo_UPD };
2878    return SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
2879  }
2880
2881  case ARMISD::VLD4_UPD: {
2882    static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD,
2883                                         ARM::VLD4d16Pseudo_UPD,
2884                                         ARM::VLD4d32Pseudo_UPD,
2885                                         ARM::VLD1q64wb_fixed};
2886    static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
2887                                          ARM::VLD4q16Pseudo_UPD,
2888                                          ARM::VLD4q32Pseudo_UPD };
2889    static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
2890                                          ARM::VLD4q16oddPseudo_UPD,
2891                                          ARM::VLD4q32oddPseudo_UPD };
2892    return SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
2893  }
2894
2895  case ARMISD::VLD2LN_UPD: {
2896    static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
2897                                         ARM::VLD2LNd16Pseudo_UPD,
2898                                         ARM::VLD2LNd32Pseudo_UPD };
2899    static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
2900                                         ARM::VLD2LNq32Pseudo_UPD };
2901    return SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes);
2902  }
2903
2904  case ARMISD::VLD3LN_UPD: {
2905    static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
2906                                         ARM::VLD3LNd16Pseudo_UPD,
2907                                         ARM::VLD3LNd32Pseudo_UPD };
2908    static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
2909                                         ARM::VLD3LNq32Pseudo_UPD };
2910    return SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes);
2911  }
2912
2913  case ARMISD::VLD4LN_UPD: {
2914    static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
2915                                         ARM::VLD4LNd16Pseudo_UPD,
2916                                         ARM::VLD4LNd32Pseudo_UPD };
2917    static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
2918                                         ARM::VLD4LNq32Pseudo_UPD };
2919    return SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes);
2920  }
2921
2922  case ARMISD::VST1_UPD: {
2923    static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
2924                                         ARM::VST1d16wb_fixed,
2925                                         ARM::VST1d32wb_fixed,
2926                                         ARM::VST1d64wb_fixed };
2927    static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
2928                                         ARM::VST1q16wb_fixed,
2929                                         ARM::VST1q32wb_fixed,
2930                                         ARM::VST1q64wb_fixed };
2931    return SelectVST(N, true, 1, DOpcodes, QOpcodes, 0);
2932  }
2933
2934  case ARMISD::VST2_UPD: {
2935    static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed,
2936                                         ARM::VST2d16wb_fixed,
2937                                         ARM::VST2d32wb_fixed,
2938                                         ARM::VST1q64wb_fixed};
2939    static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed,
2940                                         ARM::VST2q16PseudoWB_fixed,
2941                                         ARM::VST2q32PseudoWB_fixed };
2942    return SelectVST(N, true, 2, DOpcodes, QOpcodes, 0);
2943  }
2944
2945  case ARMISD::VST3_UPD: {
2946    static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
2947                                         ARM::VST3d16Pseudo_UPD,
2948                                         ARM::VST3d32Pseudo_UPD,
2949                                         ARM::VST1d64TPseudoWB_fixed};
2950    static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
2951                                          ARM::VST3q16Pseudo_UPD,
2952                                          ARM::VST3q32Pseudo_UPD };
2953    static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
2954                                          ARM::VST3q16oddPseudo_UPD,
2955                                          ARM::VST3q32oddPseudo_UPD };
2956    return SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
2957  }
2958
2959  case ARMISD::VST4_UPD: {
2960    static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD,
2961                                         ARM::VST4d16Pseudo_UPD,
2962                                         ARM::VST4d32Pseudo_UPD,
2963                                         ARM::VST1d64QPseudoWB_fixed};
2964    static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
2965                                          ARM::VST4q16Pseudo_UPD,
2966                                          ARM::VST4q32Pseudo_UPD };
2967    static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
2968                                          ARM::VST4q16oddPseudo_UPD,
2969                                          ARM::VST4q32oddPseudo_UPD };
2970    return SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
2971  }
2972
2973  case ARMISD::VST2LN_UPD: {
2974    static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
2975                                         ARM::VST2LNd16Pseudo_UPD,
2976                                         ARM::VST2LNd32Pseudo_UPD };
2977    static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
2978                                         ARM::VST2LNq32Pseudo_UPD };
2979    return SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes);
2980  }
2981
2982  case ARMISD::VST3LN_UPD: {
2983    static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
2984                                         ARM::VST3LNd16Pseudo_UPD,
2985                                         ARM::VST3LNd32Pseudo_UPD };
2986    static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
2987                                         ARM::VST3LNq32Pseudo_UPD };
2988    return SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes);
2989  }
2990
2991  case ARMISD::VST4LN_UPD: {
2992    static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
2993                                         ARM::VST4LNd16Pseudo_UPD,
2994                                         ARM::VST4LNd32Pseudo_UPD };
2995    static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
2996                                         ARM::VST4LNq32Pseudo_UPD };
2997    return SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes);
2998  }
2999
3000  case ISD::INTRINSIC_VOID:
3001  case ISD::INTRINSIC_W_CHAIN: {
3002    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
3003    switch (IntNo) {
3004    default:
3005      break;
3006
3007    case Intrinsic::arm_ldrexd: {
3008      SDValue MemAddr = N->getOperand(2);
3009      DebugLoc dl = N->getDebugLoc();
3010      SDValue Chain = N->getOperand(0);
3011
3012      unsigned NewOpc = ARM::LDREXD;
3013      if (Subtarget->isThumb() && Subtarget->hasThumb2())
3014        NewOpc = ARM::t2LDREXD;
3015
3016      // arm_ldrexd returns a i64 value in {i32, i32}
3017      std::vector<EVT> ResTys;
3018      ResTys.push_back(MVT::i32);
3019      ResTys.push_back(MVT::i32);
3020      ResTys.push_back(MVT::Other);
3021
3022      // place arguments in the right order
3023      SmallVector<SDValue, 7> Ops;
3024      Ops.push_back(MemAddr);
3025      Ops.push_back(getAL(CurDAG));
3026      Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3027      Ops.push_back(Chain);
3028      SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops.data(),
3029                                          Ops.size());
3030      // Transfer memoperands.
3031      MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
3032      MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3033      cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1);
3034
3035      // Until there's support for specifing explicit register constraints
3036      // like the use of even/odd register pair, hardcode ldrexd to always
3037      // use the pair [R0, R1] to hold the load result.
3038      Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ARM::R0,
3039                                   SDValue(Ld, 0), SDValue(0,0));
3040      Chain = CurDAG->getCopyToReg(Chain, dl, ARM::R1,
3041                                   SDValue(Ld, 1), Chain.getValue(1));
3042
3043      // Remap uses.
3044      SDValue Glue = Chain.getValue(1);
3045      if (!SDValue(N, 0).use_empty()) {
3046        SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
3047                                                ARM::R0, MVT::i32, Glue);
3048        Glue = Result.getValue(2);
3049        ReplaceUses(SDValue(N, 0), Result);
3050      }
3051      if (!SDValue(N, 1).use_empty()) {
3052        SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
3053                                                ARM::R1, MVT::i32, Glue);
3054        Glue = Result.getValue(2);
3055        ReplaceUses(SDValue(N, 1), Result);
3056      }
3057
3058      ReplaceUses(SDValue(N, 2), SDValue(Ld, 2));
3059      return NULL;
3060    }
3061
3062    case Intrinsic::arm_strexd: {
3063      DebugLoc dl = N->getDebugLoc();
3064      SDValue Chain = N->getOperand(0);
3065      SDValue Val0 = N->getOperand(2);
3066      SDValue Val1 = N->getOperand(3);
3067      SDValue MemAddr = N->getOperand(4);
3068
3069      // Until there's support for specifing explicit register constraints
3070      // like the use of even/odd register pair, hardcode strexd to always
3071      // use the pair [R2, R3] to hold the i64 (i32, i32) value to be stored.
3072      Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ARM::R2, Val0,
3073                                   SDValue(0, 0));
3074      Chain = CurDAG->getCopyToReg(Chain, dl, ARM::R3, Val1, Chain.getValue(1));
3075
3076      SDValue Glue = Chain.getValue(1);
3077      Val0 = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
3078                                    ARM::R2, MVT::i32, Glue);
3079      Glue = Val0.getValue(1);
3080      Val1 = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
3081                                    ARM::R3, MVT::i32, Glue);
3082
3083      // Store exclusive double return a i32 value which is the return status
3084      // of the issued store.
3085      std::vector<EVT> ResTys;
3086      ResTys.push_back(MVT::i32);
3087      ResTys.push_back(MVT::Other);
3088
3089      // place arguments in the right order
3090      SmallVector<SDValue, 7> Ops;
3091      Ops.push_back(Val0);
3092      Ops.push_back(Val1);
3093      Ops.push_back(MemAddr);
3094      Ops.push_back(getAL(CurDAG));
3095      Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3096      Ops.push_back(Chain);
3097
3098      unsigned NewOpc = ARM::STREXD;
3099      if (Subtarget->isThumb() && Subtarget->hasThumb2())
3100        NewOpc = ARM::t2STREXD;
3101
3102      SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops.data(),
3103                                          Ops.size());
3104      // Transfer memoperands.
3105      MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
3106      MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3107      cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1);
3108
3109      return St;
3110    }
3111
3112    case Intrinsic::arm_neon_vld1: {
3113      static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
3114                                           ARM::VLD1d32, ARM::VLD1d64 };
3115      static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
3116                                           ARM::VLD1q32, ARM::VLD1q64};
3117      return SelectVLD(N, false, 1, DOpcodes, QOpcodes, 0);
3118    }
3119
3120    case Intrinsic::arm_neon_vld2: {
3121      static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
3122                                           ARM::VLD2d32, ARM::VLD1q64 };
3123      static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
3124                                           ARM::VLD2q32Pseudo };
3125      return SelectVLD(N, false, 2, DOpcodes, QOpcodes, 0);
3126    }
3127
3128    case Intrinsic::arm_neon_vld3: {
3129      static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo,
3130                                           ARM::VLD3d16Pseudo,
3131                                           ARM::VLD3d32Pseudo,
3132                                           ARM::VLD1d64TPseudo };
3133      static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
3134                                            ARM::VLD3q16Pseudo_UPD,
3135                                            ARM::VLD3q32Pseudo_UPD };
3136      static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo,
3137                                            ARM::VLD3q16oddPseudo,
3138                                            ARM::VLD3q32oddPseudo };
3139      return SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3140    }
3141
3142    case Intrinsic::arm_neon_vld4: {
3143      static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo,
3144                                           ARM::VLD4d16Pseudo,
3145                                           ARM::VLD4d32Pseudo,
3146                                           ARM::VLD1d64QPseudo };
3147      static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
3148                                            ARM::VLD4q16Pseudo_UPD,
3149                                            ARM::VLD4q32Pseudo_UPD };
3150      static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo,
3151                                            ARM::VLD4q16oddPseudo,
3152                                            ARM::VLD4q32oddPseudo };
3153      return SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3154    }
3155
3156    case Intrinsic::arm_neon_vld2lane: {
3157      static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo,
3158                                           ARM::VLD2LNd16Pseudo,
3159                                           ARM::VLD2LNd32Pseudo };
3160      static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo,
3161                                           ARM::VLD2LNq32Pseudo };
3162      return SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes);
3163    }
3164
3165    case Intrinsic::arm_neon_vld3lane: {
3166      static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo,
3167                                           ARM::VLD3LNd16Pseudo,
3168                                           ARM::VLD3LNd32Pseudo };
3169      static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo,
3170                                           ARM::VLD3LNq32Pseudo };
3171      return SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes);
3172    }
3173
3174    case Intrinsic::arm_neon_vld4lane: {
3175      static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo,
3176                                           ARM::VLD4LNd16Pseudo,
3177                                           ARM::VLD4LNd32Pseudo };
3178      static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo,
3179                                           ARM::VLD4LNq32Pseudo };
3180      return SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes);
3181    }
3182
3183    case Intrinsic::arm_neon_vst1: {
3184      static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
3185                                           ARM::VST1d32, ARM::VST1d64 };
3186      static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
3187                                           ARM::VST1q32, ARM::VST1q64 };
3188      return SelectVST(N, false, 1, DOpcodes, QOpcodes, 0);
3189    }
3190
3191    case Intrinsic::arm_neon_vst2: {
3192      static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
3193                                           ARM::VST2d32, ARM::VST1q64 };
3194      static uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
3195                                     ARM::VST2q32Pseudo };
3196      return SelectVST(N, false, 2, DOpcodes, QOpcodes, 0);
3197    }
3198
3199    case Intrinsic::arm_neon_vst3: {
3200      static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo,
3201                                           ARM::VST3d16Pseudo,
3202                                           ARM::VST3d32Pseudo,
3203                                           ARM::VST1d64TPseudo };
3204      static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3205                                            ARM::VST3q16Pseudo_UPD,
3206                                            ARM::VST3q32Pseudo_UPD };
3207      static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo,
3208                                            ARM::VST3q16oddPseudo,
3209                                            ARM::VST3q32oddPseudo };
3210      return SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3211    }
3212
3213    case Intrinsic::arm_neon_vst4: {
3214      static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo,
3215                                           ARM::VST4d16Pseudo,
3216                                           ARM::VST4d32Pseudo,
3217                                           ARM::VST1d64QPseudo };
3218      static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3219                                            ARM::VST4q16Pseudo_UPD,
3220                                            ARM::VST4q32Pseudo_UPD };
3221      static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo,
3222                                            ARM::VST4q16oddPseudo,
3223                                            ARM::VST4q32oddPseudo };
3224      return SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3225    }
3226
3227    case Intrinsic::arm_neon_vst2lane: {
3228      static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo,
3229                                           ARM::VST2LNd16Pseudo,
3230                                           ARM::VST2LNd32Pseudo };
3231      static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo,
3232                                           ARM::VST2LNq32Pseudo };
3233      return SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes);
3234    }
3235
3236    case Intrinsic::arm_neon_vst3lane: {
3237      static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo,
3238                                           ARM::VST3LNd16Pseudo,
3239                                           ARM::VST3LNd32Pseudo };
3240      static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo,
3241                                           ARM::VST3LNq32Pseudo };
3242      return SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes);
3243    }
3244
3245    case Intrinsic::arm_neon_vst4lane: {
3246      static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo,
3247                                           ARM::VST4LNd16Pseudo,
3248                                           ARM::VST4LNd32Pseudo };
3249      static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo,
3250                                           ARM::VST4LNq32Pseudo };
3251      return SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes);
3252    }
3253    }
3254    break;
3255  }
3256
3257  case ISD::INTRINSIC_WO_CHAIN: {
3258    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
3259    switch (IntNo) {
3260    default:
3261      break;
3262
3263    case Intrinsic::arm_neon_vtbl2:
3264      return SelectVTBL(N, false, 2, ARM::VTBL2);
3265    case Intrinsic::arm_neon_vtbl3:
3266      return SelectVTBL(N, false, 3, ARM::VTBL3Pseudo);
3267    case Intrinsic::arm_neon_vtbl4:
3268      return SelectVTBL(N, false, 4, ARM::VTBL4Pseudo);
3269
3270    case Intrinsic::arm_neon_vtbx2:
3271      return SelectVTBL(N, true, 2, ARM::VTBX2);
3272    case Intrinsic::arm_neon_vtbx3:
3273      return SelectVTBL(N, true, 3, ARM::VTBX3Pseudo);
3274    case Intrinsic::arm_neon_vtbx4:
3275      return SelectVTBL(N, true, 4, ARM::VTBX4Pseudo);
3276    }
3277    break;
3278  }
3279
3280  case ARMISD::VTBL1: {
3281    DebugLoc dl = N->getDebugLoc();
3282    EVT VT = N->getValueType(0);
3283    SmallVector<SDValue, 6> Ops;
3284
3285    Ops.push_back(N->getOperand(0));
3286    Ops.push_back(N->getOperand(1));
3287    Ops.push_back(getAL(CurDAG));                    // Predicate
3288    Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
3289    return CurDAG->getMachineNode(ARM::VTBL1, dl, VT, Ops.data(), Ops.size());
3290  }
3291  case ARMISD::VTBL2: {
3292    DebugLoc dl = N->getDebugLoc();
3293    EVT VT = N->getValueType(0);
3294
3295    // Form a REG_SEQUENCE to force register allocation.
3296    SDValue V0 = N->getOperand(0);
3297    SDValue V1 = N->getOperand(1);
3298    SDValue RegSeq = SDValue(PairDRegs(MVT::v16i8, V0, V1), 0);
3299
3300    SmallVector<SDValue, 6> Ops;
3301    Ops.push_back(RegSeq);
3302    Ops.push_back(N->getOperand(2));
3303    Ops.push_back(getAL(CurDAG));                    // Predicate
3304    Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
3305    return CurDAG->getMachineNode(ARM::VTBL2, dl, VT,
3306                                  Ops.data(), Ops.size());
3307  }
3308
3309  case ISD::CONCAT_VECTORS:
3310    return SelectConcatVector(N);
3311
3312  case ARMISD::ATOMOR64_DAG:
3313    return SelectAtomic64(N, ARM::ATOMOR6432);
3314  case ARMISD::ATOMXOR64_DAG:
3315    return SelectAtomic64(N, ARM::ATOMXOR6432);
3316  case ARMISD::ATOMADD64_DAG:
3317    return SelectAtomic64(N, ARM::ATOMADD6432);
3318  case ARMISD::ATOMSUB64_DAG:
3319    return SelectAtomic64(N, ARM::ATOMSUB6432);
3320  case ARMISD::ATOMNAND64_DAG:
3321    return SelectAtomic64(N, ARM::ATOMNAND6432);
3322  case ARMISD::ATOMAND64_DAG:
3323    return SelectAtomic64(N, ARM::ATOMAND6432);
3324  case ARMISD::ATOMSWAP64_DAG:
3325    return SelectAtomic64(N, ARM::ATOMSWAP6432);
3326  case ARMISD::ATOMCMPXCHG64_DAG:
3327    return SelectAtomic64(N, ARM::ATOMCMPXCHG6432);
3328  }
3329
3330  return SelectCode(N);
3331}
3332
3333bool ARMDAGToDAGISel::
3334SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
3335                             std::vector<SDValue> &OutOps) {
3336  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
3337  // Require the address to be in a register.  That is safe for all ARM
3338  // variants and it is hard to do anything much smarter without knowing
3339  // how the operand is used.
3340  OutOps.push_back(Op);
3341  return false;
3342}
3343
3344/// createARMISelDag - This pass converts a legalized DAG into a
3345/// ARM-specific DAG, ready for instruction scheduling.
3346///
3347FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
3348                                     CodeGenOpt::Level OptLevel) {
3349  return new ARMDAGToDAGISel(TM, OptLevel);
3350}
3351