ARMISelDAGToDAG.cpp revision 198090 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARM.h"
15#include "ARMAddressingModes.h"
16#include "ARMConstantPoolValue.h"
17#include "ARMISelLowering.h"
18#include "ARMTargetMachine.h"
19#include "llvm/CallingConv.h"
20#include "llvm/Constants.h"
21#include "llvm/DerivedTypes.h"
22#include "llvm/Function.h"
23#include "llvm/Intrinsics.h"
24#include "llvm/LLVMContext.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/SelectionDAG.h"
29#include "llvm/CodeGen/SelectionDAGISel.h"
30#include "llvm/Target/TargetLowering.h"
31#include "llvm/Target/TargetOptions.h"
32#include "llvm/Support/Compiler.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/raw_ostream.h"
36
37using namespace llvm;
38
39//===--------------------------------------------------------------------===//
40/// ARMDAGToDAGISel - ARM specific code to select ARM machine
41/// instructions for SelectionDAG operations.
42///
43namespace {
44class ARMDAGToDAGISel : public SelectionDAGISel {
45  ARMBaseTargetMachine &TM;
46
47  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
48  /// make the right decision when generating code for different targets.
49  const ARMSubtarget *Subtarget;
50
51public:
52  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
53                           CodeGenOpt::Level OptLevel)
54    : SelectionDAGISel(tm, OptLevel), TM(tm),
55    Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
56  }
57
58  virtual const char *getPassName() const {
59    return "ARM Instruction Selection";
60  }
61
62  /// getI32Imm - Return a target constant of type i32 with the specified
63  /// value.
64  inline SDValue getI32Imm(unsigned Imm) {
65    return CurDAG->getTargetConstant(Imm, MVT::i32);
66  }
67
68  SDNode *Select(SDValue Op);
69  virtual void InstructionSelect();
70  bool SelectShifterOperandReg(SDValue Op, SDValue N, SDValue &A,
71                               SDValue &B, SDValue &C);
72  bool SelectAddrMode2(SDValue Op, SDValue N, SDValue &Base,
73                       SDValue &Offset, SDValue &Opc);
74  bool SelectAddrMode2Offset(SDValue Op, SDValue N,
75                             SDValue &Offset, SDValue &Opc);
76  bool SelectAddrMode3(SDValue Op, SDValue N, SDValue &Base,
77                       SDValue &Offset, SDValue &Opc);
78  bool SelectAddrMode3Offset(SDValue Op, SDValue N,
79                             SDValue &Offset, SDValue &Opc);
80  bool SelectAddrMode4(SDValue Op, SDValue N, SDValue &Addr,
81                       SDValue &Mode);
82  bool SelectAddrMode5(SDValue Op, SDValue N, SDValue &Base,
83                       SDValue &Offset);
84  bool SelectAddrMode6(SDValue Op, SDValue N, SDValue &Addr, SDValue &Update,
85                       SDValue &Opc);
86
87  bool SelectAddrModePC(SDValue Op, SDValue N, SDValue &Offset,
88                        SDValue &Label);
89
90  bool SelectThumbAddrModeRR(SDValue Op, SDValue N, SDValue &Base,
91                             SDValue &Offset);
92  bool SelectThumbAddrModeRI5(SDValue Op, SDValue N, unsigned Scale,
93                              SDValue &Base, SDValue &OffImm,
94                              SDValue &Offset);
95  bool SelectThumbAddrModeS1(SDValue Op, SDValue N, SDValue &Base,
96                             SDValue &OffImm, SDValue &Offset);
97  bool SelectThumbAddrModeS2(SDValue Op, SDValue N, SDValue &Base,
98                             SDValue &OffImm, SDValue &Offset);
99  bool SelectThumbAddrModeS4(SDValue Op, SDValue N, SDValue &Base,
100                             SDValue &OffImm, SDValue &Offset);
101  bool SelectThumbAddrModeSP(SDValue Op, SDValue N, SDValue &Base,
102                             SDValue &OffImm);
103
104  bool SelectT2ShifterOperandReg(SDValue Op, SDValue N,
105                                 SDValue &BaseReg, SDValue &Opc);
106  bool SelectT2AddrModeImm12(SDValue Op, SDValue N, SDValue &Base,
107                             SDValue &OffImm);
108  bool SelectT2AddrModeImm8(SDValue Op, SDValue N, SDValue &Base,
109                            SDValue &OffImm);
110  bool SelectT2AddrModeImm8Offset(SDValue Op, SDValue N,
111                                 SDValue &OffImm);
112  bool SelectT2AddrModeImm8s4(SDValue Op, SDValue N, SDValue &Base,
113                              SDValue &OffImm);
114  bool SelectT2AddrModeSoReg(SDValue Op, SDValue N, SDValue &Base,
115                             SDValue &OffReg, SDValue &ShImm);
116
117  // Include the pieces autogenerated from the target description.
118#include "ARMGenDAGISel.inc"
119
120private:
121  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
122  /// ARM.
123  SDNode *SelectARMIndexedLoad(SDValue Op);
124  SDNode *SelectT2IndexedLoad(SDValue Op);
125
126  /// SelectDYN_ALLOC - Select dynamic alloc for Thumb.
127  SDNode *SelectDYN_ALLOC(SDValue Op);
128
129  /// SelectVLD - Select NEON load intrinsics.  NumVecs should
130  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
131  /// loads of D registers and even subregs and odd subregs of Q registers.
132  /// For NumVecs == 2, QOpcodes1 is not used.
133  SDNode *SelectVLD(SDValue Op, unsigned NumVecs, unsigned *DOpcodes,
134                    unsigned *QOpcodes0, unsigned *QOpcodes1);
135
136  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
137  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
138  /// load/store of D registers and even subregs and odd subregs of Q registers.
139  SDNode *SelectVLDSTLane(SDValue Op, bool IsLoad, unsigned NumVecs,
140                          unsigned *DOpcodes, unsigned *QOpcodes0,
141                          unsigned *QOpcodes1);
142
143  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
144  SDNode *SelectV6T2BitfieldExtractOp(SDValue Op, unsigned Opc);
145
146  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
147  /// inline asm expressions.
148  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
149                                            char ConstraintCode,
150                                            std::vector<SDValue> &OutOps);
151
152  /// PairDRegs - Insert a pair of double registers into an implicit def to
153  /// form a quad register.
154  SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
155};
156}
157
158/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
159/// operand. If so Imm will receive the 32-bit value.
160static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
161  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
162    Imm = cast<ConstantSDNode>(N)->getZExtValue();
163    return true;
164  }
165  return false;
166}
167
168// isInt32Immediate - This method tests to see if a constant operand.
169// If so Imm will receive the 32 bit value.
170static bool isInt32Immediate(SDValue N, unsigned &Imm) {
171  return isInt32Immediate(N.getNode(), Imm);
172}
173
174// isOpcWithIntImmediate - This method tests to see if the node is a specific
175// opcode and that it has a immediate integer right operand.
176// If so Imm will receive the 32 bit value.
177static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
178  return N->getOpcode() == Opc &&
179         isInt32Immediate(N->getOperand(1).getNode(), Imm);
180}
181
182
183void ARMDAGToDAGISel::InstructionSelect() {
184  DEBUG(BB->dump());
185
186  SelectRoot(*CurDAG);
187  CurDAG->RemoveDeadNodes();
188}
189
190bool ARMDAGToDAGISel::SelectShifterOperandReg(SDValue Op,
191                                              SDValue N,
192                                              SDValue &BaseReg,
193                                              SDValue &ShReg,
194                                              SDValue &Opc) {
195  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
196
197  // Don't match base register only case. That is matched to a separate
198  // lower complexity pattern with explicit register operand.
199  if (ShOpcVal == ARM_AM::no_shift) return false;
200
201  BaseReg = N.getOperand(0);
202  unsigned ShImmVal = 0;
203  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
204    ShReg = CurDAG->getRegister(0, MVT::i32);
205    ShImmVal = RHS->getZExtValue() & 31;
206  } else {
207    ShReg = N.getOperand(1);
208  }
209  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
210                                  MVT::i32);
211  return true;
212}
213
214bool ARMDAGToDAGISel::SelectAddrMode2(SDValue Op, SDValue N,
215                                      SDValue &Base, SDValue &Offset,
216                                      SDValue &Opc) {
217  if (N.getOpcode() == ISD::MUL) {
218    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
219      // X * [3,5,9] -> X + X * [2,4,8] etc.
220      int RHSC = (int)RHS->getZExtValue();
221      if (RHSC & 1) {
222        RHSC = RHSC & ~1;
223        ARM_AM::AddrOpc AddSub = ARM_AM::add;
224        if (RHSC < 0) {
225          AddSub = ARM_AM::sub;
226          RHSC = - RHSC;
227        }
228        if (isPowerOf2_32(RHSC)) {
229          unsigned ShAmt = Log2_32(RHSC);
230          Base = Offset = N.getOperand(0);
231          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
232                                                            ARM_AM::lsl),
233                                          MVT::i32);
234          return true;
235        }
236      }
237    }
238  }
239
240  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
241    Base = N;
242    if (N.getOpcode() == ISD::FrameIndex) {
243      int FI = cast<FrameIndexSDNode>(N)->getIndex();
244      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
245    } else if (N.getOpcode() == ARMISD::Wrapper) {
246      Base = N.getOperand(0);
247    }
248    Offset = CurDAG->getRegister(0, MVT::i32);
249    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
250                                                      ARM_AM::no_shift),
251                                    MVT::i32);
252    return true;
253  }
254
255  // Match simple R +/- imm12 operands.
256  if (N.getOpcode() == ISD::ADD)
257    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
258      int RHSC = (int)RHS->getZExtValue();
259      if ((RHSC >= 0 && RHSC < 0x1000) ||
260          (RHSC < 0 && RHSC > -0x1000)) { // 12 bits.
261        Base = N.getOperand(0);
262        if (Base.getOpcode() == ISD::FrameIndex) {
263          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
264          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
265        }
266        Offset = CurDAG->getRegister(0, MVT::i32);
267
268        ARM_AM::AddrOpc AddSub = ARM_AM::add;
269        if (RHSC < 0) {
270          AddSub = ARM_AM::sub;
271          RHSC = - RHSC;
272        }
273        Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
274                                                          ARM_AM::no_shift),
275                                        MVT::i32);
276        return true;
277      }
278    }
279
280  // Otherwise this is R +/- [possibly shifted] R
281  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::ADD ? ARM_AM::add:ARM_AM::sub;
282  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1));
283  unsigned ShAmt = 0;
284
285  Base   = N.getOperand(0);
286  Offset = N.getOperand(1);
287
288  if (ShOpcVal != ARM_AM::no_shift) {
289    // Check to see if the RHS of the shift is a constant, if not, we can't fold
290    // it.
291    if (ConstantSDNode *Sh =
292           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
293      ShAmt = Sh->getZExtValue();
294      Offset = N.getOperand(1).getOperand(0);
295    } else {
296      ShOpcVal = ARM_AM::no_shift;
297    }
298  }
299
300  // Try matching (R shl C) + (R).
301  if (N.getOpcode() == ISD::ADD && ShOpcVal == ARM_AM::no_shift) {
302    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0));
303    if (ShOpcVal != ARM_AM::no_shift) {
304      // Check to see if the RHS of the shift is a constant, if not, we can't
305      // fold it.
306      if (ConstantSDNode *Sh =
307          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
308        ShAmt = Sh->getZExtValue();
309        Offset = N.getOperand(0).getOperand(0);
310        Base = N.getOperand(1);
311      } else {
312        ShOpcVal = ARM_AM::no_shift;
313      }
314    }
315  }
316
317  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
318                                  MVT::i32);
319  return true;
320}
321
322bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDValue Op, SDValue N,
323                                            SDValue &Offset, SDValue &Opc) {
324  unsigned Opcode = Op.getOpcode();
325  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
326    ? cast<LoadSDNode>(Op)->getAddressingMode()
327    : cast<StoreSDNode>(Op)->getAddressingMode();
328  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
329    ? ARM_AM::add : ARM_AM::sub;
330  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
331    int Val = (int)C->getZExtValue();
332    if (Val >= 0 && Val < 0x1000) { // 12 bits.
333      Offset = CurDAG->getRegister(0, MVT::i32);
334      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
335                                                        ARM_AM::no_shift),
336                                      MVT::i32);
337      return true;
338    }
339  }
340
341  Offset = N;
342  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
343  unsigned ShAmt = 0;
344  if (ShOpcVal != ARM_AM::no_shift) {
345    // Check to see if the RHS of the shift is a constant, if not, we can't fold
346    // it.
347    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
348      ShAmt = Sh->getZExtValue();
349      Offset = N.getOperand(0);
350    } else {
351      ShOpcVal = ARM_AM::no_shift;
352    }
353  }
354
355  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
356                                  MVT::i32);
357  return true;
358}
359
360
361bool ARMDAGToDAGISel::SelectAddrMode3(SDValue Op, SDValue N,
362                                      SDValue &Base, SDValue &Offset,
363                                      SDValue &Opc) {
364  if (N.getOpcode() == ISD::SUB) {
365    // X - C  is canonicalize to X + -C, no need to handle it here.
366    Base = N.getOperand(0);
367    Offset = N.getOperand(1);
368    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
369    return true;
370  }
371
372  if (N.getOpcode() != ISD::ADD) {
373    Base = N;
374    if (N.getOpcode() == ISD::FrameIndex) {
375      int FI = cast<FrameIndexSDNode>(N)->getIndex();
376      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
377    }
378    Offset = CurDAG->getRegister(0, MVT::i32);
379    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
380    return true;
381  }
382
383  // If the RHS is +/- imm8, fold into addr mode.
384  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
385    int RHSC = (int)RHS->getZExtValue();
386    if ((RHSC >= 0 && RHSC < 256) ||
387        (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
388      Base = N.getOperand(0);
389      if (Base.getOpcode() == ISD::FrameIndex) {
390        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
391        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
392      }
393      Offset = CurDAG->getRegister(0, MVT::i32);
394
395      ARM_AM::AddrOpc AddSub = ARM_AM::add;
396      if (RHSC < 0) {
397        AddSub = ARM_AM::sub;
398        RHSC = - RHSC;
399      }
400      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
401      return true;
402    }
403  }
404
405  Base = N.getOperand(0);
406  Offset = N.getOperand(1);
407  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
408  return true;
409}
410
411bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDValue Op, SDValue N,
412                                            SDValue &Offset, SDValue &Opc) {
413  unsigned Opcode = Op.getOpcode();
414  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
415    ? cast<LoadSDNode>(Op)->getAddressingMode()
416    : cast<StoreSDNode>(Op)->getAddressingMode();
417  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
418    ? ARM_AM::add : ARM_AM::sub;
419  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
420    int Val = (int)C->getZExtValue();
421    if (Val >= 0 && Val < 256) {
422      Offset = CurDAG->getRegister(0, MVT::i32);
423      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
424      return true;
425    }
426  }
427
428  Offset = N;
429  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
430  return true;
431}
432
433bool ARMDAGToDAGISel::SelectAddrMode4(SDValue Op, SDValue N,
434                                      SDValue &Addr, SDValue &Mode) {
435  Addr = N;
436  Mode = CurDAG->getTargetConstant(0, MVT::i32);
437  return true;
438}
439
440bool ARMDAGToDAGISel::SelectAddrMode5(SDValue Op, SDValue N,
441                                      SDValue &Base, SDValue &Offset) {
442  if (N.getOpcode() != ISD::ADD) {
443    Base = N;
444    if (N.getOpcode() == ISD::FrameIndex) {
445      int FI = cast<FrameIndexSDNode>(N)->getIndex();
446      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
447    } else if (N.getOpcode() == ARMISD::Wrapper) {
448      Base = N.getOperand(0);
449    }
450    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
451                                       MVT::i32);
452    return true;
453  }
454
455  // If the RHS is +/- imm8, fold into addr mode.
456  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
457    int RHSC = (int)RHS->getZExtValue();
458    if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied by 4.
459      RHSC >>= 2;
460      if ((RHSC >= 0 && RHSC < 256) ||
461          (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
462        Base = N.getOperand(0);
463        if (Base.getOpcode() == ISD::FrameIndex) {
464          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
465          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
466        }
467
468        ARM_AM::AddrOpc AddSub = ARM_AM::add;
469        if (RHSC < 0) {
470          AddSub = ARM_AM::sub;
471          RHSC = - RHSC;
472        }
473        Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
474                                           MVT::i32);
475        return true;
476      }
477    }
478  }
479
480  Base = N;
481  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
482                                     MVT::i32);
483  return true;
484}
485
486bool ARMDAGToDAGISel::SelectAddrMode6(SDValue Op, SDValue N,
487                                      SDValue &Addr, SDValue &Update,
488                                      SDValue &Opc) {
489  Addr = N;
490  // Default to no writeback.
491  Update = CurDAG->getRegister(0, MVT::i32);
492  Opc = CurDAG->getTargetConstant(ARM_AM::getAM6Opc(false), MVT::i32);
493  return true;
494}
495
496bool ARMDAGToDAGISel::SelectAddrModePC(SDValue Op, SDValue N,
497                                       SDValue &Offset, SDValue &Label) {
498  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
499    Offset = N.getOperand(0);
500    SDValue N1 = N.getOperand(1);
501    Label  = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
502                                       MVT::i32);
503    return true;
504  }
505  return false;
506}
507
508bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue Op, SDValue N,
509                                            SDValue &Base, SDValue &Offset){
510  // FIXME dl should come from the parent load or store, not the address
511  DebugLoc dl = Op.getDebugLoc();
512  if (N.getOpcode() != ISD::ADD) {
513    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
514    if (!NC || NC->getZExtValue() != 0)
515      return false;
516
517    Base = Offset = N;
518    return true;
519  }
520
521  Base = N.getOperand(0);
522  Offset = N.getOperand(1);
523  return true;
524}
525
526bool
527ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDValue Op, SDValue N,
528                                        unsigned Scale, SDValue &Base,
529                                        SDValue &OffImm, SDValue &Offset) {
530  if (Scale == 4) {
531    SDValue TmpBase, TmpOffImm;
532    if (SelectThumbAddrModeSP(Op, N, TmpBase, TmpOffImm))
533      return false;  // We want to select tLDRspi / tSTRspi instead.
534    if (N.getOpcode() == ARMISD::Wrapper &&
535        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
536      return false;  // We want to select tLDRpci instead.
537  }
538
539  if (N.getOpcode() != ISD::ADD) {
540    Base = (N.getOpcode() == ARMISD::Wrapper) ? N.getOperand(0) : N;
541    Offset = CurDAG->getRegister(0, MVT::i32);
542    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
543    return true;
544  }
545
546  // Thumb does not have [sp, r] address mode.
547  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
548  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
549  if ((LHSR && LHSR->getReg() == ARM::SP) ||
550      (RHSR && RHSR->getReg() == ARM::SP)) {
551    Base = N;
552    Offset = CurDAG->getRegister(0, MVT::i32);
553    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
554    return true;
555  }
556
557  // If the RHS is + imm5 * scale, fold into addr mode.
558  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
559    int RHSC = (int)RHS->getZExtValue();
560    if ((RHSC & (Scale-1)) == 0) {  // The constant is implicitly multiplied.
561      RHSC /= Scale;
562      if (RHSC >= 0 && RHSC < 32) {
563        Base = N.getOperand(0);
564        Offset = CurDAG->getRegister(0, MVT::i32);
565        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
566        return true;
567      }
568    }
569  }
570
571  Base = N.getOperand(0);
572  Offset = N.getOperand(1);
573  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
574  return true;
575}
576
577bool ARMDAGToDAGISel::SelectThumbAddrModeS1(SDValue Op, SDValue N,
578                                            SDValue &Base, SDValue &OffImm,
579                                            SDValue &Offset) {
580  return SelectThumbAddrModeRI5(Op, N, 1, Base, OffImm, Offset);
581}
582
583bool ARMDAGToDAGISel::SelectThumbAddrModeS2(SDValue Op, SDValue N,
584                                            SDValue &Base, SDValue &OffImm,
585                                            SDValue &Offset) {
586  return SelectThumbAddrModeRI5(Op, N, 2, Base, OffImm, Offset);
587}
588
589bool ARMDAGToDAGISel::SelectThumbAddrModeS4(SDValue Op, SDValue N,
590                                            SDValue &Base, SDValue &OffImm,
591                                            SDValue &Offset) {
592  return SelectThumbAddrModeRI5(Op, N, 4, Base, OffImm, Offset);
593}
594
595bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue Op, SDValue N,
596                                           SDValue &Base, SDValue &OffImm) {
597  if (N.getOpcode() == ISD::FrameIndex) {
598    int FI = cast<FrameIndexSDNode>(N)->getIndex();
599    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
600    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
601    return true;
602  }
603
604  if (N.getOpcode() != ISD::ADD)
605    return false;
606
607  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
608  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
609      (LHSR && LHSR->getReg() == ARM::SP)) {
610    // If the RHS is + imm8 * scale, fold into addr mode.
611    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
612      int RHSC = (int)RHS->getZExtValue();
613      if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied.
614        RHSC >>= 2;
615        if (RHSC >= 0 && RHSC < 256) {
616          Base = N.getOperand(0);
617          if (Base.getOpcode() == ISD::FrameIndex) {
618            int FI = cast<FrameIndexSDNode>(Base)->getIndex();
619            Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
620          }
621          OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
622          return true;
623        }
624      }
625    }
626  }
627
628  return false;
629}
630
631bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue Op, SDValue N,
632                                                SDValue &BaseReg,
633                                                SDValue &Opc) {
634  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
635
636  // Don't match base register only case. That is matched to a separate
637  // lower complexity pattern with explicit register operand.
638  if (ShOpcVal == ARM_AM::no_shift) return false;
639
640  BaseReg = N.getOperand(0);
641  unsigned ShImmVal = 0;
642  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
643    ShImmVal = RHS->getZExtValue() & 31;
644    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
645    return true;
646  }
647
648  return false;
649}
650
651bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue Op, SDValue N,
652                                            SDValue &Base, SDValue &OffImm) {
653  // Match simple R + imm12 operands.
654
655  // Base only.
656  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
657    if (N.getOpcode() == ISD::FrameIndex) {
658      // Match frame index...
659      int FI = cast<FrameIndexSDNode>(N)->getIndex();
660      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
661      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
662      return true;
663    } else if (N.getOpcode() == ARMISD::Wrapper) {
664      Base = N.getOperand(0);
665      if (Base.getOpcode() == ISD::TargetConstantPool)
666        return false;  // We want to select t2LDRpci instead.
667    } else
668      Base = N;
669    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
670    return true;
671  }
672
673  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
674    if (SelectT2AddrModeImm8(Op, N, Base, OffImm))
675      // Let t2LDRi8 handle (R - imm8).
676      return false;
677
678    int RHSC = (int)RHS->getZExtValue();
679    if (N.getOpcode() == ISD::SUB)
680      RHSC = -RHSC;
681
682    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
683      Base   = N.getOperand(0);
684      if (Base.getOpcode() == ISD::FrameIndex) {
685        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
686        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
687      }
688      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
689      return true;
690    }
691  }
692
693  // Base only.
694  Base = N;
695  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
696  return true;
697}
698
699bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue Op, SDValue N,
700                                           SDValue &Base, SDValue &OffImm) {
701  // Match simple R - imm8 operands.
702  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) {
703    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
704      int RHSC = (int)RHS->getSExtValue();
705      if (N.getOpcode() == ISD::SUB)
706        RHSC = -RHSC;
707
708      if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
709        Base = N.getOperand(0);
710        if (Base.getOpcode() == ISD::FrameIndex) {
711          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
712          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
713        }
714        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
715        return true;
716      }
717    }
718  }
719
720  return false;
721}
722
723bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDValue Op, SDValue N,
724                                                 SDValue &OffImm){
725  unsigned Opcode = Op.getOpcode();
726  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
727    ? cast<LoadSDNode>(Op)->getAddressingMode()
728    : cast<StoreSDNode>(Op)->getAddressingMode();
729  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N)) {
730    int RHSC = (int)RHS->getZExtValue();
731    if (RHSC >= 0 && RHSC < 0x100) { // 8 bits.
732      OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
733        ? CurDAG->getTargetConstant(RHSC, MVT::i32)
734        : CurDAG->getTargetConstant(-RHSC, MVT::i32);
735      return true;
736    }
737  }
738
739  return false;
740}
741
742bool ARMDAGToDAGISel::SelectT2AddrModeImm8s4(SDValue Op, SDValue N,
743                                             SDValue &Base, SDValue &OffImm) {
744  if (N.getOpcode() == ISD::ADD) {
745    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
746      int RHSC = (int)RHS->getZExtValue();
747      if (((RHSC & 0x3) == 0) &&
748          ((RHSC >= 0 && RHSC < 0x400) || (RHSC < 0 && RHSC > -0x400))) { // 8 bits.
749        Base   = N.getOperand(0);
750        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
751        return true;
752      }
753    }
754  } else if (N.getOpcode() == ISD::SUB) {
755    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
756      int RHSC = (int)RHS->getZExtValue();
757      if (((RHSC & 0x3) == 0) && (RHSC >= 0 && RHSC < 0x400)) { // 8 bits.
758        Base   = N.getOperand(0);
759        OffImm = CurDAG->getTargetConstant(-RHSC, MVT::i32);
760        return true;
761      }
762    }
763  }
764
765  return false;
766}
767
768bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue Op, SDValue N,
769                                            SDValue &Base,
770                                            SDValue &OffReg, SDValue &ShImm) {
771  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
772  if (N.getOpcode() != ISD::ADD)
773    return false;
774
775  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
776  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
777    int RHSC = (int)RHS->getZExtValue();
778    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
779      return false;
780    else if (RHSC < 0 && RHSC >= -255) // 8 bits
781      return false;
782  }
783
784  // Look for (R + R) or (R + (R << [1,2,3])).
785  unsigned ShAmt = 0;
786  Base   = N.getOperand(0);
787  OffReg = N.getOperand(1);
788
789  // Swap if it is ((R << c) + R).
790  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg);
791  if (ShOpcVal != ARM_AM::lsl) {
792    ShOpcVal = ARM_AM::getShiftOpcForNode(Base);
793    if (ShOpcVal == ARM_AM::lsl)
794      std::swap(Base, OffReg);
795  }
796
797  if (ShOpcVal == ARM_AM::lsl) {
798    // Check to see if the RHS of the shift is a constant, if not, we can't fold
799    // it.
800    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
801      ShAmt = Sh->getZExtValue();
802      if (ShAmt >= 4) {
803        ShAmt = 0;
804        ShOpcVal = ARM_AM::no_shift;
805      } else
806        OffReg = OffReg.getOperand(0);
807    } else {
808      ShOpcVal = ARM_AM::no_shift;
809    }
810  }
811
812  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
813
814  return true;
815}
816
817//===--------------------------------------------------------------------===//
818
819/// getAL - Returns a ARMCC::AL immediate node.
820static inline SDValue getAL(SelectionDAG *CurDAG) {
821  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
822}
823
824SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDValue Op) {
825  LoadSDNode *LD = cast<LoadSDNode>(Op);
826  ISD::MemIndexedMode AM = LD->getAddressingMode();
827  if (AM == ISD::UNINDEXED)
828    return NULL;
829
830  EVT LoadedVT = LD->getMemoryVT();
831  SDValue Offset, AMOpc;
832  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
833  unsigned Opcode = 0;
834  bool Match = false;
835  if (LoadedVT == MVT::i32 &&
836      SelectAddrMode2Offset(Op, LD->getOffset(), Offset, AMOpc)) {
837    Opcode = isPre ? ARM::LDR_PRE : ARM::LDR_POST;
838    Match = true;
839  } else if (LoadedVT == MVT::i16 &&
840             SelectAddrMode3Offset(Op, LD->getOffset(), Offset, AMOpc)) {
841    Match = true;
842    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
843      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
844      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
845  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
846    if (LD->getExtensionType() == ISD::SEXTLOAD) {
847      if (SelectAddrMode3Offset(Op, LD->getOffset(), Offset, AMOpc)) {
848        Match = true;
849        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
850      }
851    } else {
852      if (SelectAddrMode2Offset(Op, LD->getOffset(), Offset, AMOpc)) {
853        Match = true;
854        Opcode = isPre ? ARM::LDRB_PRE : ARM::LDRB_POST;
855      }
856    }
857  }
858
859  if (Match) {
860    SDValue Chain = LD->getChain();
861    SDValue Base = LD->getBasePtr();
862    SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
863                     CurDAG->getRegister(0, MVT::i32), Chain };
864    return CurDAG->getMachineNode(Opcode, Op.getDebugLoc(), MVT::i32, MVT::i32,
865                                  MVT::Other, Ops, 6);
866  }
867
868  return NULL;
869}
870
871SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDValue Op) {
872  LoadSDNode *LD = cast<LoadSDNode>(Op);
873  ISD::MemIndexedMode AM = LD->getAddressingMode();
874  if (AM == ISD::UNINDEXED)
875    return NULL;
876
877  EVT LoadedVT = LD->getMemoryVT();
878  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
879  SDValue Offset;
880  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
881  unsigned Opcode = 0;
882  bool Match = false;
883  if (SelectT2AddrModeImm8Offset(Op, LD->getOffset(), Offset)) {
884    switch (LoadedVT.getSimpleVT().SimpleTy) {
885    case MVT::i32:
886      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
887      break;
888    case MVT::i16:
889      if (isSExtLd)
890        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
891      else
892        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
893      break;
894    case MVT::i8:
895    case MVT::i1:
896      if (isSExtLd)
897        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
898      else
899        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
900      break;
901    default:
902      return NULL;
903    }
904    Match = true;
905  }
906
907  if (Match) {
908    SDValue Chain = LD->getChain();
909    SDValue Base = LD->getBasePtr();
910    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
911                     CurDAG->getRegister(0, MVT::i32), Chain };
912    return CurDAG->getMachineNode(Opcode, Op.getDebugLoc(), MVT::i32, MVT::i32,
913                                  MVT::Other, Ops, 5);
914  }
915
916  return NULL;
917}
918
919SDNode *ARMDAGToDAGISel::SelectDYN_ALLOC(SDValue Op) {
920  SDNode *N = Op.getNode();
921  DebugLoc dl = N->getDebugLoc();
922  EVT VT = Op.getValueType();
923  SDValue Chain = Op.getOperand(0);
924  SDValue Size = Op.getOperand(1);
925  SDValue Align = Op.getOperand(2);
926  SDValue SP = CurDAG->getRegister(ARM::SP, MVT::i32);
927  int32_t AlignVal = cast<ConstantSDNode>(Align)->getSExtValue();
928  if (AlignVal < 0)
929    // We need to align the stack. Use Thumb1 tAND which is the only thumb
930    // instruction that can read and write SP. This matches to a pseudo
931    // instruction that has a chain to ensure the result is written back to
932    // the stack pointer.
933    SP = SDValue(CurDAG->getMachineNode(ARM::tANDsp, dl, VT, SP, Align), 0);
934
935  bool isC = isa<ConstantSDNode>(Size);
936  uint32_t C = isC ? cast<ConstantSDNode>(Size)->getZExtValue() : ~0UL;
937  // Handle the most common case for both Thumb1 and Thumb2:
938  // tSUBspi - immediate is between 0 ... 508 inclusive.
939  if (C <= 508 && ((C & 3) == 0))
940    // FIXME: tSUBspi encode scale 4 implicitly.
941    return CurDAG->SelectNodeTo(N, ARM::tSUBspi_, VT, MVT::Other, SP,
942                                CurDAG->getTargetConstant(C/4, MVT::i32),
943                                Chain);
944
945  if (Subtarget->isThumb1Only()) {
946    // Use tADDspr since Thumb1 does not have a sub r, sp, r. ARMISelLowering
947    // should have negated the size operand already. FIXME: We can't insert
948    // new target independent node at this stage so we are forced to negate
949    // it earlier. Is there a better solution?
950    return CurDAG->SelectNodeTo(N, ARM::tADDspr_, VT, MVT::Other, SP, Size,
951                                Chain);
952  } else if (Subtarget->isThumb2()) {
953    if (isC && Predicate_t2_so_imm(Size.getNode())) {
954      // t2SUBrSPi
955      SDValue Ops[] = { SP, CurDAG->getTargetConstant(C, MVT::i32), Chain };
956      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPi_, VT, MVT::Other, Ops, 3);
957    } else if (isC && Predicate_imm0_4095(Size.getNode())) {
958      // t2SUBrSPi12
959      SDValue Ops[] = { SP, CurDAG->getTargetConstant(C, MVT::i32), Chain };
960      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPi12_, VT, MVT::Other, Ops, 3);
961    } else {
962      // t2SUBrSPs
963      SDValue Ops[] = { SP, Size,
964                        getI32Imm(ARM_AM::getSORegOpc(ARM_AM::lsl,0)), Chain };
965      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPs_, VT, MVT::Other, Ops, 4);
966    }
967  }
968
969  // FIXME: Add ADD / SUB sp instructions for ARM.
970  return 0;
971}
972
973/// PairDRegs - Insert a pair of double registers into an implicit def to
974/// form a quad register.
975SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
976  DebugLoc dl = V0.getNode()->getDebugLoc();
977  SDValue Undef =
978    SDValue(CurDAG->getMachineNode(TargetInstrInfo::IMPLICIT_DEF, dl, VT), 0);
979  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32);
980  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32);
981  SDNode *Pair = CurDAG->getMachineNode(TargetInstrInfo::INSERT_SUBREG, dl,
982                                        VT, Undef, V0, SubReg0);
983  return CurDAG->getMachineNode(TargetInstrInfo::INSERT_SUBREG, dl,
984                                VT, SDValue(Pair, 0), V1, SubReg1);
985}
986
987/// GetNEONSubregVT - Given a type for a 128-bit NEON vector, return the type
988/// for a 64-bit subregister of the vector.
989static EVT GetNEONSubregVT(EVT VT) {
990  switch (VT.getSimpleVT().SimpleTy) {
991  default: llvm_unreachable("unhandled NEON type");
992  case MVT::v16i8: return MVT::v8i8;
993  case MVT::v8i16: return MVT::v4i16;
994  case MVT::v4f32: return MVT::v2f32;
995  case MVT::v4i32: return MVT::v2i32;
996  case MVT::v2i64: return MVT::v1i64;
997  }
998}
999
1000SDNode *ARMDAGToDAGISel::SelectVLD(SDValue Op, unsigned NumVecs,
1001                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1002                                   unsigned *QOpcodes1) {
1003  assert(NumVecs >=2 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1004  SDNode *N = Op.getNode();
1005  DebugLoc dl = N->getDebugLoc();
1006
1007  SDValue MemAddr, MemUpdate, MemOpc;
1008  if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1009    return NULL;
1010
1011  SDValue Chain = N->getOperand(0);
1012  EVT VT = N->getValueType(0);
1013  bool is64BitVector = VT.is64BitVector();
1014
1015  unsigned OpcodeIndex;
1016  switch (VT.getSimpleVT().SimpleTy) {
1017  default: llvm_unreachable("unhandled vld type");
1018    // Double-register operations:
1019  case MVT::v8i8:  OpcodeIndex = 0; break;
1020  case MVT::v4i16: OpcodeIndex = 1; break;
1021  case MVT::v2f32:
1022  case MVT::v2i32: OpcodeIndex = 2; break;
1023  case MVT::v1i64: OpcodeIndex = 3; break;
1024    // Quad-register operations:
1025  case MVT::v16i8: OpcodeIndex = 0; break;
1026  case MVT::v8i16: OpcodeIndex = 1; break;
1027  case MVT::v4f32:
1028  case MVT::v4i32: OpcodeIndex = 2; break;
1029  }
1030
1031  if (is64BitVector) {
1032    unsigned Opc = DOpcodes[OpcodeIndex];
1033    const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc, Chain };
1034    std::vector<EVT> ResTys(NumVecs, VT);
1035    ResTys.push_back(MVT::Other);
1036    return CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 4);
1037  }
1038
1039  EVT RegVT = GetNEONSubregVT(VT);
1040  if (NumVecs == 2) {
1041    // Quad registers are directly supported for VLD2,
1042    // loading 2 pairs of D regs.
1043    unsigned Opc = QOpcodes0[OpcodeIndex];
1044    const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc, Chain };
1045    std::vector<EVT> ResTys(4, VT);
1046    ResTys.push_back(MVT::Other);
1047    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 4);
1048    Chain = SDValue(VLd, 4);
1049
1050    // Combine the even and odd subregs to produce the result.
1051    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1052      SDNode *Q = PairDRegs(VT, SDValue(VLd, 2*Vec), SDValue(VLd, 2*Vec+1));
1053      ReplaceUses(SDValue(N, Vec), SDValue(Q, 0));
1054    }
1055  } else {
1056    // Otherwise, quad registers are loaded with two separate instructions,
1057    // where one loads the even registers and the other loads the odd registers.
1058
1059    // Enable writeback to the address register.
1060    MemOpc = CurDAG->getTargetConstant(ARM_AM::getAM6Opc(true), MVT::i32);
1061
1062    std::vector<EVT> ResTys(NumVecs, RegVT);
1063    ResTys.push_back(MemAddr.getValueType());
1064    ResTys.push_back(MVT::Other);
1065
1066    // Load the even subreg.
1067    unsigned Opc = QOpcodes0[OpcodeIndex];
1068    const SDValue OpsA[] = { MemAddr, MemUpdate, MemOpc, Chain };
1069    SDNode *VLdA = CurDAG->getMachineNode(Opc, dl, ResTys, OpsA, 4);
1070    Chain = SDValue(VLdA, NumVecs+1);
1071
1072    // Load the odd subreg.
1073    Opc = QOpcodes1[OpcodeIndex];
1074    const SDValue OpsB[] = { SDValue(VLdA, NumVecs), MemUpdate, MemOpc, Chain };
1075    SDNode *VLdB = CurDAG->getMachineNode(Opc, dl, ResTys, OpsB, 4);
1076    Chain = SDValue(VLdB, NumVecs+1);
1077
1078    // Combine the even and odd subregs to produce the result.
1079    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1080      SDNode *Q = PairDRegs(VT, SDValue(VLdA, Vec), SDValue(VLdB, Vec));
1081      ReplaceUses(SDValue(N, Vec), SDValue(Q, 0));
1082    }
1083  }
1084  ReplaceUses(SDValue(N, NumVecs), Chain);
1085  return NULL;
1086}
1087
1088SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDValue Op, bool IsLoad,
1089                                         unsigned NumVecs, unsigned *DOpcodes,
1090                                         unsigned *QOpcodes0,
1091                                         unsigned *QOpcodes1) {
1092  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
1093  SDNode *N = Op.getNode();
1094  DebugLoc dl = N->getDebugLoc();
1095
1096  SDValue MemAddr, MemUpdate, MemOpc;
1097  if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1098    return NULL;
1099
1100  SDValue Chain = N->getOperand(0);
1101  unsigned Lane =
1102    cast<ConstantSDNode>(N->getOperand(NumVecs+3))->getZExtValue();
1103  EVT VT = IsLoad ? N->getValueType(0) : N->getOperand(3).getValueType();
1104  bool is64BitVector = VT.is64BitVector();
1105
1106  // Quad registers are handled by load/store of subregs. Find the subreg info.
1107  unsigned NumElts = 0;
1108  int SubregIdx = 0;
1109  EVT RegVT = VT;
1110  if (!is64BitVector) {
1111    RegVT = GetNEONSubregVT(VT);
1112    NumElts = RegVT.getVectorNumElements();
1113    SubregIdx = (Lane < NumElts) ? ARM::DSUBREG_0 : ARM::DSUBREG_1;
1114  }
1115
1116  unsigned OpcodeIndex;
1117  switch (VT.getSimpleVT().SimpleTy) {
1118  default: llvm_unreachable("unhandled vld/vst lane type");
1119    // Double-register operations:
1120  case MVT::v8i8:  OpcodeIndex = 0; break;
1121  case MVT::v4i16: OpcodeIndex = 1; break;
1122  case MVT::v2f32:
1123  case MVT::v2i32: OpcodeIndex = 2; break;
1124    // Quad-register operations:
1125  case MVT::v8i16: OpcodeIndex = 0; break;
1126  case MVT::v4f32:
1127  case MVT::v4i32: OpcodeIndex = 1; break;
1128  }
1129
1130  SmallVector<SDValue, 9> Ops;
1131  Ops.push_back(MemAddr);
1132  Ops.push_back(MemUpdate);
1133  Ops.push_back(MemOpc);
1134
1135  unsigned Opc = 0;
1136  if (is64BitVector) {
1137    Opc = DOpcodes[OpcodeIndex];
1138    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1139      Ops.push_back(N->getOperand(Vec+3));
1140  } else {
1141    // Check if this is loading the even or odd subreg of a Q register.
1142    if (Lane < NumElts) {
1143      Opc = QOpcodes0[OpcodeIndex];
1144    } else {
1145      Lane -= NumElts;
1146      Opc = QOpcodes1[OpcodeIndex];
1147    }
1148    // Extract the subregs of the input vector.
1149    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1150      Ops.push_back(CurDAG->getTargetExtractSubreg(SubregIdx, dl, RegVT,
1151                                                   N->getOperand(Vec+3)));
1152  }
1153  Ops.push_back(getI32Imm(Lane));
1154  Ops.push_back(Chain);
1155
1156  if (!IsLoad)
1157    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), NumVecs+5);
1158
1159  std::vector<EVT> ResTys(NumVecs, RegVT);
1160  ResTys.push_back(MVT::Other);
1161  SDNode *VLdLn =
1162    CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), NumVecs+5);
1163  // For a 64-bit vector load to D registers, nothing more needs to be done.
1164  if (is64BitVector)
1165    return VLdLn;
1166
1167  // For 128-bit vectors, take the 64-bit results of the load and insert them
1168  // as subregs into the result.
1169  for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1170    SDValue QuadVec = CurDAG->getTargetInsertSubreg(SubregIdx, dl, VT,
1171                                                    N->getOperand(Vec+3),
1172                                                    SDValue(VLdLn, Vec));
1173    ReplaceUses(SDValue(N, Vec), QuadVec);
1174  }
1175
1176  Chain = SDValue(VLdLn, NumVecs);
1177  ReplaceUses(SDValue(N, NumVecs), Chain);
1178  return NULL;
1179}
1180
1181SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDValue Op,
1182                                                     unsigned Opc) {
1183  if (!Subtarget->hasV6T2Ops())
1184    return NULL;
1185
1186  unsigned Shl_imm = 0;
1187  if (isOpcWithIntImmediate(Op.getOperand(0).getNode(), ISD::SHL, Shl_imm)){
1188    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
1189    unsigned Srl_imm = 0;
1190    if (isInt32Immediate(Op.getOperand(1), Srl_imm)) {
1191      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1192      unsigned Width = 32 - Srl_imm;
1193      int LSB = Srl_imm - Shl_imm;
1194      if ((LSB + Width) > 32)
1195        return NULL;
1196      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1197      SDValue Ops[] = { Op.getOperand(0).getOperand(0),
1198                        CurDAG->getTargetConstant(LSB, MVT::i32),
1199                        CurDAG->getTargetConstant(Width, MVT::i32),
1200                        getAL(CurDAG), Reg0 };
1201      return CurDAG->SelectNodeTo(Op.getNode(), Opc, MVT::i32, Ops, 5);
1202    }
1203  }
1204  return NULL;
1205}
1206
1207SDNode *ARMDAGToDAGISel::Select(SDValue Op) {
1208  SDNode *N = Op.getNode();
1209  DebugLoc dl = N->getDebugLoc();
1210
1211  if (N->isMachineOpcode())
1212    return NULL;   // Already selected.
1213
1214  switch (N->getOpcode()) {
1215  default: break;
1216  case ISD::Constant: {
1217    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
1218    bool UseCP = true;
1219    if (Subtarget->hasThumb2())
1220      // Thumb2-aware targets have the MOVT instruction, so all immediates can
1221      // be done with MOV + MOVT, at worst.
1222      UseCP = 0;
1223    else {
1224      if (Subtarget->isThumb()) {
1225        UseCP = (Val > 255 &&                          // MOV
1226                 ~Val > 255 &&                         // MOV + MVN
1227                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
1228      } else
1229        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
1230                 ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
1231                 !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
1232    }
1233
1234    if (UseCP) {
1235      SDValue CPIdx =
1236        CurDAG->getTargetConstantPool(ConstantInt::get(
1237                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
1238                                      TLI.getPointerTy());
1239
1240      SDNode *ResNode;
1241      if (Subtarget->isThumb1Only()) {
1242        SDValue Pred = CurDAG->getTargetConstant(0xEULL, MVT::i32);
1243        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1244        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
1245        ResNode = CurDAG->getMachineNode(ARM::tLDRcp, dl, MVT::i32, MVT::Other,
1246                                         Ops, 4);
1247      } else {
1248        SDValue Ops[] = {
1249          CPIdx,
1250          CurDAG->getRegister(0, MVT::i32),
1251          CurDAG->getTargetConstant(0, MVT::i32),
1252          getAL(CurDAG),
1253          CurDAG->getRegister(0, MVT::i32),
1254          CurDAG->getEntryNode()
1255        };
1256        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
1257                                       Ops, 6);
1258      }
1259      ReplaceUses(Op, SDValue(ResNode, 0));
1260      return NULL;
1261    }
1262
1263    // Other cases are autogenerated.
1264    break;
1265  }
1266  case ISD::FrameIndex: {
1267    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
1268    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1269    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1270    if (Subtarget->isThumb1Only()) {
1271      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, TFI,
1272                                  CurDAG->getTargetConstant(0, MVT::i32));
1273    } else {
1274      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
1275                      ARM::t2ADDri : ARM::ADDri);
1276      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
1277                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1278                        CurDAG->getRegister(0, MVT::i32) };
1279      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1280    }
1281  }
1282  case ARMISD::DYN_ALLOC:
1283    return SelectDYN_ALLOC(Op);
1284  case ISD::SRL:
1285    if (SDNode *I = SelectV6T2BitfieldExtractOp(Op,
1286                      Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX))
1287      return I;
1288    break;
1289  case ISD::SRA:
1290    if (SDNode *I = SelectV6T2BitfieldExtractOp(Op,
1291                      Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX))
1292      return I;
1293    break;
1294  case ISD::MUL:
1295    if (Subtarget->isThumb1Only())
1296      break;
1297    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
1298      unsigned RHSV = C->getZExtValue();
1299      if (!RHSV) break;
1300      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
1301        unsigned ShImm = Log2_32(RHSV-1);
1302        if (ShImm >= 32)
1303          break;
1304        SDValue V = Op.getOperand(0);
1305        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1306        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1307        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1308        if (Subtarget->isThumb()) {
1309          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1310          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
1311        } else {
1312          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1313          return CurDAG->SelectNodeTo(N, ARM::ADDrs, MVT::i32, Ops, 7);
1314        }
1315      }
1316      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
1317        unsigned ShImm = Log2_32(RHSV+1);
1318        if (ShImm >= 32)
1319          break;
1320        SDValue V = Op.getOperand(0);
1321        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1322        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1323        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1324        if (Subtarget->isThumb()) {
1325          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0 };
1326          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 5);
1327        } else {
1328          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1329          return CurDAG->SelectNodeTo(N, ARM::RSBrs, MVT::i32, Ops, 7);
1330        }
1331      }
1332    }
1333    break;
1334  case ARMISD::FMRRD:
1335    return CurDAG->getMachineNode(ARM::FMRRD, dl, MVT::i32, MVT::i32,
1336                                  Op.getOperand(0), getAL(CurDAG),
1337                                  CurDAG->getRegister(0, MVT::i32));
1338  case ISD::UMUL_LOHI: {
1339    if (Subtarget->isThumb1Only())
1340      break;
1341    if (Subtarget->isThumb()) {
1342      SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1),
1343                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1344                        CurDAG->getRegister(0, MVT::i32) };
1345      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops,4);
1346    } else {
1347      SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1),
1348                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1349                        CurDAG->getRegister(0, MVT::i32) };
1350      return CurDAG->getMachineNode(ARM::UMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1351    }
1352  }
1353  case ISD::SMUL_LOHI: {
1354    if (Subtarget->isThumb1Only())
1355      break;
1356    if (Subtarget->isThumb()) {
1357      SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1),
1358                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1359      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops,4);
1360    } else {
1361      SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1),
1362                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1363                        CurDAG->getRegister(0, MVT::i32) };
1364      return CurDAG->getMachineNode(ARM::SMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1365    }
1366  }
1367  case ISD::LOAD: {
1368    SDNode *ResNode = 0;
1369    if (Subtarget->isThumb() && Subtarget->hasThumb2())
1370      ResNode = SelectT2IndexedLoad(Op);
1371    else
1372      ResNode = SelectARMIndexedLoad(Op);
1373    if (ResNode)
1374      return ResNode;
1375    // Other cases are autogenerated.
1376    break;
1377  }
1378  case ARMISD::BRCOND: {
1379    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1380    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1381    // Pattern complexity = 6  cost = 1  size = 0
1382
1383    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1384    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
1385    // Pattern complexity = 6  cost = 1  size = 0
1386
1387    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1388    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1389    // Pattern complexity = 6  cost = 1  size = 0
1390
1391    unsigned Opc = Subtarget->isThumb() ?
1392      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
1393    SDValue Chain = Op.getOperand(0);
1394    SDValue N1 = Op.getOperand(1);
1395    SDValue N2 = Op.getOperand(2);
1396    SDValue N3 = Op.getOperand(3);
1397    SDValue InFlag = Op.getOperand(4);
1398    assert(N1.getOpcode() == ISD::BasicBlock);
1399    assert(N2.getOpcode() == ISD::Constant);
1400    assert(N3.getOpcode() == ISD::Register);
1401
1402    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1403                               cast<ConstantSDNode>(N2)->getZExtValue()),
1404                               MVT::i32);
1405    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
1406    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
1407                                             MVT::Flag, Ops, 5);
1408    Chain = SDValue(ResNode, 0);
1409    if (Op.getNode()->getNumValues() == 2) {
1410      InFlag = SDValue(ResNode, 1);
1411      ReplaceUses(SDValue(Op.getNode(), 1), InFlag);
1412    }
1413    ReplaceUses(SDValue(Op.getNode(), 0), SDValue(Chain.getNode(), Chain.getResNo()));
1414    return NULL;
1415  }
1416  case ARMISD::CMOV: {
1417    EVT VT = Op.getValueType();
1418    SDValue N0 = Op.getOperand(0);
1419    SDValue N1 = Op.getOperand(1);
1420    SDValue N2 = Op.getOperand(2);
1421    SDValue N3 = Op.getOperand(3);
1422    SDValue InFlag = Op.getOperand(4);
1423    assert(N2.getOpcode() == ISD::Constant);
1424    assert(N3.getOpcode() == ISD::Register);
1425
1426    if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
1427      // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1428      // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1429      // Pattern complexity = 18  cost = 1  size = 0
1430      SDValue CPTmp0;
1431      SDValue CPTmp1;
1432      SDValue CPTmp2;
1433      if (Subtarget->isThumb()) {
1434        if (SelectT2ShifterOperandReg(Op, N1, CPTmp0, CPTmp1)) {
1435          unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
1436          unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
1437          unsigned Opc = 0;
1438          switch (SOShOp) {
1439          case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
1440          case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
1441          case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
1442          case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
1443          default:
1444            llvm_unreachable("Unknown so_reg opcode!");
1445            break;
1446          }
1447          SDValue SOShImm =
1448            CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
1449          SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1450                                   cast<ConstantSDNode>(N2)->getZExtValue()),
1451                                   MVT::i32);
1452          SDValue Ops[] = { N0, CPTmp0, SOShImm, Tmp2, N3, InFlag };
1453          return CurDAG->SelectNodeTo(Op.getNode(), Opc, MVT::i32,Ops, 6);
1454        }
1455      } else {
1456        if (SelectShifterOperandReg(Op, N1, CPTmp0, CPTmp1, CPTmp2)) {
1457          SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1458                                   cast<ConstantSDNode>(N2)->getZExtValue()),
1459                                   MVT::i32);
1460          SDValue Ops[] = { N0, CPTmp0, CPTmp1, CPTmp2, Tmp2, N3, InFlag };
1461          return CurDAG->SelectNodeTo(Op.getNode(),
1462                                      ARM::MOVCCs, MVT::i32, Ops, 7);
1463        }
1464      }
1465
1466      // Pattern: (ARMcmov:i32 GPR:i32:$false,
1467      //             (imm:i32)<<P:Predicate_so_imm>>:$true,
1468      //             (imm:i32):$cc)
1469      // Emits: (MOVCCi:i32 GPR:i32:$false,
1470      //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
1471      // Pattern complexity = 10  cost = 1  size = 0
1472      if (N3.getOpcode() == ISD::Constant) {
1473        if (Subtarget->isThumb()) {
1474          if (Predicate_t2_so_imm(N3.getNode())) {
1475            SDValue Tmp1 = CurDAG->getTargetConstant(((unsigned)
1476                                     cast<ConstantSDNode>(N1)->getZExtValue()),
1477                                     MVT::i32);
1478            SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1479                                     cast<ConstantSDNode>(N2)->getZExtValue()),
1480                                     MVT::i32);
1481            SDValue Ops[] = { N0, Tmp1, Tmp2, N3, InFlag };
1482            return CurDAG->SelectNodeTo(Op.getNode(),
1483                                        ARM::t2MOVCCi, MVT::i32, Ops, 5);
1484          }
1485        } else {
1486          if (Predicate_so_imm(N3.getNode())) {
1487            SDValue Tmp1 = CurDAG->getTargetConstant(((unsigned)
1488                                     cast<ConstantSDNode>(N1)->getZExtValue()),
1489                                     MVT::i32);
1490            SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1491                                     cast<ConstantSDNode>(N2)->getZExtValue()),
1492                                     MVT::i32);
1493            SDValue Ops[] = { N0, Tmp1, Tmp2, N3, InFlag };
1494            return CurDAG->SelectNodeTo(Op.getNode(),
1495                                        ARM::MOVCCi, MVT::i32, Ops, 5);
1496          }
1497        }
1498      }
1499    }
1500
1501    // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1502    // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1503    // Pattern complexity = 6  cost = 1  size = 0
1504    //
1505    // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1506    // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1507    // Pattern complexity = 6  cost = 11  size = 0
1508    //
1509    // Also FCPYScc and FCPYDcc.
1510    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1511                               cast<ConstantSDNode>(N2)->getZExtValue()),
1512                               MVT::i32);
1513    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
1514    unsigned Opc = 0;
1515    switch (VT.getSimpleVT().SimpleTy) {
1516    default: assert(false && "Illegal conditional move type!");
1517      break;
1518    case MVT::i32:
1519      Opc = Subtarget->isThumb()
1520        ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
1521        : ARM::MOVCCr;
1522      break;
1523    case MVT::f32:
1524      Opc = ARM::FCPYScc;
1525      break;
1526    case MVT::f64:
1527      Opc = ARM::FCPYDcc;
1528      break;
1529    }
1530    return CurDAG->SelectNodeTo(Op.getNode(), Opc, VT, Ops, 5);
1531  }
1532  case ARMISD::CNEG: {
1533    EVT VT = Op.getValueType();
1534    SDValue N0 = Op.getOperand(0);
1535    SDValue N1 = Op.getOperand(1);
1536    SDValue N2 = Op.getOperand(2);
1537    SDValue N3 = Op.getOperand(3);
1538    SDValue InFlag = Op.getOperand(4);
1539    assert(N2.getOpcode() == ISD::Constant);
1540    assert(N3.getOpcode() == ISD::Register);
1541
1542    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1543                               cast<ConstantSDNode>(N2)->getZExtValue()),
1544                               MVT::i32);
1545    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
1546    unsigned Opc = 0;
1547    switch (VT.getSimpleVT().SimpleTy) {
1548    default: assert(false && "Illegal conditional move type!");
1549      break;
1550    case MVT::f32:
1551      Opc = ARM::FNEGScc;
1552      break;
1553    case MVT::f64:
1554      Opc = ARM::FNEGDcc;
1555      break;
1556    }
1557    return CurDAG->SelectNodeTo(Op.getNode(), Opc, VT, Ops, 5);
1558  }
1559
1560  case ARMISD::VZIP: {
1561    unsigned Opc = 0;
1562    EVT VT = N->getValueType(0);
1563    switch (VT.getSimpleVT().SimpleTy) {
1564    default: return NULL;
1565    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
1566    case MVT::v4i16: Opc = ARM::VZIPd16; break;
1567    case MVT::v2f32:
1568    case MVT::v2i32: Opc = ARM::VZIPd32; break;
1569    case MVT::v16i8: Opc = ARM::VZIPq8; break;
1570    case MVT::v8i16: Opc = ARM::VZIPq16; break;
1571    case MVT::v4f32:
1572    case MVT::v4i32: Opc = ARM::VZIPq32; break;
1573    }
1574    return CurDAG->getMachineNode(Opc, dl, VT, VT,
1575                                  N->getOperand(0), N->getOperand(1));
1576  }
1577  case ARMISD::VUZP: {
1578    unsigned Opc = 0;
1579    EVT VT = N->getValueType(0);
1580    switch (VT.getSimpleVT().SimpleTy) {
1581    default: return NULL;
1582    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
1583    case MVT::v4i16: Opc = ARM::VUZPd16; break;
1584    case MVT::v2f32:
1585    case MVT::v2i32: Opc = ARM::VUZPd32; break;
1586    case MVT::v16i8: Opc = ARM::VUZPq8; break;
1587    case MVT::v8i16: Opc = ARM::VUZPq16; break;
1588    case MVT::v4f32:
1589    case MVT::v4i32: Opc = ARM::VUZPq32; break;
1590    }
1591    return CurDAG->getMachineNode(Opc, dl, VT, VT,
1592                                  N->getOperand(0), N->getOperand(1));
1593  }
1594  case ARMISD::VTRN: {
1595    unsigned Opc = 0;
1596    EVT VT = N->getValueType(0);
1597    switch (VT.getSimpleVT().SimpleTy) {
1598    default: return NULL;
1599    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
1600    case MVT::v4i16: Opc = ARM::VTRNd16; break;
1601    case MVT::v2f32:
1602    case MVT::v2i32: Opc = ARM::VTRNd32; break;
1603    case MVT::v16i8: Opc = ARM::VTRNq8; break;
1604    case MVT::v8i16: Opc = ARM::VTRNq16; break;
1605    case MVT::v4f32:
1606    case MVT::v4i32: Opc = ARM::VTRNq32; break;
1607    }
1608    return CurDAG->getMachineNode(Opc, dl, VT, VT,
1609                                  N->getOperand(0), N->getOperand(1));
1610  }
1611
1612  case ISD::INTRINSIC_VOID:
1613  case ISD::INTRINSIC_W_CHAIN: {
1614    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1615    EVT VT = N->getValueType(0);
1616    unsigned Opc = 0;
1617
1618    switch (IntNo) {
1619    default:
1620      break;
1621
1622    case Intrinsic::arm_neon_vld2: {
1623      unsigned DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
1624                              ARM::VLD2d32, ARM::VLD2d64 };
1625      unsigned QOpcodes[] = { ARM::VLD2q8, ARM::VLD2q16, ARM::VLD2q32 };
1626      return SelectVLD(Op, 2, DOpcodes, QOpcodes, 0);
1627    }
1628
1629    case Intrinsic::arm_neon_vld3: {
1630      unsigned DOpcodes[] = { ARM::VLD3d8, ARM::VLD3d16,
1631                              ARM::VLD3d32, ARM::VLD3d64 };
1632      unsigned QOpcodes0[] = { ARM::VLD3q8a, ARM::VLD3q16a, ARM::VLD3q32a };
1633      unsigned QOpcodes1[] = { ARM::VLD3q8b, ARM::VLD3q16b, ARM::VLD3q32b };
1634      return SelectVLD(Op, 3, DOpcodes, QOpcodes0, QOpcodes1);
1635    }
1636
1637    case Intrinsic::arm_neon_vld4: {
1638      unsigned DOpcodes[] = { ARM::VLD4d8, ARM::VLD4d16,
1639                              ARM::VLD4d32, ARM::VLD4d64 };
1640      unsigned QOpcodes0[] = { ARM::VLD4q8a, ARM::VLD4q16a, ARM::VLD4q32a };
1641      unsigned QOpcodes1[] = { ARM::VLD4q8b, ARM::VLD4q16b, ARM::VLD4q32b };
1642      return SelectVLD(Op, 4, DOpcodes, QOpcodes0, QOpcodes1);
1643    }
1644
1645    case Intrinsic::arm_neon_vld2lane: {
1646      unsigned DOpcodes[] = { ARM::VLD2LNd8, ARM::VLD2LNd16, ARM::VLD2LNd32 };
1647      unsigned QOpcodes0[] = { ARM::VLD2LNq16a, ARM::VLD2LNq32a };
1648      unsigned QOpcodes1[] = { ARM::VLD2LNq16b, ARM::VLD2LNq32b };
1649      return SelectVLDSTLane(Op, true, 2, DOpcodes, QOpcodes0, QOpcodes1);
1650    }
1651
1652    case Intrinsic::arm_neon_vld3lane: {
1653      unsigned DOpcodes[] = { ARM::VLD3LNd8, ARM::VLD3LNd16, ARM::VLD3LNd32 };
1654      unsigned QOpcodes0[] = { ARM::VLD3LNq16a, ARM::VLD3LNq32a };
1655      unsigned QOpcodes1[] = { ARM::VLD3LNq16b, ARM::VLD3LNq32b };
1656      return SelectVLDSTLane(Op, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
1657    }
1658
1659    case Intrinsic::arm_neon_vld4lane: {
1660      unsigned DOpcodes[] = { ARM::VLD4LNd8, ARM::VLD4LNd16, ARM::VLD4LNd32 };
1661      unsigned QOpcodes0[] = { ARM::VLD4LNq16a, ARM::VLD4LNq32a };
1662      unsigned QOpcodes1[] = { ARM::VLD4LNq16b, ARM::VLD4LNq32b };
1663      return SelectVLDSTLane(Op, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
1664    }
1665
1666    case Intrinsic::arm_neon_vst2: {
1667      SDValue MemAddr, MemUpdate, MemOpc;
1668      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1669        return NULL;
1670      SDValue Chain = N->getOperand(0);
1671      VT = N->getOperand(3).getValueType();
1672      if (VT.is64BitVector()) {
1673        switch (VT.getSimpleVT().SimpleTy) {
1674        default: llvm_unreachable("unhandled vst2 type");
1675        case MVT::v8i8:  Opc = ARM::VST2d8; break;
1676        case MVT::v4i16: Opc = ARM::VST2d16; break;
1677        case MVT::v2f32:
1678        case MVT::v2i32: Opc = ARM::VST2d32; break;
1679        case MVT::v1i64: Opc = ARM::VST2d64; break;
1680        }
1681        const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1682                                N->getOperand(3), N->getOperand(4), Chain };
1683        return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 6);
1684      }
1685      // Quad registers are stored as pairs of double registers.
1686      EVT RegVT;
1687      switch (VT.getSimpleVT().SimpleTy) {
1688      default: llvm_unreachable("unhandled vst2 type");
1689      case MVT::v16i8: Opc = ARM::VST2q8; RegVT = MVT::v8i8; break;
1690      case MVT::v8i16: Opc = ARM::VST2q16; RegVT = MVT::v4i16; break;
1691      case MVT::v4f32: Opc = ARM::VST2q32; RegVT = MVT::v2f32; break;
1692      case MVT::v4i32: Opc = ARM::VST2q32; RegVT = MVT::v2i32; break;
1693      }
1694      SDValue D0 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1695                                                  N->getOperand(3));
1696      SDValue D1 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1697                                                  N->getOperand(3));
1698      SDValue D2 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1699                                                  N->getOperand(4));
1700      SDValue D3 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1701                                                  N->getOperand(4));
1702      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1703                              D0, D1, D2, D3, Chain };
1704      return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 8);
1705    }
1706
1707    case Intrinsic::arm_neon_vst3: {
1708      SDValue MemAddr, MemUpdate, MemOpc;
1709      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1710        return NULL;
1711      SDValue Chain = N->getOperand(0);
1712      VT = N->getOperand(3).getValueType();
1713      if (VT.is64BitVector()) {
1714        switch (VT.getSimpleVT().SimpleTy) {
1715        default: llvm_unreachable("unhandled vst3 type");
1716        case MVT::v8i8:  Opc = ARM::VST3d8; break;
1717        case MVT::v4i16: Opc = ARM::VST3d16; break;
1718        case MVT::v2f32:
1719        case MVT::v2i32: Opc = ARM::VST3d32; break;
1720        case MVT::v1i64: Opc = ARM::VST3d64; break;
1721        }
1722        const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1723                                N->getOperand(3), N->getOperand(4),
1724                                N->getOperand(5), Chain };
1725        return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 7);
1726      }
1727      // Quad registers are stored with two separate instructions, where one
1728      // stores the even registers and the other stores the odd registers.
1729      EVT RegVT;
1730      unsigned Opc2 = 0;
1731      switch (VT.getSimpleVT().SimpleTy) {
1732      default: llvm_unreachable("unhandled vst3 type");
1733      case MVT::v16i8:
1734        Opc = ARM::VST3q8a;  Opc2 = ARM::VST3q8b;  RegVT = MVT::v8i8; break;
1735      case MVT::v8i16:
1736        Opc = ARM::VST3q16a; Opc2 = ARM::VST3q16b; RegVT = MVT::v4i16; break;
1737      case MVT::v4f32:
1738        Opc = ARM::VST3q32a; Opc2 = ARM::VST3q32b; RegVT = MVT::v2f32; break;
1739      case MVT::v4i32:
1740        Opc = ARM::VST3q32a; Opc2 = ARM::VST3q32b; RegVT = MVT::v2i32; break;
1741      }
1742      // Enable writeback to the address register.
1743      MemOpc = CurDAG->getTargetConstant(ARM_AM::getAM6Opc(true), MVT::i32);
1744
1745      SDValue D0 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1746                                                  N->getOperand(3));
1747      SDValue D2 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1748                                                  N->getOperand(4));
1749      SDValue D4 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1750                                                  N->getOperand(5));
1751      const SDValue OpsA[] = { MemAddr, MemUpdate, MemOpc, D0, D2, D4, Chain };
1752      SDNode *VStA = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1753                                            MVT::Other, OpsA, 7);
1754      Chain = SDValue(VStA, 1);
1755
1756      SDValue D1 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1757                                                  N->getOperand(3));
1758      SDValue D3 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1759                                                  N->getOperand(4));
1760      SDValue D5 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1761                                                  N->getOperand(5));
1762      MemAddr = SDValue(VStA, 0);
1763      const SDValue OpsB[] = { MemAddr, MemUpdate, MemOpc, D1, D3, D5, Chain };
1764      SDNode *VStB = CurDAG->getMachineNode(Opc2, dl, MemAddr.getValueType(),
1765                                            MVT::Other, OpsB, 7);
1766      Chain = SDValue(VStB, 1);
1767      ReplaceUses(SDValue(N, 0), Chain);
1768      return NULL;
1769    }
1770
1771    case Intrinsic::arm_neon_vst4: {
1772      SDValue MemAddr, MemUpdate, MemOpc;
1773      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1774        return NULL;
1775      SDValue Chain = N->getOperand(0);
1776      VT = N->getOperand(3).getValueType();
1777      if (VT.is64BitVector()) {
1778        switch (VT.getSimpleVT().SimpleTy) {
1779        default: llvm_unreachable("unhandled vst4 type");
1780        case MVT::v8i8:  Opc = ARM::VST4d8; break;
1781        case MVT::v4i16: Opc = ARM::VST4d16; break;
1782        case MVT::v2f32:
1783        case MVT::v2i32: Opc = ARM::VST4d32; break;
1784        case MVT::v1i64: Opc = ARM::VST4d64; break;
1785        }
1786        const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1787                                N->getOperand(3), N->getOperand(4),
1788                                N->getOperand(5), N->getOperand(6), Chain };
1789        return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 8);
1790      }
1791      // Quad registers are stored with two separate instructions, where one
1792      // stores the even registers and the other stores the odd registers.
1793      EVT RegVT;
1794      unsigned Opc2 = 0;
1795      switch (VT.getSimpleVT().SimpleTy) {
1796      default: llvm_unreachable("unhandled vst4 type");
1797      case MVT::v16i8:
1798        Opc = ARM::VST4q8a;  Opc2 = ARM::VST4q8b;  RegVT = MVT::v8i8; break;
1799      case MVT::v8i16:
1800        Opc = ARM::VST4q16a; Opc2 = ARM::VST4q16b; RegVT = MVT::v4i16; break;
1801      case MVT::v4f32:
1802        Opc = ARM::VST4q32a; Opc2 = ARM::VST4q32b; RegVT = MVT::v2f32; break;
1803      case MVT::v4i32:
1804        Opc = ARM::VST4q32a; Opc2 = ARM::VST4q32b; RegVT = MVT::v2i32; break;
1805      }
1806      // Enable writeback to the address register.
1807      MemOpc = CurDAG->getTargetConstant(ARM_AM::getAM6Opc(true), MVT::i32);
1808
1809      SDValue D0 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1810                                                  N->getOperand(3));
1811      SDValue D2 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1812                                                  N->getOperand(4));
1813      SDValue D4 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1814                                                  N->getOperand(5));
1815      SDValue D6 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1816                                                  N->getOperand(6));
1817      const SDValue OpsA[] = { MemAddr, MemUpdate, MemOpc,
1818                               D0, D2, D4, D6, Chain };
1819      SDNode *VStA = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1820                                            MVT::Other, OpsA, 8);
1821      Chain = SDValue(VStA, 1);
1822
1823      SDValue D1 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1824                                                  N->getOperand(3));
1825      SDValue D3 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1826                                                  N->getOperand(4));
1827      SDValue D5 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1828                                                  N->getOperand(5));
1829      SDValue D7 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1830                                                  N->getOperand(6));
1831      MemAddr = SDValue(VStA, 0);
1832      const SDValue OpsB[] = { MemAddr, MemUpdate, MemOpc,
1833                               D1, D3, D5, D7, Chain };
1834      SDNode *VStB = CurDAG->getMachineNode(Opc2, dl, MemAddr.getValueType(),
1835                                            MVT::Other, OpsB, 8);
1836      Chain = SDValue(VStB, 1);
1837      ReplaceUses(SDValue(N, 0), Chain);
1838      return NULL;
1839    }
1840
1841    case Intrinsic::arm_neon_vst2lane: {
1842      unsigned DOpcodes[] = { ARM::VST2LNd8, ARM::VST2LNd16, ARM::VST2LNd32 };
1843      unsigned QOpcodes0[] = { ARM::VST2LNq16a, ARM::VST2LNq32a };
1844      unsigned QOpcodes1[] = { ARM::VST2LNq16b, ARM::VST2LNq32b };
1845      return SelectVLDSTLane(Op, false, 2, DOpcodes, QOpcodes0, QOpcodes1);
1846    }
1847
1848    case Intrinsic::arm_neon_vst3lane: {
1849      unsigned DOpcodes[] = { ARM::VST3LNd8, ARM::VST3LNd16, ARM::VST3LNd32 };
1850      unsigned QOpcodes0[] = { ARM::VST3LNq16a, ARM::VST3LNq32a };
1851      unsigned QOpcodes1[] = { ARM::VST3LNq16b, ARM::VST3LNq32b };
1852      return SelectVLDSTLane(Op, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
1853    }
1854
1855    case Intrinsic::arm_neon_vst4lane: {
1856      unsigned DOpcodes[] = { ARM::VST4LNd8, ARM::VST4LNd16, ARM::VST4LNd32 };
1857      unsigned QOpcodes0[] = { ARM::VST4LNq16a, ARM::VST4LNq32a };
1858      unsigned QOpcodes1[] = { ARM::VST4LNq16b, ARM::VST4LNq32b };
1859      return SelectVLDSTLane(Op, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
1860    }
1861    }
1862  }
1863  }
1864
1865  return SelectCode(Op);
1866}
1867
1868bool ARMDAGToDAGISel::
1869SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
1870                             std::vector<SDValue> &OutOps) {
1871  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
1872  // Require the address to be in a register.  That is safe for all ARM
1873  // variants and it is hard to do anything much smarter without knowing
1874  // how the operand is used.
1875  OutOps.push_back(Op);
1876  return false;
1877}
1878
1879/// createARMISelDag - This pass converts a legalized DAG into a
1880/// ARM-specific DAG, ready for instruction scheduling.
1881///
1882FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
1883                                     CodeGenOpt::Level OptLevel) {
1884  return new ARMDAGToDAGISel(TM, OptLevel);
1885}
1886