ARMISelDAGToDAG.cpp revision 255804
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines an instruction selector for the ARM target. 11// 12//===----------------------------------------------------------------------===// 13 14#define DEBUG_TYPE "arm-isel" 15#include "ARM.h" 16#include "ARMBaseInstrInfo.h" 17#include "ARMTargetMachine.h" 18#include "MCTargetDesc/ARMAddressingModes.h" 19#include "llvm/CodeGen/MachineFrameInfo.h" 20#include "llvm/CodeGen/MachineFunction.h" 21#include "llvm/CodeGen/MachineInstrBuilder.h" 22#include "llvm/CodeGen/MachineRegisterInfo.h" 23#include "llvm/CodeGen/SelectionDAG.h" 24#include "llvm/CodeGen/SelectionDAGISel.h" 25#include "llvm/IR/CallingConv.h" 26#include "llvm/IR/Constants.h" 27#include "llvm/IR/DerivedTypes.h" 28#include "llvm/IR/Function.h" 29#include "llvm/IR/Intrinsics.h" 30#include "llvm/IR/LLVMContext.h" 31#include "llvm/Support/CommandLine.h" 32#include "llvm/Support/Compiler.h" 33#include "llvm/Support/Debug.h" 34#include "llvm/Support/ErrorHandling.h" 35#include "llvm/Support/raw_ostream.h" 36#include "llvm/Target/TargetLowering.h" 37#include "llvm/Target/TargetOptions.h" 38 39using namespace llvm; 40 41static cl::opt<bool> 42DisableShifterOp("disable-shifter-op", cl::Hidden, 43 cl::desc("Disable isel of shifter-op"), 44 cl::init(false)); 45 46static cl::opt<bool> 47CheckVMLxHazard("check-vmlx-hazard", cl::Hidden, 48 cl::desc("Check fp vmla / vmls hazard at isel time"), 49 cl::init(true)); 50 51//===--------------------------------------------------------------------===// 52/// ARMDAGToDAGISel - ARM specific code to select ARM machine 53/// instructions for SelectionDAG operations. 54/// 55namespace { 56 57enum AddrMode2Type { 58 AM2_BASE, // Simple AM2 (+-imm12) 59 AM2_SHOP // Shifter-op AM2 60}; 61 62class ARMDAGToDAGISel : public SelectionDAGISel { 63 ARMBaseTargetMachine &TM; 64 const ARMBaseInstrInfo *TII; 65 66 /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can 67 /// make the right decision when generating code for different targets. 68 const ARMSubtarget *Subtarget; 69 70public: 71 explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm, 72 CodeGenOpt::Level OptLevel) 73 : SelectionDAGISel(tm, OptLevel), TM(tm), 74 TII(static_cast<const ARMBaseInstrInfo*>(TM.getInstrInfo())), 75 Subtarget(&TM.getSubtarget<ARMSubtarget>()) { 76 } 77 78 virtual const char *getPassName() const { 79 return "ARM Instruction Selection"; 80 } 81 82 virtual void PreprocessISelDAG(); 83 84 /// getI32Imm - Return a target constant of type i32 with the specified 85 /// value. 86 inline SDValue getI32Imm(unsigned Imm) { 87 return CurDAG->getTargetConstant(Imm, MVT::i32); 88 } 89 90 SDNode *Select(SDNode *N); 91 92 93 bool hasNoVMLxHazardUse(SDNode *N) const; 94 bool isShifterOpProfitable(const SDValue &Shift, 95 ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt); 96 bool SelectRegShifterOperand(SDValue N, SDValue &A, 97 SDValue &B, SDValue &C, 98 bool CheckProfitability = true); 99 bool SelectImmShifterOperand(SDValue N, SDValue &A, 100 SDValue &B, bool CheckProfitability = true); 101 bool SelectShiftRegShifterOperand(SDValue N, SDValue &A, 102 SDValue &B, SDValue &C) { 103 // Don't apply the profitability check 104 return SelectRegShifterOperand(N, A, B, C, false); 105 } 106 bool SelectShiftImmShifterOperand(SDValue N, SDValue &A, 107 SDValue &B) { 108 // Don't apply the profitability check 109 return SelectImmShifterOperand(N, A, B, false); 110 } 111 112 bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm); 113 bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc); 114 115 AddrMode2Type SelectAddrMode2Worker(SDValue N, SDValue &Base, 116 SDValue &Offset, SDValue &Opc); 117 bool SelectAddrMode2Base(SDValue N, SDValue &Base, SDValue &Offset, 118 SDValue &Opc) { 119 return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_BASE; 120 } 121 122 bool SelectAddrMode2ShOp(SDValue N, SDValue &Base, SDValue &Offset, 123 SDValue &Opc) { 124 return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_SHOP; 125 } 126 127 bool SelectAddrMode2(SDValue N, SDValue &Base, SDValue &Offset, 128 SDValue &Opc) { 129 SelectAddrMode2Worker(N, Base, Offset, Opc); 130// return SelectAddrMode2ShOp(N, Base, Offset, Opc); 131 // This always matches one way or another. 132 return true; 133 } 134 135 bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N, 136 SDValue &Offset, SDValue &Opc); 137 bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N, 138 SDValue &Offset, SDValue &Opc); 139 bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N, 140 SDValue &Offset, SDValue &Opc); 141 bool SelectAddrOffsetNone(SDValue N, SDValue &Base); 142 bool SelectAddrMode3(SDValue N, SDValue &Base, 143 SDValue &Offset, SDValue &Opc); 144 bool SelectAddrMode3Offset(SDNode *Op, SDValue N, 145 SDValue &Offset, SDValue &Opc); 146 bool SelectAddrMode5(SDValue N, SDValue &Base, 147 SDValue &Offset); 148 bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align); 149 bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset); 150 151 bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label); 152 153 // Thumb Addressing Modes: 154 bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset); 155 bool SelectThumbAddrModeRI(SDValue N, SDValue &Base, SDValue &Offset, 156 unsigned Scale); 157 bool SelectThumbAddrModeRI5S1(SDValue N, SDValue &Base, SDValue &Offset); 158 bool SelectThumbAddrModeRI5S2(SDValue N, SDValue &Base, SDValue &Offset); 159 bool SelectThumbAddrModeRI5S4(SDValue N, SDValue &Base, SDValue &Offset); 160 bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base, 161 SDValue &OffImm); 162 bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base, 163 SDValue &OffImm); 164 bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base, 165 SDValue &OffImm); 166 bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base, 167 SDValue &OffImm); 168 bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm); 169 170 // Thumb 2 Addressing Modes: 171 bool SelectT2ShifterOperandReg(SDValue N, 172 SDValue &BaseReg, SDValue &Opc); 173 bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm); 174 bool SelectT2AddrModeImm8(SDValue N, SDValue &Base, 175 SDValue &OffImm); 176 bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N, 177 SDValue &OffImm); 178 bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base, 179 SDValue &OffReg, SDValue &ShImm); 180 181 inline bool is_so_imm(unsigned Imm) const { 182 return ARM_AM::getSOImmVal(Imm) != -1; 183 } 184 185 inline bool is_so_imm_not(unsigned Imm) const { 186 return ARM_AM::getSOImmVal(~Imm) != -1; 187 } 188 189 inline bool is_t2_so_imm(unsigned Imm) const { 190 return ARM_AM::getT2SOImmVal(Imm) != -1; 191 } 192 193 inline bool is_t2_so_imm_not(unsigned Imm) const { 194 return ARM_AM::getT2SOImmVal(~Imm) != -1; 195 } 196 197 // Include the pieces autogenerated from the target description. 198#include "ARMGenDAGISel.inc" 199 200private: 201 /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for 202 /// ARM. 203 SDNode *SelectARMIndexedLoad(SDNode *N); 204 SDNode *SelectT2IndexedLoad(SDNode *N); 205 206 /// SelectVLD - Select NEON load intrinsics. NumVecs should be 207 /// 1, 2, 3 or 4. The opcode arrays specify the instructions used for 208 /// loads of D registers and even subregs and odd subregs of Q registers. 209 /// For NumVecs <= 2, QOpcodes1 is not used. 210 SDNode *SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs, 211 const uint16_t *DOpcodes, 212 const uint16_t *QOpcodes0, const uint16_t *QOpcodes1); 213 214 /// SelectVST - Select NEON store intrinsics. NumVecs should 215 /// be 1, 2, 3 or 4. The opcode arrays specify the instructions used for 216 /// stores of D registers and even subregs and odd subregs of Q registers. 217 /// For NumVecs <= 2, QOpcodes1 is not used. 218 SDNode *SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, 219 const uint16_t *DOpcodes, 220 const uint16_t *QOpcodes0, const uint16_t *QOpcodes1); 221 222 /// SelectVLDSTLane - Select NEON load/store lane intrinsics. NumVecs should 223 /// be 2, 3 or 4. The opcode arrays specify the instructions used for 224 /// load/store of D registers and Q registers. 225 SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, 226 bool isUpdating, unsigned NumVecs, 227 const uint16_t *DOpcodes, const uint16_t *QOpcodes); 228 229 /// SelectVLDDup - Select NEON load-duplicate intrinsics. NumVecs 230 /// should be 2, 3 or 4. The opcode array specifies the instructions used 231 /// for loading D registers. (Q registers are not supported.) 232 SDNode *SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs, 233 const uint16_t *Opcodes); 234 235 /// SelectVTBL - Select NEON VTBL and VTBX intrinsics. NumVecs should be 2, 236 /// 3 or 4. These are custom-selected so that a REG_SEQUENCE can be 237 /// generated to force the table registers to be consecutive. 238 SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc); 239 240 /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM. 241 SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned); 242 243 /// SelectCMOVOp - Select CMOV instructions for ARM. 244 SDNode *SelectCMOVOp(SDNode *N); 245 SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, 246 ARMCC::CondCodes CCVal, SDValue CCR, 247 SDValue InFlag); 248 SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, 249 ARMCC::CondCodes CCVal, SDValue CCR, 250 SDValue InFlag); 251 SDNode *SelectT2CMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, 252 ARMCC::CondCodes CCVal, SDValue CCR, 253 SDValue InFlag); 254 SDNode *SelectARMCMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, 255 ARMCC::CondCodes CCVal, SDValue CCR, 256 SDValue InFlag); 257 258 // Select special operations if node forms integer ABS pattern 259 SDNode *SelectABSOp(SDNode *N); 260 261 SDNode *SelectInlineAsm(SDNode *N); 262 263 SDNode *SelectConcatVector(SDNode *N); 264 265 SDNode *SelectAtomic64(SDNode *Node, unsigned Opc); 266 267 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 268 /// inline asm expressions. 269 virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, 270 char ConstraintCode, 271 std::vector<SDValue> &OutOps); 272 273 // Form pairs of consecutive R, S, D, or Q registers. 274 SDNode *createGPRPairNode(EVT VT, SDValue V0, SDValue V1); 275 SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1); 276 SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1); 277 SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1); 278 279 // Form sequences of 4 consecutive S, D, or Q registers. 280 SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); 281 SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); 282 SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); 283 284 // Get the alignment operand for a NEON VLD or VST instruction. 285 SDValue GetVLDSTAlign(SDValue Align, unsigned NumVecs, bool is64BitVector); 286}; 287} 288 289/// isInt32Immediate - This method tests to see if the node is a 32-bit constant 290/// operand. If so Imm will receive the 32-bit value. 291static bool isInt32Immediate(SDNode *N, unsigned &Imm) { 292 if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) { 293 Imm = cast<ConstantSDNode>(N)->getZExtValue(); 294 return true; 295 } 296 return false; 297} 298 299// isInt32Immediate - This method tests to see if a constant operand. 300// If so Imm will receive the 32 bit value. 301static bool isInt32Immediate(SDValue N, unsigned &Imm) { 302 return isInt32Immediate(N.getNode(), Imm); 303} 304 305// isOpcWithIntImmediate - This method tests to see if the node is a specific 306// opcode and that it has a immediate integer right operand. 307// If so Imm will receive the 32 bit value. 308static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) { 309 return N->getOpcode() == Opc && 310 isInt32Immediate(N->getOperand(1).getNode(), Imm); 311} 312 313/// \brief Check whether a particular node is a constant value representable as 314/// (N * Scale) where (N in [\p RangeMin, \p RangeMax). 315/// 316/// \param ScaledConstant [out] - On success, the pre-scaled constant value. 317static bool isScaledConstantInRange(SDValue Node, int Scale, 318 int RangeMin, int RangeMax, 319 int &ScaledConstant) { 320 assert(Scale > 0 && "Invalid scale!"); 321 322 // Check that this is a constant. 323 const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node); 324 if (!C) 325 return false; 326 327 ScaledConstant = (int) C->getZExtValue(); 328 if ((ScaledConstant % Scale) != 0) 329 return false; 330 331 ScaledConstant /= Scale; 332 return ScaledConstant >= RangeMin && ScaledConstant < RangeMax; 333} 334 335void ARMDAGToDAGISel::PreprocessISelDAG() { 336 if (!Subtarget->hasV6T2Ops()) 337 return; 338 339 bool isThumb2 = Subtarget->isThumb(); 340 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 341 E = CurDAG->allnodes_end(); I != E; ) { 342 SDNode *N = I++; // Preincrement iterator to avoid invalidation issues. 343 344 if (N->getOpcode() != ISD::ADD) 345 continue; 346 347 // Look for (add X1, (and (srl X2, c1), c2)) where c2 is constant with 348 // leading zeros, followed by consecutive set bits, followed by 1 or 2 349 // trailing zeros, e.g. 1020. 350 // Transform the expression to 351 // (add X1, (shl (and (srl X2, c1), (c2>>tz)), tz)) where tz is the number 352 // of trailing zeros of c2. The left shift would be folded as an shifter 353 // operand of 'add' and the 'and' and 'srl' would become a bits extraction 354 // node (UBFX). 355 356 SDValue N0 = N->getOperand(0); 357 SDValue N1 = N->getOperand(1); 358 unsigned And_imm = 0; 359 if (!isOpcWithIntImmediate(N1.getNode(), ISD::AND, And_imm)) { 360 if (isOpcWithIntImmediate(N0.getNode(), ISD::AND, And_imm)) 361 std::swap(N0, N1); 362 } 363 if (!And_imm) 364 continue; 365 366 // Check if the AND mask is an immediate of the form: 000.....1111111100 367 unsigned TZ = CountTrailingZeros_32(And_imm); 368 if (TZ != 1 && TZ != 2) 369 // Be conservative here. Shifter operands aren't always free. e.g. On 370 // Swift, left shifter operand of 1 / 2 for free but others are not. 371 // e.g. 372 // ubfx r3, r1, #16, #8 373 // ldr.w r3, [r0, r3, lsl #2] 374 // vs. 375 // mov.w r9, #1020 376 // and.w r2, r9, r1, lsr #14 377 // ldr r2, [r0, r2] 378 continue; 379 And_imm >>= TZ; 380 if (And_imm & (And_imm + 1)) 381 continue; 382 383 // Look for (and (srl X, c1), c2). 384 SDValue Srl = N1.getOperand(0); 385 unsigned Srl_imm = 0; 386 if (!isOpcWithIntImmediate(Srl.getNode(), ISD::SRL, Srl_imm) || 387 (Srl_imm <= 2)) 388 continue; 389 390 // Make sure first operand is not a shifter operand which would prevent 391 // folding of the left shift. 392 SDValue CPTmp0; 393 SDValue CPTmp1; 394 SDValue CPTmp2; 395 if (isThumb2) { 396 if (SelectT2ShifterOperandReg(N0, CPTmp0, CPTmp1)) 397 continue; 398 } else { 399 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) || 400 SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2)) 401 continue; 402 } 403 404 // Now make the transformation. 405 Srl = CurDAG->getNode(ISD::SRL, Srl.getDebugLoc(), MVT::i32, 406 Srl.getOperand(0), 407 CurDAG->getConstant(Srl_imm+TZ, MVT::i32)); 408 N1 = CurDAG->getNode(ISD::AND, N1.getDebugLoc(), MVT::i32, 409 Srl, CurDAG->getConstant(And_imm, MVT::i32)); 410 N1 = CurDAG->getNode(ISD::SHL, N1.getDebugLoc(), MVT::i32, 411 N1, CurDAG->getConstant(TZ, MVT::i32)); 412 CurDAG->UpdateNodeOperands(N, N0, N1); 413 } 414} 415 416/// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS 417/// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at 418/// least on current ARM implementations) which should be avoidded. 419bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const { 420 if (OptLevel == CodeGenOpt::None) 421 return true; 422 423 if (!CheckVMLxHazard) 424 return true; 425 426 if (!Subtarget->isCortexA8() && !Subtarget->isLikeA9() && 427 !Subtarget->isSwift()) 428 return true; 429 430 if (!N->hasOneUse()) 431 return false; 432 433 SDNode *Use = *N->use_begin(); 434 if (Use->getOpcode() == ISD::CopyToReg) 435 return true; 436 if (Use->isMachineOpcode()) { 437 const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode()); 438 if (MCID.mayStore()) 439 return true; 440 unsigned Opcode = MCID.getOpcode(); 441 if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD) 442 return true; 443 // vmlx feeding into another vmlx. We actually want to unfold 444 // the use later in the MLxExpansion pass. e.g. 445 // vmla 446 // vmla (stall 8 cycles) 447 // 448 // vmul (5 cycles) 449 // vadd (5 cycles) 450 // vmla 451 // This adds up to about 18 - 19 cycles. 452 // 453 // vmla 454 // vmul (stall 4 cycles) 455 // vadd adds up to about 14 cycles. 456 return TII->isFpMLxInstruction(Opcode); 457 } 458 459 return false; 460} 461 462bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift, 463 ARM_AM::ShiftOpc ShOpcVal, 464 unsigned ShAmt) { 465 if (!Subtarget->isLikeA9() && !Subtarget->isSwift()) 466 return true; 467 if (Shift.hasOneUse()) 468 return true; 469 // R << 2 is free. 470 return ShOpcVal == ARM_AM::lsl && 471 (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1)); 472} 473 474bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N, 475 SDValue &BaseReg, 476 SDValue &Opc, 477 bool CheckProfitability) { 478 if (DisableShifterOp) 479 return false; 480 481 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 482 483 // Don't match base register only case. That is matched to a separate 484 // lower complexity pattern with explicit register operand. 485 if (ShOpcVal == ARM_AM::no_shift) return false; 486 487 BaseReg = N.getOperand(0); 488 unsigned ShImmVal = 0; 489 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); 490 if (!RHS) return false; 491 ShImmVal = RHS->getZExtValue() & 31; 492 Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal), 493 MVT::i32); 494 return true; 495} 496 497bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N, 498 SDValue &BaseReg, 499 SDValue &ShReg, 500 SDValue &Opc, 501 bool CheckProfitability) { 502 if (DisableShifterOp) 503 return false; 504 505 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 506 507 // Don't match base register only case. That is matched to a separate 508 // lower complexity pattern with explicit register operand. 509 if (ShOpcVal == ARM_AM::no_shift) return false; 510 511 BaseReg = N.getOperand(0); 512 unsigned ShImmVal = 0; 513 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); 514 if (RHS) return false; 515 516 ShReg = N.getOperand(1); 517 if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal)) 518 return false; 519 Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal), 520 MVT::i32); 521 return true; 522} 523 524 525bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N, 526 SDValue &Base, 527 SDValue &OffImm) { 528 // Match simple R + imm12 operands. 529 530 // Base only. 531 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 532 !CurDAG->isBaseWithConstantOffset(N)) { 533 if (N.getOpcode() == ISD::FrameIndex) { 534 // Match frame index. 535 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 536 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 537 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 538 return true; 539 } 540 541 if (N.getOpcode() == ARMISD::Wrapper && 542 !(Subtarget->useMovt() && 543 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) { 544 Base = N.getOperand(0); 545 } else 546 Base = N; 547 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 548 return true; 549 } 550 551 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 552 int RHSC = (int)RHS->getZExtValue(); 553 if (N.getOpcode() == ISD::SUB) 554 RHSC = -RHSC; 555 556 if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned) 557 Base = N.getOperand(0); 558 if (Base.getOpcode() == ISD::FrameIndex) { 559 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 560 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 561 } 562 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 563 return true; 564 } 565 } 566 567 // Base only. 568 Base = N; 569 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 570 return true; 571} 572 573 574 575bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, 576 SDValue &Opc) { 577 if (N.getOpcode() == ISD::MUL && 578 ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) { 579 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 580 // X * [3,5,9] -> X + X * [2,4,8] etc. 581 int RHSC = (int)RHS->getZExtValue(); 582 if (RHSC & 1) { 583 RHSC = RHSC & ~1; 584 ARM_AM::AddrOpc AddSub = ARM_AM::add; 585 if (RHSC < 0) { 586 AddSub = ARM_AM::sub; 587 RHSC = - RHSC; 588 } 589 if (isPowerOf2_32(RHSC)) { 590 unsigned ShAmt = Log2_32(RHSC); 591 Base = Offset = N.getOperand(0); 592 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, 593 ARM_AM::lsl), 594 MVT::i32); 595 return true; 596 } 597 } 598 } 599 } 600 601 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 602 // ISD::OR that is equivalent to an ISD::ADD. 603 !CurDAG->isBaseWithConstantOffset(N)) 604 return false; 605 606 // Leave simple R +/- imm12 operands for LDRi12 607 if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) { 608 int RHSC; 609 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, 610 -0x1000+1, 0x1000, RHSC)) // 12 bits. 611 return false; 612 } 613 614 // Otherwise this is R +/- [possibly shifted] R. 615 ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add; 616 ARM_AM::ShiftOpc ShOpcVal = 617 ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode()); 618 unsigned ShAmt = 0; 619 620 Base = N.getOperand(0); 621 Offset = N.getOperand(1); 622 623 if (ShOpcVal != ARM_AM::no_shift) { 624 // Check to see if the RHS of the shift is a constant, if not, we can't fold 625 // it. 626 if (ConstantSDNode *Sh = 627 dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) { 628 ShAmt = Sh->getZExtValue(); 629 if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt)) 630 Offset = N.getOperand(1).getOperand(0); 631 else { 632 ShAmt = 0; 633 ShOpcVal = ARM_AM::no_shift; 634 } 635 } else { 636 ShOpcVal = ARM_AM::no_shift; 637 } 638 } 639 640 // Try matching (R shl C) + (R). 641 if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift && 642 !(Subtarget->isLikeA9() || Subtarget->isSwift() || 643 N.getOperand(0).hasOneUse())) { 644 ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode()); 645 if (ShOpcVal != ARM_AM::no_shift) { 646 // Check to see if the RHS of the shift is a constant, if not, we can't 647 // fold it. 648 if (ConstantSDNode *Sh = 649 dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) { 650 ShAmt = Sh->getZExtValue(); 651 if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) { 652 Offset = N.getOperand(0).getOperand(0); 653 Base = N.getOperand(1); 654 } else { 655 ShAmt = 0; 656 ShOpcVal = ARM_AM::no_shift; 657 } 658 } else { 659 ShOpcVal = ARM_AM::no_shift; 660 } 661 } 662 } 663 664 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), 665 MVT::i32); 666 return true; 667} 668 669 670//----- 671 672AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N, 673 SDValue &Base, 674 SDValue &Offset, 675 SDValue &Opc) { 676 if (N.getOpcode() == ISD::MUL && 677 (!(Subtarget->isLikeA9() || Subtarget->isSwift()) || N.hasOneUse())) { 678 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 679 // X * [3,5,9] -> X + X * [2,4,8] etc. 680 int RHSC = (int)RHS->getZExtValue(); 681 if (RHSC & 1) { 682 RHSC = RHSC & ~1; 683 ARM_AM::AddrOpc AddSub = ARM_AM::add; 684 if (RHSC < 0) { 685 AddSub = ARM_AM::sub; 686 RHSC = - RHSC; 687 } 688 if (isPowerOf2_32(RHSC)) { 689 unsigned ShAmt = Log2_32(RHSC); 690 Base = Offset = N.getOperand(0); 691 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, 692 ARM_AM::lsl), 693 MVT::i32); 694 return AM2_SHOP; 695 } 696 } 697 } 698 } 699 700 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 701 // ISD::OR that is equivalent to an ADD. 702 !CurDAG->isBaseWithConstantOffset(N)) { 703 Base = N; 704 if (N.getOpcode() == ISD::FrameIndex) { 705 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 706 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 707 } else if (N.getOpcode() == ARMISD::Wrapper && 708 !(Subtarget->useMovt() && 709 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) { 710 Base = N.getOperand(0); 711 } 712 Offset = CurDAG->getRegister(0, MVT::i32); 713 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0, 714 ARM_AM::no_shift), 715 MVT::i32); 716 return AM2_BASE; 717 } 718 719 // Match simple R +/- imm12 operands. 720 if (N.getOpcode() != ISD::SUB) { 721 int RHSC; 722 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, 723 -0x1000+1, 0x1000, RHSC)) { // 12 bits. 724 Base = N.getOperand(0); 725 if (Base.getOpcode() == ISD::FrameIndex) { 726 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 727 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 728 } 729 Offset = CurDAG->getRegister(0, MVT::i32); 730 731 ARM_AM::AddrOpc AddSub = ARM_AM::add; 732 if (RHSC < 0) { 733 AddSub = ARM_AM::sub; 734 RHSC = - RHSC; 735 } 736 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC, 737 ARM_AM::no_shift), 738 MVT::i32); 739 return AM2_BASE; 740 } 741 } 742 743 if ((Subtarget->isLikeA9() || Subtarget->isSwift()) && !N.hasOneUse()) { 744 // Compute R +/- (R << N) and reuse it. 745 Base = N; 746 Offset = CurDAG->getRegister(0, MVT::i32); 747 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0, 748 ARM_AM::no_shift), 749 MVT::i32); 750 return AM2_BASE; 751 } 752 753 // Otherwise this is R +/- [possibly shifted] R. 754 ARM_AM::AddrOpc AddSub = N.getOpcode() != ISD::SUB ? ARM_AM::add:ARM_AM::sub; 755 ARM_AM::ShiftOpc ShOpcVal = 756 ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode()); 757 unsigned ShAmt = 0; 758 759 Base = N.getOperand(0); 760 Offset = N.getOperand(1); 761 762 if (ShOpcVal != ARM_AM::no_shift) { 763 // Check to see if the RHS of the shift is a constant, if not, we can't fold 764 // it. 765 if (ConstantSDNode *Sh = 766 dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) { 767 ShAmt = Sh->getZExtValue(); 768 if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt)) 769 Offset = N.getOperand(1).getOperand(0); 770 else { 771 ShAmt = 0; 772 ShOpcVal = ARM_AM::no_shift; 773 } 774 } else { 775 ShOpcVal = ARM_AM::no_shift; 776 } 777 } 778 779 // Try matching (R shl C) + (R). 780 if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift && 781 !(Subtarget->isLikeA9() || Subtarget->isSwift() || 782 N.getOperand(0).hasOneUse())) { 783 ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode()); 784 if (ShOpcVal != ARM_AM::no_shift) { 785 // Check to see if the RHS of the shift is a constant, if not, we can't 786 // fold it. 787 if (ConstantSDNode *Sh = 788 dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) { 789 ShAmt = Sh->getZExtValue(); 790 if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) { 791 Offset = N.getOperand(0).getOperand(0); 792 Base = N.getOperand(1); 793 } else { 794 ShAmt = 0; 795 ShOpcVal = ARM_AM::no_shift; 796 } 797 } else { 798 ShOpcVal = ARM_AM::no_shift; 799 } 800 } 801 } 802 803 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), 804 MVT::i32); 805 return AM2_SHOP; 806} 807 808bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N, 809 SDValue &Offset, SDValue &Opc) { 810 unsigned Opcode = Op->getOpcode(); 811 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 812 ? cast<LoadSDNode>(Op)->getAddressingMode() 813 : cast<StoreSDNode>(Op)->getAddressingMode(); 814 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 815 ? ARM_AM::add : ARM_AM::sub; 816 int Val; 817 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) 818 return false; 819 820 Offset = N; 821 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 822 unsigned ShAmt = 0; 823 if (ShOpcVal != ARM_AM::no_shift) { 824 // Check to see if the RHS of the shift is a constant, if not, we can't fold 825 // it. 826 if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 827 ShAmt = Sh->getZExtValue(); 828 if (isShifterOpProfitable(N, ShOpcVal, ShAmt)) 829 Offset = N.getOperand(0); 830 else { 831 ShAmt = 0; 832 ShOpcVal = ARM_AM::no_shift; 833 } 834 } else { 835 ShOpcVal = ARM_AM::no_shift; 836 } 837 } 838 839 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), 840 MVT::i32); 841 return true; 842} 843 844bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N, 845 SDValue &Offset, SDValue &Opc) { 846 unsigned Opcode = Op->getOpcode(); 847 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 848 ? cast<LoadSDNode>(Op)->getAddressingMode() 849 : cast<StoreSDNode>(Op)->getAddressingMode(); 850 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 851 ? ARM_AM::add : ARM_AM::sub; 852 int Val; 853 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits. 854 if (AddSub == ARM_AM::sub) Val *= -1; 855 Offset = CurDAG->getRegister(0, MVT::i32); 856 Opc = CurDAG->getTargetConstant(Val, MVT::i32); 857 return true; 858 } 859 860 return false; 861} 862 863 864bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N, 865 SDValue &Offset, SDValue &Opc) { 866 unsigned Opcode = Op->getOpcode(); 867 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 868 ? cast<LoadSDNode>(Op)->getAddressingMode() 869 : cast<StoreSDNode>(Op)->getAddressingMode(); 870 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 871 ? ARM_AM::add : ARM_AM::sub; 872 int Val; 873 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits. 874 Offset = CurDAG->getRegister(0, MVT::i32); 875 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val, 876 ARM_AM::no_shift), 877 MVT::i32); 878 return true; 879 } 880 881 return false; 882} 883 884bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) { 885 Base = N; 886 return true; 887} 888 889bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N, 890 SDValue &Base, SDValue &Offset, 891 SDValue &Opc) { 892 if (N.getOpcode() == ISD::SUB) { 893 // X - C is canonicalize to X + -C, no need to handle it here. 894 Base = N.getOperand(0); 895 Offset = N.getOperand(1); 896 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32); 897 return true; 898 } 899 900 if (!CurDAG->isBaseWithConstantOffset(N)) { 901 Base = N; 902 if (N.getOpcode() == ISD::FrameIndex) { 903 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 904 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 905 } 906 Offset = CurDAG->getRegister(0, MVT::i32); 907 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32); 908 return true; 909 } 910 911 // If the RHS is +/- imm8, fold into addr mode. 912 int RHSC; 913 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, 914 -256 + 1, 256, RHSC)) { // 8 bits. 915 Base = N.getOperand(0); 916 if (Base.getOpcode() == ISD::FrameIndex) { 917 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 918 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 919 } 920 Offset = CurDAG->getRegister(0, MVT::i32); 921 922 ARM_AM::AddrOpc AddSub = ARM_AM::add; 923 if (RHSC < 0) { 924 AddSub = ARM_AM::sub; 925 RHSC = -RHSC; 926 } 927 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32); 928 return true; 929 } 930 931 Base = N.getOperand(0); 932 Offset = N.getOperand(1); 933 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32); 934 return true; 935} 936 937bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N, 938 SDValue &Offset, SDValue &Opc) { 939 unsigned Opcode = Op->getOpcode(); 940 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 941 ? cast<LoadSDNode>(Op)->getAddressingMode() 942 : cast<StoreSDNode>(Op)->getAddressingMode(); 943 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 944 ? ARM_AM::add : ARM_AM::sub; 945 int Val; 946 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits. 947 Offset = CurDAG->getRegister(0, MVT::i32); 948 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32); 949 return true; 950 } 951 952 Offset = N; 953 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32); 954 return true; 955} 956 957bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N, 958 SDValue &Base, SDValue &Offset) { 959 if (!CurDAG->isBaseWithConstantOffset(N)) { 960 Base = N; 961 if (N.getOpcode() == ISD::FrameIndex) { 962 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 963 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 964 } else if (N.getOpcode() == ARMISD::Wrapper && 965 !(Subtarget->useMovt() && 966 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) { 967 Base = N.getOperand(0); 968 } 969 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0), 970 MVT::i32); 971 return true; 972 } 973 974 // If the RHS is +/- imm8, fold into addr mode. 975 int RHSC; 976 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 977 -256 + 1, 256, RHSC)) { 978 Base = N.getOperand(0); 979 if (Base.getOpcode() == ISD::FrameIndex) { 980 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 981 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 982 } 983 984 ARM_AM::AddrOpc AddSub = ARM_AM::add; 985 if (RHSC < 0) { 986 AddSub = ARM_AM::sub; 987 RHSC = -RHSC; 988 } 989 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC), 990 MVT::i32); 991 return true; 992 } 993 994 Base = N; 995 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0), 996 MVT::i32); 997 return true; 998} 999 1000bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr, 1001 SDValue &Align) { 1002 Addr = N; 1003 1004 unsigned Alignment = 0; 1005 if (LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(Parent)) { 1006 // This case occurs only for VLD1-lane/dup and VST1-lane instructions. 1007 // The maximum alignment is equal to the memory size being referenced. 1008 unsigned LSNAlign = LSN->getAlignment(); 1009 unsigned MemSize = LSN->getMemoryVT().getSizeInBits() / 8; 1010 if (LSNAlign >= MemSize && MemSize > 1) 1011 Alignment = MemSize; 1012 } else { 1013 // All other uses of addrmode6 are for intrinsics. For now just record 1014 // the raw alignment value; it will be refined later based on the legal 1015 // alignment operands for the intrinsic. 1016 Alignment = cast<MemIntrinsicSDNode>(Parent)->getAlignment(); 1017 } 1018 1019 Align = CurDAG->getTargetConstant(Alignment, MVT::i32); 1020 return true; 1021} 1022 1023bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N, 1024 SDValue &Offset) { 1025 LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op); 1026 ISD::MemIndexedMode AM = LdSt->getAddressingMode(); 1027 if (AM != ISD::POST_INC) 1028 return false; 1029 Offset = N; 1030 if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) { 1031 if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits()) 1032 Offset = CurDAG->getRegister(0, MVT::i32); 1033 } 1034 return true; 1035} 1036 1037bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N, 1038 SDValue &Offset, SDValue &Label) { 1039 if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) { 1040 Offset = N.getOperand(0); 1041 SDValue N1 = N.getOperand(1); 1042 Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(), 1043 MVT::i32); 1044 return true; 1045 } 1046 1047 return false; 1048} 1049 1050 1051//===----------------------------------------------------------------------===// 1052// Thumb Addressing Modes 1053//===----------------------------------------------------------------------===// 1054 1055bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N, 1056 SDValue &Base, SDValue &Offset){ 1057 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) { 1058 ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N); 1059 if (!NC || !NC->isNullValue()) 1060 return false; 1061 1062 Base = Offset = N; 1063 return true; 1064 } 1065 1066 Base = N.getOperand(0); 1067 Offset = N.getOperand(1); 1068 return true; 1069} 1070 1071bool 1072ARMDAGToDAGISel::SelectThumbAddrModeRI(SDValue N, SDValue &Base, 1073 SDValue &Offset, unsigned Scale) { 1074 if (Scale == 4) { 1075 SDValue TmpBase, TmpOffImm; 1076 if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm)) 1077 return false; // We want to select tLDRspi / tSTRspi instead. 1078 1079 if (N.getOpcode() == ARMISD::Wrapper && 1080 N.getOperand(0).getOpcode() == ISD::TargetConstantPool) 1081 return false; // We want to select tLDRpci instead. 1082 } 1083 1084 if (!CurDAG->isBaseWithConstantOffset(N)) 1085 return false; 1086 1087 // Thumb does not have [sp, r] address mode. 1088 RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0)); 1089 RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1)); 1090 if ((LHSR && LHSR->getReg() == ARM::SP) || 1091 (RHSR && RHSR->getReg() == ARM::SP)) 1092 return false; 1093 1094 // FIXME: Why do we explicitly check for a match here and then return false? 1095 // Presumably to allow something else to match, but shouldn't this be 1096 // documented? 1097 int RHSC; 1098 if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) 1099 return false; 1100 1101 Base = N.getOperand(0); 1102 Offset = N.getOperand(1); 1103 return true; 1104} 1105 1106bool 1107ARMDAGToDAGISel::SelectThumbAddrModeRI5S1(SDValue N, 1108 SDValue &Base, 1109 SDValue &Offset) { 1110 return SelectThumbAddrModeRI(N, Base, Offset, 1); 1111} 1112 1113bool 1114ARMDAGToDAGISel::SelectThumbAddrModeRI5S2(SDValue N, 1115 SDValue &Base, 1116 SDValue &Offset) { 1117 return SelectThumbAddrModeRI(N, Base, Offset, 2); 1118} 1119 1120bool 1121ARMDAGToDAGISel::SelectThumbAddrModeRI5S4(SDValue N, 1122 SDValue &Base, 1123 SDValue &Offset) { 1124 return SelectThumbAddrModeRI(N, Base, Offset, 4); 1125} 1126 1127bool 1128ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, 1129 SDValue &Base, SDValue &OffImm) { 1130 if (Scale == 4) { 1131 SDValue TmpBase, TmpOffImm; 1132 if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm)) 1133 return false; // We want to select tLDRspi / tSTRspi instead. 1134 1135 if (N.getOpcode() == ARMISD::Wrapper && 1136 N.getOperand(0).getOpcode() == ISD::TargetConstantPool) 1137 return false; // We want to select tLDRpci instead. 1138 } 1139 1140 if (!CurDAG->isBaseWithConstantOffset(N)) { 1141 if (N.getOpcode() == ARMISD::Wrapper && 1142 !(Subtarget->useMovt() && 1143 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) { 1144 Base = N.getOperand(0); 1145 } else { 1146 Base = N; 1147 } 1148 1149 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1150 return true; 1151 } 1152 1153 RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0)); 1154 RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1)); 1155 if ((LHSR && LHSR->getReg() == ARM::SP) || 1156 (RHSR && RHSR->getReg() == ARM::SP)) { 1157 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(N.getOperand(0)); 1158 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); 1159 unsigned LHSC = LHS ? LHS->getZExtValue() : 0; 1160 unsigned RHSC = RHS ? RHS->getZExtValue() : 0; 1161 1162 // Thumb does not have [sp, #imm5] address mode for non-zero imm5. 1163 if (LHSC != 0 || RHSC != 0) return false; 1164 1165 Base = N; 1166 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1167 return true; 1168 } 1169 1170 // If the RHS is + imm5 * scale, fold into addr mode. 1171 int RHSC; 1172 if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) { 1173 Base = N.getOperand(0); 1174 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 1175 return true; 1176 } 1177 1178 Base = N.getOperand(0); 1179 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1180 return true; 1181} 1182 1183bool 1184ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base, 1185 SDValue &OffImm) { 1186 return SelectThumbAddrModeImm5S(N, 4, Base, OffImm); 1187} 1188 1189bool 1190ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base, 1191 SDValue &OffImm) { 1192 return SelectThumbAddrModeImm5S(N, 2, Base, OffImm); 1193} 1194 1195bool 1196ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base, 1197 SDValue &OffImm) { 1198 return SelectThumbAddrModeImm5S(N, 1, Base, OffImm); 1199} 1200 1201bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N, 1202 SDValue &Base, SDValue &OffImm) { 1203 if (N.getOpcode() == ISD::FrameIndex) { 1204 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 1205 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 1206 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1207 return true; 1208 } 1209 1210 if (!CurDAG->isBaseWithConstantOffset(N)) 1211 return false; 1212 1213 RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0)); 1214 if (N.getOperand(0).getOpcode() == ISD::FrameIndex || 1215 (LHSR && LHSR->getReg() == ARM::SP)) { 1216 // If the RHS is + imm8 * scale, fold into addr mode. 1217 int RHSC; 1218 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) { 1219 Base = N.getOperand(0); 1220 if (Base.getOpcode() == ISD::FrameIndex) { 1221 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1222 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 1223 } 1224 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 1225 return true; 1226 } 1227 } 1228 1229 return false; 1230} 1231 1232 1233//===----------------------------------------------------------------------===// 1234// Thumb 2 Addressing Modes 1235//===----------------------------------------------------------------------===// 1236 1237 1238bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg, 1239 SDValue &Opc) { 1240 if (DisableShifterOp) 1241 return false; 1242 1243 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 1244 1245 // Don't match base register only case. That is matched to a separate 1246 // lower complexity pattern with explicit register operand. 1247 if (ShOpcVal == ARM_AM::no_shift) return false; 1248 1249 BaseReg = N.getOperand(0); 1250 unsigned ShImmVal = 0; 1251 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1252 ShImmVal = RHS->getZExtValue() & 31; 1253 Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal)); 1254 return true; 1255 } 1256 1257 return false; 1258} 1259 1260bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N, 1261 SDValue &Base, SDValue &OffImm) { 1262 // Match simple R + imm12 operands. 1263 1264 // Base only. 1265 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 1266 !CurDAG->isBaseWithConstantOffset(N)) { 1267 if (N.getOpcode() == ISD::FrameIndex) { 1268 // Match frame index. 1269 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 1270 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 1271 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1272 return true; 1273 } 1274 1275 if (N.getOpcode() == ARMISD::Wrapper && 1276 !(Subtarget->useMovt() && 1277 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) { 1278 Base = N.getOperand(0); 1279 if (Base.getOpcode() == ISD::TargetConstantPool) 1280 return false; // We want to select t2LDRpci instead. 1281 } else 1282 Base = N; 1283 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1284 return true; 1285 } 1286 1287 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1288 if (SelectT2AddrModeImm8(N, Base, OffImm)) 1289 // Let t2LDRi8 handle (R - imm8). 1290 return false; 1291 1292 int RHSC = (int)RHS->getZExtValue(); 1293 if (N.getOpcode() == ISD::SUB) 1294 RHSC = -RHSC; 1295 1296 if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned) 1297 Base = N.getOperand(0); 1298 if (Base.getOpcode() == ISD::FrameIndex) { 1299 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1300 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 1301 } 1302 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 1303 return true; 1304 } 1305 } 1306 1307 // Base only. 1308 Base = N; 1309 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1310 return true; 1311} 1312 1313bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N, 1314 SDValue &Base, SDValue &OffImm) { 1315 // Match simple R - imm8 operands. 1316 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 1317 !CurDAG->isBaseWithConstantOffset(N)) 1318 return false; 1319 1320 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1321 int RHSC = (int)RHS->getSExtValue(); 1322 if (N.getOpcode() == ISD::SUB) 1323 RHSC = -RHSC; 1324 1325 if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative) 1326 Base = N.getOperand(0); 1327 if (Base.getOpcode() == ISD::FrameIndex) { 1328 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1329 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 1330 } 1331 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 1332 return true; 1333 } 1334 } 1335 1336 return false; 1337} 1338 1339bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N, 1340 SDValue &OffImm){ 1341 unsigned Opcode = Op->getOpcode(); 1342 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 1343 ? cast<LoadSDNode>(Op)->getAddressingMode() 1344 : cast<StoreSDNode>(Op)->getAddressingMode(); 1345 int RHSC; 1346 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits. 1347 OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC)) 1348 ? CurDAG->getTargetConstant(RHSC, MVT::i32) 1349 : CurDAG->getTargetConstant(-RHSC, MVT::i32); 1350 return true; 1351 } 1352 1353 return false; 1354} 1355 1356bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N, 1357 SDValue &Base, 1358 SDValue &OffReg, SDValue &ShImm) { 1359 // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12. 1360 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) 1361 return false; 1362 1363 // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8. 1364 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1365 int RHSC = (int)RHS->getZExtValue(); 1366 if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned) 1367 return false; 1368 else if (RHSC < 0 && RHSC >= -255) // 8 bits 1369 return false; 1370 } 1371 1372 // Look for (R + R) or (R + (R << [1,2,3])). 1373 unsigned ShAmt = 0; 1374 Base = N.getOperand(0); 1375 OffReg = N.getOperand(1); 1376 1377 // Swap if it is ((R << c) + R). 1378 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode()); 1379 if (ShOpcVal != ARM_AM::lsl) { 1380 ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode()); 1381 if (ShOpcVal == ARM_AM::lsl) 1382 std::swap(Base, OffReg); 1383 } 1384 1385 if (ShOpcVal == ARM_AM::lsl) { 1386 // Check to see if the RHS of the shift is a constant, if not, we can't fold 1387 // it. 1388 if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) { 1389 ShAmt = Sh->getZExtValue(); 1390 if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt)) 1391 OffReg = OffReg.getOperand(0); 1392 else { 1393 ShAmt = 0; 1394 ShOpcVal = ARM_AM::no_shift; 1395 } 1396 } else { 1397 ShOpcVal = ARM_AM::no_shift; 1398 } 1399 } 1400 1401 ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32); 1402 1403 return true; 1404} 1405 1406//===--------------------------------------------------------------------===// 1407 1408/// getAL - Returns a ARMCC::AL immediate node. 1409static inline SDValue getAL(SelectionDAG *CurDAG) { 1410 return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32); 1411} 1412 1413SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) { 1414 LoadSDNode *LD = cast<LoadSDNode>(N); 1415 ISD::MemIndexedMode AM = LD->getAddressingMode(); 1416 if (AM == ISD::UNINDEXED) 1417 return NULL; 1418 1419 EVT LoadedVT = LD->getMemoryVT(); 1420 SDValue Offset, AMOpc; 1421 bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC); 1422 unsigned Opcode = 0; 1423 bool Match = false; 1424 if (LoadedVT == MVT::i32 && isPre && 1425 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) { 1426 Opcode = ARM::LDR_PRE_IMM; 1427 Match = true; 1428 } else if (LoadedVT == MVT::i32 && !isPre && 1429 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) { 1430 Opcode = ARM::LDR_POST_IMM; 1431 Match = true; 1432 } else if (LoadedVT == MVT::i32 && 1433 SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) { 1434 Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG; 1435 Match = true; 1436 1437 } else if (LoadedVT == MVT::i16 && 1438 SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { 1439 Match = true; 1440 Opcode = (LD->getExtensionType() == ISD::SEXTLOAD) 1441 ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST) 1442 : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST); 1443 } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) { 1444 if (LD->getExtensionType() == ISD::SEXTLOAD) { 1445 if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { 1446 Match = true; 1447 Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST; 1448 } 1449 } else { 1450 if (isPre && 1451 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) { 1452 Match = true; 1453 Opcode = ARM::LDRB_PRE_IMM; 1454 } else if (!isPre && 1455 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) { 1456 Match = true; 1457 Opcode = ARM::LDRB_POST_IMM; 1458 } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) { 1459 Match = true; 1460 Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG; 1461 } 1462 } 1463 } 1464 1465 if (Match) { 1466 if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) { 1467 SDValue Chain = LD->getChain(); 1468 SDValue Base = LD->getBasePtr(); 1469 SDValue Ops[]= { Base, AMOpc, getAL(CurDAG), 1470 CurDAG->getRegister(0, MVT::i32), Chain }; 1471 return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, 1472 MVT::i32, MVT::Other, Ops); 1473 } else { 1474 SDValue Chain = LD->getChain(); 1475 SDValue Base = LD->getBasePtr(); 1476 SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG), 1477 CurDAG->getRegister(0, MVT::i32), Chain }; 1478 return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, 1479 MVT::i32, MVT::Other, Ops); 1480 } 1481 } 1482 1483 return NULL; 1484} 1485 1486SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) { 1487 LoadSDNode *LD = cast<LoadSDNode>(N); 1488 ISD::MemIndexedMode AM = LD->getAddressingMode(); 1489 if (AM == ISD::UNINDEXED) 1490 return NULL; 1491 1492 EVT LoadedVT = LD->getMemoryVT(); 1493 bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD; 1494 SDValue Offset; 1495 bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC); 1496 unsigned Opcode = 0; 1497 bool Match = false; 1498 if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) { 1499 switch (LoadedVT.getSimpleVT().SimpleTy) { 1500 case MVT::i32: 1501 Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST; 1502 break; 1503 case MVT::i16: 1504 if (isSExtLd) 1505 Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST; 1506 else 1507 Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST; 1508 break; 1509 case MVT::i8: 1510 case MVT::i1: 1511 if (isSExtLd) 1512 Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST; 1513 else 1514 Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST; 1515 break; 1516 default: 1517 return NULL; 1518 } 1519 Match = true; 1520 } 1521 1522 if (Match) { 1523 SDValue Chain = LD->getChain(); 1524 SDValue Base = LD->getBasePtr(); 1525 SDValue Ops[]= { Base, Offset, getAL(CurDAG), 1526 CurDAG->getRegister(0, MVT::i32), Chain }; 1527 return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32, 1528 MVT::Other, Ops); 1529 } 1530 1531 return NULL; 1532} 1533 1534/// \brief Form a GPRPair pseudo register from a pair of GPR regs. 1535SDNode *ARMDAGToDAGISel::createGPRPairNode(EVT VT, SDValue V0, SDValue V1) { 1536 DebugLoc dl = V0.getNode()->getDebugLoc(); 1537 SDValue RegClass = 1538 CurDAG->getTargetConstant(ARM::GPRPairRegClassID, MVT::i32); 1539 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, MVT::i32); 1540 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, MVT::i32); 1541 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1542 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1543} 1544 1545/// \brief Form a D register from a pair of S registers. 1546SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) { 1547 DebugLoc dl = V0.getNode()->getDebugLoc(); 1548 SDValue RegClass = 1549 CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, MVT::i32); 1550 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32); 1551 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32); 1552 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1553 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1554} 1555 1556/// \brief Form a quad register from a pair of D registers. 1557SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) { 1558 DebugLoc dl = V0.getNode()->getDebugLoc(); 1559 SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, MVT::i32); 1560 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32); 1561 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32); 1562 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1563 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1564} 1565 1566/// \brief Form 4 consecutive D registers from a pair of Q registers. 1567SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) { 1568 DebugLoc dl = V0.getNode()->getDebugLoc(); 1569 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32); 1570 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32); 1571 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32); 1572 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1573 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1574} 1575 1576/// \brief Form 4 consecutive S registers. 1577SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, 1578 SDValue V2, SDValue V3) { 1579 DebugLoc dl = V0.getNode()->getDebugLoc(); 1580 SDValue RegClass = 1581 CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, MVT::i32); 1582 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32); 1583 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32); 1584 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32); 1585 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32); 1586 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, 1587 V2, SubReg2, V3, SubReg3 }; 1588 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1589} 1590 1591/// \brief Form 4 consecutive D registers. 1592SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, 1593 SDValue V2, SDValue V3) { 1594 DebugLoc dl = V0.getNode()->getDebugLoc(); 1595 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32); 1596 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32); 1597 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32); 1598 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32); 1599 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32); 1600 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, 1601 V2, SubReg2, V3, SubReg3 }; 1602 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1603} 1604 1605/// \brief Form 4 consecutive Q registers. 1606SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, 1607 SDValue V2, SDValue V3) { 1608 DebugLoc dl = V0.getNode()->getDebugLoc(); 1609 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, MVT::i32); 1610 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32); 1611 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32); 1612 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32); 1613 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32); 1614 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, 1615 V2, SubReg2, V3, SubReg3 }; 1616 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1617} 1618 1619/// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand 1620/// of a NEON VLD or VST instruction. The supported values depend on the 1621/// number of registers being loaded. 1622SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, unsigned NumVecs, 1623 bool is64BitVector) { 1624 unsigned NumRegs = NumVecs; 1625 if (!is64BitVector && NumVecs < 3) 1626 NumRegs *= 2; 1627 1628 unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); 1629 if (Alignment >= 32 && NumRegs == 4) 1630 Alignment = 32; 1631 else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4)) 1632 Alignment = 16; 1633 else if (Alignment >= 8) 1634 Alignment = 8; 1635 else 1636 Alignment = 0; 1637 1638 return CurDAG->getTargetConstant(Alignment, MVT::i32); 1639} 1640 1641// Get the register stride update opcode of a VLD/VST instruction that 1642// is otherwise equivalent to the given fixed stride updating instruction. 1643static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) { 1644 switch (Opc) { 1645 default: break; 1646 case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register; 1647 case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register; 1648 case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register; 1649 case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register; 1650 case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register; 1651 case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register; 1652 case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register; 1653 case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register; 1654 1655 case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register; 1656 case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register; 1657 case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register; 1658 case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register; 1659 case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register; 1660 case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register; 1661 case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register; 1662 case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register; 1663 case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register; 1664 case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register; 1665 1666 case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register; 1667 case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register; 1668 case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register; 1669 case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register; 1670 case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register; 1671 case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register; 1672 1673 case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register; 1674 case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register; 1675 case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register; 1676 case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register; 1677 case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register; 1678 case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register; 1679 1680 case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register; 1681 case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register; 1682 case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register; 1683 } 1684 return Opc; // If not one we handle, return it unchanged. 1685} 1686 1687SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs, 1688 const uint16_t *DOpcodes, 1689 const uint16_t *QOpcodes0, 1690 const uint16_t *QOpcodes1) { 1691 assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range"); 1692 DebugLoc dl = N->getDebugLoc(); 1693 1694 SDValue MemAddr, Align; 1695 unsigned AddrOpIdx = isUpdating ? 1 : 2; 1696 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) 1697 return NULL; 1698 1699 SDValue Chain = N->getOperand(0); 1700 EVT VT = N->getValueType(0); 1701 bool is64BitVector = VT.is64BitVector(); 1702 Align = GetVLDSTAlign(Align, NumVecs, is64BitVector); 1703 1704 unsigned OpcodeIndex; 1705 switch (VT.getSimpleVT().SimpleTy) { 1706 default: llvm_unreachable("unhandled vld type"); 1707 // Double-register operations: 1708 case MVT::v8i8: OpcodeIndex = 0; break; 1709 case MVT::v4i16: OpcodeIndex = 1; break; 1710 case MVT::v2f32: 1711 case MVT::v2i32: OpcodeIndex = 2; break; 1712 case MVT::v1i64: OpcodeIndex = 3; break; 1713 // Quad-register operations: 1714 case MVT::v16i8: OpcodeIndex = 0; break; 1715 case MVT::v8i16: OpcodeIndex = 1; break; 1716 case MVT::v4f32: 1717 case MVT::v4i32: OpcodeIndex = 2; break; 1718 case MVT::v2i64: OpcodeIndex = 3; 1719 assert(NumVecs == 1 && "v2i64 type only supported for VLD1"); 1720 break; 1721 } 1722 1723 EVT ResTy; 1724 if (NumVecs == 1) 1725 ResTy = VT; 1726 else { 1727 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; 1728 if (!is64BitVector) 1729 ResTyElts *= 2; 1730 ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts); 1731 } 1732 std::vector<EVT> ResTys; 1733 ResTys.push_back(ResTy); 1734 if (isUpdating) 1735 ResTys.push_back(MVT::i32); 1736 ResTys.push_back(MVT::Other); 1737 1738 SDValue Pred = getAL(CurDAG); 1739 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 1740 SDNode *VLd; 1741 SmallVector<SDValue, 7> Ops; 1742 1743 // Double registers and VLD1/VLD2 quad registers are directly supported. 1744 if (is64BitVector || NumVecs <= 2) { 1745 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : 1746 QOpcodes0[OpcodeIndex]); 1747 Ops.push_back(MemAddr); 1748 Ops.push_back(Align); 1749 if (isUpdating) { 1750 SDValue Inc = N->getOperand(AddrOpIdx + 1); 1751 // FIXME: VLD1/VLD2 fixed increment doesn't need Reg0. Remove the reg0 1752 // case entirely when the rest are updated to that form, too. 1753 if ((NumVecs == 1 || NumVecs == 2) && !isa<ConstantSDNode>(Inc.getNode())) 1754 Opc = getVLDSTRegisterUpdateOpcode(Opc); 1755 // We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so 1756 // check for that explicitly too. Horribly hacky, but temporary. 1757 if ((NumVecs != 1 && NumVecs != 2 && Opc != ARM::VLD1q64wb_fixed) || 1758 !isa<ConstantSDNode>(Inc.getNode())) 1759 Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc); 1760 } 1761 Ops.push_back(Pred); 1762 Ops.push_back(Reg0); 1763 Ops.push_back(Chain); 1764 VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 1765 1766 } else { 1767 // Otherwise, quad registers are loaded with two separate instructions, 1768 // where one loads the even registers and the other loads the odd registers. 1769 EVT AddrTy = MemAddr.getValueType(); 1770 1771 // Load the even subregs. This is always an updating load, so that it 1772 // provides the address to the second load for the odd subregs. 1773 SDValue ImplDef = 1774 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0); 1775 const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain }; 1776 SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, 1777 ResTy, AddrTy, MVT::Other, OpsA); 1778 Chain = SDValue(VLdA, 2); 1779 1780 // Load the odd subregs. 1781 Ops.push_back(SDValue(VLdA, 1)); 1782 Ops.push_back(Align); 1783 if (isUpdating) { 1784 SDValue Inc = N->getOperand(AddrOpIdx + 1); 1785 assert(isa<ConstantSDNode>(Inc.getNode()) && 1786 "only constant post-increment update allowed for VLD3/4"); 1787 (void)Inc; 1788 Ops.push_back(Reg0); 1789 } 1790 Ops.push_back(SDValue(VLdA, 0)); 1791 Ops.push_back(Pred); 1792 Ops.push_back(Reg0); 1793 Ops.push_back(Chain); 1794 VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops); 1795 } 1796 1797 // Transfer memoperands. 1798 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1799 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 1800 cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1); 1801 1802 if (NumVecs == 1) 1803 return VLd; 1804 1805 // Extract out the subregisters. 1806 SDValue SuperReg = SDValue(VLd, 0); 1807 assert(ARM::dsub_7 == ARM::dsub_0+7 && 1808 ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering"); 1809 unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0); 1810 for (unsigned Vec = 0; Vec < NumVecs; ++Vec) 1811 ReplaceUses(SDValue(N, Vec), 1812 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg)); 1813 ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1)); 1814 if (isUpdating) 1815 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2)); 1816 return NULL; 1817} 1818 1819SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, 1820 const uint16_t *DOpcodes, 1821 const uint16_t *QOpcodes0, 1822 const uint16_t *QOpcodes1) { 1823 assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range"); 1824 DebugLoc dl = N->getDebugLoc(); 1825 1826 SDValue MemAddr, Align; 1827 unsigned AddrOpIdx = isUpdating ? 1 : 2; 1828 unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1) 1829 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) 1830 return NULL; 1831 1832 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1833 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 1834 1835 SDValue Chain = N->getOperand(0); 1836 EVT VT = N->getOperand(Vec0Idx).getValueType(); 1837 bool is64BitVector = VT.is64BitVector(); 1838 Align = GetVLDSTAlign(Align, NumVecs, is64BitVector); 1839 1840 unsigned OpcodeIndex; 1841 switch (VT.getSimpleVT().SimpleTy) { 1842 default: llvm_unreachable("unhandled vst type"); 1843 // Double-register operations: 1844 case MVT::v8i8: OpcodeIndex = 0; break; 1845 case MVT::v4i16: OpcodeIndex = 1; break; 1846 case MVT::v2f32: 1847 case MVT::v2i32: OpcodeIndex = 2; break; 1848 case MVT::v1i64: OpcodeIndex = 3; break; 1849 // Quad-register operations: 1850 case MVT::v16i8: OpcodeIndex = 0; break; 1851 case MVT::v8i16: OpcodeIndex = 1; break; 1852 case MVT::v4f32: 1853 case MVT::v4i32: OpcodeIndex = 2; break; 1854 case MVT::v2i64: OpcodeIndex = 3; 1855 assert(NumVecs == 1 && "v2i64 type only supported for VST1"); 1856 break; 1857 } 1858 1859 std::vector<EVT> ResTys; 1860 if (isUpdating) 1861 ResTys.push_back(MVT::i32); 1862 ResTys.push_back(MVT::Other); 1863 1864 SDValue Pred = getAL(CurDAG); 1865 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 1866 SmallVector<SDValue, 7> Ops; 1867 1868 // Double registers and VST1/VST2 quad registers are directly supported. 1869 if (is64BitVector || NumVecs <= 2) { 1870 SDValue SrcReg; 1871 if (NumVecs == 1) { 1872 SrcReg = N->getOperand(Vec0Idx); 1873 } else if (is64BitVector) { 1874 // Form a REG_SEQUENCE to force register allocation. 1875 SDValue V0 = N->getOperand(Vec0Idx + 0); 1876 SDValue V1 = N->getOperand(Vec0Idx + 1); 1877 if (NumVecs == 2) 1878 SrcReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0); 1879 else { 1880 SDValue V2 = N->getOperand(Vec0Idx + 2); 1881 // If it's a vst3, form a quad D-register and leave the last part as 1882 // an undef. 1883 SDValue V3 = (NumVecs == 3) 1884 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0) 1885 : N->getOperand(Vec0Idx + 3); 1886 SrcReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0); 1887 } 1888 } else { 1889 // Form a QQ register. 1890 SDValue Q0 = N->getOperand(Vec0Idx); 1891 SDValue Q1 = N->getOperand(Vec0Idx + 1); 1892 SrcReg = SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0); 1893 } 1894 1895 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : 1896 QOpcodes0[OpcodeIndex]); 1897 Ops.push_back(MemAddr); 1898 Ops.push_back(Align); 1899 if (isUpdating) { 1900 SDValue Inc = N->getOperand(AddrOpIdx + 1); 1901 // FIXME: VST1/VST2 fixed increment doesn't need Reg0. Remove the reg0 1902 // case entirely when the rest are updated to that form, too. 1903 if (NumVecs <= 2 && !isa<ConstantSDNode>(Inc.getNode())) 1904 Opc = getVLDSTRegisterUpdateOpcode(Opc); 1905 // We use a VST1 for v1i64 even if the pseudo says vld2/3/4, so 1906 // check for that explicitly too. Horribly hacky, but temporary. 1907 if ((NumVecs > 2 && Opc != ARM::VST1q64wb_fixed) || 1908 !isa<ConstantSDNode>(Inc.getNode())) 1909 Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc); 1910 } 1911 Ops.push_back(SrcReg); 1912 Ops.push_back(Pred); 1913 Ops.push_back(Reg0); 1914 Ops.push_back(Chain); 1915 SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 1916 1917 // Transfer memoperands. 1918 cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1); 1919 1920 return VSt; 1921 } 1922 1923 // Otherwise, quad registers are stored with two separate instructions, 1924 // where one stores the even registers and the other stores the odd registers. 1925 1926 // Form the QQQQ REG_SEQUENCE. 1927 SDValue V0 = N->getOperand(Vec0Idx + 0); 1928 SDValue V1 = N->getOperand(Vec0Idx + 1); 1929 SDValue V2 = N->getOperand(Vec0Idx + 2); 1930 SDValue V3 = (NumVecs == 3) 1931 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) 1932 : N->getOperand(Vec0Idx + 3); 1933 SDValue RegSeq = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0); 1934 1935 // Store the even D registers. This is always an updating store, so that it 1936 // provides the address to the second store for the odd subregs. 1937 const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain }; 1938 SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, 1939 MemAddr.getValueType(), 1940 MVT::Other, OpsA); 1941 cast<MachineSDNode>(VStA)->setMemRefs(MemOp, MemOp + 1); 1942 Chain = SDValue(VStA, 1); 1943 1944 // Store the odd D registers. 1945 Ops.push_back(SDValue(VStA, 0)); 1946 Ops.push_back(Align); 1947 if (isUpdating) { 1948 SDValue Inc = N->getOperand(AddrOpIdx + 1); 1949 assert(isa<ConstantSDNode>(Inc.getNode()) && 1950 "only constant post-increment update allowed for VST3/4"); 1951 (void)Inc; 1952 Ops.push_back(Reg0); 1953 } 1954 Ops.push_back(RegSeq); 1955 Ops.push_back(Pred); 1956 Ops.push_back(Reg0); 1957 Ops.push_back(Chain); 1958 SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, 1959 Ops); 1960 cast<MachineSDNode>(VStB)->setMemRefs(MemOp, MemOp + 1); 1961 return VStB; 1962} 1963 1964SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, 1965 bool isUpdating, unsigned NumVecs, 1966 const uint16_t *DOpcodes, 1967 const uint16_t *QOpcodes) { 1968 assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range"); 1969 DebugLoc dl = N->getDebugLoc(); 1970 1971 SDValue MemAddr, Align; 1972 unsigned AddrOpIdx = isUpdating ? 1 : 2; 1973 unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1) 1974 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) 1975 return NULL; 1976 1977 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1978 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 1979 1980 SDValue Chain = N->getOperand(0); 1981 unsigned Lane = 1982 cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue(); 1983 EVT VT = N->getOperand(Vec0Idx).getValueType(); 1984 bool is64BitVector = VT.is64BitVector(); 1985 1986 unsigned Alignment = 0; 1987 if (NumVecs != 3) { 1988 Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); 1989 unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8; 1990 if (Alignment > NumBytes) 1991 Alignment = NumBytes; 1992 if (Alignment < 8 && Alignment < NumBytes) 1993 Alignment = 0; 1994 // Alignment must be a power of two; make sure of that. 1995 Alignment = (Alignment & -Alignment); 1996 if (Alignment == 1) 1997 Alignment = 0; 1998 } 1999 Align = CurDAG->getTargetConstant(Alignment, MVT::i32); 2000 2001 unsigned OpcodeIndex; 2002 switch (VT.getSimpleVT().SimpleTy) { 2003 default: llvm_unreachable("unhandled vld/vst lane type"); 2004 // Double-register operations: 2005 case MVT::v8i8: OpcodeIndex = 0; break; 2006 case MVT::v4i16: OpcodeIndex = 1; break; 2007 case MVT::v2f32: 2008 case MVT::v2i32: OpcodeIndex = 2; break; 2009 // Quad-register operations: 2010 case MVT::v8i16: OpcodeIndex = 0; break; 2011 case MVT::v4f32: 2012 case MVT::v4i32: OpcodeIndex = 1; break; 2013 } 2014 2015 std::vector<EVT> ResTys; 2016 if (IsLoad) { 2017 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; 2018 if (!is64BitVector) 2019 ResTyElts *= 2; 2020 ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), 2021 MVT::i64, ResTyElts)); 2022 } 2023 if (isUpdating) 2024 ResTys.push_back(MVT::i32); 2025 ResTys.push_back(MVT::Other); 2026 2027 SDValue Pred = getAL(CurDAG); 2028 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2029 2030 SmallVector<SDValue, 8> Ops; 2031 Ops.push_back(MemAddr); 2032 Ops.push_back(Align); 2033 if (isUpdating) { 2034 SDValue Inc = N->getOperand(AddrOpIdx + 1); 2035 Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc); 2036 } 2037 2038 SDValue SuperReg; 2039 SDValue V0 = N->getOperand(Vec0Idx + 0); 2040 SDValue V1 = N->getOperand(Vec0Idx + 1); 2041 if (NumVecs == 2) { 2042 if (is64BitVector) 2043 SuperReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0); 2044 else 2045 SuperReg = SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0); 2046 } else { 2047 SDValue V2 = N->getOperand(Vec0Idx + 2); 2048 SDValue V3 = (NumVecs == 3) 2049 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) 2050 : N->getOperand(Vec0Idx + 3); 2051 if (is64BitVector) 2052 SuperReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0); 2053 else 2054 SuperReg = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0); 2055 } 2056 Ops.push_back(SuperReg); 2057 Ops.push_back(getI32Imm(Lane)); 2058 Ops.push_back(Pred); 2059 Ops.push_back(Reg0); 2060 Ops.push_back(Chain); 2061 2062 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : 2063 QOpcodes[OpcodeIndex]); 2064 SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 2065 cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1); 2066 if (!IsLoad) 2067 return VLdLn; 2068 2069 // Extract the subregisters. 2070 SuperReg = SDValue(VLdLn, 0); 2071 assert(ARM::dsub_7 == ARM::dsub_0+7 && 2072 ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering"); 2073 unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0; 2074 for (unsigned Vec = 0; Vec < NumVecs; ++Vec) 2075 ReplaceUses(SDValue(N, Vec), 2076 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg)); 2077 ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1)); 2078 if (isUpdating) 2079 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2)); 2080 return NULL; 2081} 2082 2083SDNode *ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating, 2084 unsigned NumVecs, 2085 const uint16_t *Opcodes) { 2086 assert(NumVecs >=2 && NumVecs <= 4 && "VLDDup NumVecs out-of-range"); 2087 DebugLoc dl = N->getDebugLoc(); 2088 2089 SDValue MemAddr, Align; 2090 if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align)) 2091 return NULL; 2092 2093 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 2094 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 2095 2096 SDValue Chain = N->getOperand(0); 2097 EVT VT = N->getValueType(0); 2098 2099 unsigned Alignment = 0; 2100 if (NumVecs != 3) { 2101 Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); 2102 unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8; 2103 if (Alignment > NumBytes) 2104 Alignment = NumBytes; 2105 if (Alignment < 8 && Alignment < NumBytes) 2106 Alignment = 0; 2107 // Alignment must be a power of two; make sure of that. 2108 Alignment = (Alignment & -Alignment); 2109 if (Alignment == 1) 2110 Alignment = 0; 2111 } 2112 Align = CurDAG->getTargetConstant(Alignment, MVT::i32); 2113 2114 unsigned OpcodeIndex; 2115 switch (VT.getSimpleVT().SimpleTy) { 2116 default: llvm_unreachable("unhandled vld-dup type"); 2117 case MVT::v8i8: OpcodeIndex = 0; break; 2118 case MVT::v4i16: OpcodeIndex = 1; break; 2119 case MVT::v2f32: 2120 case MVT::v2i32: OpcodeIndex = 2; break; 2121 } 2122 2123 SDValue Pred = getAL(CurDAG); 2124 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2125 SDValue SuperReg; 2126 unsigned Opc = Opcodes[OpcodeIndex]; 2127 SmallVector<SDValue, 6> Ops; 2128 Ops.push_back(MemAddr); 2129 Ops.push_back(Align); 2130 if (isUpdating) { 2131 // fixed-stride update instructions don't have an explicit writeback 2132 // operand. It's implicit in the opcode itself. 2133 SDValue Inc = N->getOperand(2); 2134 if (!isa<ConstantSDNode>(Inc.getNode())) 2135 Ops.push_back(Inc); 2136 // FIXME: VLD3 and VLD4 haven't been updated to that form yet. 2137 else if (NumVecs > 2) 2138 Ops.push_back(Reg0); 2139 } 2140 Ops.push_back(Pred); 2141 Ops.push_back(Reg0); 2142 Ops.push_back(Chain); 2143 2144 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; 2145 std::vector<EVT> ResTys; 2146 ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,ResTyElts)); 2147 if (isUpdating) 2148 ResTys.push_back(MVT::i32); 2149 ResTys.push_back(MVT::Other); 2150 SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 2151 cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1); 2152 SuperReg = SDValue(VLdDup, 0); 2153 2154 // Extract the subregisters. 2155 assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering"); 2156 unsigned SubIdx = ARM::dsub_0; 2157 for (unsigned Vec = 0; Vec < NumVecs; ++Vec) 2158 ReplaceUses(SDValue(N, Vec), 2159 CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg)); 2160 ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1)); 2161 if (isUpdating) 2162 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2)); 2163 return NULL; 2164} 2165 2166SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, 2167 unsigned Opc) { 2168 assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range"); 2169 DebugLoc dl = N->getDebugLoc(); 2170 EVT VT = N->getValueType(0); 2171 unsigned FirstTblReg = IsExt ? 2 : 1; 2172 2173 // Form a REG_SEQUENCE to force register allocation. 2174 SDValue RegSeq; 2175 SDValue V0 = N->getOperand(FirstTblReg + 0); 2176 SDValue V1 = N->getOperand(FirstTblReg + 1); 2177 if (NumVecs == 2) 2178 RegSeq = SDValue(createDRegPairNode(MVT::v16i8, V0, V1), 0); 2179 else { 2180 SDValue V2 = N->getOperand(FirstTblReg + 2); 2181 // If it's a vtbl3, form a quad D-register and leave the last part as 2182 // an undef. 2183 SDValue V3 = (NumVecs == 3) 2184 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) 2185 : N->getOperand(FirstTblReg + 3); 2186 RegSeq = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0); 2187 } 2188 2189 SmallVector<SDValue, 6> Ops; 2190 if (IsExt) 2191 Ops.push_back(N->getOperand(1)); 2192 Ops.push_back(RegSeq); 2193 Ops.push_back(N->getOperand(FirstTblReg + NumVecs)); 2194 Ops.push_back(getAL(CurDAG)); // predicate 2195 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register 2196 return CurDAG->getMachineNode(Opc, dl, VT, Ops); 2197} 2198 2199SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N, 2200 bool isSigned) { 2201 if (!Subtarget->hasV6T2Ops()) 2202 return NULL; 2203 2204 unsigned Opc = isSigned 2205 ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX) 2206 : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX); 2207 2208 // For unsigned extracts, check for a shift right and mask 2209 unsigned And_imm = 0; 2210 if (N->getOpcode() == ISD::AND) { 2211 if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) { 2212 2213 // The immediate is a mask of the low bits iff imm & (imm+1) == 0 2214 if (And_imm & (And_imm + 1)) 2215 return NULL; 2216 2217 unsigned Srl_imm = 0; 2218 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, 2219 Srl_imm)) { 2220 assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!"); 2221 2222 // Note: The width operand is encoded as width-1. 2223 unsigned Width = CountTrailingOnes_32(And_imm) - 1; 2224 unsigned LSB = Srl_imm; 2225 2226 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2227 2228 if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) { 2229 // It's cheaper to use a right shift to extract the top bits. 2230 if (Subtarget->isThumb()) { 2231 Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri; 2232 SDValue Ops[] = { N->getOperand(0).getOperand(0), 2233 CurDAG->getTargetConstant(LSB, MVT::i32), 2234 getAL(CurDAG), Reg0, Reg0 }; 2235 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5); 2236 } 2237 2238 // ARM models shift instructions as MOVsi with shifter operand. 2239 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(ISD::SRL); 2240 SDValue ShOpc = 2241 CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB), 2242 MVT::i32); 2243 SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc, 2244 getAL(CurDAG), Reg0, Reg0 }; 2245 return CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops, 5); 2246 } 2247 2248 SDValue Ops[] = { N->getOperand(0).getOperand(0), 2249 CurDAG->getTargetConstant(LSB, MVT::i32), 2250 CurDAG->getTargetConstant(Width, MVT::i32), 2251 getAL(CurDAG), Reg0 }; 2252 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5); 2253 } 2254 } 2255 return NULL; 2256 } 2257 2258 // Otherwise, we're looking for a shift of a shift 2259 unsigned Shl_imm = 0; 2260 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) { 2261 assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!"); 2262 unsigned Srl_imm = 0; 2263 if (isInt32Immediate(N->getOperand(1), Srl_imm)) { 2264 assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!"); 2265 // Note: The width operand is encoded as width-1. 2266 unsigned Width = 32 - Srl_imm - 1; 2267 int LSB = Srl_imm - Shl_imm; 2268 if (LSB < 0) 2269 return NULL; 2270 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2271 SDValue Ops[] = { N->getOperand(0).getOperand(0), 2272 CurDAG->getTargetConstant(LSB, MVT::i32), 2273 CurDAG->getTargetConstant(Width, MVT::i32), 2274 getAL(CurDAG), Reg0 }; 2275 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5); 2276 } 2277 } 2278 return NULL; 2279} 2280 2281SDNode *ARMDAGToDAGISel:: 2282SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, 2283 ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) { 2284 SDValue CPTmp0; 2285 SDValue CPTmp1; 2286 if (SelectT2ShifterOperandReg(TrueVal, CPTmp0, CPTmp1)) { 2287 unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue(); 2288 unsigned SOShOp = ARM_AM::getSORegShOp(SOVal); 2289 unsigned Opc = 0; 2290 switch (SOShOp) { 2291 case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break; 2292 case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break; 2293 case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break; 2294 case ARM_AM::ror: Opc = ARM::t2MOVCCror; break; 2295 default: 2296 llvm_unreachable("Unknown so_reg opcode!"); 2297 } 2298 SDValue SOShImm = 2299 CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32); 2300 SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32); 2301 SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag }; 2302 return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6); 2303 } 2304 return 0; 2305} 2306 2307SDNode *ARMDAGToDAGISel:: 2308SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, 2309 ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) { 2310 SDValue CPTmp0; 2311 SDValue CPTmp1; 2312 SDValue CPTmp2; 2313 if (SelectImmShifterOperand(TrueVal, CPTmp0, CPTmp2)) { 2314 SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32); 2315 SDValue Ops[] = { FalseVal, CPTmp0, CPTmp2, CC, CCR, InFlag }; 2316 return CurDAG->SelectNodeTo(N, ARM::MOVCCsi, MVT::i32, Ops, 6); 2317 } 2318 2319 if (SelectRegShifterOperand(TrueVal, CPTmp0, CPTmp1, CPTmp2)) { 2320 SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32); 2321 SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag }; 2322 return CurDAG->SelectNodeTo(N, ARM::MOVCCsr, MVT::i32, Ops, 7); 2323 } 2324 return 0; 2325} 2326 2327SDNode *ARMDAGToDAGISel:: 2328SelectT2CMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, 2329 ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) { 2330 ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal); 2331 if (!T) 2332 return 0; 2333 2334 unsigned Opc = 0; 2335 unsigned TrueImm = T->getZExtValue(); 2336 if (is_t2_so_imm(TrueImm)) { 2337 Opc = ARM::t2MOVCCi; 2338 } else if (TrueImm <= 0xffff) { 2339 Opc = ARM::t2MOVCCi16; 2340 } else if (is_t2_so_imm_not(TrueImm)) { 2341 TrueImm = ~TrueImm; 2342 Opc = ARM::t2MVNCCi; 2343 } else if (TrueVal.getNode()->hasOneUse() && Subtarget->hasV6T2Ops()) { 2344 // Large immediate. 2345 Opc = ARM::t2MOVCCi32imm; 2346 } 2347 2348 if (Opc) { 2349 SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32); 2350 SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32); 2351 SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag }; 2352 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5); 2353 } 2354 2355 return 0; 2356} 2357 2358SDNode *ARMDAGToDAGISel:: 2359SelectARMCMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal, 2360 ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) { 2361 ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal); 2362 if (!T) 2363 return 0; 2364 2365 unsigned Opc = 0; 2366 unsigned TrueImm = T->getZExtValue(); 2367 bool isSoImm = is_so_imm(TrueImm); 2368 if (isSoImm) { 2369 Opc = ARM::MOVCCi; 2370 } else if (Subtarget->hasV6T2Ops() && TrueImm <= 0xffff) { 2371 Opc = ARM::MOVCCi16; 2372 } else if (is_so_imm_not(TrueImm)) { 2373 TrueImm = ~TrueImm; 2374 Opc = ARM::MVNCCi; 2375 } else if (TrueVal.getNode()->hasOneUse() && 2376 (Subtarget->hasV6T2Ops() || ARM_AM::isSOImmTwoPartVal(TrueImm))) { 2377 // Large immediate. 2378 Opc = ARM::MOVCCi32imm; 2379 } 2380 2381 if (Opc) { 2382 SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32); 2383 SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32); 2384 SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag }; 2385 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5); 2386 } 2387 2388 return 0; 2389} 2390 2391SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) { 2392 EVT VT = N->getValueType(0); 2393 SDValue FalseVal = N->getOperand(0); 2394 SDValue TrueVal = N->getOperand(1); 2395 SDValue CC = N->getOperand(2); 2396 SDValue CCR = N->getOperand(3); 2397 SDValue InFlag = N->getOperand(4); 2398 assert(CC.getOpcode() == ISD::Constant); 2399 assert(CCR.getOpcode() == ISD::Register); 2400 ARMCC::CondCodes CCVal = 2401 (ARMCC::CondCodes)cast<ConstantSDNode>(CC)->getZExtValue(); 2402 2403 if (!Subtarget->isThumb1Only() && VT == MVT::i32) { 2404 // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc) 2405 // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc) 2406 // Pattern complexity = 18 cost = 1 size = 0 2407 if (Subtarget->isThumb()) { 2408 SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal, 2409 CCVal, CCR, InFlag); 2410 if (!Res) 2411 Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal, 2412 ARMCC::getOppositeCondition(CCVal), CCR, InFlag); 2413 if (Res) 2414 return Res; 2415 } else { 2416 SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal, 2417 CCVal, CCR, InFlag); 2418 if (!Res) 2419 Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal, 2420 ARMCC::getOppositeCondition(CCVal), CCR, InFlag); 2421 if (Res) 2422 return Res; 2423 } 2424 2425 // Pattern: (ARMcmov:i32 GPR:i32:$false, 2426 // (imm:i32)<<P:Pred_so_imm>>:$true, 2427 // (imm:i32):$cc) 2428 // Emits: (MOVCCi:i32 GPR:i32:$false, 2429 // (so_imm:i32 (imm:i32):$true), (imm:i32):$cc) 2430 // Pattern complexity = 10 cost = 1 size = 0 2431 if (Subtarget->isThumb()) { 2432 SDNode *Res = SelectT2CMOVImmOp(N, FalseVal, TrueVal, 2433 CCVal, CCR, InFlag); 2434 if (!Res) 2435 Res = SelectT2CMOVImmOp(N, TrueVal, FalseVal, 2436 ARMCC::getOppositeCondition(CCVal), CCR, InFlag); 2437 if (Res) 2438 return Res; 2439 } else { 2440 SDNode *Res = SelectARMCMOVImmOp(N, FalseVal, TrueVal, 2441 CCVal, CCR, InFlag); 2442 if (!Res) 2443 Res = SelectARMCMOVImmOp(N, TrueVal, FalseVal, 2444 ARMCC::getOppositeCondition(CCVal), CCR, InFlag); 2445 if (Res) 2446 return Res; 2447 } 2448 } 2449 2450 // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc) 2451 // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc) 2452 // Pattern complexity = 6 cost = 1 size = 0 2453 // 2454 // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc) 2455 // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc) 2456 // Pattern complexity = 6 cost = 11 size = 0 2457 // 2458 // Also VMOVScc and VMOVDcc. 2459 SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32); 2460 SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag }; 2461 unsigned Opc = 0; 2462 switch (VT.getSimpleVT().SimpleTy) { 2463 default: llvm_unreachable("Illegal conditional move type!"); 2464 case MVT::i32: 2465 Opc = Subtarget->isThumb() 2466 ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo) 2467 : ARM::MOVCCr; 2468 break; 2469 case MVT::f32: 2470 Opc = ARM::VMOVScc; 2471 break; 2472 case MVT::f64: 2473 Opc = ARM::VMOVDcc; 2474 break; 2475 } 2476 return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5); 2477} 2478 2479/// Target-specific DAG combining for ISD::XOR. 2480/// Target-independent combining lowers SELECT_CC nodes of the form 2481/// select_cc setg[ge] X, 0, X, -X 2482/// select_cc setgt X, -1, X, -X 2483/// select_cc setl[te] X, 0, -X, X 2484/// select_cc setlt X, 1, -X, X 2485/// which represent Integer ABS into: 2486/// Y = sra (X, size(X)-1); xor (add (X, Y), Y) 2487/// ARM instruction selection detects the latter and matches it to 2488/// ARM::ABS or ARM::t2ABS machine node. 2489SDNode *ARMDAGToDAGISel::SelectABSOp(SDNode *N){ 2490 SDValue XORSrc0 = N->getOperand(0); 2491 SDValue XORSrc1 = N->getOperand(1); 2492 EVT VT = N->getValueType(0); 2493 2494 if (Subtarget->isThumb1Only()) 2495 return NULL; 2496 2497 if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA) 2498 return NULL; 2499 2500 SDValue ADDSrc0 = XORSrc0.getOperand(0); 2501 SDValue ADDSrc1 = XORSrc0.getOperand(1); 2502 SDValue SRASrc0 = XORSrc1.getOperand(0); 2503 SDValue SRASrc1 = XORSrc1.getOperand(1); 2504 ConstantSDNode *SRAConstant = dyn_cast<ConstantSDNode>(SRASrc1); 2505 EVT XType = SRASrc0.getValueType(); 2506 unsigned Size = XType.getSizeInBits() - 1; 2507 2508 if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 && 2509 XType.isInteger() && SRAConstant != NULL && 2510 Size == SRAConstant->getZExtValue()) { 2511 unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS; 2512 return CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0); 2513 } 2514 2515 return NULL; 2516} 2517 2518SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) { 2519 // The only time a CONCAT_VECTORS operation can have legal types is when 2520 // two 64-bit vectors are concatenated to a 128-bit vector. 2521 EVT VT = N->getValueType(0); 2522 if (!VT.is128BitVector() || N->getNumOperands() != 2) 2523 llvm_unreachable("unexpected CONCAT_VECTORS"); 2524 return createDRegPairNode(VT, N->getOperand(0), N->getOperand(1)); 2525} 2526 2527SDNode *ARMDAGToDAGISel::SelectAtomic64(SDNode *Node, unsigned Opc) { 2528 SmallVector<SDValue, 6> Ops; 2529 Ops.push_back(Node->getOperand(1)); // Ptr 2530 Ops.push_back(Node->getOperand(2)); // Low part of Val1 2531 Ops.push_back(Node->getOperand(3)); // High part of Val1 2532 if (Opc == ARM::ATOMCMPXCHG6432) { 2533 Ops.push_back(Node->getOperand(4)); // Low part of Val2 2534 Ops.push_back(Node->getOperand(5)); // High part of Val2 2535 } 2536 Ops.push_back(Node->getOperand(0)); // Chain 2537 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 2538 MemOp[0] = cast<MemSDNode>(Node)->getMemOperand(); 2539 SDNode *ResNode = CurDAG->getMachineNode(Opc, Node->getDebugLoc(), 2540 MVT::i32, MVT::i32, MVT::Other, 2541 Ops); 2542 cast<MachineSDNode>(ResNode)->setMemRefs(MemOp, MemOp + 1); 2543 return ResNode; 2544} 2545 2546SDNode *ARMDAGToDAGISel::Select(SDNode *N) { 2547 DebugLoc dl = N->getDebugLoc(); 2548 2549 if (N->isMachineOpcode()) { 2550 N->setNodeId(-1); 2551 return NULL; // Already selected. 2552 } 2553 2554 switch (N->getOpcode()) { 2555 default: break; 2556 case ISD::INLINEASM: { 2557 SDNode *ResNode = SelectInlineAsm(N); 2558 if (ResNode) 2559 return ResNode; 2560 break; 2561 } 2562 case ISD::XOR: { 2563 // Select special operations if XOR node forms integer ABS pattern 2564 SDNode *ResNode = SelectABSOp(N); 2565 if (ResNode) 2566 return ResNode; 2567 // Other cases are autogenerated. 2568 break; 2569 } 2570 case ISD::Constant: { 2571 unsigned Val = cast<ConstantSDNode>(N)->getZExtValue(); 2572 bool UseCP = true; 2573 if (Subtarget->hasThumb2()) 2574 // Thumb2-aware targets have the MOVT instruction, so all immediates can 2575 // be done with MOV + MOVT, at worst. 2576 UseCP = 0; 2577 else { 2578 if (Subtarget->isThumb()) { 2579 UseCP = (Val > 255 && // MOV 2580 ~Val > 255 && // MOV + MVN 2581 !ARM_AM::isThumbImmShiftedVal(Val)); // MOV + LSL 2582 } else 2583 UseCP = (ARM_AM::getSOImmVal(Val) == -1 && // MOV 2584 ARM_AM::getSOImmVal(~Val) == -1 && // MVN 2585 !ARM_AM::isSOImmTwoPartVal(Val)); // two instrs. 2586 } 2587 2588 if (UseCP) { 2589 SDValue CPIdx = 2590 CurDAG->getTargetConstantPool(ConstantInt::get( 2591 Type::getInt32Ty(*CurDAG->getContext()), Val), 2592 TLI.getPointerTy()); 2593 2594 SDNode *ResNode; 2595 if (Subtarget->isThumb1Only()) { 2596 SDValue Pred = getAL(CurDAG); 2597 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2598 SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() }; 2599 ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other, 2600 Ops); 2601 } else { 2602 SDValue Ops[] = { 2603 CPIdx, 2604 CurDAG->getTargetConstant(0, MVT::i32), 2605 getAL(CurDAG), 2606 CurDAG->getRegister(0, MVT::i32), 2607 CurDAG->getEntryNode() 2608 }; 2609 ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other, 2610 Ops); 2611 } 2612 ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0)); 2613 return NULL; 2614 } 2615 2616 // Other cases are autogenerated. 2617 break; 2618 } 2619 case ISD::FrameIndex: { 2620 // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm. 2621 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 2622 SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 2623 if (Subtarget->isThumb1Only()) { 2624 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32), 2625 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; 2626 return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, Ops, 4); 2627 } else { 2628 unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ? 2629 ARM::t2ADDri : ARM::ADDri); 2630 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32), 2631 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), 2632 CurDAG->getRegister(0, MVT::i32) }; 2633 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5); 2634 } 2635 } 2636 case ISD::SRL: 2637 if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false)) 2638 return I; 2639 break; 2640 case ISD::SRA: 2641 if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true)) 2642 return I; 2643 break; 2644 case ISD::MUL: 2645 if (Subtarget->isThumb1Only()) 2646 break; 2647 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 2648 unsigned RHSV = C->getZExtValue(); 2649 if (!RHSV) break; 2650 if (isPowerOf2_32(RHSV-1)) { // 2^n+1? 2651 unsigned ShImm = Log2_32(RHSV-1); 2652 if (ShImm >= 32) 2653 break; 2654 SDValue V = N->getOperand(0); 2655 ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm); 2656 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32); 2657 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2658 if (Subtarget->isThumb()) { 2659 SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; 2660 return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6); 2661 } else { 2662 SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; 2663 return CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops, 7); 2664 } 2665 } 2666 if (isPowerOf2_32(RHSV+1)) { // 2^n-1? 2667 unsigned ShImm = Log2_32(RHSV+1); 2668 if (ShImm >= 32) 2669 break; 2670 SDValue V = N->getOperand(0); 2671 ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm); 2672 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32); 2673 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2674 if (Subtarget->isThumb()) { 2675 SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; 2676 return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 6); 2677 } else { 2678 SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; 2679 return CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops, 7); 2680 } 2681 } 2682 } 2683 break; 2684 case ISD::AND: { 2685 // Check for unsigned bitfield extract 2686 if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false)) 2687 return I; 2688 2689 // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits 2690 // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits 2691 // are entirely contributed by c2 and lower 16-bits are entirely contributed 2692 // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)). 2693 // Select it to: "movt x, ((c1 & 0xffff) >> 16) 2694 EVT VT = N->getValueType(0); 2695 if (VT != MVT::i32) 2696 break; 2697 unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2()) 2698 ? ARM::t2MOVTi16 2699 : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0); 2700 if (!Opc) 2701 break; 2702 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); 2703 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 2704 if (!N1C) 2705 break; 2706 if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) { 2707 SDValue N2 = N0.getOperand(1); 2708 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2); 2709 if (!N2C) 2710 break; 2711 unsigned N1CVal = N1C->getZExtValue(); 2712 unsigned N2CVal = N2C->getZExtValue(); 2713 if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) && 2714 (N1CVal & 0xffffU) == 0xffffU && 2715 (N2CVal & 0xffffU) == 0x0U) { 2716 SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16, 2717 MVT::i32); 2718 SDValue Ops[] = { N0.getOperand(0), Imm16, 2719 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; 2720 return CurDAG->getMachineNode(Opc, dl, VT, Ops); 2721 } 2722 } 2723 break; 2724 } 2725 case ARMISD::VMOVRRD: 2726 return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32, 2727 N->getOperand(0), getAL(CurDAG), 2728 CurDAG->getRegister(0, MVT::i32)); 2729 case ISD::UMUL_LOHI: { 2730 if (Subtarget->isThumb1Only()) 2731 break; 2732 if (Subtarget->isThumb()) { 2733 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2734 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; 2735 return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops); 2736 } else { 2737 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2738 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), 2739 CurDAG->getRegister(0, MVT::i32) }; 2740 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2741 ARM::UMULL : ARM::UMULLv5, 2742 dl, MVT::i32, MVT::i32, Ops); 2743 } 2744 } 2745 case ISD::SMUL_LOHI: { 2746 if (Subtarget->isThumb1Only()) 2747 break; 2748 if (Subtarget->isThumb()) { 2749 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2750 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; 2751 return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops); 2752 } else { 2753 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2754 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), 2755 CurDAG->getRegister(0, MVT::i32) }; 2756 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2757 ARM::SMULL : ARM::SMULLv5, 2758 dl, MVT::i32, MVT::i32, Ops); 2759 } 2760 } 2761 case ARMISD::UMLAL:{ 2762 if (Subtarget->isThumb()) { 2763 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 2764 N->getOperand(3), getAL(CurDAG), 2765 CurDAG->getRegister(0, MVT::i32)}; 2766 return CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops); 2767 }else{ 2768 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 2769 N->getOperand(3), getAL(CurDAG), 2770 CurDAG->getRegister(0, MVT::i32), 2771 CurDAG->getRegister(0, MVT::i32) }; 2772 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2773 ARM::UMLAL : ARM::UMLALv5, 2774 dl, MVT::i32, MVT::i32, Ops); 2775 } 2776 } 2777 case ARMISD::SMLAL:{ 2778 if (Subtarget->isThumb()) { 2779 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 2780 N->getOperand(3), getAL(CurDAG), 2781 CurDAG->getRegister(0, MVT::i32)}; 2782 return CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops); 2783 }else{ 2784 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 2785 N->getOperand(3), getAL(CurDAG), 2786 CurDAG->getRegister(0, MVT::i32), 2787 CurDAG->getRegister(0, MVT::i32) }; 2788 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2789 ARM::SMLAL : ARM::SMLALv5, 2790 dl, MVT::i32, MVT::i32, Ops); 2791 } 2792 } 2793 case ISD::LOAD: { 2794 SDNode *ResNode = 0; 2795 if (Subtarget->isThumb() && Subtarget->hasThumb2()) 2796 ResNode = SelectT2IndexedLoad(N); 2797 else 2798 ResNode = SelectARMIndexedLoad(N); 2799 if (ResNode) 2800 return ResNode; 2801 // Other cases are autogenerated. 2802 break; 2803 } 2804 case ARMISD::BRCOND: { 2805 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) 2806 // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc) 2807 // Pattern complexity = 6 cost = 1 size = 0 2808 2809 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) 2810 // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc) 2811 // Pattern complexity = 6 cost = 1 size = 0 2812 2813 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) 2814 // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc) 2815 // Pattern complexity = 6 cost = 1 size = 0 2816 2817 unsigned Opc = Subtarget->isThumb() ? 2818 ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc; 2819 SDValue Chain = N->getOperand(0); 2820 SDValue N1 = N->getOperand(1); 2821 SDValue N2 = N->getOperand(2); 2822 SDValue N3 = N->getOperand(3); 2823 SDValue InFlag = N->getOperand(4); 2824 assert(N1.getOpcode() == ISD::BasicBlock); 2825 assert(N2.getOpcode() == ISD::Constant); 2826 assert(N3.getOpcode() == ISD::Register); 2827 2828 SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned) 2829 cast<ConstantSDNode>(N2)->getZExtValue()), 2830 MVT::i32); 2831 SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag }; 2832 SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, 2833 MVT::Glue, Ops); 2834 Chain = SDValue(ResNode, 0); 2835 if (N->getNumValues() == 2) { 2836 InFlag = SDValue(ResNode, 1); 2837 ReplaceUses(SDValue(N, 1), InFlag); 2838 } 2839 ReplaceUses(SDValue(N, 0), 2840 SDValue(Chain.getNode(), Chain.getResNo())); 2841 return NULL; 2842 } 2843 case ARMISD::CMOV: 2844 return SelectCMOVOp(N); 2845 case ARMISD::VZIP: { 2846 unsigned Opc = 0; 2847 EVT VT = N->getValueType(0); 2848 switch (VT.getSimpleVT().SimpleTy) { 2849 default: return NULL; 2850 case MVT::v8i8: Opc = ARM::VZIPd8; break; 2851 case MVT::v4i16: Opc = ARM::VZIPd16; break; 2852 case MVT::v2f32: 2853 // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm. 2854 case MVT::v2i32: Opc = ARM::VTRNd32; break; 2855 case MVT::v16i8: Opc = ARM::VZIPq8; break; 2856 case MVT::v8i16: Opc = ARM::VZIPq16; break; 2857 case MVT::v4f32: 2858 case MVT::v4i32: Opc = ARM::VZIPq32; break; 2859 } 2860 SDValue Pred = getAL(CurDAG); 2861 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2862 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; 2863 return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops); 2864 } 2865 case ARMISD::VUZP: { 2866 unsigned Opc = 0; 2867 EVT VT = N->getValueType(0); 2868 switch (VT.getSimpleVT().SimpleTy) { 2869 default: return NULL; 2870 case MVT::v8i8: Opc = ARM::VUZPd8; break; 2871 case MVT::v4i16: Opc = ARM::VUZPd16; break; 2872 case MVT::v2f32: 2873 // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm. 2874 case MVT::v2i32: Opc = ARM::VTRNd32; break; 2875 case MVT::v16i8: Opc = ARM::VUZPq8; break; 2876 case MVT::v8i16: Opc = ARM::VUZPq16; break; 2877 case MVT::v4f32: 2878 case MVT::v4i32: Opc = ARM::VUZPq32; break; 2879 } 2880 SDValue Pred = getAL(CurDAG); 2881 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2882 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; 2883 return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops); 2884 } 2885 case ARMISD::VTRN: { 2886 unsigned Opc = 0; 2887 EVT VT = N->getValueType(0); 2888 switch (VT.getSimpleVT().SimpleTy) { 2889 default: return NULL; 2890 case MVT::v8i8: Opc = ARM::VTRNd8; break; 2891 case MVT::v4i16: Opc = ARM::VTRNd16; break; 2892 case MVT::v2f32: 2893 case MVT::v2i32: Opc = ARM::VTRNd32; break; 2894 case MVT::v16i8: Opc = ARM::VTRNq8; break; 2895 case MVT::v8i16: Opc = ARM::VTRNq16; break; 2896 case MVT::v4f32: 2897 case MVT::v4i32: Opc = ARM::VTRNq32; break; 2898 } 2899 SDValue Pred = getAL(CurDAG); 2900 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2901 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; 2902 return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops); 2903 } 2904 case ARMISD::BUILD_VECTOR: { 2905 EVT VecVT = N->getValueType(0); 2906 EVT EltVT = VecVT.getVectorElementType(); 2907 unsigned NumElts = VecVT.getVectorNumElements(); 2908 if (EltVT == MVT::f64) { 2909 assert(NumElts == 2 && "unexpected type for BUILD_VECTOR"); 2910 return createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)); 2911 } 2912 assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR"); 2913 if (NumElts == 2) 2914 return createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)); 2915 assert(NumElts == 4 && "unexpected type for BUILD_VECTOR"); 2916 return createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1), 2917 N->getOperand(2), N->getOperand(3)); 2918 } 2919 2920 case ARMISD::VLD2DUP: { 2921 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16, 2922 ARM::VLD2DUPd32 }; 2923 return SelectVLDDup(N, false, 2, Opcodes); 2924 } 2925 2926 case ARMISD::VLD3DUP: { 2927 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo, 2928 ARM::VLD3DUPd16Pseudo, 2929 ARM::VLD3DUPd32Pseudo }; 2930 return SelectVLDDup(N, false, 3, Opcodes); 2931 } 2932 2933 case ARMISD::VLD4DUP: { 2934 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo, 2935 ARM::VLD4DUPd16Pseudo, 2936 ARM::VLD4DUPd32Pseudo }; 2937 return SelectVLDDup(N, false, 4, Opcodes); 2938 } 2939 2940 case ARMISD::VLD2DUP_UPD: { 2941 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed, 2942 ARM::VLD2DUPd16wb_fixed, 2943 ARM::VLD2DUPd32wb_fixed }; 2944 return SelectVLDDup(N, true, 2, Opcodes); 2945 } 2946 2947 case ARMISD::VLD3DUP_UPD: { 2948 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD, 2949 ARM::VLD3DUPd16Pseudo_UPD, 2950 ARM::VLD3DUPd32Pseudo_UPD }; 2951 return SelectVLDDup(N, true, 3, Opcodes); 2952 } 2953 2954 case ARMISD::VLD4DUP_UPD: { 2955 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD, 2956 ARM::VLD4DUPd16Pseudo_UPD, 2957 ARM::VLD4DUPd32Pseudo_UPD }; 2958 return SelectVLDDup(N, true, 4, Opcodes); 2959 } 2960 2961 case ARMISD::VLD1_UPD: { 2962 static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed, 2963 ARM::VLD1d16wb_fixed, 2964 ARM::VLD1d32wb_fixed, 2965 ARM::VLD1d64wb_fixed }; 2966 static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed, 2967 ARM::VLD1q16wb_fixed, 2968 ARM::VLD1q32wb_fixed, 2969 ARM::VLD1q64wb_fixed }; 2970 return SelectVLD(N, true, 1, DOpcodes, QOpcodes, 0); 2971 } 2972 2973 case ARMISD::VLD2_UPD: { 2974 static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed, 2975 ARM::VLD2d16wb_fixed, 2976 ARM::VLD2d32wb_fixed, 2977 ARM::VLD1q64wb_fixed}; 2978 static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed, 2979 ARM::VLD2q16PseudoWB_fixed, 2980 ARM::VLD2q32PseudoWB_fixed }; 2981 return SelectVLD(N, true, 2, DOpcodes, QOpcodes, 0); 2982 } 2983 2984 case ARMISD::VLD3_UPD: { 2985 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD, 2986 ARM::VLD3d16Pseudo_UPD, 2987 ARM::VLD3d32Pseudo_UPD, 2988 ARM::VLD1q64wb_fixed}; 2989 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD, 2990 ARM::VLD3q16Pseudo_UPD, 2991 ARM::VLD3q32Pseudo_UPD }; 2992 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD, 2993 ARM::VLD3q16oddPseudo_UPD, 2994 ARM::VLD3q32oddPseudo_UPD }; 2995 return SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); 2996 } 2997 2998 case ARMISD::VLD4_UPD: { 2999 static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD, 3000 ARM::VLD4d16Pseudo_UPD, 3001 ARM::VLD4d32Pseudo_UPD, 3002 ARM::VLD1q64wb_fixed}; 3003 static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD, 3004 ARM::VLD4q16Pseudo_UPD, 3005 ARM::VLD4q32Pseudo_UPD }; 3006 static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD, 3007 ARM::VLD4q16oddPseudo_UPD, 3008 ARM::VLD4q32oddPseudo_UPD }; 3009 return SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); 3010 } 3011 3012 case ARMISD::VLD2LN_UPD: { 3013 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD, 3014 ARM::VLD2LNd16Pseudo_UPD, 3015 ARM::VLD2LNd32Pseudo_UPD }; 3016 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD, 3017 ARM::VLD2LNq32Pseudo_UPD }; 3018 return SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes); 3019 } 3020 3021 case ARMISD::VLD3LN_UPD: { 3022 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD, 3023 ARM::VLD3LNd16Pseudo_UPD, 3024 ARM::VLD3LNd32Pseudo_UPD }; 3025 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD, 3026 ARM::VLD3LNq32Pseudo_UPD }; 3027 return SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes); 3028 } 3029 3030 case ARMISD::VLD4LN_UPD: { 3031 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD, 3032 ARM::VLD4LNd16Pseudo_UPD, 3033 ARM::VLD4LNd32Pseudo_UPD }; 3034 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD, 3035 ARM::VLD4LNq32Pseudo_UPD }; 3036 return SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes); 3037 } 3038 3039 case ARMISD::VST1_UPD: { 3040 static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed, 3041 ARM::VST1d16wb_fixed, 3042 ARM::VST1d32wb_fixed, 3043 ARM::VST1d64wb_fixed }; 3044 static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed, 3045 ARM::VST1q16wb_fixed, 3046 ARM::VST1q32wb_fixed, 3047 ARM::VST1q64wb_fixed }; 3048 return SelectVST(N, true, 1, DOpcodes, QOpcodes, 0); 3049 } 3050 3051 case ARMISD::VST2_UPD: { 3052 static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed, 3053 ARM::VST2d16wb_fixed, 3054 ARM::VST2d32wb_fixed, 3055 ARM::VST1q64wb_fixed}; 3056 static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed, 3057 ARM::VST2q16PseudoWB_fixed, 3058 ARM::VST2q32PseudoWB_fixed }; 3059 return SelectVST(N, true, 2, DOpcodes, QOpcodes, 0); 3060 } 3061 3062 case ARMISD::VST3_UPD: { 3063 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD, 3064 ARM::VST3d16Pseudo_UPD, 3065 ARM::VST3d32Pseudo_UPD, 3066 ARM::VST1d64TPseudoWB_fixed}; 3067 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD, 3068 ARM::VST3q16Pseudo_UPD, 3069 ARM::VST3q32Pseudo_UPD }; 3070 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD, 3071 ARM::VST3q16oddPseudo_UPD, 3072 ARM::VST3q32oddPseudo_UPD }; 3073 return SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); 3074 } 3075 3076 case ARMISD::VST4_UPD: { 3077 static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD, 3078 ARM::VST4d16Pseudo_UPD, 3079 ARM::VST4d32Pseudo_UPD, 3080 ARM::VST1d64QPseudoWB_fixed}; 3081 static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD, 3082 ARM::VST4q16Pseudo_UPD, 3083 ARM::VST4q32Pseudo_UPD }; 3084 static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD, 3085 ARM::VST4q16oddPseudo_UPD, 3086 ARM::VST4q32oddPseudo_UPD }; 3087 return SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); 3088 } 3089 3090 case ARMISD::VST2LN_UPD: { 3091 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD, 3092 ARM::VST2LNd16Pseudo_UPD, 3093 ARM::VST2LNd32Pseudo_UPD }; 3094 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD, 3095 ARM::VST2LNq32Pseudo_UPD }; 3096 return SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes); 3097 } 3098 3099 case ARMISD::VST3LN_UPD: { 3100 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD, 3101 ARM::VST3LNd16Pseudo_UPD, 3102 ARM::VST3LNd32Pseudo_UPD }; 3103 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD, 3104 ARM::VST3LNq32Pseudo_UPD }; 3105 return SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes); 3106 } 3107 3108 case ARMISD::VST4LN_UPD: { 3109 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD, 3110 ARM::VST4LNd16Pseudo_UPD, 3111 ARM::VST4LNd32Pseudo_UPD }; 3112 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD, 3113 ARM::VST4LNq32Pseudo_UPD }; 3114 return SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes); 3115 } 3116 3117 case ISD::INTRINSIC_VOID: 3118 case ISD::INTRINSIC_W_CHAIN: { 3119 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 3120 switch (IntNo) { 3121 default: 3122 break; 3123 3124 case Intrinsic::arm_ldrexd: { 3125 SDValue MemAddr = N->getOperand(2); 3126 DebugLoc dl = N->getDebugLoc(); 3127 SDValue Chain = N->getOperand(0); 3128 3129 bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2(); 3130 unsigned NewOpc = isThumb ? ARM::t2LDREXD :ARM::LDREXD; 3131 3132 // arm_ldrexd returns a i64 value in {i32, i32} 3133 std::vector<EVT> ResTys; 3134 if (isThumb) { 3135 ResTys.push_back(MVT::i32); 3136 ResTys.push_back(MVT::i32); 3137 } else 3138 ResTys.push_back(MVT::Untyped); 3139 ResTys.push_back(MVT::Other); 3140 3141 // Place arguments in the right order. 3142 SmallVector<SDValue, 7> Ops; 3143 Ops.push_back(MemAddr); 3144 Ops.push_back(getAL(CurDAG)); 3145 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); 3146 Ops.push_back(Chain); 3147 SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops); 3148 // Transfer memoperands. 3149 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 3150 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 3151 cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1); 3152 3153 // Remap uses. 3154 SDValue OutChain = isThumb ? SDValue(Ld, 2) : SDValue(Ld, 1); 3155 if (!SDValue(N, 0).use_empty()) { 3156 SDValue Result; 3157 if (isThumb) 3158 Result = SDValue(Ld, 0); 3159 else { 3160 SDValue SubRegIdx = CurDAG->getTargetConstant(ARM::gsub_0, MVT::i32); 3161 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 3162 dl, MVT::i32, SDValue(Ld, 0), SubRegIdx); 3163 Result = SDValue(ResNode,0); 3164 } 3165 ReplaceUses(SDValue(N, 0), Result); 3166 } 3167 if (!SDValue(N, 1).use_empty()) { 3168 SDValue Result; 3169 if (isThumb) 3170 Result = SDValue(Ld, 1); 3171 else { 3172 SDValue SubRegIdx = CurDAG->getTargetConstant(ARM::gsub_1, MVT::i32); 3173 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 3174 dl, MVT::i32, SDValue(Ld, 0), SubRegIdx); 3175 Result = SDValue(ResNode,0); 3176 } 3177 ReplaceUses(SDValue(N, 1), Result); 3178 } 3179 ReplaceUses(SDValue(N, 2), OutChain); 3180 return NULL; 3181 } 3182 3183 case Intrinsic::arm_strexd: { 3184 DebugLoc dl = N->getDebugLoc(); 3185 SDValue Chain = N->getOperand(0); 3186 SDValue Val0 = N->getOperand(2); 3187 SDValue Val1 = N->getOperand(3); 3188 SDValue MemAddr = N->getOperand(4); 3189 3190 // Store exclusive double return a i32 value which is the return status 3191 // of the issued store. 3192 EVT ResTys[] = { MVT::i32, MVT::Other }; 3193 3194 bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2(); 3195 // Place arguments in the right order. 3196 SmallVector<SDValue, 7> Ops; 3197 if (isThumb) { 3198 Ops.push_back(Val0); 3199 Ops.push_back(Val1); 3200 } else 3201 // arm_strexd uses GPRPair. 3202 Ops.push_back(SDValue(createGPRPairNode(MVT::Untyped, Val0, Val1), 0)); 3203 Ops.push_back(MemAddr); 3204 Ops.push_back(getAL(CurDAG)); 3205 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); 3206 Ops.push_back(Chain); 3207 3208 unsigned NewOpc = isThumb ? ARM::t2STREXD : ARM::STREXD; 3209 3210 SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops); 3211 // Transfer memoperands. 3212 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 3213 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 3214 cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1); 3215 3216 return St; 3217 } 3218 3219 case Intrinsic::arm_neon_vld1: { 3220 static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16, 3221 ARM::VLD1d32, ARM::VLD1d64 }; 3222 static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16, 3223 ARM::VLD1q32, ARM::VLD1q64}; 3224 return SelectVLD(N, false, 1, DOpcodes, QOpcodes, 0); 3225 } 3226 3227 case Intrinsic::arm_neon_vld2: { 3228 static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16, 3229 ARM::VLD2d32, ARM::VLD1q64 }; 3230 static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo, 3231 ARM::VLD2q32Pseudo }; 3232 return SelectVLD(N, false, 2, DOpcodes, QOpcodes, 0); 3233 } 3234 3235 case Intrinsic::arm_neon_vld3: { 3236 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo, 3237 ARM::VLD3d16Pseudo, 3238 ARM::VLD3d32Pseudo, 3239 ARM::VLD1d64TPseudo }; 3240 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD, 3241 ARM::VLD3q16Pseudo_UPD, 3242 ARM::VLD3q32Pseudo_UPD }; 3243 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo, 3244 ARM::VLD3q16oddPseudo, 3245 ARM::VLD3q32oddPseudo }; 3246 return SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1); 3247 } 3248 3249 case Intrinsic::arm_neon_vld4: { 3250 static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo, 3251 ARM::VLD4d16Pseudo, 3252 ARM::VLD4d32Pseudo, 3253 ARM::VLD1d64QPseudo }; 3254 static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD, 3255 ARM::VLD4q16Pseudo_UPD, 3256 ARM::VLD4q32Pseudo_UPD }; 3257 static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo, 3258 ARM::VLD4q16oddPseudo, 3259 ARM::VLD4q32oddPseudo }; 3260 return SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1); 3261 } 3262 3263 case Intrinsic::arm_neon_vld2lane: { 3264 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo, 3265 ARM::VLD2LNd16Pseudo, 3266 ARM::VLD2LNd32Pseudo }; 3267 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo, 3268 ARM::VLD2LNq32Pseudo }; 3269 return SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes); 3270 } 3271 3272 case Intrinsic::arm_neon_vld3lane: { 3273 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo, 3274 ARM::VLD3LNd16Pseudo, 3275 ARM::VLD3LNd32Pseudo }; 3276 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo, 3277 ARM::VLD3LNq32Pseudo }; 3278 return SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes); 3279 } 3280 3281 case Intrinsic::arm_neon_vld4lane: { 3282 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo, 3283 ARM::VLD4LNd16Pseudo, 3284 ARM::VLD4LNd32Pseudo }; 3285 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo, 3286 ARM::VLD4LNq32Pseudo }; 3287 return SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes); 3288 } 3289 3290 case Intrinsic::arm_neon_vst1: { 3291 static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16, 3292 ARM::VST1d32, ARM::VST1d64 }; 3293 static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16, 3294 ARM::VST1q32, ARM::VST1q64 }; 3295 return SelectVST(N, false, 1, DOpcodes, QOpcodes, 0); 3296 } 3297 3298 case Intrinsic::arm_neon_vst2: { 3299 static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16, 3300 ARM::VST2d32, ARM::VST1q64 }; 3301 static uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo, 3302 ARM::VST2q32Pseudo }; 3303 return SelectVST(N, false, 2, DOpcodes, QOpcodes, 0); 3304 } 3305 3306 case Intrinsic::arm_neon_vst3: { 3307 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo, 3308 ARM::VST3d16Pseudo, 3309 ARM::VST3d32Pseudo, 3310 ARM::VST1d64TPseudo }; 3311 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD, 3312 ARM::VST3q16Pseudo_UPD, 3313 ARM::VST3q32Pseudo_UPD }; 3314 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo, 3315 ARM::VST3q16oddPseudo, 3316 ARM::VST3q32oddPseudo }; 3317 return SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1); 3318 } 3319 3320 case Intrinsic::arm_neon_vst4: { 3321 static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo, 3322 ARM::VST4d16Pseudo, 3323 ARM::VST4d32Pseudo, 3324 ARM::VST1d64QPseudo }; 3325 static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD, 3326 ARM::VST4q16Pseudo_UPD, 3327 ARM::VST4q32Pseudo_UPD }; 3328 static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo, 3329 ARM::VST4q16oddPseudo, 3330 ARM::VST4q32oddPseudo }; 3331 return SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1); 3332 } 3333 3334 case Intrinsic::arm_neon_vst2lane: { 3335 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo, 3336 ARM::VST2LNd16Pseudo, 3337 ARM::VST2LNd32Pseudo }; 3338 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo, 3339 ARM::VST2LNq32Pseudo }; 3340 return SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes); 3341 } 3342 3343 case Intrinsic::arm_neon_vst3lane: { 3344 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo, 3345 ARM::VST3LNd16Pseudo, 3346 ARM::VST3LNd32Pseudo }; 3347 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo, 3348 ARM::VST3LNq32Pseudo }; 3349 return SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes); 3350 } 3351 3352 case Intrinsic::arm_neon_vst4lane: { 3353 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo, 3354 ARM::VST4LNd16Pseudo, 3355 ARM::VST4LNd32Pseudo }; 3356 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo, 3357 ARM::VST4LNq32Pseudo }; 3358 return SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes); 3359 } 3360 } 3361 break; 3362 } 3363 3364 case ISD::INTRINSIC_WO_CHAIN: { 3365 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 3366 switch (IntNo) { 3367 default: 3368 break; 3369 3370 case Intrinsic::arm_neon_vtbl2: 3371 return SelectVTBL(N, false, 2, ARM::VTBL2); 3372 case Intrinsic::arm_neon_vtbl3: 3373 return SelectVTBL(N, false, 3, ARM::VTBL3Pseudo); 3374 case Intrinsic::arm_neon_vtbl4: 3375 return SelectVTBL(N, false, 4, ARM::VTBL4Pseudo); 3376 3377 case Intrinsic::arm_neon_vtbx2: 3378 return SelectVTBL(N, true, 2, ARM::VTBX2); 3379 case Intrinsic::arm_neon_vtbx3: 3380 return SelectVTBL(N, true, 3, ARM::VTBX3Pseudo); 3381 case Intrinsic::arm_neon_vtbx4: 3382 return SelectVTBL(N, true, 4, ARM::VTBX4Pseudo); 3383 } 3384 break; 3385 } 3386 3387 case ARMISD::VTBL1: { 3388 DebugLoc dl = N->getDebugLoc(); 3389 EVT VT = N->getValueType(0); 3390 SmallVector<SDValue, 6> Ops; 3391 3392 Ops.push_back(N->getOperand(0)); 3393 Ops.push_back(N->getOperand(1)); 3394 Ops.push_back(getAL(CurDAG)); // Predicate 3395 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register 3396 return CurDAG->getMachineNode(ARM::VTBL1, dl, VT, Ops); 3397 } 3398 case ARMISD::VTBL2: { 3399 DebugLoc dl = N->getDebugLoc(); 3400 EVT VT = N->getValueType(0); 3401 3402 // Form a REG_SEQUENCE to force register allocation. 3403 SDValue V0 = N->getOperand(0); 3404 SDValue V1 = N->getOperand(1); 3405 SDValue RegSeq = SDValue(createDRegPairNode(MVT::v16i8, V0, V1), 0); 3406 3407 SmallVector<SDValue, 6> Ops; 3408 Ops.push_back(RegSeq); 3409 Ops.push_back(N->getOperand(2)); 3410 Ops.push_back(getAL(CurDAG)); // Predicate 3411 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register 3412 return CurDAG->getMachineNode(ARM::VTBL2, dl, VT, Ops); 3413 } 3414 3415 case ISD::CONCAT_VECTORS: 3416 return SelectConcatVector(N); 3417 3418 case ARMISD::ATOMOR64_DAG: 3419 return SelectAtomic64(N, ARM::ATOMOR6432); 3420 case ARMISD::ATOMXOR64_DAG: 3421 return SelectAtomic64(N, ARM::ATOMXOR6432); 3422 case ARMISD::ATOMADD64_DAG: 3423 return SelectAtomic64(N, ARM::ATOMADD6432); 3424 case ARMISD::ATOMSUB64_DAG: 3425 return SelectAtomic64(N, ARM::ATOMSUB6432); 3426 case ARMISD::ATOMNAND64_DAG: 3427 return SelectAtomic64(N, ARM::ATOMNAND6432); 3428 case ARMISD::ATOMAND64_DAG: 3429 return SelectAtomic64(N, ARM::ATOMAND6432); 3430 case ARMISD::ATOMSWAP64_DAG: 3431 return SelectAtomic64(N, ARM::ATOMSWAP6432); 3432 case ARMISD::ATOMCMPXCHG64_DAG: 3433 return SelectAtomic64(N, ARM::ATOMCMPXCHG6432); 3434 3435 case ARMISD::ATOMMIN64_DAG: 3436 return SelectAtomic64(N, ARM::ATOMMIN6432); 3437 case ARMISD::ATOMUMIN64_DAG: 3438 return SelectAtomic64(N, ARM::ATOMUMIN6432); 3439 case ARMISD::ATOMMAX64_DAG: 3440 return SelectAtomic64(N, ARM::ATOMMAX6432); 3441 case ARMISD::ATOMUMAX64_DAG: 3442 return SelectAtomic64(N, ARM::ATOMUMAX6432); 3443 } 3444 3445 return SelectCode(N); 3446} 3447 3448SDNode *ARMDAGToDAGISel::SelectInlineAsm(SDNode *N){ 3449 std::vector<SDValue> AsmNodeOperands; 3450 unsigned Flag, Kind; 3451 bool Changed = false; 3452 unsigned NumOps = N->getNumOperands(); 3453 3454 ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>( 3455 N->getOperand(InlineAsm::Op_AsmString)); 3456 StringRef AsmString = StringRef(S->getSymbol()); 3457 3458 // Normally, i64 data is bounded to two arbitrary GRPs for "%r" constraint. 3459 // However, some instrstions (e.g. ldrexd/strexd in ARM mode) require 3460 // (even/even+1) GPRs and use %n and %Hn to refer to the individual regs 3461 // respectively. Since there is no constraint to explicitly specify a 3462 // reg pair, we search %H operand inside the asm string. If it is found, the 3463 // transformation below enforces a GPRPair reg class for "%r" for 64-bit data. 3464 if (AsmString.find(":H}") == StringRef::npos) 3465 return NULL; 3466 3467 DebugLoc dl = N->getDebugLoc(); 3468 SDValue Glue = N->getOperand(NumOps-1); 3469 3470 // Glue node will be appended late. 3471 for(unsigned i = 0; i < NumOps -1; ++i) { 3472 SDValue op = N->getOperand(i); 3473 AsmNodeOperands.push_back(op); 3474 3475 if (i < InlineAsm::Op_FirstOperand) 3476 continue; 3477 3478 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(i))) { 3479 Flag = C->getZExtValue(); 3480 Kind = InlineAsm::getKind(Flag); 3481 } 3482 else 3483 continue; 3484 3485 if (Kind != InlineAsm::Kind_RegUse && Kind != InlineAsm::Kind_RegDef 3486 && Kind != InlineAsm::Kind_RegDefEarlyClobber) 3487 continue; 3488 3489 unsigned RegNum = InlineAsm::getNumOperandRegisters(Flag); 3490 unsigned RC; 3491 bool HasRC = InlineAsm::hasRegClassConstraint(Flag, RC); 3492 if (!HasRC || RC != ARM::GPRRegClassID || RegNum != 2) 3493 continue; 3494 3495 assert((i+2 < NumOps-1) && "Invalid number of operands in inline asm"); 3496 SDValue V0 = N->getOperand(i+1); 3497 SDValue V1 = N->getOperand(i+2); 3498 unsigned Reg0 = cast<RegisterSDNode>(V0)->getReg(); 3499 unsigned Reg1 = cast<RegisterSDNode>(V1)->getReg(); 3500 SDValue PairedReg; 3501 MachineRegisterInfo &MRI = MF->getRegInfo(); 3502 3503 if (Kind == InlineAsm::Kind_RegDef || 3504 Kind == InlineAsm::Kind_RegDefEarlyClobber) { 3505 // Replace the two GPRs with 1 GPRPair and copy values from GPRPair to 3506 // the original GPRs. 3507 3508 unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass); 3509 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped); 3510 SDValue Chain = SDValue(N,0); 3511 3512 SDNode *GU = N->getGluedUser(); 3513 SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped, 3514 Chain.getValue(1)); 3515 3516 // Extract values from a GPRPair reg and copy to the original GPR reg. 3517 SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32, 3518 RegCopy); 3519 SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32, 3520 RegCopy); 3521 SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0, 3522 RegCopy.getValue(1)); 3523 SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1)); 3524 3525 // Update the original glue user. 3526 std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1); 3527 Ops.push_back(T1.getValue(1)); 3528 CurDAG->UpdateNodeOperands(GU, &Ops[0], Ops.size()); 3529 GU = T1.getNode(); 3530 } 3531 else { 3532 // For Kind == InlineAsm::Kind_RegUse, we first copy two GPRs into a 3533 // GPRPair and then pass the GPRPair to the inline asm. 3534 SDValue Chain = AsmNodeOperands[InlineAsm::Op_InputChain]; 3535 3536 // As REG_SEQ doesn't take RegisterSDNode, we copy them first. 3537 SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32, 3538 Chain.getValue(1)); 3539 SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32, 3540 T0.getValue(1)); 3541 SDValue Pair = SDValue(createGPRPairNode(MVT::Untyped, T0, T1), 0); 3542 3543 // Copy REG_SEQ into a GPRPair-typed VR and replace the original two 3544 // i32 VRs of inline asm with it. 3545 unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass); 3546 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped); 3547 Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair, T1.getValue(1)); 3548 3549 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain; 3550 Glue = Chain.getValue(1); 3551 } 3552 3553 Changed = true; 3554 3555 if(PairedReg.getNode()) { 3556 Flag = InlineAsm::getFlagWord(Kind, 1 /* RegNum*/); 3557 Flag = InlineAsm::getFlagWordForRegClass(Flag, ARM::GPRPairRegClassID); 3558 // Replace the current flag. 3559 AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant( 3560 Flag, MVT::i32); 3561 // Add the new register node and skip the original two GPRs. 3562 AsmNodeOperands.push_back(PairedReg); 3563 // Skip the next two GPRs. 3564 i += 2; 3565 } 3566 } 3567 3568 AsmNodeOperands.push_back(Glue); 3569 if (!Changed) 3570 return NULL; 3571 3572 SDValue New = CurDAG->getNode(ISD::INLINEASM, N->getDebugLoc(), 3573 CurDAG->getVTList(MVT::Other, MVT::Glue), &AsmNodeOperands[0], 3574 AsmNodeOperands.size()); 3575 New->setNodeId(-1); 3576 return New.getNode(); 3577} 3578 3579 3580bool ARMDAGToDAGISel:: 3581SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode, 3582 std::vector<SDValue> &OutOps) { 3583 assert(ConstraintCode == 'm' && "unexpected asm memory constraint"); 3584 // Require the address to be in a register. That is safe for all ARM 3585 // variants and it is hard to do anything much smarter without knowing 3586 // how the operand is used. 3587 OutOps.push_back(Op); 3588 return false; 3589} 3590 3591/// createARMISelDag - This pass converts a legalized DAG into a 3592/// ARM-specific DAG, ready for instruction scheduling. 3593/// 3594FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM, 3595 CodeGenOpt::Level OptLevel) { 3596 return new ARMDAGToDAGISel(TM, OptLevel); 3597} 3598