NVPTXISelDAGToDAG.cpp revision 249423
1239310Sdim//===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===// 2239310Sdim// 3239310Sdim// The LLVM Compiler Infrastructure 4239310Sdim// 5239310Sdim// This file is distributed under the University of Illinois Open Source 6239310Sdim// License. See LICENSE.TXT for details. 7239310Sdim// 8239310Sdim//===----------------------------------------------------------------------===// 9239310Sdim// 10239310Sdim// This file defines an instruction selector for the NVPTX target. 11239310Sdim// 12239310Sdim//===----------------------------------------------------------------------===// 13239310Sdim 14239310Sdim#include "NVPTXISelDAGToDAG.h" 15249423Sdim#include "llvm/IR/GlobalValue.h" 16249423Sdim#include "llvm/IR/Instructions.h" 17249423Sdim#include "llvm/Support/CommandLine.h" 18239310Sdim#include "llvm/Support/Debug.h" 19239310Sdim#include "llvm/Support/ErrorHandling.h" 20249423Sdim#include "llvm/Support/raw_ostream.h" 21239310Sdim#include "llvm/Target/TargetIntrinsicInfo.h" 22239310Sdim 23239310Sdim#undef DEBUG_TYPE 24239310Sdim#define DEBUG_TYPE "nvptx-isel" 25239310Sdim 26239310Sdimusing namespace llvm; 27239310Sdim 28249423Sdimstatic cl::opt<bool> UseFMADInstruction( 29249423Sdim "nvptx-mad-enable", cl::ZeroOrMore, 30249423Sdim cl::desc("NVPTX Specific: Enable generating FMAD instructions"), 31249423Sdim cl::init(false)); 32239310Sdim 33239310Sdimstatic cl::opt<int> 34249423SdimFMAContractLevel("nvptx-fma-level", cl::ZeroOrMore, 35239310Sdim cl::desc("NVPTX Specific: FMA contraction (0: don't do it" 36249423Sdim " 1: do it 2: do it aggressively"), 37249423Sdim cl::init(2)); 38239310Sdim 39249423Sdimstatic cl::opt<int> UsePrecDivF32( 40249423Sdim "nvptx-prec-divf32", cl::ZeroOrMore, 41249423Sdim cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use" 42249423Sdim " IEEE Compliant F32 div.rnd if avaiable."), 43249423Sdim cl::init(2)); 44239310Sdim 45239310Sdim/// createNVPTXISelDag - This pass converts a legalized DAG into a 46239310Sdim/// NVPTX-specific DAG, ready for instruction scheduling. 47239310SdimFunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM, 48239310Sdim llvm::CodeGenOpt::Level OptLevel) { 49239310Sdim return new NVPTXDAGToDAGISel(TM, OptLevel); 50239310Sdim} 51239310Sdim 52239310SdimNVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm, 53239310Sdim CodeGenOpt::Level OptLevel) 54249423Sdim : SelectionDAGISel(tm, OptLevel), 55249423Sdim Subtarget(tm.getSubtarget<NVPTXSubtarget>()) { 56239310Sdim // Always do fma.f32 fpcontract if the target supports the instruction. 57239310Sdim // Always do fma.f64 fpcontract if the target supports the instruction. 58239310Sdim // Do mad.f32 is nvptx-mad-enable is specified and the target does not 59239310Sdim // support fma.f32. 60239310Sdim 61239310Sdim doFMADF32 = (OptLevel > 0) && UseFMADInstruction && !Subtarget.hasFMAF32(); 62249423Sdim doFMAF32 = (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel >= 1); 63249423Sdim doFMAF64 = (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel >= 1); 64249423Sdim doFMAF32AGG = 65249423Sdim (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel == 2); 66249423Sdim doFMAF64AGG = 67249423Sdim (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel == 2); 68239310Sdim 69239310Sdim allowFMA = (FMAContractLevel >= 1) || UseFMADInstruction; 70239310Sdim 71239310Sdim UseF32FTZ = false; 72239310Sdim 73239310Sdim doMulWide = (OptLevel > 0); 74239310Sdim 75239310Sdim // Decide how to translate f32 div 76239310Sdim do_DIVF32_PREC = UsePrecDivF32; 77239310Sdim // sm less than sm_20 does not support div.rnd. Use div.full. 78239310Sdim if (do_DIVF32_PREC == 2 && !Subtarget.reqPTX20()) 79239310Sdim do_DIVF32_PREC = 1; 80239310Sdim 81239310Sdim} 82239310Sdim 83239310Sdim/// Select - Select instructions not customized! Used for 84239310Sdim/// expanded, promoted and normal instructions. 85249423SdimSDNode *NVPTXDAGToDAGISel::Select(SDNode *N) { 86239310Sdim 87239310Sdim if (N->isMachineOpcode()) 88249423Sdim return NULL; // Already selected. 89239310Sdim 90239310Sdim SDNode *ResNode = NULL; 91239310Sdim switch (N->getOpcode()) { 92239310Sdim case ISD::LOAD: 93239310Sdim ResNode = SelectLoad(N); 94239310Sdim break; 95239310Sdim case ISD::STORE: 96239310Sdim ResNode = SelectStore(N); 97239310Sdim break; 98249423Sdim case NVPTXISD::LoadV2: 99249423Sdim case NVPTXISD::LoadV4: 100249423Sdim ResNode = SelectLoadVector(N); 101249423Sdim break; 102249423Sdim case NVPTXISD::LDGV2: 103249423Sdim case NVPTXISD::LDGV4: 104249423Sdim case NVPTXISD::LDUV2: 105249423Sdim case NVPTXISD::LDUV4: 106249423Sdim ResNode = SelectLDGLDUVector(N); 107249423Sdim break; 108249423Sdim case NVPTXISD::StoreV2: 109249423Sdim case NVPTXISD::StoreV4: 110249423Sdim ResNode = SelectStoreVector(N); 111249423Sdim break; 112249423Sdim default: 113249423Sdim break; 114239310Sdim } 115239310Sdim if (ResNode) 116239310Sdim return ResNode; 117239310Sdim return SelectCode(N); 118239310Sdim} 119239310Sdim 120249423Sdimstatic unsigned int getCodeAddrSpace(MemSDNode *N, 121249423Sdim const NVPTXSubtarget &Subtarget) { 122239310Sdim const Value *Src = N->getSrcValue(); 123239310Sdim if (!Src) 124239310Sdim return NVPTX::PTXLdStInstCode::LOCAL; 125239310Sdim 126239310Sdim if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) { 127239310Sdim switch (PT->getAddressSpace()) { 128249423Sdim case llvm::ADDRESS_SPACE_LOCAL: 129249423Sdim return NVPTX::PTXLdStInstCode::LOCAL; 130249423Sdim case llvm::ADDRESS_SPACE_GLOBAL: 131249423Sdim return NVPTX::PTXLdStInstCode::GLOBAL; 132249423Sdim case llvm::ADDRESS_SPACE_SHARED: 133249423Sdim return NVPTX::PTXLdStInstCode::SHARED; 134239310Sdim case llvm::ADDRESS_SPACE_CONST_NOT_GEN: 135239310Sdim return NVPTX::PTXLdStInstCode::CONSTANT; 136249423Sdim case llvm::ADDRESS_SPACE_GENERIC: 137249423Sdim return NVPTX::PTXLdStInstCode::GENERIC; 138249423Sdim case llvm::ADDRESS_SPACE_PARAM: 139249423Sdim return NVPTX::PTXLdStInstCode::PARAM; 140239310Sdim case llvm::ADDRESS_SPACE_CONST: 141239310Sdim // If the arch supports generic address space, translate it to GLOBAL 142239310Sdim // for correctness. 143239310Sdim // If the arch does not support generic address space, then the arch 144239310Sdim // does not really support ADDRESS_SPACE_CONST, translate it to 145239310Sdim // to CONSTANT for better performance. 146239310Sdim if (Subtarget.hasGenericLdSt()) 147239310Sdim return NVPTX::PTXLdStInstCode::GLOBAL; 148239310Sdim else 149239310Sdim return NVPTX::PTXLdStInstCode::CONSTANT; 150249423Sdim default: 151249423Sdim break; 152239310Sdim } 153239310Sdim } 154239310Sdim return NVPTX::PTXLdStInstCode::LOCAL; 155239310Sdim} 156239310Sdim 157249423SdimSDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) { 158239310Sdim DebugLoc dl = N->getDebugLoc(); 159239310Sdim LoadSDNode *LD = cast<LoadSDNode>(N); 160239310Sdim EVT LoadedVT = LD->getMemoryVT(); 161249423Sdim SDNode *NVPTXLD = NULL; 162239310Sdim 163239310Sdim // do not support pre/post inc/dec 164239310Sdim if (LD->isIndexed()) 165239310Sdim return NULL; 166239310Sdim 167239310Sdim if (!LoadedVT.isSimple()) 168239310Sdim return NULL; 169239310Sdim 170239310Sdim // Address Space Setting 171239310Sdim unsigned int codeAddrSpace = getCodeAddrSpace(LD, Subtarget); 172239310Sdim 173239310Sdim // Volatile Setting 174239310Sdim // - .volatile is only availalble for .global and .shared 175239310Sdim bool isVolatile = LD->isVolatile(); 176239310Sdim if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 177239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 178239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 179239310Sdim isVolatile = false; 180239310Sdim 181239310Sdim // Vector Setting 182239310Sdim MVT SimpleVT = LoadedVT.getSimpleVT(); 183239310Sdim unsigned vecType = NVPTX::PTXLdStInstCode::Scalar; 184239310Sdim if (SimpleVT.isVector()) { 185239310Sdim unsigned num = SimpleVT.getVectorNumElements(); 186239310Sdim if (num == 2) 187239310Sdim vecType = NVPTX::PTXLdStInstCode::V2; 188239310Sdim else if (num == 4) 189239310Sdim vecType = NVPTX::PTXLdStInstCode::V4; 190239310Sdim else 191239310Sdim return NULL; 192239310Sdim } 193239310Sdim 194239310Sdim // Type Setting: fromType + fromTypeWidth 195239310Sdim // 196239310Sdim // Sign : ISD::SEXTLOAD 197239310Sdim // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the 198239310Sdim // type is integer 199239310Sdim // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float 200239310Sdim MVT ScalarVT = SimpleVT.getScalarType(); 201249423Sdim unsigned fromTypeWidth = ScalarVT.getSizeInBits(); 202239310Sdim unsigned int fromType; 203239310Sdim if ((LD->getExtensionType() == ISD::SEXTLOAD)) 204239310Sdim fromType = NVPTX::PTXLdStInstCode::Signed; 205239310Sdim else if (ScalarVT.isFloatingPoint()) 206239310Sdim fromType = NVPTX::PTXLdStInstCode::Float; 207239310Sdim else 208239310Sdim fromType = NVPTX::PTXLdStInstCode::Unsigned; 209239310Sdim 210239310Sdim // Create the machine instruction DAG 211239310Sdim SDValue Chain = N->getOperand(0); 212239310Sdim SDValue N1 = N->getOperand(1); 213239310Sdim SDValue Addr; 214239310Sdim SDValue Offset, Base; 215239310Sdim unsigned Opcode; 216239310Sdim MVT::SimpleValueType TargetVT = LD->getValueType(0).getSimpleVT().SimpleTy; 217239310Sdim 218239310Sdim if (SelectDirectAddr(N1, Addr)) { 219239310Sdim switch (TargetVT) { 220249423Sdim case MVT::i8: 221249423Sdim Opcode = NVPTX::LD_i8_avar; 222249423Sdim break; 223249423Sdim case MVT::i16: 224249423Sdim Opcode = NVPTX::LD_i16_avar; 225249423Sdim break; 226249423Sdim case MVT::i32: 227249423Sdim Opcode = NVPTX::LD_i32_avar; 228249423Sdim break; 229249423Sdim case MVT::i64: 230249423Sdim Opcode = NVPTX::LD_i64_avar; 231249423Sdim break; 232249423Sdim case MVT::f32: 233249423Sdim Opcode = NVPTX::LD_f32_avar; 234249423Sdim break; 235249423Sdim case MVT::f64: 236249423Sdim Opcode = NVPTX::LD_f64_avar; 237249423Sdim break; 238249423Sdim default: 239249423Sdim return NULL; 240239310Sdim } 241249423Sdim SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 242249423Sdim getI32Imm(vecType), getI32Imm(fromType), 243249423Sdim getI32Imm(fromTypeWidth), Addr, Chain }; 244249423Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops, 7); 245249423Sdim } else if (Subtarget.is64Bit() 246249423Sdim ? SelectADDRsi64(N1.getNode(), N1, Base, Offset) 247249423Sdim : SelectADDRsi(N1.getNode(), N1, Base, Offset)) { 248239310Sdim switch (TargetVT) { 249249423Sdim case MVT::i8: 250249423Sdim Opcode = NVPTX::LD_i8_asi; 251249423Sdim break; 252249423Sdim case MVT::i16: 253249423Sdim Opcode = NVPTX::LD_i16_asi; 254249423Sdim break; 255249423Sdim case MVT::i32: 256249423Sdim Opcode = NVPTX::LD_i32_asi; 257249423Sdim break; 258249423Sdim case MVT::i64: 259249423Sdim Opcode = NVPTX::LD_i64_asi; 260249423Sdim break; 261249423Sdim case MVT::f32: 262249423Sdim Opcode = NVPTX::LD_f32_asi; 263249423Sdim break; 264249423Sdim case MVT::f64: 265249423Sdim Opcode = NVPTX::LD_f64_asi; 266249423Sdim break; 267249423Sdim default: 268249423Sdim return NULL; 269239310Sdim } 270249423Sdim SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 271249423Sdim getI32Imm(vecType), getI32Imm(fromType), 272249423Sdim getI32Imm(fromTypeWidth), Base, Offset, Chain }; 273249423Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops, 8); 274249423Sdim } else if (Subtarget.is64Bit() 275249423Sdim ? SelectADDRri64(N1.getNode(), N1, Base, Offset) 276249423Sdim : SelectADDRri(N1.getNode(), N1, Base, Offset)) { 277249423Sdim if (Subtarget.is64Bit()) { 278249423Sdim switch (TargetVT) { 279249423Sdim case MVT::i8: 280249423Sdim Opcode = NVPTX::LD_i8_ari_64; 281249423Sdim break; 282249423Sdim case MVT::i16: 283249423Sdim Opcode = NVPTX::LD_i16_ari_64; 284249423Sdim break; 285249423Sdim case MVT::i32: 286249423Sdim Opcode = NVPTX::LD_i32_ari_64; 287249423Sdim break; 288249423Sdim case MVT::i64: 289249423Sdim Opcode = NVPTX::LD_i64_ari_64; 290249423Sdim break; 291249423Sdim case MVT::f32: 292249423Sdim Opcode = NVPTX::LD_f32_ari_64; 293249423Sdim break; 294249423Sdim case MVT::f64: 295249423Sdim Opcode = NVPTX::LD_f64_ari_64; 296249423Sdim break; 297249423Sdim default: 298249423Sdim return NULL; 299249423Sdim } 300249423Sdim } else { 301249423Sdim switch (TargetVT) { 302249423Sdim case MVT::i8: 303249423Sdim Opcode = NVPTX::LD_i8_ari; 304249423Sdim break; 305249423Sdim case MVT::i16: 306249423Sdim Opcode = NVPTX::LD_i16_ari; 307249423Sdim break; 308249423Sdim case MVT::i32: 309249423Sdim Opcode = NVPTX::LD_i32_ari; 310249423Sdim break; 311249423Sdim case MVT::i64: 312249423Sdim Opcode = NVPTX::LD_i64_ari; 313249423Sdim break; 314249423Sdim case MVT::f32: 315249423Sdim Opcode = NVPTX::LD_f32_ari; 316249423Sdim break; 317249423Sdim case MVT::f64: 318249423Sdim Opcode = NVPTX::LD_f64_ari; 319249423Sdim break; 320249423Sdim default: 321249423Sdim return NULL; 322249423Sdim } 323239310Sdim } 324249423Sdim SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 325249423Sdim getI32Imm(vecType), getI32Imm(fromType), 326249423Sdim getI32Imm(fromTypeWidth), Base, Offset, Chain }; 327249423Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops, 8); 328249423Sdim } else { 329249423Sdim if (Subtarget.is64Bit()) { 330249423Sdim switch (TargetVT) { 331249423Sdim case MVT::i8: 332249423Sdim Opcode = NVPTX::LD_i8_areg_64; 333249423Sdim break; 334249423Sdim case MVT::i16: 335249423Sdim Opcode = NVPTX::LD_i16_areg_64; 336249423Sdim break; 337249423Sdim case MVT::i32: 338249423Sdim Opcode = NVPTX::LD_i32_areg_64; 339249423Sdim break; 340249423Sdim case MVT::i64: 341249423Sdim Opcode = NVPTX::LD_i64_areg_64; 342249423Sdim break; 343249423Sdim case MVT::f32: 344249423Sdim Opcode = NVPTX::LD_f32_areg_64; 345249423Sdim break; 346249423Sdim case MVT::f64: 347249423Sdim Opcode = NVPTX::LD_f64_areg_64; 348249423Sdim break; 349249423Sdim default: 350249423Sdim return NULL; 351249423Sdim } 352249423Sdim } else { 353249423Sdim switch (TargetVT) { 354249423Sdim case MVT::i8: 355249423Sdim Opcode = NVPTX::LD_i8_areg; 356249423Sdim break; 357249423Sdim case MVT::i16: 358249423Sdim Opcode = NVPTX::LD_i16_areg; 359249423Sdim break; 360249423Sdim case MVT::i32: 361249423Sdim Opcode = NVPTX::LD_i32_areg; 362249423Sdim break; 363249423Sdim case MVT::i64: 364249423Sdim Opcode = NVPTX::LD_i64_areg; 365249423Sdim break; 366249423Sdim case MVT::f32: 367249423Sdim Opcode = NVPTX::LD_f32_areg; 368249423Sdim break; 369249423Sdim case MVT::f64: 370249423Sdim Opcode = NVPTX::LD_f64_areg; 371249423Sdim break; 372249423Sdim default: 373249423Sdim return NULL; 374249423Sdim } 375239310Sdim } 376249423Sdim SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 377249423Sdim getI32Imm(vecType), getI32Imm(fromType), 378249423Sdim getI32Imm(fromTypeWidth), N1, Chain }; 379249423Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops, 7); 380239310Sdim } 381239310Sdim 382239310Sdim if (NVPTXLD != NULL) { 383239310Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 384239310Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 385239310Sdim cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1); 386239310Sdim } 387239310Sdim 388239310Sdim return NVPTXLD; 389239310Sdim} 390239310Sdim 391249423SdimSDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) { 392249423Sdim 393249423Sdim SDValue Chain = N->getOperand(0); 394249423Sdim SDValue Op1 = N->getOperand(1); 395249423Sdim SDValue Addr, Offset, Base; 396249423Sdim unsigned Opcode; 397249423Sdim DebugLoc DL = N->getDebugLoc(); 398249423Sdim SDNode *LD; 399249423Sdim MemSDNode *MemSD = cast<MemSDNode>(N); 400249423Sdim EVT LoadedVT = MemSD->getMemoryVT(); 401249423Sdim 402249423Sdim if (!LoadedVT.isSimple()) 403249423Sdim return NULL; 404249423Sdim 405249423Sdim // Address Space Setting 406249423Sdim unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget); 407249423Sdim 408249423Sdim // Volatile Setting 409249423Sdim // - .volatile is only availalble for .global and .shared 410249423Sdim bool IsVolatile = MemSD->isVolatile(); 411249423Sdim if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 412249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 413249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 414249423Sdim IsVolatile = false; 415249423Sdim 416249423Sdim // Vector Setting 417249423Sdim MVT SimpleVT = LoadedVT.getSimpleVT(); 418249423Sdim 419249423Sdim // Type Setting: fromType + fromTypeWidth 420249423Sdim // 421249423Sdim // Sign : ISD::SEXTLOAD 422249423Sdim // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the 423249423Sdim // type is integer 424249423Sdim // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float 425249423Sdim MVT ScalarVT = SimpleVT.getScalarType(); 426249423Sdim unsigned FromTypeWidth = ScalarVT.getSizeInBits(); 427249423Sdim unsigned int FromType; 428249423Sdim // The last operand holds the original LoadSDNode::getExtensionType() value 429249423Sdim unsigned ExtensionType = cast<ConstantSDNode>( 430249423Sdim N->getOperand(N->getNumOperands() - 1))->getZExtValue(); 431249423Sdim if (ExtensionType == ISD::SEXTLOAD) 432249423Sdim FromType = NVPTX::PTXLdStInstCode::Signed; 433249423Sdim else if (ScalarVT.isFloatingPoint()) 434249423Sdim FromType = NVPTX::PTXLdStInstCode::Float; 435249423Sdim else 436249423Sdim FromType = NVPTX::PTXLdStInstCode::Unsigned; 437249423Sdim 438249423Sdim unsigned VecType; 439249423Sdim 440249423Sdim switch (N->getOpcode()) { 441249423Sdim case NVPTXISD::LoadV2: 442249423Sdim VecType = NVPTX::PTXLdStInstCode::V2; 443249423Sdim break; 444249423Sdim case NVPTXISD::LoadV4: 445249423Sdim VecType = NVPTX::PTXLdStInstCode::V4; 446249423Sdim break; 447249423Sdim default: 448249423Sdim return NULL; 449249423Sdim } 450249423Sdim 451249423Sdim EVT EltVT = N->getValueType(0); 452249423Sdim 453249423Sdim if (SelectDirectAddr(Op1, Addr)) { 454249423Sdim switch (N->getOpcode()) { 455249423Sdim default: 456249423Sdim return NULL; 457249423Sdim case NVPTXISD::LoadV2: 458249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 459249423Sdim default: 460249423Sdim return NULL; 461249423Sdim case MVT::i8: 462249423Sdim Opcode = NVPTX::LDV_i8_v2_avar; 463249423Sdim break; 464249423Sdim case MVT::i16: 465249423Sdim Opcode = NVPTX::LDV_i16_v2_avar; 466249423Sdim break; 467249423Sdim case MVT::i32: 468249423Sdim Opcode = NVPTX::LDV_i32_v2_avar; 469249423Sdim break; 470249423Sdim case MVT::i64: 471249423Sdim Opcode = NVPTX::LDV_i64_v2_avar; 472249423Sdim break; 473249423Sdim case MVT::f32: 474249423Sdim Opcode = NVPTX::LDV_f32_v2_avar; 475249423Sdim break; 476249423Sdim case MVT::f64: 477249423Sdim Opcode = NVPTX::LDV_f64_v2_avar; 478249423Sdim break; 479249423Sdim } 480249423Sdim break; 481249423Sdim case NVPTXISD::LoadV4: 482249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 483249423Sdim default: 484249423Sdim return NULL; 485249423Sdim case MVT::i8: 486249423Sdim Opcode = NVPTX::LDV_i8_v4_avar; 487249423Sdim break; 488249423Sdim case MVT::i16: 489249423Sdim Opcode = NVPTX::LDV_i16_v4_avar; 490249423Sdim break; 491249423Sdim case MVT::i32: 492249423Sdim Opcode = NVPTX::LDV_i32_v4_avar; 493249423Sdim break; 494249423Sdim case MVT::f32: 495249423Sdim Opcode = NVPTX::LDV_f32_v4_avar; 496249423Sdim break; 497249423Sdim } 498249423Sdim break; 499249423Sdim } 500249423Sdim 501249423Sdim SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 502249423Sdim getI32Imm(VecType), getI32Imm(FromType), 503249423Sdim getI32Imm(FromTypeWidth), Addr, Chain }; 504249423Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops, 7); 505249423Sdim } else if (Subtarget.is64Bit() 506249423Sdim ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset) 507249423Sdim : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) { 508249423Sdim switch (N->getOpcode()) { 509249423Sdim default: 510249423Sdim return NULL; 511249423Sdim case NVPTXISD::LoadV2: 512249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 513249423Sdim default: 514249423Sdim return NULL; 515249423Sdim case MVT::i8: 516249423Sdim Opcode = NVPTX::LDV_i8_v2_asi; 517249423Sdim break; 518249423Sdim case MVT::i16: 519249423Sdim Opcode = NVPTX::LDV_i16_v2_asi; 520249423Sdim break; 521249423Sdim case MVT::i32: 522249423Sdim Opcode = NVPTX::LDV_i32_v2_asi; 523249423Sdim break; 524249423Sdim case MVT::i64: 525249423Sdim Opcode = NVPTX::LDV_i64_v2_asi; 526249423Sdim break; 527249423Sdim case MVT::f32: 528249423Sdim Opcode = NVPTX::LDV_f32_v2_asi; 529249423Sdim break; 530249423Sdim case MVT::f64: 531249423Sdim Opcode = NVPTX::LDV_f64_v2_asi; 532249423Sdim break; 533249423Sdim } 534249423Sdim break; 535249423Sdim case NVPTXISD::LoadV4: 536249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 537249423Sdim default: 538249423Sdim return NULL; 539249423Sdim case MVT::i8: 540249423Sdim Opcode = NVPTX::LDV_i8_v4_asi; 541249423Sdim break; 542249423Sdim case MVT::i16: 543249423Sdim Opcode = NVPTX::LDV_i16_v4_asi; 544249423Sdim break; 545249423Sdim case MVT::i32: 546249423Sdim Opcode = NVPTX::LDV_i32_v4_asi; 547249423Sdim break; 548249423Sdim case MVT::f32: 549249423Sdim Opcode = NVPTX::LDV_f32_v4_asi; 550249423Sdim break; 551249423Sdim } 552249423Sdim break; 553249423Sdim } 554249423Sdim 555249423Sdim SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 556249423Sdim getI32Imm(VecType), getI32Imm(FromType), 557249423Sdim getI32Imm(FromTypeWidth), Base, Offset, Chain }; 558249423Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops, 8); 559249423Sdim } else if (Subtarget.is64Bit() 560249423Sdim ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) 561249423Sdim : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { 562249423Sdim if (Subtarget.is64Bit()) { 563249423Sdim switch (N->getOpcode()) { 564249423Sdim default: 565249423Sdim return NULL; 566249423Sdim case NVPTXISD::LoadV2: 567249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 568249423Sdim default: 569249423Sdim return NULL; 570249423Sdim case MVT::i8: 571249423Sdim Opcode = NVPTX::LDV_i8_v2_ari_64; 572249423Sdim break; 573249423Sdim case MVT::i16: 574249423Sdim Opcode = NVPTX::LDV_i16_v2_ari_64; 575249423Sdim break; 576249423Sdim case MVT::i32: 577249423Sdim Opcode = NVPTX::LDV_i32_v2_ari_64; 578249423Sdim break; 579249423Sdim case MVT::i64: 580249423Sdim Opcode = NVPTX::LDV_i64_v2_ari_64; 581249423Sdim break; 582249423Sdim case MVT::f32: 583249423Sdim Opcode = NVPTX::LDV_f32_v2_ari_64; 584249423Sdim break; 585249423Sdim case MVT::f64: 586249423Sdim Opcode = NVPTX::LDV_f64_v2_ari_64; 587249423Sdim break; 588249423Sdim } 589249423Sdim break; 590249423Sdim case NVPTXISD::LoadV4: 591249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 592249423Sdim default: 593249423Sdim return NULL; 594249423Sdim case MVT::i8: 595249423Sdim Opcode = NVPTX::LDV_i8_v4_ari_64; 596249423Sdim break; 597249423Sdim case MVT::i16: 598249423Sdim Opcode = NVPTX::LDV_i16_v4_ari_64; 599249423Sdim break; 600249423Sdim case MVT::i32: 601249423Sdim Opcode = NVPTX::LDV_i32_v4_ari_64; 602249423Sdim break; 603249423Sdim case MVT::f32: 604249423Sdim Opcode = NVPTX::LDV_f32_v4_ari_64; 605249423Sdim break; 606249423Sdim } 607249423Sdim break; 608249423Sdim } 609249423Sdim } else { 610249423Sdim switch (N->getOpcode()) { 611249423Sdim default: 612249423Sdim return NULL; 613249423Sdim case NVPTXISD::LoadV2: 614249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 615249423Sdim default: 616249423Sdim return NULL; 617249423Sdim case MVT::i8: 618249423Sdim Opcode = NVPTX::LDV_i8_v2_ari; 619249423Sdim break; 620249423Sdim case MVT::i16: 621249423Sdim Opcode = NVPTX::LDV_i16_v2_ari; 622249423Sdim break; 623249423Sdim case MVT::i32: 624249423Sdim Opcode = NVPTX::LDV_i32_v2_ari; 625249423Sdim break; 626249423Sdim case MVT::i64: 627249423Sdim Opcode = NVPTX::LDV_i64_v2_ari; 628249423Sdim break; 629249423Sdim case MVT::f32: 630249423Sdim Opcode = NVPTX::LDV_f32_v2_ari; 631249423Sdim break; 632249423Sdim case MVT::f64: 633249423Sdim Opcode = NVPTX::LDV_f64_v2_ari; 634249423Sdim break; 635249423Sdim } 636249423Sdim break; 637249423Sdim case NVPTXISD::LoadV4: 638249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 639249423Sdim default: 640249423Sdim return NULL; 641249423Sdim case MVT::i8: 642249423Sdim Opcode = NVPTX::LDV_i8_v4_ari; 643249423Sdim break; 644249423Sdim case MVT::i16: 645249423Sdim Opcode = NVPTX::LDV_i16_v4_ari; 646249423Sdim break; 647249423Sdim case MVT::i32: 648249423Sdim Opcode = NVPTX::LDV_i32_v4_ari; 649249423Sdim break; 650249423Sdim case MVT::f32: 651249423Sdim Opcode = NVPTX::LDV_f32_v4_ari; 652249423Sdim break; 653249423Sdim } 654249423Sdim break; 655249423Sdim } 656249423Sdim } 657249423Sdim 658249423Sdim SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 659249423Sdim getI32Imm(VecType), getI32Imm(FromType), 660249423Sdim getI32Imm(FromTypeWidth), Base, Offset, Chain }; 661249423Sdim 662249423Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops, 8); 663249423Sdim } else { 664249423Sdim if (Subtarget.is64Bit()) { 665249423Sdim switch (N->getOpcode()) { 666249423Sdim default: 667249423Sdim return NULL; 668249423Sdim case NVPTXISD::LoadV2: 669249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 670249423Sdim default: 671249423Sdim return NULL; 672249423Sdim case MVT::i8: 673249423Sdim Opcode = NVPTX::LDV_i8_v2_areg_64; 674249423Sdim break; 675249423Sdim case MVT::i16: 676249423Sdim Opcode = NVPTX::LDV_i16_v2_areg_64; 677249423Sdim break; 678249423Sdim case MVT::i32: 679249423Sdim Opcode = NVPTX::LDV_i32_v2_areg_64; 680249423Sdim break; 681249423Sdim case MVT::i64: 682249423Sdim Opcode = NVPTX::LDV_i64_v2_areg_64; 683249423Sdim break; 684249423Sdim case MVT::f32: 685249423Sdim Opcode = NVPTX::LDV_f32_v2_areg_64; 686249423Sdim break; 687249423Sdim case MVT::f64: 688249423Sdim Opcode = NVPTX::LDV_f64_v2_areg_64; 689249423Sdim break; 690249423Sdim } 691249423Sdim break; 692249423Sdim case NVPTXISD::LoadV4: 693249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 694249423Sdim default: 695249423Sdim return NULL; 696249423Sdim case MVT::i8: 697249423Sdim Opcode = NVPTX::LDV_i8_v4_areg_64; 698249423Sdim break; 699249423Sdim case MVT::i16: 700249423Sdim Opcode = NVPTX::LDV_i16_v4_areg_64; 701249423Sdim break; 702249423Sdim case MVT::i32: 703249423Sdim Opcode = NVPTX::LDV_i32_v4_areg_64; 704249423Sdim break; 705249423Sdim case MVT::f32: 706249423Sdim Opcode = NVPTX::LDV_f32_v4_areg_64; 707249423Sdim break; 708249423Sdim } 709249423Sdim break; 710249423Sdim } 711249423Sdim } else { 712249423Sdim switch (N->getOpcode()) { 713249423Sdim default: 714249423Sdim return NULL; 715249423Sdim case NVPTXISD::LoadV2: 716249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 717249423Sdim default: 718249423Sdim return NULL; 719249423Sdim case MVT::i8: 720249423Sdim Opcode = NVPTX::LDV_i8_v2_areg; 721249423Sdim break; 722249423Sdim case MVT::i16: 723249423Sdim Opcode = NVPTX::LDV_i16_v2_areg; 724249423Sdim break; 725249423Sdim case MVT::i32: 726249423Sdim Opcode = NVPTX::LDV_i32_v2_areg; 727249423Sdim break; 728249423Sdim case MVT::i64: 729249423Sdim Opcode = NVPTX::LDV_i64_v2_areg; 730249423Sdim break; 731249423Sdim case MVT::f32: 732249423Sdim Opcode = NVPTX::LDV_f32_v2_areg; 733249423Sdim break; 734249423Sdim case MVT::f64: 735249423Sdim Opcode = NVPTX::LDV_f64_v2_areg; 736249423Sdim break; 737249423Sdim } 738249423Sdim break; 739249423Sdim case NVPTXISD::LoadV4: 740249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 741249423Sdim default: 742249423Sdim return NULL; 743249423Sdim case MVT::i8: 744249423Sdim Opcode = NVPTX::LDV_i8_v4_areg; 745249423Sdim break; 746249423Sdim case MVT::i16: 747249423Sdim Opcode = NVPTX::LDV_i16_v4_areg; 748249423Sdim break; 749249423Sdim case MVT::i32: 750249423Sdim Opcode = NVPTX::LDV_i32_v4_areg; 751249423Sdim break; 752249423Sdim case MVT::f32: 753249423Sdim Opcode = NVPTX::LDV_f32_v4_areg; 754249423Sdim break; 755249423Sdim } 756249423Sdim break; 757249423Sdim } 758249423Sdim } 759249423Sdim 760249423Sdim SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 761249423Sdim getI32Imm(VecType), getI32Imm(FromType), 762249423Sdim getI32Imm(FromTypeWidth), Op1, Chain }; 763249423Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops, 7); 764249423Sdim } 765249423Sdim 766249423Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 767249423Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 768249423Sdim cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); 769249423Sdim 770249423Sdim return LD; 771249423Sdim} 772249423Sdim 773249423SdimSDNode *NVPTXDAGToDAGISel::SelectLDGLDUVector(SDNode *N) { 774249423Sdim 775249423Sdim SDValue Chain = N->getOperand(0); 776249423Sdim SDValue Op1 = N->getOperand(1); 777249423Sdim unsigned Opcode; 778249423Sdim DebugLoc DL = N->getDebugLoc(); 779249423Sdim SDNode *LD; 780249423Sdim 781249423Sdim EVT RetVT = N->getValueType(0); 782249423Sdim 783249423Sdim // Select opcode 784249423Sdim if (Subtarget.is64Bit()) { 785249423Sdim switch (N->getOpcode()) { 786249423Sdim default: 787249423Sdim return NULL; 788249423Sdim case NVPTXISD::LDGV2: 789249423Sdim switch (RetVT.getSimpleVT().SimpleTy) { 790249423Sdim default: 791249423Sdim return NULL; 792249423Sdim case MVT::i8: 793249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_64; 794249423Sdim break; 795249423Sdim case MVT::i16: 796249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_64; 797249423Sdim break; 798249423Sdim case MVT::i32: 799249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_64; 800249423Sdim break; 801249423Sdim case MVT::i64: 802249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_64; 803249423Sdim break; 804249423Sdim case MVT::f32: 805249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_64; 806249423Sdim break; 807249423Sdim case MVT::f64: 808249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_64; 809249423Sdim break; 810249423Sdim } 811249423Sdim break; 812249423Sdim case NVPTXISD::LDGV4: 813249423Sdim switch (RetVT.getSimpleVT().SimpleTy) { 814249423Sdim default: 815249423Sdim return NULL; 816249423Sdim case MVT::i8: 817249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_64; 818249423Sdim break; 819249423Sdim case MVT::i16: 820249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_64; 821249423Sdim break; 822249423Sdim case MVT::i32: 823249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_64; 824249423Sdim break; 825249423Sdim case MVT::f32: 826249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_64; 827249423Sdim break; 828249423Sdim } 829249423Sdim break; 830249423Sdim case NVPTXISD::LDUV2: 831249423Sdim switch (RetVT.getSimpleVT().SimpleTy) { 832249423Sdim default: 833249423Sdim return NULL; 834249423Sdim case MVT::i8: 835249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_64; 836249423Sdim break; 837249423Sdim case MVT::i16: 838249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_64; 839249423Sdim break; 840249423Sdim case MVT::i32: 841249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_64; 842249423Sdim break; 843249423Sdim case MVT::i64: 844249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_64; 845249423Sdim break; 846249423Sdim case MVT::f32: 847249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_64; 848249423Sdim break; 849249423Sdim case MVT::f64: 850249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_64; 851249423Sdim break; 852249423Sdim } 853249423Sdim break; 854249423Sdim case NVPTXISD::LDUV4: 855249423Sdim switch (RetVT.getSimpleVT().SimpleTy) { 856249423Sdim default: 857249423Sdim return NULL; 858249423Sdim case MVT::i8: 859249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_64; 860249423Sdim break; 861249423Sdim case MVT::i16: 862249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_64; 863249423Sdim break; 864249423Sdim case MVT::i32: 865249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_64; 866249423Sdim break; 867249423Sdim case MVT::f32: 868249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_64; 869249423Sdim break; 870249423Sdim } 871249423Sdim break; 872249423Sdim } 873249423Sdim } else { 874249423Sdim switch (N->getOpcode()) { 875249423Sdim default: 876249423Sdim return NULL; 877249423Sdim case NVPTXISD::LDGV2: 878249423Sdim switch (RetVT.getSimpleVT().SimpleTy) { 879249423Sdim default: 880249423Sdim return NULL; 881249423Sdim case MVT::i8: 882249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_32; 883249423Sdim break; 884249423Sdim case MVT::i16: 885249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_32; 886249423Sdim break; 887249423Sdim case MVT::i32: 888249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_32; 889249423Sdim break; 890249423Sdim case MVT::i64: 891249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_32; 892249423Sdim break; 893249423Sdim case MVT::f32: 894249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_32; 895249423Sdim break; 896249423Sdim case MVT::f64: 897249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_32; 898249423Sdim break; 899249423Sdim } 900249423Sdim break; 901249423Sdim case NVPTXISD::LDGV4: 902249423Sdim switch (RetVT.getSimpleVT().SimpleTy) { 903249423Sdim default: 904249423Sdim return NULL; 905249423Sdim case MVT::i8: 906249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_32; 907249423Sdim break; 908249423Sdim case MVT::i16: 909249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_32; 910249423Sdim break; 911249423Sdim case MVT::i32: 912249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_32; 913249423Sdim break; 914249423Sdim case MVT::f32: 915249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_32; 916249423Sdim break; 917249423Sdim } 918249423Sdim break; 919249423Sdim case NVPTXISD::LDUV2: 920249423Sdim switch (RetVT.getSimpleVT().SimpleTy) { 921249423Sdim default: 922249423Sdim return NULL; 923249423Sdim case MVT::i8: 924249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_32; 925249423Sdim break; 926249423Sdim case MVT::i16: 927249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_32; 928249423Sdim break; 929249423Sdim case MVT::i32: 930249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_32; 931249423Sdim break; 932249423Sdim case MVT::i64: 933249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_32; 934249423Sdim break; 935249423Sdim case MVT::f32: 936249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_32; 937249423Sdim break; 938249423Sdim case MVT::f64: 939249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_32; 940249423Sdim break; 941249423Sdim } 942249423Sdim break; 943249423Sdim case NVPTXISD::LDUV4: 944249423Sdim switch (RetVT.getSimpleVT().SimpleTy) { 945249423Sdim default: 946249423Sdim return NULL; 947249423Sdim case MVT::i8: 948249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_32; 949249423Sdim break; 950249423Sdim case MVT::i16: 951249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_32; 952249423Sdim break; 953249423Sdim case MVT::i32: 954249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_32; 955249423Sdim break; 956249423Sdim case MVT::f32: 957249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_32; 958249423Sdim break; 959249423Sdim } 960249423Sdim break; 961249423Sdim } 962249423Sdim } 963249423Sdim 964249423Sdim SDValue Ops[] = { Op1, Chain }; 965249423Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), &Ops[0], 2); 966249423Sdim 967249423Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 968249423Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 969249423Sdim cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); 970249423Sdim 971249423Sdim return LD; 972249423Sdim} 973249423Sdim 974249423SdimSDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) { 975239310Sdim DebugLoc dl = N->getDebugLoc(); 976239310Sdim StoreSDNode *ST = cast<StoreSDNode>(N); 977239310Sdim EVT StoreVT = ST->getMemoryVT(); 978239310Sdim SDNode *NVPTXST = NULL; 979239310Sdim 980239310Sdim // do not support pre/post inc/dec 981239310Sdim if (ST->isIndexed()) 982239310Sdim return NULL; 983239310Sdim 984239310Sdim if (!StoreVT.isSimple()) 985239310Sdim return NULL; 986239310Sdim 987239310Sdim // Address Space Setting 988239310Sdim unsigned int codeAddrSpace = getCodeAddrSpace(ST, Subtarget); 989239310Sdim 990239310Sdim // Volatile Setting 991239310Sdim // - .volatile is only availalble for .global and .shared 992239310Sdim bool isVolatile = ST->isVolatile(); 993239310Sdim if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 994239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 995239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 996239310Sdim isVolatile = false; 997239310Sdim 998239310Sdim // Vector Setting 999239310Sdim MVT SimpleVT = StoreVT.getSimpleVT(); 1000239310Sdim unsigned vecType = NVPTX::PTXLdStInstCode::Scalar; 1001239310Sdim if (SimpleVT.isVector()) { 1002239310Sdim unsigned num = SimpleVT.getVectorNumElements(); 1003239310Sdim if (num == 2) 1004239310Sdim vecType = NVPTX::PTXLdStInstCode::V2; 1005239310Sdim else if (num == 4) 1006239310Sdim vecType = NVPTX::PTXLdStInstCode::V4; 1007239310Sdim else 1008239310Sdim return NULL; 1009239310Sdim } 1010239310Sdim 1011239310Sdim // Type Setting: toType + toTypeWidth 1012239310Sdim // - for integer type, always use 'u' 1013239310Sdim // 1014239310Sdim MVT ScalarVT = SimpleVT.getScalarType(); 1015249423Sdim unsigned toTypeWidth = ScalarVT.getSizeInBits(); 1016239310Sdim unsigned int toType; 1017239310Sdim if (ScalarVT.isFloatingPoint()) 1018239310Sdim toType = NVPTX::PTXLdStInstCode::Float; 1019239310Sdim else 1020239310Sdim toType = NVPTX::PTXLdStInstCode::Unsigned; 1021239310Sdim 1022239310Sdim // Create the machine instruction DAG 1023239310Sdim SDValue Chain = N->getOperand(0); 1024239310Sdim SDValue N1 = N->getOperand(1); 1025239310Sdim SDValue N2 = N->getOperand(2); 1026239310Sdim SDValue Addr; 1027239310Sdim SDValue Offset, Base; 1028239310Sdim unsigned Opcode; 1029239310Sdim MVT::SimpleValueType SourceVT = 1030239310Sdim N1.getNode()->getValueType(0).getSimpleVT().SimpleTy; 1031239310Sdim 1032239310Sdim if (SelectDirectAddr(N2, Addr)) { 1033239310Sdim switch (SourceVT) { 1034249423Sdim case MVT::i8: 1035249423Sdim Opcode = NVPTX::ST_i8_avar; 1036249423Sdim break; 1037249423Sdim case MVT::i16: 1038249423Sdim Opcode = NVPTX::ST_i16_avar; 1039249423Sdim break; 1040249423Sdim case MVT::i32: 1041249423Sdim Opcode = NVPTX::ST_i32_avar; 1042249423Sdim break; 1043249423Sdim case MVT::i64: 1044249423Sdim Opcode = NVPTX::ST_i64_avar; 1045249423Sdim break; 1046249423Sdim case MVT::f32: 1047249423Sdim Opcode = NVPTX::ST_f32_avar; 1048249423Sdim break; 1049249423Sdim case MVT::f64: 1050249423Sdim Opcode = NVPTX::ST_f64_avar; 1051249423Sdim break; 1052249423Sdim default: 1053249423Sdim return NULL; 1054239310Sdim } 1055249423Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1056249423Sdim getI32Imm(vecType), getI32Imm(toType), 1057249423Sdim getI32Imm(toTypeWidth), Addr, Chain }; 1058249423Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops, 8); 1059249423Sdim } else if (Subtarget.is64Bit() 1060249423Sdim ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) 1061249423Sdim : SelectADDRsi(N2.getNode(), N2, Base, Offset)) { 1062239310Sdim switch (SourceVT) { 1063249423Sdim case MVT::i8: 1064249423Sdim Opcode = NVPTX::ST_i8_asi; 1065249423Sdim break; 1066249423Sdim case MVT::i16: 1067249423Sdim Opcode = NVPTX::ST_i16_asi; 1068249423Sdim break; 1069249423Sdim case MVT::i32: 1070249423Sdim Opcode = NVPTX::ST_i32_asi; 1071249423Sdim break; 1072249423Sdim case MVT::i64: 1073249423Sdim Opcode = NVPTX::ST_i64_asi; 1074249423Sdim break; 1075249423Sdim case MVT::f32: 1076249423Sdim Opcode = NVPTX::ST_f32_asi; 1077249423Sdim break; 1078249423Sdim case MVT::f64: 1079249423Sdim Opcode = NVPTX::ST_f64_asi; 1080249423Sdim break; 1081249423Sdim default: 1082249423Sdim return NULL; 1083239310Sdim } 1084249423Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1085249423Sdim getI32Imm(vecType), getI32Imm(toType), 1086249423Sdim getI32Imm(toTypeWidth), Base, Offset, Chain }; 1087249423Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops, 9); 1088249423Sdim } else if (Subtarget.is64Bit() 1089249423Sdim ? SelectADDRri64(N2.getNode(), N2, Base, Offset) 1090249423Sdim : SelectADDRri(N2.getNode(), N2, Base, Offset)) { 1091249423Sdim if (Subtarget.is64Bit()) { 1092249423Sdim switch (SourceVT) { 1093249423Sdim case MVT::i8: 1094249423Sdim Opcode = NVPTX::ST_i8_ari_64; 1095249423Sdim break; 1096249423Sdim case MVT::i16: 1097249423Sdim Opcode = NVPTX::ST_i16_ari_64; 1098249423Sdim break; 1099249423Sdim case MVT::i32: 1100249423Sdim Opcode = NVPTX::ST_i32_ari_64; 1101249423Sdim break; 1102249423Sdim case MVT::i64: 1103249423Sdim Opcode = NVPTX::ST_i64_ari_64; 1104249423Sdim break; 1105249423Sdim case MVT::f32: 1106249423Sdim Opcode = NVPTX::ST_f32_ari_64; 1107249423Sdim break; 1108249423Sdim case MVT::f64: 1109249423Sdim Opcode = NVPTX::ST_f64_ari_64; 1110249423Sdim break; 1111249423Sdim default: 1112249423Sdim return NULL; 1113249423Sdim } 1114249423Sdim } else { 1115249423Sdim switch (SourceVT) { 1116249423Sdim case MVT::i8: 1117249423Sdim Opcode = NVPTX::ST_i8_ari; 1118249423Sdim break; 1119249423Sdim case MVT::i16: 1120249423Sdim Opcode = NVPTX::ST_i16_ari; 1121249423Sdim break; 1122249423Sdim case MVT::i32: 1123249423Sdim Opcode = NVPTX::ST_i32_ari; 1124249423Sdim break; 1125249423Sdim case MVT::i64: 1126249423Sdim Opcode = NVPTX::ST_i64_ari; 1127249423Sdim break; 1128249423Sdim case MVT::f32: 1129249423Sdim Opcode = NVPTX::ST_f32_ari; 1130249423Sdim break; 1131249423Sdim case MVT::f64: 1132249423Sdim Opcode = NVPTX::ST_f64_ari; 1133249423Sdim break; 1134249423Sdim default: 1135249423Sdim return NULL; 1136249423Sdim } 1137239310Sdim } 1138249423Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1139249423Sdim getI32Imm(vecType), getI32Imm(toType), 1140249423Sdim getI32Imm(toTypeWidth), Base, Offset, Chain }; 1141249423Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops, 9); 1142239310Sdim } else { 1143249423Sdim if (Subtarget.is64Bit()) { 1144249423Sdim switch (SourceVT) { 1145249423Sdim case MVT::i8: 1146249423Sdim Opcode = NVPTX::ST_i8_areg_64; 1147249423Sdim break; 1148249423Sdim case MVT::i16: 1149249423Sdim Opcode = NVPTX::ST_i16_areg_64; 1150249423Sdim break; 1151249423Sdim case MVT::i32: 1152249423Sdim Opcode = NVPTX::ST_i32_areg_64; 1153249423Sdim break; 1154249423Sdim case MVT::i64: 1155249423Sdim Opcode = NVPTX::ST_i64_areg_64; 1156249423Sdim break; 1157249423Sdim case MVT::f32: 1158249423Sdim Opcode = NVPTX::ST_f32_areg_64; 1159249423Sdim break; 1160249423Sdim case MVT::f64: 1161249423Sdim Opcode = NVPTX::ST_f64_areg_64; 1162249423Sdim break; 1163249423Sdim default: 1164249423Sdim return NULL; 1165249423Sdim } 1166249423Sdim } else { 1167249423Sdim switch (SourceVT) { 1168249423Sdim case MVT::i8: 1169249423Sdim Opcode = NVPTX::ST_i8_areg; 1170249423Sdim break; 1171249423Sdim case MVT::i16: 1172249423Sdim Opcode = NVPTX::ST_i16_areg; 1173249423Sdim break; 1174249423Sdim case MVT::i32: 1175249423Sdim Opcode = NVPTX::ST_i32_areg; 1176249423Sdim break; 1177249423Sdim case MVT::i64: 1178249423Sdim Opcode = NVPTX::ST_i64_areg; 1179249423Sdim break; 1180249423Sdim case MVT::f32: 1181249423Sdim Opcode = NVPTX::ST_f32_areg; 1182249423Sdim break; 1183249423Sdim case MVT::f64: 1184249423Sdim Opcode = NVPTX::ST_f64_areg; 1185249423Sdim break; 1186249423Sdim default: 1187249423Sdim return NULL; 1188249423Sdim } 1189239310Sdim } 1190249423Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1191249423Sdim getI32Imm(vecType), getI32Imm(toType), 1192249423Sdim getI32Imm(toTypeWidth), N2, Chain }; 1193249423Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops, 8); 1194239310Sdim } 1195239310Sdim 1196239310Sdim if (NVPTXST != NULL) { 1197239310Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 1198239310Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 1199239310Sdim cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1); 1200239310Sdim } 1201239310Sdim 1202239310Sdim return NVPTXST; 1203239310Sdim} 1204239310Sdim 1205249423SdimSDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) { 1206249423Sdim SDValue Chain = N->getOperand(0); 1207249423Sdim SDValue Op1 = N->getOperand(1); 1208249423Sdim SDValue Addr, Offset, Base; 1209249423Sdim unsigned Opcode; 1210249423Sdim DebugLoc DL = N->getDebugLoc(); 1211249423Sdim SDNode *ST; 1212249423Sdim EVT EltVT = Op1.getValueType(); 1213249423Sdim MemSDNode *MemSD = cast<MemSDNode>(N); 1214249423Sdim EVT StoreVT = MemSD->getMemoryVT(); 1215249423Sdim 1216249423Sdim // Address Space Setting 1217249423Sdim unsigned CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget); 1218249423Sdim 1219249423Sdim if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) { 1220249423Sdim report_fatal_error("Cannot store to pointer that points to constant " 1221249423Sdim "memory space"); 1222249423Sdim } 1223249423Sdim 1224249423Sdim // Volatile Setting 1225249423Sdim // - .volatile is only availalble for .global and .shared 1226249423Sdim bool IsVolatile = MemSD->isVolatile(); 1227249423Sdim if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 1228249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 1229249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 1230249423Sdim IsVolatile = false; 1231249423Sdim 1232249423Sdim // Type Setting: toType + toTypeWidth 1233249423Sdim // - for integer type, always use 'u' 1234249423Sdim assert(StoreVT.isSimple() && "Store value is not simple"); 1235249423Sdim MVT ScalarVT = StoreVT.getSimpleVT().getScalarType(); 1236249423Sdim unsigned ToTypeWidth = ScalarVT.getSizeInBits(); 1237249423Sdim unsigned ToType; 1238249423Sdim if (ScalarVT.isFloatingPoint()) 1239249423Sdim ToType = NVPTX::PTXLdStInstCode::Float; 1240249423Sdim else 1241249423Sdim ToType = NVPTX::PTXLdStInstCode::Unsigned; 1242249423Sdim 1243249423Sdim SmallVector<SDValue, 12> StOps; 1244249423Sdim SDValue N2; 1245249423Sdim unsigned VecType; 1246249423Sdim 1247249423Sdim switch (N->getOpcode()) { 1248249423Sdim case NVPTXISD::StoreV2: 1249249423Sdim VecType = NVPTX::PTXLdStInstCode::V2; 1250249423Sdim StOps.push_back(N->getOperand(1)); 1251249423Sdim StOps.push_back(N->getOperand(2)); 1252249423Sdim N2 = N->getOperand(3); 1253249423Sdim break; 1254249423Sdim case NVPTXISD::StoreV4: 1255249423Sdim VecType = NVPTX::PTXLdStInstCode::V4; 1256249423Sdim StOps.push_back(N->getOperand(1)); 1257249423Sdim StOps.push_back(N->getOperand(2)); 1258249423Sdim StOps.push_back(N->getOperand(3)); 1259249423Sdim StOps.push_back(N->getOperand(4)); 1260249423Sdim N2 = N->getOperand(5); 1261249423Sdim break; 1262249423Sdim default: 1263249423Sdim return NULL; 1264249423Sdim } 1265249423Sdim 1266249423Sdim StOps.push_back(getI32Imm(IsVolatile)); 1267249423Sdim StOps.push_back(getI32Imm(CodeAddrSpace)); 1268249423Sdim StOps.push_back(getI32Imm(VecType)); 1269249423Sdim StOps.push_back(getI32Imm(ToType)); 1270249423Sdim StOps.push_back(getI32Imm(ToTypeWidth)); 1271249423Sdim 1272249423Sdim if (SelectDirectAddr(N2, Addr)) { 1273249423Sdim switch (N->getOpcode()) { 1274249423Sdim default: 1275249423Sdim return NULL; 1276249423Sdim case NVPTXISD::StoreV2: 1277249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1278249423Sdim default: 1279249423Sdim return NULL; 1280249423Sdim case MVT::i8: 1281249423Sdim Opcode = NVPTX::STV_i8_v2_avar; 1282249423Sdim break; 1283249423Sdim case MVT::i16: 1284249423Sdim Opcode = NVPTX::STV_i16_v2_avar; 1285249423Sdim break; 1286249423Sdim case MVT::i32: 1287249423Sdim Opcode = NVPTX::STV_i32_v2_avar; 1288249423Sdim break; 1289249423Sdim case MVT::i64: 1290249423Sdim Opcode = NVPTX::STV_i64_v2_avar; 1291249423Sdim break; 1292249423Sdim case MVT::f32: 1293249423Sdim Opcode = NVPTX::STV_f32_v2_avar; 1294249423Sdim break; 1295249423Sdim case MVT::f64: 1296249423Sdim Opcode = NVPTX::STV_f64_v2_avar; 1297249423Sdim break; 1298249423Sdim } 1299249423Sdim break; 1300249423Sdim case NVPTXISD::StoreV4: 1301249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1302249423Sdim default: 1303249423Sdim return NULL; 1304249423Sdim case MVT::i8: 1305249423Sdim Opcode = NVPTX::STV_i8_v4_avar; 1306249423Sdim break; 1307249423Sdim case MVT::i16: 1308249423Sdim Opcode = NVPTX::STV_i16_v4_avar; 1309249423Sdim break; 1310249423Sdim case MVT::i32: 1311249423Sdim Opcode = NVPTX::STV_i32_v4_avar; 1312249423Sdim break; 1313249423Sdim case MVT::f32: 1314249423Sdim Opcode = NVPTX::STV_f32_v4_avar; 1315249423Sdim break; 1316249423Sdim } 1317249423Sdim break; 1318249423Sdim } 1319249423Sdim StOps.push_back(Addr); 1320249423Sdim } else if (Subtarget.is64Bit() 1321249423Sdim ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) 1322249423Sdim : SelectADDRsi(N2.getNode(), N2, Base, Offset)) { 1323249423Sdim switch (N->getOpcode()) { 1324249423Sdim default: 1325249423Sdim return NULL; 1326249423Sdim case NVPTXISD::StoreV2: 1327249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1328249423Sdim default: 1329249423Sdim return NULL; 1330249423Sdim case MVT::i8: 1331249423Sdim Opcode = NVPTX::STV_i8_v2_asi; 1332249423Sdim break; 1333249423Sdim case MVT::i16: 1334249423Sdim Opcode = NVPTX::STV_i16_v2_asi; 1335249423Sdim break; 1336249423Sdim case MVT::i32: 1337249423Sdim Opcode = NVPTX::STV_i32_v2_asi; 1338249423Sdim break; 1339249423Sdim case MVT::i64: 1340249423Sdim Opcode = NVPTX::STV_i64_v2_asi; 1341249423Sdim break; 1342249423Sdim case MVT::f32: 1343249423Sdim Opcode = NVPTX::STV_f32_v2_asi; 1344249423Sdim break; 1345249423Sdim case MVT::f64: 1346249423Sdim Opcode = NVPTX::STV_f64_v2_asi; 1347249423Sdim break; 1348249423Sdim } 1349249423Sdim break; 1350249423Sdim case NVPTXISD::StoreV4: 1351249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1352249423Sdim default: 1353249423Sdim return NULL; 1354249423Sdim case MVT::i8: 1355249423Sdim Opcode = NVPTX::STV_i8_v4_asi; 1356249423Sdim break; 1357249423Sdim case MVT::i16: 1358249423Sdim Opcode = NVPTX::STV_i16_v4_asi; 1359249423Sdim break; 1360249423Sdim case MVT::i32: 1361249423Sdim Opcode = NVPTX::STV_i32_v4_asi; 1362249423Sdim break; 1363249423Sdim case MVT::f32: 1364249423Sdim Opcode = NVPTX::STV_f32_v4_asi; 1365249423Sdim break; 1366249423Sdim } 1367249423Sdim break; 1368249423Sdim } 1369249423Sdim StOps.push_back(Base); 1370249423Sdim StOps.push_back(Offset); 1371249423Sdim } else if (Subtarget.is64Bit() 1372249423Sdim ? SelectADDRri64(N2.getNode(), N2, Base, Offset) 1373249423Sdim : SelectADDRri(N2.getNode(), N2, Base, Offset)) { 1374249423Sdim if (Subtarget.is64Bit()) { 1375249423Sdim switch (N->getOpcode()) { 1376249423Sdim default: 1377249423Sdim return NULL; 1378249423Sdim case NVPTXISD::StoreV2: 1379249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1380249423Sdim default: 1381249423Sdim return NULL; 1382249423Sdim case MVT::i8: 1383249423Sdim Opcode = NVPTX::STV_i8_v2_ari_64; 1384249423Sdim break; 1385249423Sdim case MVT::i16: 1386249423Sdim Opcode = NVPTX::STV_i16_v2_ari_64; 1387249423Sdim break; 1388249423Sdim case MVT::i32: 1389249423Sdim Opcode = NVPTX::STV_i32_v2_ari_64; 1390249423Sdim break; 1391249423Sdim case MVT::i64: 1392249423Sdim Opcode = NVPTX::STV_i64_v2_ari_64; 1393249423Sdim break; 1394249423Sdim case MVT::f32: 1395249423Sdim Opcode = NVPTX::STV_f32_v2_ari_64; 1396249423Sdim break; 1397249423Sdim case MVT::f64: 1398249423Sdim Opcode = NVPTX::STV_f64_v2_ari_64; 1399249423Sdim break; 1400249423Sdim } 1401249423Sdim break; 1402249423Sdim case NVPTXISD::StoreV4: 1403249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1404249423Sdim default: 1405249423Sdim return NULL; 1406249423Sdim case MVT::i8: 1407249423Sdim Opcode = NVPTX::STV_i8_v4_ari_64; 1408249423Sdim break; 1409249423Sdim case MVT::i16: 1410249423Sdim Opcode = NVPTX::STV_i16_v4_ari_64; 1411249423Sdim break; 1412249423Sdim case MVT::i32: 1413249423Sdim Opcode = NVPTX::STV_i32_v4_ari_64; 1414249423Sdim break; 1415249423Sdim case MVT::f32: 1416249423Sdim Opcode = NVPTX::STV_f32_v4_ari_64; 1417249423Sdim break; 1418249423Sdim } 1419249423Sdim break; 1420249423Sdim } 1421249423Sdim } else { 1422249423Sdim switch (N->getOpcode()) { 1423249423Sdim default: 1424249423Sdim return NULL; 1425249423Sdim case NVPTXISD::StoreV2: 1426249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1427249423Sdim default: 1428249423Sdim return NULL; 1429249423Sdim case MVT::i8: 1430249423Sdim Opcode = NVPTX::STV_i8_v2_ari; 1431249423Sdim break; 1432249423Sdim case MVT::i16: 1433249423Sdim Opcode = NVPTX::STV_i16_v2_ari; 1434249423Sdim break; 1435249423Sdim case MVT::i32: 1436249423Sdim Opcode = NVPTX::STV_i32_v2_ari; 1437249423Sdim break; 1438249423Sdim case MVT::i64: 1439249423Sdim Opcode = NVPTX::STV_i64_v2_ari; 1440249423Sdim break; 1441249423Sdim case MVT::f32: 1442249423Sdim Opcode = NVPTX::STV_f32_v2_ari; 1443249423Sdim break; 1444249423Sdim case MVT::f64: 1445249423Sdim Opcode = NVPTX::STV_f64_v2_ari; 1446249423Sdim break; 1447249423Sdim } 1448249423Sdim break; 1449249423Sdim case NVPTXISD::StoreV4: 1450249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1451249423Sdim default: 1452249423Sdim return NULL; 1453249423Sdim case MVT::i8: 1454249423Sdim Opcode = NVPTX::STV_i8_v4_ari; 1455249423Sdim break; 1456249423Sdim case MVT::i16: 1457249423Sdim Opcode = NVPTX::STV_i16_v4_ari; 1458249423Sdim break; 1459249423Sdim case MVT::i32: 1460249423Sdim Opcode = NVPTX::STV_i32_v4_ari; 1461249423Sdim break; 1462249423Sdim case MVT::f32: 1463249423Sdim Opcode = NVPTX::STV_f32_v4_ari; 1464249423Sdim break; 1465249423Sdim } 1466249423Sdim break; 1467249423Sdim } 1468249423Sdim } 1469249423Sdim StOps.push_back(Base); 1470249423Sdim StOps.push_back(Offset); 1471249423Sdim } else { 1472249423Sdim if (Subtarget.is64Bit()) { 1473249423Sdim switch (N->getOpcode()) { 1474249423Sdim default: 1475249423Sdim return NULL; 1476249423Sdim case NVPTXISD::StoreV2: 1477249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1478249423Sdim default: 1479249423Sdim return NULL; 1480249423Sdim case MVT::i8: 1481249423Sdim Opcode = NVPTX::STV_i8_v2_areg_64; 1482249423Sdim break; 1483249423Sdim case MVT::i16: 1484249423Sdim Opcode = NVPTX::STV_i16_v2_areg_64; 1485249423Sdim break; 1486249423Sdim case MVT::i32: 1487249423Sdim Opcode = NVPTX::STV_i32_v2_areg_64; 1488249423Sdim break; 1489249423Sdim case MVT::i64: 1490249423Sdim Opcode = NVPTX::STV_i64_v2_areg_64; 1491249423Sdim break; 1492249423Sdim case MVT::f32: 1493249423Sdim Opcode = NVPTX::STV_f32_v2_areg_64; 1494249423Sdim break; 1495249423Sdim case MVT::f64: 1496249423Sdim Opcode = NVPTX::STV_f64_v2_areg_64; 1497249423Sdim break; 1498249423Sdim } 1499249423Sdim break; 1500249423Sdim case NVPTXISD::StoreV4: 1501249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1502249423Sdim default: 1503249423Sdim return NULL; 1504249423Sdim case MVT::i8: 1505249423Sdim Opcode = NVPTX::STV_i8_v4_areg_64; 1506249423Sdim break; 1507249423Sdim case MVT::i16: 1508249423Sdim Opcode = NVPTX::STV_i16_v4_areg_64; 1509249423Sdim break; 1510249423Sdim case MVT::i32: 1511249423Sdim Opcode = NVPTX::STV_i32_v4_areg_64; 1512249423Sdim break; 1513249423Sdim case MVT::f32: 1514249423Sdim Opcode = NVPTX::STV_f32_v4_areg_64; 1515249423Sdim break; 1516249423Sdim } 1517249423Sdim break; 1518249423Sdim } 1519249423Sdim } else { 1520249423Sdim switch (N->getOpcode()) { 1521249423Sdim default: 1522249423Sdim return NULL; 1523249423Sdim case NVPTXISD::StoreV2: 1524249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1525249423Sdim default: 1526249423Sdim return NULL; 1527249423Sdim case MVT::i8: 1528249423Sdim Opcode = NVPTX::STV_i8_v2_areg; 1529249423Sdim break; 1530249423Sdim case MVT::i16: 1531249423Sdim Opcode = NVPTX::STV_i16_v2_areg; 1532249423Sdim break; 1533249423Sdim case MVT::i32: 1534249423Sdim Opcode = NVPTX::STV_i32_v2_areg; 1535249423Sdim break; 1536249423Sdim case MVT::i64: 1537249423Sdim Opcode = NVPTX::STV_i64_v2_areg; 1538249423Sdim break; 1539249423Sdim case MVT::f32: 1540249423Sdim Opcode = NVPTX::STV_f32_v2_areg; 1541249423Sdim break; 1542249423Sdim case MVT::f64: 1543249423Sdim Opcode = NVPTX::STV_f64_v2_areg; 1544249423Sdim break; 1545249423Sdim } 1546249423Sdim break; 1547249423Sdim case NVPTXISD::StoreV4: 1548249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1549249423Sdim default: 1550249423Sdim return NULL; 1551249423Sdim case MVT::i8: 1552249423Sdim Opcode = NVPTX::STV_i8_v4_areg; 1553249423Sdim break; 1554249423Sdim case MVT::i16: 1555249423Sdim Opcode = NVPTX::STV_i16_v4_areg; 1556249423Sdim break; 1557249423Sdim case MVT::i32: 1558249423Sdim Opcode = NVPTX::STV_i32_v4_areg; 1559249423Sdim break; 1560249423Sdim case MVT::f32: 1561249423Sdim Opcode = NVPTX::STV_f32_v4_areg; 1562249423Sdim break; 1563249423Sdim } 1564249423Sdim break; 1565249423Sdim } 1566249423Sdim } 1567249423Sdim StOps.push_back(N2); 1568249423Sdim } 1569249423Sdim 1570249423Sdim StOps.push_back(Chain); 1571249423Sdim 1572249423Sdim ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, &StOps[0], StOps.size()); 1573249423Sdim 1574249423Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 1575249423Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 1576249423Sdim cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1); 1577249423Sdim 1578249423Sdim return ST; 1579249423Sdim} 1580249423Sdim 1581239310Sdim// SelectDirectAddr - Match a direct address for DAG. 1582239310Sdim// A direct address could be a globaladdress or externalsymbol. 1583239310Sdimbool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) { 1584239310Sdim // Return true if TGA or ES. 1585249423Sdim if (N.getOpcode() == ISD::TargetGlobalAddress || 1586249423Sdim N.getOpcode() == ISD::TargetExternalSymbol) { 1587239310Sdim Address = N; 1588239310Sdim return true; 1589239310Sdim } 1590239310Sdim if (N.getOpcode() == NVPTXISD::Wrapper) { 1591239310Sdim Address = N.getOperand(0); 1592239310Sdim return true; 1593239310Sdim } 1594239310Sdim if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) { 1595239310Sdim unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue(); 1596239310Sdim if (IID == Intrinsic::nvvm_ptr_gen_to_param) 1597239310Sdim if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam) 1598239310Sdim return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address)); 1599239310Sdim } 1600239310Sdim return false; 1601239310Sdim} 1602239310Sdim 1603239310Sdim// symbol+offset 1604249423Sdimbool NVPTXDAGToDAGISel::SelectADDRsi_imp( 1605249423Sdim SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { 1606239310Sdim if (Addr.getOpcode() == ISD::ADD) { 1607239310Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) { 1608249423Sdim SDValue base = Addr.getOperand(0); 1609239310Sdim if (SelectDirectAddr(base, Base)) { 1610239310Sdim Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt); 1611239310Sdim return true; 1612239310Sdim } 1613239310Sdim } 1614239310Sdim } 1615239310Sdim return false; 1616239310Sdim} 1617239310Sdim 1618239310Sdim// symbol+offset 1619239310Sdimbool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr, 1620239310Sdim SDValue &Base, SDValue &Offset) { 1621239310Sdim return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32); 1622239310Sdim} 1623239310Sdim 1624239310Sdim// symbol+offset 1625239310Sdimbool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr, 1626239310Sdim SDValue &Base, SDValue &Offset) { 1627239310Sdim return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64); 1628239310Sdim} 1629239310Sdim 1630239310Sdim// register+offset 1631249423Sdimbool NVPTXDAGToDAGISel::SelectADDRri_imp( 1632249423Sdim SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { 1633239310Sdim if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) { 1634239310Sdim Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt); 1635239310Sdim Offset = CurDAG->getTargetConstant(0, mvt); 1636239310Sdim return true; 1637239310Sdim } 1638239310Sdim if (Addr.getOpcode() == ISD::TargetExternalSymbol || 1639239310Sdim Addr.getOpcode() == ISD::TargetGlobalAddress) 1640249423Sdim return false; // direct calls. 1641239310Sdim 1642239310Sdim if (Addr.getOpcode() == ISD::ADD) { 1643239310Sdim if (SelectDirectAddr(Addr.getOperand(0), Addr)) { 1644239310Sdim return false; 1645239310Sdim } 1646239310Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) { 1647239310Sdim if (FrameIndexSDNode *FIN = 1648249423Sdim dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) 1649239310Sdim // Constant offset from frame ref. 1650239310Sdim Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt); 1651239310Sdim else 1652239310Sdim Base = Addr.getOperand(0); 1653239310Sdim Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt); 1654239310Sdim return true; 1655239310Sdim } 1656239310Sdim } 1657239310Sdim return false; 1658239310Sdim} 1659239310Sdim 1660239310Sdim// register+offset 1661239310Sdimbool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr, 1662239310Sdim SDValue &Base, SDValue &Offset) { 1663239310Sdim return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32); 1664239310Sdim} 1665239310Sdim 1666239310Sdim// register+offset 1667239310Sdimbool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr, 1668239310Sdim SDValue &Base, SDValue &Offset) { 1669239310Sdim return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64); 1670239310Sdim} 1671239310Sdim 1672239310Sdimbool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N, 1673239310Sdim unsigned int spN) const { 1674239310Sdim const Value *Src = NULL; 1675239310Sdim // Even though MemIntrinsicSDNode is a subclas of MemSDNode, 1676239310Sdim // the classof() for MemSDNode does not include MemIntrinsicSDNode 1677239310Sdim // (See SelectionDAGNodes.h). So we need to check for both. 1678239310Sdim if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) { 1679239310Sdim Src = mN->getSrcValue(); 1680249423Sdim } else if (MemSDNode *mN = dyn_cast<MemIntrinsicSDNode>(N)) { 1681239310Sdim Src = mN->getSrcValue(); 1682239310Sdim } 1683239310Sdim if (!Src) 1684239310Sdim return false; 1685239310Sdim if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) 1686239310Sdim return (PT->getAddressSpace() == spN); 1687239310Sdim return false; 1688239310Sdim} 1689239310Sdim 1690239310Sdim/// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 1691239310Sdim/// inline asm expressions. 1692249423Sdimbool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand( 1693249423Sdim const SDValue &Op, char ConstraintCode, std::vector<SDValue> &OutOps) { 1694239310Sdim SDValue Op0, Op1; 1695239310Sdim switch (ConstraintCode) { 1696249423Sdim default: 1697249423Sdim return true; 1698249423Sdim case 'm': // memory 1699239310Sdim if (SelectDirectAddr(Op, Op0)) { 1700239310Sdim OutOps.push_back(Op0); 1701239310Sdim OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32)); 1702239310Sdim return false; 1703239310Sdim } 1704239310Sdim if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) { 1705239310Sdim OutOps.push_back(Op0); 1706239310Sdim OutOps.push_back(Op1); 1707239310Sdim return false; 1708239310Sdim } 1709239310Sdim break; 1710239310Sdim } 1711239310Sdim return true; 1712239310Sdim} 1713239310Sdim 1714239310Sdim// Return true if N is a undef or a constant. 1715239310Sdim// If N was undef, return a (i8imm 0) in Retval 1716239310Sdim// If N was imm, convert it to i8imm and return in Retval 1717239310Sdim// Note: The convert to i8imm is required, otherwise the 1718239310Sdim// pattern matcher inserts a bunch of IMOVi8rr to convert 1719239310Sdim// the imm to i8imm, and this causes instruction selection 1720239310Sdim// to fail. 1721249423Sdimbool NVPTXDAGToDAGISel::UndefOrImm(SDValue Op, SDValue N, SDValue &Retval) { 1722249423Sdim if (!(N.getOpcode() == ISD::UNDEF) && !(N.getOpcode() == ISD::Constant)) 1723239310Sdim return false; 1724239310Sdim 1725239310Sdim if (N.getOpcode() == ISD::UNDEF) 1726239310Sdim Retval = CurDAG->getTargetConstant(0, MVT::i8); 1727239310Sdim else { 1728239310Sdim ConstantSDNode *cn = cast<ConstantSDNode>(N.getNode()); 1729239310Sdim unsigned retval = cn->getZExtValue(); 1730239310Sdim Retval = CurDAG->getTargetConstant(retval, MVT::i8); 1731239310Sdim } 1732239310Sdim return true; 1733239310Sdim} 1734