1239310Sdim//===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===// 2239310Sdim// 3239310Sdim// The LLVM Compiler Infrastructure 4239310Sdim// 5239310Sdim// This file is distributed under the University of Illinois Open Source 6239310Sdim// License. See LICENSE.TXT for details. 7239310Sdim// 8239310Sdim//===----------------------------------------------------------------------===// 9239310Sdim// 10239310Sdim// This file defines an instruction selector for the NVPTX target. 11239310Sdim// 12239310Sdim//===----------------------------------------------------------------------===// 13239310Sdim 14239310Sdim#include "NVPTXISelDAGToDAG.h" 15296417Sdim#include "NVPTXUtilities.h" 16296417Sdim#include "llvm/Analysis/ValueTracking.h" 17249423Sdim#include "llvm/IR/GlobalValue.h" 18249423Sdim#include "llvm/IR/Instructions.h" 19249423Sdim#include "llvm/Support/CommandLine.h" 20239310Sdim#include "llvm/Support/Debug.h" 21239310Sdim#include "llvm/Support/ErrorHandling.h" 22249423Sdim#include "llvm/Support/raw_ostream.h" 23239310Sdim#include "llvm/Target/TargetIntrinsicInfo.h" 24239310Sdim 25239310Sdimusing namespace llvm; 26239310Sdim 27276479Sdim#define DEBUG_TYPE "nvptx-isel" 28239310Sdim 29249423Sdimstatic cl::opt<int> UsePrecDivF32( 30261991Sdim "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden, 31249423Sdim cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use" 32280031Sdim " IEEE Compliant F32 div.rnd if available."), 33249423Sdim cl::init(2)); 34239310Sdim 35251662Sdimstatic cl::opt<bool> 36261991SdimUsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden, 37251662Sdim cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."), 38251662Sdim cl::init(true)); 39251662Sdim 40261991Sdimstatic cl::opt<bool> 41261991SdimFtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden, 42261991Sdim cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."), 43261991Sdim cl::init(false)); 44261991Sdim 45261991Sdim 46239310Sdim/// createNVPTXISelDag - This pass converts a legalized DAG into a 47239310Sdim/// NVPTX-specific DAG, ready for instruction scheduling. 48239310SdimFunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM, 49239310Sdim llvm::CodeGenOpt::Level OptLevel) { 50239310Sdim return new NVPTXDAGToDAGISel(TM, OptLevel); 51239310Sdim} 52239310Sdim 53239310SdimNVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm, 54239310Sdim CodeGenOpt::Level OptLevel) 55288943Sdim : SelectionDAGISel(tm, OptLevel), TM(tm) { 56239310Sdim doMulWide = (OptLevel > 0); 57261991Sdim} 58239310Sdim 59288943Sdimbool NVPTXDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { 60288943Sdim Subtarget = &static_cast<const NVPTXSubtarget &>(MF.getSubtarget()); 61288943Sdim return SelectionDAGISel::runOnMachineFunction(MF); 62288943Sdim} 63288943Sdim 64261991Sdimint NVPTXDAGToDAGISel::getDivF32Level() const { 65261991Sdim if (UsePrecDivF32.getNumOccurrences() > 0) { 66261991Sdim // If nvptx-prec-div32=N is used on the command-line, always honor it 67261991Sdim return UsePrecDivF32; 68261991Sdim } else { 69261991Sdim // Otherwise, use div.approx if fast math is enabled 70261991Sdim if (TM.Options.UnsafeFPMath) 71261991Sdim return 0; 72261991Sdim else 73261991Sdim return 2; 74261991Sdim } 75261991Sdim} 76239310Sdim 77261991Sdimbool NVPTXDAGToDAGISel::usePrecSqrtF32() const { 78261991Sdim if (UsePrecSqrtF32.getNumOccurrences() > 0) { 79261991Sdim // If nvptx-prec-sqrtf32 is used on the command-line, always honor it 80261991Sdim return UsePrecSqrtF32; 81261991Sdim } else { 82261991Sdim // Otherwise, use sqrt.approx if fast math is enabled 83288943Sdim return !TM.Options.UnsafeFPMath; 84261991Sdim } 85239310Sdim} 86239310Sdim 87261991Sdimbool NVPTXDAGToDAGISel::useF32FTZ() const { 88261991Sdim if (FtzEnabled.getNumOccurrences() > 0) { 89261991Sdim // If nvptx-f32ftz is used on the command-line, always honor it 90261991Sdim return FtzEnabled; 91261991Sdim } else { 92261991Sdim const Function *F = MF->getFunction(); 93261991Sdim // Otherwise, check for an nvptx-f32ftz attribute on the function 94261991Sdim if (F->hasFnAttribute("nvptx-f32ftz")) 95288943Sdim return F->getFnAttribute("nvptx-f32ftz").getValueAsString() == "true"; 96261991Sdim else 97261991Sdim return false; 98261991Sdim } 99261991Sdim} 100261991Sdim 101276479Sdimbool NVPTXDAGToDAGISel::allowFMA() const { 102288943Sdim const NVPTXTargetLowering *TL = Subtarget->getTargetLowering(); 103276479Sdim return TL->allowFMA(*MF, OptLevel); 104276479Sdim} 105276479Sdim 106239310Sdim/// Select - Select instructions not customized! Used for 107239310Sdim/// expanded, promoted and normal instructions. 108249423SdimSDNode *NVPTXDAGToDAGISel::Select(SDNode *N) { 109239310Sdim 110255804Sdim if (N->isMachineOpcode()) { 111255804Sdim N->setNodeId(-1); 112276479Sdim return nullptr; // Already selected. 113255804Sdim } 114239310Sdim 115276479Sdim SDNode *ResNode = nullptr; 116239310Sdim switch (N->getOpcode()) { 117239310Sdim case ISD::LOAD: 118239310Sdim ResNode = SelectLoad(N); 119239310Sdim break; 120239310Sdim case ISD::STORE: 121239310Sdim ResNode = SelectStore(N); 122239310Sdim break; 123249423Sdim case NVPTXISD::LoadV2: 124249423Sdim case NVPTXISD::LoadV4: 125249423Sdim ResNode = SelectLoadVector(N); 126249423Sdim break; 127249423Sdim case NVPTXISD::LDGV2: 128249423Sdim case NVPTXISD::LDGV4: 129249423Sdim case NVPTXISD::LDUV2: 130249423Sdim case NVPTXISD::LDUV4: 131276479Sdim ResNode = SelectLDGLDU(N); 132249423Sdim break; 133249423Sdim case NVPTXISD::StoreV2: 134249423Sdim case NVPTXISD::StoreV4: 135249423Sdim ResNode = SelectStoreVector(N); 136249423Sdim break; 137261991Sdim case NVPTXISD::LoadParam: 138261991Sdim case NVPTXISD::LoadParamV2: 139261991Sdim case NVPTXISD::LoadParamV4: 140261991Sdim ResNode = SelectLoadParam(N); 141261991Sdim break; 142261991Sdim case NVPTXISD::StoreRetval: 143261991Sdim case NVPTXISD::StoreRetvalV2: 144261991Sdim case NVPTXISD::StoreRetvalV4: 145261991Sdim ResNode = SelectStoreRetval(N); 146261991Sdim break; 147261991Sdim case NVPTXISD::StoreParam: 148261991Sdim case NVPTXISD::StoreParamV2: 149261991Sdim case NVPTXISD::StoreParamV4: 150261991Sdim case NVPTXISD::StoreParamS32: 151261991Sdim case NVPTXISD::StoreParamU32: 152261991Sdim ResNode = SelectStoreParam(N); 153261991Sdim break; 154276479Sdim case ISD::INTRINSIC_WO_CHAIN: 155276479Sdim ResNode = SelectIntrinsicNoChain(N); 156276479Sdim break; 157276479Sdim case ISD::INTRINSIC_W_CHAIN: 158276479Sdim ResNode = SelectIntrinsicChain(N); 159276479Sdim break; 160276479Sdim case NVPTXISD::Tex1DFloatS32: 161276479Sdim case NVPTXISD::Tex1DFloatFloat: 162276479Sdim case NVPTXISD::Tex1DFloatFloatLevel: 163276479Sdim case NVPTXISD::Tex1DFloatFloatGrad: 164276479Sdim case NVPTXISD::Tex1DS32S32: 165276479Sdim case NVPTXISD::Tex1DS32Float: 166276479Sdim case NVPTXISD::Tex1DS32FloatLevel: 167276479Sdim case NVPTXISD::Tex1DS32FloatGrad: 168276479Sdim case NVPTXISD::Tex1DU32S32: 169276479Sdim case NVPTXISD::Tex1DU32Float: 170276479Sdim case NVPTXISD::Tex1DU32FloatLevel: 171276479Sdim case NVPTXISD::Tex1DU32FloatGrad: 172276479Sdim case NVPTXISD::Tex1DArrayFloatS32: 173276479Sdim case NVPTXISD::Tex1DArrayFloatFloat: 174276479Sdim case NVPTXISD::Tex1DArrayFloatFloatLevel: 175276479Sdim case NVPTXISD::Tex1DArrayFloatFloatGrad: 176276479Sdim case NVPTXISD::Tex1DArrayS32S32: 177276479Sdim case NVPTXISD::Tex1DArrayS32Float: 178276479Sdim case NVPTXISD::Tex1DArrayS32FloatLevel: 179276479Sdim case NVPTXISD::Tex1DArrayS32FloatGrad: 180276479Sdim case NVPTXISD::Tex1DArrayU32S32: 181276479Sdim case NVPTXISD::Tex1DArrayU32Float: 182276479Sdim case NVPTXISD::Tex1DArrayU32FloatLevel: 183276479Sdim case NVPTXISD::Tex1DArrayU32FloatGrad: 184276479Sdim case NVPTXISD::Tex2DFloatS32: 185276479Sdim case NVPTXISD::Tex2DFloatFloat: 186276479Sdim case NVPTXISD::Tex2DFloatFloatLevel: 187276479Sdim case NVPTXISD::Tex2DFloatFloatGrad: 188276479Sdim case NVPTXISD::Tex2DS32S32: 189276479Sdim case NVPTXISD::Tex2DS32Float: 190276479Sdim case NVPTXISD::Tex2DS32FloatLevel: 191276479Sdim case NVPTXISD::Tex2DS32FloatGrad: 192276479Sdim case NVPTXISD::Tex2DU32S32: 193276479Sdim case NVPTXISD::Tex2DU32Float: 194276479Sdim case NVPTXISD::Tex2DU32FloatLevel: 195276479Sdim case NVPTXISD::Tex2DU32FloatGrad: 196276479Sdim case NVPTXISD::Tex2DArrayFloatS32: 197276479Sdim case NVPTXISD::Tex2DArrayFloatFloat: 198276479Sdim case NVPTXISD::Tex2DArrayFloatFloatLevel: 199276479Sdim case NVPTXISD::Tex2DArrayFloatFloatGrad: 200276479Sdim case NVPTXISD::Tex2DArrayS32S32: 201276479Sdim case NVPTXISD::Tex2DArrayS32Float: 202276479Sdim case NVPTXISD::Tex2DArrayS32FloatLevel: 203276479Sdim case NVPTXISD::Tex2DArrayS32FloatGrad: 204276479Sdim case NVPTXISD::Tex2DArrayU32S32: 205276479Sdim case NVPTXISD::Tex2DArrayU32Float: 206276479Sdim case NVPTXISD::Tex2DArrayU32FloatLevel: 207276479Sdim case NVPTXISD::Tex2DArrayU32FloatGrad: 208276479Sdim case NVPTXISD::Tex3DFloatS32: 209276479Sdim case NVPTXISD::Tex3DFloatFloat: 210276479Sdim case NVPTXISD::Tex3DFloatFloatLevel: 211276479Sdim case NVPTXISD::Tex3DFloatFloatGrad: 212276479Sdim case NVPTXISD::Tex3DS32S32: 213276479Sdim case NVPTXISD::Tex3DS32Float: 214276479Sdim case NVPTXISD::Tex3DS32FloatLevel: 215276479Sdim case NVPTXISD::Tex3DS32FloatGrad: 216276479Sdim case NVPTXISD::Tex3DU32S32: 217276479Sdim case NVPTXISD::Tex3DU32Float: 218276479Sdim case NVPTXISD::Tex3DU32FloatLevel: 219276479Sdim case NVPTXISD::Tex3DU32FloatGrad: 220276479Sdim case NVPTXISD::TexCubeFloatFloat: 221276479Sdim case NVPTXISD::TexCubeFloatFloatLevel: 222276479Sdim case NVPTXISD::TexCubeS32Float: 223276479Sdim case NVPTXISD::TexCubeS32FloatLevel: 224276479Sdim case NVPTXISD::TexCubeU32Float: 225276479Sdim case NVPTXISD::TexCubeU32FloatLevel: 226276479Sdim case NVPTXISD::TexCubeArrayFloatFloat: 227276479Sdim case NVPTXISD::TexCubeArrayFloatFloatLevel: 228276479Sdim case NVPTXISD::TexCubeArrayS32Float: 229276479Sdim case NVPTXISD::TexCubeArrayS32FloatLevel: 230276479Sdim case NVPTXISD::TexCubeArrayU32Float: 231276479Sdim case NVPTXISD::TexCubeArrayU32FloatLevel: 232276479Sdim case NVPTXISD::Tld4R2DFloatFloat: 233276479Sdim case NVPTXISD::Tld4G2DFloatFloat: 234276479Sdim case NVPTXISD::Tld4B2DFloatFloat: 235276479Sdim case NVPTXISD::Tld4A2DFloatFloat: 236276479Sdim case NVPTXISD::Tld4R2DS64Float: 237276479Sdim case NVPTXISD::Tld4G2DS64Float: 238276479Sdim case NVPTXISD::Tld4B2DS64Float: 239276479Sdim case NVPTXISD::Tld4A2DS64Float: 240276479Sdim case NVPTXISD::Tld4R2DU64Float: 241276479Sdim case NVPTXISD::Tld4G2DU64Float: 242276479Sdim case NVPTXISD::Tld4B2DU64Float: 243276479Sdim case NVPTXISD::Tld4A2DU64Float: 244276479Sdim case NVPTXISD::TexUnified1DFloatS32: 245276479Sdim case NVPTXISD::TexUnified1DFloatFloat: 246276479Sdim case NVPTXISD::TexUnified1DFloatFloatLevel: 247276479Sdim case NVPTXISD::TexUnified1DFloatFloatGrad: 248276479Sdim case NVPTXISD::TexUnified1DS32S32: 249276479Sdim case NVPTXISD::TexUnified1DS32Float: 250276479Sdim case NVPTXISD::TexUnified1DS32FloatLevel: 251276479Sdim case NVPTXISD::TexUnified1DS32FloatGrad: 252276479Sdim case NVPTXISD::TexUnified1DU32S32: 253276479Sdim case NVPTXISD::TexUnified1DU32Float: 254276479Sdim case NVPTXISD::TexUnified1DU32FloatLevel: 255276479Sdim case NVPTXISD::TexUnified1DU32FloatGrad: 256276479Sdim case NVPTXISD::TexUnified1DArrayFloatS32: 257276479Sdim case NVPTXISD::TexUnified1DArrayFloatFloat: 258276479Sdim case NVPTXISD::TexUnified1DArrayFloatFloatLevel: 259276479Sdim case NVPTXISD::TexUnified1DArrayFloatFloatGrad: 260276479Sdim case NVPTXISD::TexUnified1DArrayS32S32: 261276479Sdim case NVPTXISD::TexUnified1DArrayS32Float: 262276479Sdim case NVPTXISD::TexUnified1DArrayS32FloatLevel: 263276479Sdim case NVPTXISD::TexUnified1DArrayS32FloatGrad: 264276479Sdim case NVPTXISD::TexUnified1DArrayU32S32: 265276479Sdim case NVPTXISD::TexUnified1DArrayU32Float: 266276479Sdim case NVPTXISD::TexUnified1DArrayU32FloatLevel: 267276479Sdim case NVPTXISD::TexUnified1DArrayU32FloatGrad: 268276479Sdim case NVPTXISD::TexUnified2DFloatS32: 269276479Sdim case NVPTXISD::TexUnified2DFloatFloat: 270276479Sdim case NVPTXISD::TexUnified2DFloatFloatLevel: 271276479Sdim case NVPTXISD::TexUnified2DFloatFloatGrad: 272276479Sdim case NVPTXISD::TexUnified2DS32S32: 273276479Sdim case NVPTXISD::TexUnified2DS32Float: 274276479Sdim case NVPTXISD::TexUnified2DS32FloatLevel: 275276479Sdim case NVPTXISD::TexUnified2DS32FloatGrad: 276276479Sdim case NVPTXISD::TexUnified2DU32S32: 277276479Sdim case NVPTXISD::TexUnified2DU32Float: 278276479Sdim case NVPTXISD::TexUnified2DU32FloatLevel: 279276479Sdim case NVPTXISD::TexUnified2DU32FloatGrad: 280276479Sdim case NVPTXISD::TexUnified2DArrayFloatS32: 281276479Sdim case NVPTXISD::TexUnified2DArrayFloatFloat: 282276479Sdim case NVPTXISD::TexUnified2DArrayFloatFloatLevel: 283276479Sdim case NVPTXISD::TexUnified2DArrayFloatFloatGrad: 284276479Sdim case NVPTXISD::TexUnified2DArrayS32S32: 285276479Sdim case NVPTXISD::TexUnified2DArrayS32Float: 286276479Sdim case NVPTXISD::TexUnified2DArrayS32FloatLevel: 287276479Sdim case NVPTXISD::TexUnified2DArrayS32FloatGrad: 288276479Sdim case NVPTXISD::TexUnified2DArrayU32S32: 289276479Sdim case NVPTXISD::TexUnified2DArrayU32Float: 290276479Sdim case NVPTXISD::TexUnified2DArrayU32FloatLevel: 291276479Sdim case NVPTXISD::TexUnified2DArrayU32FloatGrad: 292276479Sdim case NVPTXISD::TexUnified3DFloatS32: 293276479Sdim case NVPTXISD::TexUnified3DFloatFloat: 294276479Sdim case NVPTXISD::TexUnified3DFloatFloatLevel: 295276479Sdim case NVPTXISD::TexUnified3DFloatFloatGrad: 296276479Sdim case NVPTXISD::TexUnified3DS32S32: 297276479Sdim case NVPTXISD::TexUnified3DS32Float: 298276479Sdim case NVPTXISD::TexUnified3DS32FloatLevel: 299276479Sdim case NVPTXISD::TexUnified3DS32FloatGrad: 300276479Sdim case NVPTXISD::TexUnified3DU32S32: 301276479Sdim case NVPTXISD::TexUnified3DU32Float: 302276479Sdim case NVPTXISD::TexUnified3DU32FloatLevel: 303276479Sdim case NVPTXISD::TexUnified3DU32FloatGrad: 304276479Sdim case NVPTXISD::TexUnifiedCubeFloatFloat: 305276479Sdim case NVPTXISD::TexUnifiedCubeFloatFloatLevel: 306276479Sdim case NVPTXISD::TexUnifiedCubeS32Float: 307276479Sdim case NVPTXISD::TexUnifiedCubeS32FloatLevel: 308276479Sdim case NVPTXISD::TexUnifiedCubeU32Float: 309276479Sdim case NVPTXISD::TexUnifiedCubeU32FloatLevel: 310276479Sdim case NVPTXISD::TexUnifiedCubeArrayFloatFloat: 311276479Sdim case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel: 312276479Sdim case NVPTXISD::TexUnifiedCubeArrayS32Float: 313276479Sdim case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel: 314276479Sdim case NVPTXISD::TexUnifiedCubeArrayU32Float: 315276479Sdim case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel: 316276479Sdim case NVPTXISD::Tld4UnifiedR2DFloatFloat: 317276479Sdim case NVPTXISD::Tld4UnifiedG2DFloatFloat: 318276479Sdim case NVPTXISD::Tld4UnifiedB2DFloatFloat: 319276479Sdim case NVPTXISD::Tld4UnifiedA2DFloatFloat: 320276479Sdim case NVPTXISD::Tld4UnifiedR2DS64Float: 321276479Sdim case NVPTXISD::Tld4UnifiedG2DS64Float: 322276479Sdim case NVPTXISD::Tld4UnifiedB2DS64Float: 323276479Sdim case NVPTXISD::Tld4UnifiedA2DS64Float: 324276479Sdim case NVPTXISD::Tld4UnifiedR2DU64Float: 325276479Sdim case NVPTXISD::Tld4UnifiedG2DU64Float: 326276479Sdim case NVPTXISD::Tld4UnifiedB2DU64Float: 327276479Sdim case NVPTXISD::Tld4UnifiedA2DU64Float: 328276479Sdim ResNode = SelectTextureIntrinsic(N); 329276479Sdim break; 330276479Sdim case NVPTXISD::Suld1DI8Clamp: 331276479Sdim case NVPTXISD::Suld1DI16Clamp: 332276479Sdim case NVPTXISD::Suld1DI32Clamp: 333276479Sdim case NVPTXISD::Suld1DI64Clamp: 334276479Sdim case NVPTXISD::Suld1DV2I8Clamp: 335276479Sdim case NVPTXISD::Suld1DV2I16Clamp: 336276479Sdim case NVPTXISD::Suld1DV2I32Clamp: 337276479Sdim case NVPTXISD::Suld1DV2I64Clamp: 338276479Sdim case NVPTXISD::Suld1DV4I8Clamp: 339276479Sdim case NVPTXISD::Suld1DV4I16Clamp: 340276479Sdim case NVPTXISD::Suld1DV4I32Clamp: 341276479Sdim case NVPTXISD::Suld1DArrayI8Clamp: 342276479Sdim case NVPTXISD::Suld1DArrayI16Clamp: 343276479Sdim case NVPTXISD::Suld1DArrayI32Clamp: 344276479Sdim case NVPTXISD::Suld1DArrayI64Clamp: 345276479Sdim case NVPTXISD::Suld1DArrayV2I8Clamp: 346276479Sdim case NVPTXISD::Suld1DArrayV2I16Clamp: 347276479Sdim case NVPTXISD::Suld1DArrayV2I32Clamp: 348276479Sdim case NVPTXISD::Suld1DArrayV2I64Clamp: 349276479Sdim case NVPTXISD::Suld1DArrayV4I8Clamp: 350276479Sdim case NVPTXISD::Suld1DArrayV4I16Clamp: 351276479Sdim case NVPTXISD::Suld1DArrayV4I32Clamp: 352276479Sdim case NVPTXISD::Suld2DI8Clamp: 353276479Sdim case NVPTXISD::Suld2DI16Clamp: 354276479Sdim case NVPTXISD::Suld2DI32Clamp: 355276479Sdim case NVPTXISD::Suld2DI64Clamp: 356276479Sdim case NVPTXISD::Suld2DV2I8Clamp: 357276479Sdim case NVPTXISD::Suld2DV2I16Clamp: 358276479Sdim case NVPTXISD::Suld2DV2I32Clamp: 359276479Sdim case NVPTXISD::Suld2DV2I64Clamp: 360276479Sdim case NVPTXISD::Suld2DV4I8Clamp: 361276479Sdim case NVPTXISD::Suld2DV4I16Clamp: 362276479Sdim case NVPTXISD::Suld2DV4I32Clamp: 363276479Sdim case NVPTXISD::Suld2DArrayI8Clamp: 364276479Sdim case NVPTXISD::Suld2DArrayI16Clamp: 365276479Sdim case NVPTXISD::Suld2DArrayI32Clamp: 366276479Sdim case NVPTXISD::Suld2DArrayI64Clamp: 367276479Sdim case NVPTXISD::Suld2DArrayV2I8Clamp: 368276479Sdim case NVPTXISD::Suld2DArrayV2I16Clamp: 369276479Sdim case NVPTXISD::Suld2DArrayV2I32Clamp: 370276479Sdim case NVPTXISD::Suld2DArrayV2I64Clamp: 371276479Sdim case NVPTXISD::Suld2DArrayV4I8Clamp: 372276479Sdim case NVPTXISD::Suld2DArrayV4I16Clamp: 373276479Sdim case NVPTXISD::Suld2DArrayV4I32Clamp: 374276479Sdim case NVPTXISD::Suld3DI8Clamp: 375276479Sdim case NVPTXISD::Suld3DI16Clamp: 376276479Sdim case NVPTXISD::Suld3DI32Clamp: 377276479Sdim case NVPTXISD::Suld3DI64Clamp: 378276479Sdim case NVPTXISD::Suld3DV2I8Clamp: 379276479Sdim case NVPTXISD::Suld3DV2I16Clamp: 380276479Sdim case NVPTXISD::Suld3DV2I32Clamp: 381276479Sdim case NVPTXISD::Suld3DV2I64Clamp: 382276479Sdim case NVPTXISD::Suld3DV4I8Clamp: 383276479Sdim case NVPTXISD::Suld3DV4I16Clamp: 384276479Sdim case NVPTXISD::Suld3DV4I32Clamp: 385276479Sdim case NVPTXISD::Suld1DI8Trap: 386276479Sdim case NVPTXISD::Suld1DI16Trap: 387276479Sdim case NVPTXISD::Suld1DI32Trap: 388276479Sdim case NVPTXISD::Suld1DI64Trap: 389276479Sdim case NVPTXISD::Suld1DV2I8Trap: 390276479Sdim case NVPTXISD::Suld1DV2I16Trap: 391276479Sdim case NVPTXISD::Suld1DV2I32Trap: 392276479Sdim case NVPTXISD::Suld1DV2I64Trap: 393276479Sdim case NVPTXISD::Suld1DV4I8Trap: 394276479Sdim case NVPTXISD::Suld1DV4I16Trap: 395276479Sdim case NVPTXISD::Suld1DV4I32Trap: 396276479Sdim case NVPTXISD::Suld1DArrayI8Trap: 397276479Sdim case NVPTXISD::Suld1DArrayI16Trap: 398276479Sdim case NVPTXISD::Suld1DArrayI32Trap: 399276479Sdim case NVPTXISD::Suld1DArrayI64Trap: 400276479Sdim case NVPTXISD::Suld1DArrayV2I8Trap: 401276479Sdim case NVPTXISD::Suld1DArrayV2I16Trap: 402276479Sdim case NVPTXISD::Suld1DArrayV2I32Trap: 403276479Sdim case NVPTXISD::Suld1DArrayV2I64Trap: 404276479Sdim case NVPTXISD::Suld1DArrayV4I8Trap: 405276479Sdim case NVPTXISD::Suld1DArrayV4I16Trap: 406276479Sdim case NVPTXISD::Suld1DArrayV4I32Trap: 407276479Sdim case NVPTXISD::Suld2DI8Trap: 408276479Sdim case NVPTXISD::Suld2DI16Trap: 409276479Sdim case NVPTXISD::Suld2DI32Trap: 410276479Sdim case NVPTXISD::Suld2DI64Trap: 411276479Sdim case NVPTXISD::Suld2DV2I8Trap: 412276479Sdim case NVPTXISD::Suld2DV2I16Trap: 413276479Sdim case NVPTXISD::Suld2DV2I32Trap: 414276479Sdim case NVPTXISD::Suld2DV2I64Trap: 415276479Sdim case NVPTXISD::Suld2DV4I8Trap: 416276479Sdim case NVPTXISD::Suld2DV4I16Trap: 417276479Sdim case NVPTXISD::Suld2DV4I32Trap: 418276479Sdim case NVPTXISD::Suld2DArrayI8Trap: 419276479Sdim case NVPTXISD::Suld2DArrayI16Trap: 420276479Sdim case NVPTXISD::Suld2DArrayI32Trap: 421276479Sdim case NVPTXISD::Suld2DArrayI64Trap: 422276479Sdim case NVPTXISD::Suld2DArrayV2I8Trap: 423276479Sdim case NVPTXISD::Suld2DArrayV2I16Trap: 424276479Sdim case NVPTXISD::Suld2DArrayV2I32Trap: 425276479Sdim case NVPTXISD::Suld2DArrayV2I64Trap: 426276479Sdim case NVPTXISD::Suld2DArrayV4I8Trap: 427276479Sdim case NVPTXISD::Suld2DArrayV4I16Trap: 428276479Sdim case NVPTXISD::Suld2DArrayV4I32Trap: 429276479Sdim case NVPTXISD::Suld3DI8Trap: 430276479Sdim case NVPTXISD::Suld3DI16Trap: 431276479Sdim case NVPTXISD::Suld3DI32Trap: 432276479Sdim case NVPTXISD::Suld3DI64Trap: 433276479Sdim case NVPTXISD::Suld3DV2I8Trap: 434276479Sdim case NVPTXISD::Suld3DV2I16Trap: 435276479Sdim case NVPTXISD::Suld3DV2I32Trap: 436276479Sdim case NVPTXISD::Suld3DV2I64Trap: 437276479Sdim case NVPTXISD::Suld3DV4I8Trap: 438276479Sdim case NVPTXISD::Suld3DV4I16Trap: 439276479Sdim case NVPTXISD::Suld3DV4I32Trap: 440276479Sdim case NVPTXISD::Suld1DI8Zero: 441276479Sdim case NVPTXISD::Suld1DI16Zero: 442276479Sdim case NVPTXISD::Suld1DI32Zero: 443276479Sdim case NVPTXISD::Suld1DI64Zero: 444276479Sdim case NVPTXISD::Suld1DV2I8Zero: 445276479Sdim case NVPTXISD::Suld1DV2I16Zero: 446276479Sdim case NVPTXISD::Suld1DV2I32Zero: 447276479Sdim case NVPTXISD::Suld1DV2I64Zero: 448276479Sdim case NVPTXISD::Suld1DV4I8Zero: 449276479Sdim case NVPTXISD::Suld1DV4I16Zero: 450276479Sdim case NVPTXISD::Suld1DV4I32Zero: 451276479Sdim case NVPTXISD::Suld1DArrayI8Zero: 452276479Sdim case NVPTXISD::Suld1DArrayI16Zero: 453276479Sdim case NVPTXISD::Suld1DArrayI32Zero: 454276479Sdim case NVPTXISD::Suld1DArrayI64Zero: 455276479Sdim case NVPTXISD::Suld1DArrayV2I8Zero: 456276479Sdim case NVPTXISD::Suld1DArrayV2I16Zero: 457276479Sdim case NVPTXISD::Suld1DArrayV2I32Zero: 458276479Sdim case NVPTXISD::Suld1DArrayV2I64Zero: 459276479Sdim case NVPTXISD::Suld1DArrayV4I8Zero: 460276479Sdim case NVPTXISD::Suld1DArrayV4I16Zero: 461276479Sdim case NVPTXISD::Suld1DArrayV4I32Zero: 462276479Sdim case NVPTXISD::Suld2DI8Zero: 463276479Sdim case NVPTXISD::Suld2DI16Zero: 464276479Sdim case NVPTXISD::Suld2DI32Zero: 465276479Sdim case NVPTXISD::Suld2DI64Zero: 466276479Sdim case NVPTXISD::Suld2DV2I8Zero: 467276479Sdim case NVPTXISD::Suld2DV2I16Zero: 468276479Sdim case NVPTXISD::Suld2DV2I32Zero: 469276479Sdim case NVPTXISD::Suld2DV2I64Zero: 470276479Sdim case NVPTXISD::Suld2DV4I8Zero: 471276479Sdim case NVPTXISD::Suld2DV4I16Zero: 472276479Sdim case NVPTXISD::Suld2DV4I32Zero: 473276479Sdim case NVPTXISD::Suld2DArrayI8Zero: 474276479Sdim case NVPTXISD::Suld2DArrayI16Zero: 475276479Sdim case NVPTXISD::Suld2DArrayI32Zero: 476276479Sdim case NVPTXISD::Suld2DArrayI64Zero: 477276479Sdim case NVPTXISD::Suld2DArrayV2I8Zero: 478276479Sdim case NVPTXISD::Suld2DArrayV2I16Zero: 479276479Sdim case NVPTXISD::Suld2DArrayV2I32Zero: 480276479Sdim case NVPTXISD::Suld2DArrayV2I64Zero: 481276479Sdim case NVPTXISD::Suld2DArrayV4I8Zero: 482276479Sdim case NVPTXISD::Suld2DArrayV4I16Zero: 483276479Sdim case NVPTXISD::Suld2DArrayV4I32Zero: 484276479Sdim case NVPTXISD::Suld3DI8Zero: 485276479Sdim case NVPTXISD::Suld3DI16Zero: 486276479Sdim case NVPTXISD::Suld3DI32Zero: 487276479Sdim case NVPTXISD::Suld3DI64Zero: 488276479Sdim case NVPTXISD::Suld3DV2I8Zero: 489276479Sdim case NVPTXISD::Suld3DV2I16Zero: 490276479Sdim case NVPTXISD::Suld3DV2I32Zero: 491276479Sdim case NVPTXISD::Suld3DV2I64Zero: 492276479Sdim case NVPTXISD::Suld3DV4I8Zero: 493276479Sdim case NVPTXISD::Suld3DV4I16Zero: 494276479Sdim case NVPTXISD::Suld3DV4I32Zero: 495276479Sdim ResNode = SelectSurfaceIntrinsic(N); 496276479Sdim break; 497276479Sdim case ISD::AND: 498276479Sdim case ISD::SRA: 499276479Sdim case ISD::SRL: 500276479Sdim // Try to select BFE 501276479Sdim ResNode = SelectBFE(N); 502276479Sdim break; 503276479Sdim case ISD::ADDRSPACECAST: 504276479Sdim ResNode = SelectAddrSpaceCast(N); 505276479Sdim break; 506249423Sdim default: 507249423Sdim break; 508239310Sdim } 509239310Sdim if (ResNode) 510239310Sdim return ResNode; 511239310Sdim return SelectCode(N); 512239310Sdim} 513239310Sdim 514276479SdimSDNode *NVPTXDAGToDAGISel::SelectIntrinsicChain(SDNode *N) { 515276479Sdim unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 516276479Sdim switch (IID) { 517276479Sdim default: 518276479Sdim return NULL; 519276479Sdim case Intrinsic::nvvm_ldg_global_f: 520276479Sdim case Intrinsic::nvvm_ldg_global_i: 521276479Sdim case Intrinsic::nvvm_ldg_global_p: 522276479Sdim case Intrinsic::nvvm_ldu_global_f: 523276479Sdim case Intrinsic::nvvm_ldu_global_i: 524276479Sdim case Intrinsic::nvvm_ldu_global_p: 525276479Sdim return SelectLDGLDU(N); 526276479Sdim } 527276479Sdim} 528276479Sdim 529288943Sdimstatic unsigned int getCodeAddrSpace(MemSDNode *N) { 530276479Sdim const Value *Src = N->getMemOperand()->getValue(); 531261991Sdim 532239310Sdim if (!Src) 533261991Sdim return NVPTX::PTXLdStInstCode::GENERIC; 534239310Sdim 535296417Sdim if (auto *PT = dyn_cast<PointerType>(Src->getType())) { 536239310Sdim switch (PT->getAddressSpace()) { 537261991Sdim case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL; 538261991Sdim case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL; 539261991Sdim case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED; 540261991Sdim case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC; 541261991Sdim case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM; 542261991Sdim case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT; 543261991Sdim default: break; 544239310Sdim } 545239310Sdim } 546261991Sdim return NVPTX::PTXLdStInstCode::GENERIC; 547239310Sdim} 548239310Sdim 549296417Sdimstatic bool canLowerToLDG(MemSDNode *N, const NVPTXSubtarget &Subtarget, 550296417Sdim unsigned CodeAddrSpace, MachineFunction *F) { 551296417Sdim // To use non-coherent caching, the load has to be from global 552296417Sdim // memory and we have to prove that the memory area is not written 553296417Sdim // to anywhere for the duration of the kernel call, not even after 554296417Sdim // the load. 555296417Sdim // 556296417Sdim // To ensure that there are no writes to the memory, we require the 557296417Sdim // underlying pointer to be a noalias (__restrict) kernel parameter 558296417Sdim // that is never used for a write. We can only do this for kernel 559296417Sdim // functions since from within a device function, we cannot know if 560296417Sdim // there were or will be writes to the memory from the caller - or we 561296417Sdim // could, but then we would have to do inter-procedural analysis. 562296417Sdim if (!Subtarget.hasLDG() || CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL || 563296417Sdim !isKernelFunction(*F->getFunction())) { 564296417Sdim return false; 565296417Sdim } 566296417Sdim 567296417Sdim // We use GetUnderlyingObjects() here instead of 568296417Sdim // GetUnderlyingObject() mainly because the former looks through phi 569296417Sdim // nodes while the latter does not. We need to look through phi 570296417Sdim // nodes to handle pointer induction variables. 571296417Sdim SmallVector<Value *, 8> Objs; 572296417Sdim GetUnderlyingObjects(const_cast<Value *>(N->getMemOperand()->getValue()), 573296417Sdim Objs, F->getDataLayout()); 574296417Sdim for (Value *Obj : Objs) { 575296417Sdim auto *A = dyn_cast<const Argument>(Obj); 576296417Sdim if (!A || !A->onlyReadsMemory() || !A->hasNoAliasAttr()) return false; 577296417Sdim } 578296417Sdim 579296417Sdim return true; 580296417Sdim} 581296417Sdim 582276479SdimSDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) { 583276479Sdim unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 584276479Sdim switch (IID) { 585276479Sdim default: 586276479Sdim return nullptr; 587276479Sdim case Intrinsic::nvvm_texsurf_handle_internal: 588276479Sdim return SelectTexSurfHandle(N); 589276479Sdim } 590276479Sdim} 591276479Sdim 592276479SdimSDNode *NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) { 593276479Sdim // Op 0 is the intrinsic ID 594276479Sdim SDValue Wrapper = N->getOperand(1); 595276479Sdim SDValue GlobalVal = Wrapper.getOperand(0); 596276479Sdim return CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N), MVT::i64, 597276479Sdim GlobalVal); 598276479Sdim} 599276479Sdim 600276479SdimSDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) { 601276479Sdim SDValue Src = N->getOperand(0); 602276479Sdim AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N); 603276479Sdim unsigned SrcAddrSpace = CastN->getSrcAddressSpace(); 604276479Sdim unsigned DstAddrSpace = CastN->getDestAddressSpace(); 605276479Sdim 606276479Sdim assert(SrcAddrSpace != DstAddrSpace && 607276479Sdim "addrspacecast must be between different address spaces"); 608276479Sdim 609276479Sdim if (DstAddrSpace == ADDRESS_SPACE_GENERIC) { 610276479Sdim // Specific to generic 611276479Sdim unsigned Opc; 612276479Sdim switch (SrcAddrSpace) { 613276479Sdim default: report_fatal_error("Bad address space in addrspacecast"); 614276479Sdim case ADDRESS_SPACE_GLOBAL: 615288943Sdim Opc = TM.is64Bit() ? NVPTX::cvta_global_yes_64 : NVPTX::cvta_global_yes; 616276479Sdim break; 617276479Sdim case ADDRESS_SPACE_SHARED: 618288943Sdim Opc = TM.is64Bit() ? NVPTX::cvta_shared_yes_64 : NVPTX::cvta_shared_yes; 619276479Sdim break; 620276479Sdim case ADDRESS_SPACE_CONST: 621288943Sdim Opc = TM.is64Bit() ? NVPTX::cvta_const_yes_64 : NVPTX::cvta_const_yes; 622276479Sdim break; 623276479Sdim case ADDRESS_SPACE_LOCAL: 624288943Sdim Opc = TM.is64Bit() ? NVPTX::cvta_local_yes_64 : NVPTX::cvta_local_yes; 625276479Sdim break; 626276479Sdim } 627276479Sdim return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src); 628276479Sdim } else { 629276479Sdim // Generic to specific 630276479Sdim if (SrcAddrSpace != 0) 631276479Sdim report_fatal_error("Cannot cast between two non-generic address spaces"); 632276479Sdim unsigned Opc; 633276479Sdim switch (DstAddrSpace) { 634276479Sdim default: report_fatal_error("Bad address space in addrspacecast"); 635276479Sdim case ADDRESS_SPACE_GLOBAL: 636288943Sdim Opc = TM.is64Bit() ? NVPTX::cvta_to_global_yes_64 637288943Sdim : NVPTX::cvta_to_global_yes; 638276479Sdim break; 639276479Sdim case ADDRESS_SPACE_SHARED: 640288943Sdim Opc = TM.is64Bit() ? NVPTX::cvta_to_shared_yes_64 641288943Sdim : NVPTX::cvta_to_shared_yes; 642276479Sdim break; 643276479Sdim case ADDRESS_SPACE_CONST: 644288943Sdim Opc = 645288943Sdim TM.is64Bit() ? NVPTX::cvta_to_const_yes_64 : NVPTX::cvta_to_const_yes; 646276479Sdim break; 647276479Sdim case ADDRESS_SPACE_LOCAL: 648288943Sdim Opc = 649288943Sdim TM.is64Bit() ? NVPTX::cvta_to_local_yes_64 : NVPTX::cvta_to_local_yes; 650276479Sdim break; 651288943Sdim case ADDRESS_SPACE_PARAM: 652288943Sdim Opc = TM.is64Bit() ? NVPTX::nvvm_ptr_gen_to_param_64 653288943Sdim : NVPTX::nvvm_ptr_gen_to_param; 654288943Sdim break; 655276479Sdim } 656276479Sdim return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src); 657276479Sdim } 658276479Sdim} 659276479Sdim 660249423SdimSDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) { 661261991Sdim SDLoc dl(N); 662239310Sdim LoadSDNode *LD = cast<LoadSDNode>(N); 663239310Sdim EVT LoadedVT = LD->getMemoryVT(); 664276479Sdim SDNode *NVPTXLD = nullptr; 665239310Sdim 666239310Sdim // do not support pre/post inc/dec 667239310Sdim if (LD->isIndexed()) 668276479Sdim return nullptr; 669239310Sdim 670239310Sdim if (!LoadedVT.isSimple()) 671276479Sdim return nullptr; 672239310Sdim 673239310Sdim // Address Space Setting 674288943Sdim unsigned int codeAddrSpace = getCodeAddrSpace(LD); 675239310Sdim 676296417Sdim if (canLowerToLDG(LD, *Subtarget, codeAddrSpace, MF)) { 677296417Sdim return SelectLDGLDU(N); 678296417Sdim } 679296417Sdim 680239310Sdim // Volatile Setting 681239310Sdim // - .volatile is only availalble for .global and .shared 682239310Sdim bool isVolatile = LD->isVolatile(); 683239310Sdim if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 684239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 685239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 686239310Sdim isVolatile = false; 687239310Sdim 688239310Sdim // Vector Setting 689239310Sdim MVT SimpleVT = LoadedVT.getSimpleVT(); 690239310Sdim unsigned vecType = NVPTX::PTXLdStInstCode::Scalar; 691239310Sdim if (SimpleVT.isVector()) { 692239310Sdim unsigned num = SimpleVT.getVectorNumElements(); 693239310Sdim if (num == 2) 694239310Sdim vecType = NVPTX::PTXLdStInstCode::V2; 695239310Sdim else if (num == 4) 696239310Sdim vecType = NVPTX::PTXLdStInstCode::V4; 697239310Sdim else 698276479Sdim return nullptr; 699239310Sdim } 700239310Sdim 701239310Sdim // Type Setting: fromType + fromTypeWidth 702239310Sdim // 703239310Sdim // Sign : ISD::SEXTLOAD 704239310Sdim // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the 705239310Sdim // type is integer 706239310Sdim // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float 707239310Sdim MVT ScalarVT = SimpleVT.getScalarType(); 708261991Sdim // Read at least 8 bits (predicates are stored as 8-bit values) 709261991Sdim unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits()); 710239310Sdim unsigned int fromType; 711239310Sdim if ((LD->getExtensionType() == ISD::SEXTLOAD)) 712239310Sdim fromType = NVPTX::PTXLdStInstCode::Signed; 713239310Sdim else if (ScalarVT.isFloatingPoint()) 714239310Sdim fromType = NVPTX::PTXLdStInstCode::Float; 715239310Sdim else 716239310Sdim fromType = NVPTX::PTXLdStInstCode::Unsigned; 717239310Sdim 718239310Sdim // Create the machine instruction DAG 719239310Sdim SDValue Chain = N->getOperand(0); 720239310Sdim SDValue N1 = N->getOperand(1); 721239310Sdim SDValue Addr; 722239310Sdim SDValue Offset, Base; 723239310Sdim unsigned Opcode; 724261991Sdim MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy; 725239310Sdim 726239310Sdim if (SelectDirectAddr(N1, Addr)) { 727239310Sdim switch (TargetVT) { 728249423Sdim case MVT::i8: 729249423Sdim Opcode = NVPTX::LD_i8_avar; 730249423Sdim break; 731249423Sdim case MVT::i16: 732249423Sdim Opcode = NVPTX::LD_i16_avar; 733249423Sdim break; 734249423Sdim case MVT::i32: 735249423Sdim Opcode = NVPTX::LD_i32_avar; 736249423Sdim break; 737249423Sdim case MVT::i64: 738249423Sdim Opcode = NVPTX::LD_i64_avar; 739249423Sdim break; 740249423Sdim case MVT::f32: 741249423Sdim Opcode = NVPTX::LD_f32_avar; 742249423Sdim break; 743249423Sdim case MVT::f64: 744249423Sdim Opcode = NVPTX::LD_f64_avar; 745249423Sdim break; 746249423Sdim default: 747276479Sdim return nullptr; 748239310Sdim } 749288943Sdim SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl), 750288943Sdim getI32Imm(vecType, dl), getI32Imm(fromType, dl), 751288943Sdim getI32Imm(fromTypeWidth, dl), Addr, Chain }; 752251662Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 753288943Sdim } else if (TM.is64Bit() ? SelectADDRsi64(N1.getNode(), N1, Base, Offset) 754288943Sdim : SelectADDRsi(N1.getNode(), N1, Base, Offset)) { 755239310Sdim switch (TargetVT) { 756249423Sdim case MVT::i8: 757249423Sdim Opcode = NVPTX::LD_i8_asi; 758249423Sdim break; 759249423Sdim case MVT::i16: 760249423Sdim Opcode = NVPTX::LD_i16_asi; 761249423Sdim break; 762249423Sdim case MVT::i32: 763249423Sdim Opcode = NVPTX::LD_i32_asi; 764249423Sdim break; 765249423Sdim case MVT::i64: 766249423Sdim Opcode = NVPTX::LD_i64_asi; 767249423Sdim break; 768249423Sdim case MVT::f32: 769249423Sdim Opcode = NVPTX::LD_f32_asi; 770249423Sdim break; 771249423Sdim case MVT::f64: 772249423Sdim Opcode = NVPTX::LD_f64_asi; 773249423Sdim break; 774249423Sdim default: 775276479Sdim return nullptr; 776239310Sdim } 777288943Sdim SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl), 778288943Sdim getI32Imm(vecType, dl), getI32Imm(fromType, dl), 779288943Sdim getI32Imm(fromTypeWidth, dl), Base, Offset, Chain }; 780251662Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 781288943Sdim } else if (TM.is64Bit() ? SelectADDRri64(N1.getNode(), N1, Base, Offset) 782288943Sdim : SelectADDRri(N1.getNode(), N1, Base, Offset)) { 783288943Sdim if (TM.is64Bit()) { 784249423Sdim switch (TargetVT) { 785249423Sdim case MVT::i8: 786249423Sdim Opcode = NVPTX::LD_i8_ari_64; 787249423Sdim break; 788249423Sdim case MVT::i16: 789249423Sdim Opcode = NVPTX::LD_i16_ari_64; 790249423Sdim break; 791249423Sdim case MVT::i32: 792249423Sdim Opcode = NVPTX::LD_i32_ari_64; 793249423Sdim break; 794249423Sdim case MVT::i64: 795249423Sdim Opcode = NVPTX::LD_i64_ari_64; 796249423Sdim break; 797249423Sdim case MVT::f32: 798249423Sdim Opcode = NVPTX::LD_f32_ari_64; 799249423Sdim break; 800249423Sdim case MVT::f64: 801249423Sdim Opcode = NVPTX::LD_f64_ari_64; 802249423Sdim break; 803249423Sdim default: 804276479Sdim return nullptr; 805249423Sdim } 806249423Sdim } else { 807249423Sdim switch (TargetVT) { 808249423Sdim case MVT::i8: 809249423Sdim Opcode = NVPTX::LD_i8_ari; 810249423Sdim break; 811249423Sdim case MVT::i16: 812249423Sdim Opcode = NVPTX::LD_i16_ari; 813249423Sdim break; 814249423Sdim case MVT::i32: 815249423Sdim Opcode = NVPTX::LD_i32_ari; 816249423Sdim break; 817249423Sdim case MVT::i64: 818249423Sdim Opcode = NVPTX::LD_i64_ari; 819249423Sdim break; 820249423Sdim case MVT::f32: 821249423Sdim Opcode = NVPTX::LD_f32_ari; 822249423Sdim break; 823249423Sdim case MVT::f64: 824249423Sdim Opcode = NVPTX::LD_f64_ari; 825249423Sdim break; 826249423Sdim default: 827276479Sdim return nullptr; 828249423Sdim } 829239310Sdim } 830288943Sdim SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl), 831288943Sdim getI32Imm(vecType, dl), getI32Imm(fromType, dl), 832288943Sdim getI32Imm(fromTypeWidth, dl), Base, Offset, Chain }; 833251662Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 834249423Sdim } else { 835288943Sdim if (TM.is64Bit()) { 836249423Sdim switch (TargetVT) { 837249423Sdim case MVT::i8: 838249423Sdim Opcode = NVPTX::LD_i8_areg_64; 839249423Sdim break; 840249423Sdim case MVT::i16: 841249423Sdim Opcode = NVPTX::LD_i16_areg_64; 842249423Sdim break; 843249423Sdim case MVT::i32: 844249423Sdim Opcode = NVPTX::LD_i32_areg_64; 845249423Sdim break; 846249423Sdim case MVT::i64: 847249423Sdim Opcode = NVPTX::LD_i64_areg_64; 848249423Sdim break; 849249423Sdim case MVT::f32: 850249423Sdim Opcode = NVPTX::LD_f32_areg_64; 851249423Sdim break; 852249423Sdim case MVT::f64: 853249423Sdim Opcode = NVPTX::LD_f64_areg_64; 854249423Sdim break; 855249423Sdim default: 856276479Sdim return nullptr; 857249423Sdim } 858249423Sdim } else { 859249423Sdim switch (TargetVT) { 860249423Sdim case MVT::i8: 861249423Sdim Opcode = NVPTX::LD_i8_areg; 862249423Sdim break; 863249423Sdim case MVT::i16: 864249423Sdim Opcode = NVPTX::LD_i16_areg; 865249423Sdim break; 866249423Sdim case MVT::i32: 867249423Sdim Opcode = NVPTX::LD_i32_areg; 868249423Sdim break; 869249423Sdim case MVT::i64: 870249423Sdim Opcode = NVPTX::LD_i64_areg; 871249423Sdim break; 872249423Sdim case MVT::f32: 873249423Sdim Opcode = NVPTX::LD_f32_areg; 874249423Sdim break; 875249423Sdim case MVT::f64: 876249423Sdim Opcode = NVPTX::LD_f64_areg; 877249423Sdim break; 878249423Sdim default: 879276479Sdim return nullptr; 880249423Sdim } 881239310Sdim } 882288943Sdim SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl), 883288943Sdim getI32Imm(vecType, dl), getI32Imm(fromType, dl), 884288943Sdim getI32Imm(fromTypeWidth, dl), N1, Chain }; 885251662Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 886239310Sdim } 887239310Sdim 888276479Sdim if (NVPTXLD) { 889239310Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 890239310Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 891239310Sdim cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1); 892239310Sdim } 893239310Sdim 894239310Sdim return NVPTXLD; 895239310Sdim} 896239310Sdim 897249423SdimSDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) { 898249423Sdim 899249423Sdim SDValue Chain = N->getOperand(0); 900249423Sdim SDValue Op1 = N->getOperand(1); 901249423Sdim SDValue Addr, Offset, Base; 902249423Sdim unsigned Opcode; 903261991Sdim SDLoc DL(N); 904249423Sdim SDNode *LD; 905249423Sdim MemSDNode *MemSD = cast<MemSDNode>(N); 906249423Sdim EVT LoadedVT = MemSD->getMemoryVT(); 907249423Sdim 908249423Sdim if (!LoadedVT.isSimple()) 909276479Sdim return nullptr; 910249423Sdim 911249423Sdim // Address Space Setting 912288943Sdim unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD); 913249423Sdim 914296417Sdim if (canLowerToLDG(MemSD, *Subtarget, CodeAddrSpace, MF)) { 915296417Sdim return SelectLDGLDU(N); 916296417Sdim } 917296417Sdim 918249423Sdim // Volatile Setting 919249423Sdim // - .volatile is only availalble for .global and .shared 920249423Sdim bool IsVolatile = MemSD->isVolatile(); 921249423Sdim if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 922249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 923249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 924249423Sdim IsVolatile = false; 925249423Sdim 926249423Sdim // Vector Setting 927249423Sdim MVT SimpleVT = LoadedVT.getSimpleVT(); 928249423Sdim 929249423Sdim // Type Setting: fromType + fromTypeWidth 930249423Sdim // 931249423Sdim // Sign : ISD::SEXTLOAD 932249423Sdim // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the 933249423Sdim // type is integer 934249423Sdim // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float 935249423Sdim MVT ScalarVT = SimpleVT.getScalarType(); 936261991Sdim // Read at least 8 bits (predicates are stored as 8-bit values) 937261991Sdim unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits()); 938249423Sdim unsigned int FromType; 939249423Sdim // The last operand holds the original LoadSDNode::getExtensionType() value 940249423Sdim unsigned ExtensionType = cast<ConstantSDNode>( 941249423Sdim N->getOperand(N->getNumOperands() - 1))->getZExtValue(); 942249423Sdim if (ExtensionType == ISD::SEXTLOAD) 943249423Sdim FromType = NVPTX::PTXLdStInstCode::Signed; 944249423Sdim else if (ScalarVT.isFloatingPoint()) 945249423Sdim FromType = NVPTX::PTXLdStInstCode::Float; 946249423Sdim else 947249423Sdim FromType = NVPTX::PTXLdStInstCode::Unsigned; 948249423Sdim 949249423Sdim unsigned VecType; 950249423Sdim 951249423Sdim switch (N->getOpcode()) { 952249423Sdim case NVPTXISD::LoadV2: 953249423Sdim VecType = NVPTX::PTXLdStInstCode::V2; 954249423Sdim break; 955249423Sdim case NVPTXISD::LoadV4: 956249423Sdim VecType = NVPTX::PTXLdStInstCode::V4; 957249423Sdim break; 958249423Sdim default: 959276479Sdim return nullptr; 960249423Sdim } 961249423Sdim 962249423Sdim EVT EltVT = N->getValueType(0); 963249423Sdim 964249423Sdim if (SelectDirectAddr(Op1, Addr)) { 965249423Sdim switch (N->getOpcode()) { 966249423Sdim default: 967276479Sdim return nullptr; 968249423Sdim case NVPTXISD::LoadV2: 969249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 970249423Sdim default: 971276479Sdim return nullptr; 972249423Sdim case MVT::i8: 973249423Sdim Opcode = NVPTX::LDV_i8_v2_avar; 974249423Sdim break; 975249423Sdim case MVT::i16: 976249423Sdim Opcode = NVPTX::LDV_i16_v2_avar; 977249423Sdim break; 978249423Sdim case MVT::i32: 979249423Sdim Opcode = NVPTX::LDV_i32_v2_avar; 980249423Sdim break; 981249423Sdim case MVT::i64: 982249423Sdim Opcode = NVPTX::LDV_i64_v2_avar; 983249423Sdim break; 984249423Sdim case MVT::f32: 985249423Sdim Opcode = NVPTX::LDV_f32_v2_avar; 986249423Sdim break; 987249423Sdim case MVT::f64: 988249423Sdim Opcode = NVPTX::LDV_f64_v2_avar; 989249423Sdim break; 990249423Sdim } 991249423Sdim break; 992249423Sdim case NVPTXISD::LoadV4: 993249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 994249423Sdim default: 995276479Sdim return nullptr; 996249423Sdim case MVT::i8: 997249423Sdim Opcode = NVPTX::LDV_i8_v4_avar; 998249423Sdim break; 999249423Sdim case MVT::i16: 1000249423Sdim Opcode = NVPTX::LDV_i16_v4_avar; 1001249423Sdim break; 1002249423Sdim case MVT::i32: 1003249423Sdim Opcode = NVPTX::LDV_i32_v4_avar; 1004249423Sdim break; 1005249423Sdim case MVT::f32: 1006249423Sdim Opcode = NVPTX::LDV_f32_v4_avar; 1007249423Sdim break; 1008249423Sdim } 1009249423Sdim break; 1010249423Sdim } 1011249423Sdim 1012288943Sdim SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL), 1013288943Sdim getI32Imm(VecType, DL), getI32Imm(FromType, DL), 1014288943Sdim getI32Imm(FromTypeWidth, DL), Addr, Chain }; 1015251662Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 1016288943Sdim } else if (TM.is64Bit() ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset) 1017288943Sdim : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) { 1018249423Sdim switch (N->getOpcode()) { 1019249423Sdim default: 1020276479Sdim return nullptr; 1021249423Sdim case NVPTXISD::LoadV2: 1022249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1023249423Sdim default: 1024276479Sdim return nullptr; 1025249423Sdim case MVT::i8: 1026249423Sdim Opcode = NVPTX::LDV_i8_v2_asi; 1027249423Sdim break; 1028249423Sdim case MVT::i16: 1029249423Sdim Opcode = NVPTX::LDV_i16_v2_asi; 1030249423Sdim break; 1031249423Sdim case MVT::i32: 1032249423Sdim Opcode = NVPTX::LDV_i32_v2_asi; 1033249423Sdim break; 1034249423Sdim case MVT::i64: 1035249423Sdim Opcode = NVPTX::LDV_i64_v2_asi; 1036249423Sdim break; 1037249423Sdim case MVT::f32: 1038249423Sdim Opcode = NVPTX::LDV_f32_v2_asi; 1039249423Sdim break; 1040249423Sdim case MVT::f64: 1041249423Sdim Opcode = NVPTX::LDV_f64_v2_asi; 1042249423Sdim break; 1043249423Sdim } 1044249423Sdim break; 1045249423Sdim case NVPTXISD::LoadV4: 1046249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1047249423Sdim default: 1048276479Sdim return nullptr; 1049249423Sdim case MVT::i8: 1050249423Sdim Opcode = NVPTX::LDV_i8_v4_asi; 1051249423Sdim break; 1052249423Sdim case MVT::i16: 1053249423Sdim Opcode = NVPTX::LDV_i16_v4_asi; 1054249423Sdim break; 1055249423Sdim case MVT::i32: 1056249423Sdim Opcode = NVPTX::LDV_i32_v4_asi; 1057249423Sdim break; 1058249423Sdim case MVT::f32: 1059249423Sdim Opcode = NVPTX::LDV_f32_v4_asi; 1060249423Sdim break; 1061249423Sdim } 1062249423Sdim break; 1063249423Sdim } 1064249423Sdim 1065288943Sdim SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL), 1066288943Sdim getI32Imm(VecType, DL), getI32Imm(FromType, DL), 1067288943Sdim getI32Imm(FromTypeWidth, DL), Base, Offset, Chain }; 1068251662Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 1069288943Sdim } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) 1070288943Sdim : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { 1071288943Sdim if (TM.is64Bit()) { 1072249423Sdim switch (N->getOpcode()) { 1073249423Sdim default: 1074276479Sdim return nullptr; 1075249423Sdim case NVPTXISD::LoadV2: 1076249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1077249423Sdim default: 1078276479Sdim return nullptr; 1079249423Sdim case MVT::i8: 1080249423Sdim Opcode = NVPTX::LDV_i8_v2_ari_64; 1081249423Sdim break; 1082249423Sdim case MVT::i16: 1083249423Sdim Opcode = NVPTX::LDV_i16_v2_ari_64; 1084249423Sdim break; 1085249423Sdim case MVT::i32: 1086249423Sdim Opcode = NVPTX::LDV_i32_v2_ari_64; 1087249423Sdim break; 1088249423Sdim case MVT::i64: 1089249423Sdim Opcode = NVPTX::LDV_i64_v2_ari_64; 1090249423Sdim break; 1091249423Sdim case MVT::f32: 1092249423Sdim Opcode = NVPTX::LDV_f32_v2_ari_64; 1093249423Sdim break; 1094249423Sdim case MVT::f64: 1095249423Sdim Opcode = NVPTX::LDV_f64_v2_ari_64; 1096249423Sdim break; 1097249423Sdim } 1098249423Sdim break; 1099249423Sdim case NVPTXISD::LoadV4: 1100249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1101249423Sdim default: 1102276479Sdim return nullptr; 1103249423Sdim case MVT::i8: 1104249423Sdim Opcode = NVPTX::LDV_i8_v4_ari_64; 1105249423Sdim break; 1106249423Sdim case MVT::i16: 1107249423Sdim Opcode = NVPTX::LDV_i16_v4_ari_64; 1108249423Sdim break; 1109249423Sdim case MVT::i32: 1110249423Sdim Opcode = NVPTX::LDV_i32_v4_ari_64; 1111249423Sdim break; 1112249423Sdim case MVT::f32: 1113249423Sdim Opcode = NVPTX::LDV_f32_v4_ari_64; 1114249423Sdim break; 1115249423Sdim } 1116249423Sdim break; 1117249423Sdim } 1118249423Sdim } else { 1119249423Sdim switch (N->getOpcode()) { 1120249423Sdim default: 1121276479Sdim return nullptr; 1122249423Sdim case NVPTXISD::LoadV2: 1123249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1124249423Sdim default: 1125276479Sdim return nullptr; 1126249423Sdim case MVT::i8: 1127249423Sdim Opcode = NVPTX::LDV_i8_v2_ari; 1128249423Sdim break; 1129249423Sdim case MVT::i16: 1130249423Sdim Opcode = NVPTX::LDV_i16_v2_ari; 1131249423Sdim break; 1132249423Sdim case MVT::i32: 1133249423Sdim Opcode = NVPTX::LDV_i32_v2_ari; 1134249423Sdim break; 1135249423Sdim case MVT::i64: 1136249423Sdim Opcode = NVPTX::LDV_i64_v2_ari; 1137249423Sdim break; 1138249423Sdim case MVT::f32: 1139249423Sdim Opcode = NVPTX::LDV_f32_v2_ari; 1140249423Sdim break; 1141249423Sdim case MVT::f64: 1142249423Sdim Opcode = NVPTX::LDV_f64_v2_ari; 1143249423Sdim break; 1144249423Sdim } 1145249423Sdim break; 1146249423Sdim case NVPTXISD::LoadV4: 1147249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1148249423Sdim default: 1149276479Sdim return nullptr; 1150249423Sdim case MVT::i8: 1151249423Sdim Opcode = NVPTX::LDV_i8_v4_ari; 1152249423Sdim break; 1153249423Sdim case MVT::i16: 1154249423Sdim Opcode = NVPTX::LDV_i16_v4_ari; 1155249423Sdim break; 1156249423Sdim case MVT::i32: 1157249423Sdim Opcode = NVPTX::LDV_i32_v4_ari; 1158249423Sdim break; 1159249423Sdim case MVT::f32: 1160249423Sdim Opcode = NVPTX::LDV_f32_v4_ari; 1161249423Sdim break; 1162249423Sdim } 1163249423Sdim break; 1164249423Sdim } 1165249423Sdim } 1166249423Sdim 1167288943Sdim SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL), 1168288943Sdim getI32Imm(VecType, DL), getI32Imm(FromType, DL), 1169288943Sdim getI32Imm(FromTypeWidth, DL), Base, Offset, Chain }; 1170249423Sdim 1171251662Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 1172249423Sdim } else { 1173288943Sdim if (TM.is64Bit()) { 1174249423Sdim switch (N->getOpcode()) { 1175249423Sdim default: 1176276479Sdim return nullptr; 1177249423Sdim case NVPTXISD::LoadV2: 1178249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1179249423Sdim default: 1180276479Sdim return nullptr; 1181249423Sdim case MVT::i8: 1182249423Sdim Opcode = NVPTX::LDV_i8_v2_areg_64; 1183249423Sdim break; 1184249423Sdim case MVT::i16: 1185249423Sdim Opcode = NVPTX::LDV_i16_v2_areg_64; 1186249423Sdim break; 1187249423Sdim case MVT::i32: 1188249423Sdim Opcode = NVPTX::LDV_i32_v2_areg_64; 1189249423Sdim break; 1190249423Sdim case MVT::i64: 1191249423Sdim Opcode = NVPTX::LDV_i64_v2_areg_64; 1192249423Sdim break; 1193249423Sdim case MVT::f32: 1194249423Sdim Opcode = NVPTX::LDV_f32_v2_areg_64; 1195249423Sdim break; 1196249423Sdim case MVT::f64: 1197249423Sdim Opcode = NVPTX::LDV_f64_v2_areg_64; 1198249423Sdim break; 1199249423Sdim } 1200249423Sdim break; 1201249423Sdim case NVPTXISD::LoadV4: 1202249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1203249423Sdim default: 1204276479Sdim return nullptr; 1205249423Sdim case MVT::i8: 1206249423Sdim Opcode = NVPTX::LDV_i8_v4_areg_64; 1207249423Sdim break; 1208249423Sdim case MVT::i16: 1209249423Sdim Opcode = NVPTX::LDV_i16_v4_areg_64; 1210249423Sdim break; 1211249423Sdim case MVT::i32: 1212249423Sdim Opcode = NVPTX::LDV_i32_v4_areg_64; 1213249423Sdim break; 1214249423Sdim case MVT::f32: 1215249423Sdim Opcode = NVPTX::LDV_f32_v4_areg_64; 1216249423Sdim break; 1217249423Sdim } 1218249423Sdim break; 1219249423Sdim } 1220249423Sdim } else { 1221249423Sdim switch (N->getOpcode()) { 1222249423Sdim default: 1223276479Sdim return nullptr; 1224249423Sdim case NVPTXISD::LoadV2: 1225249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1226249423Sdim default: 1227276479Sdim return nullptr; 1228249423Sdim case MVT::i8: 1229249423Sdim Opcode = NVPTX::LDV_i8_v2_areg; 1230249423Sdim break; 1231249423Sdim case MVT::i16: 1232249423Sdim Opcode = NVPTX::LDV_i16_v2_areg; 1233249423Sdim break; 1234249423Sdim case MVT::i32: 1235249423Sdim Opcode = NVPTX::LDV_i32_v2_areg; 1236249423Sdim break; 1237249423Sdim case MVT::i64: 1238249423Sdim Opcode = NVPTX::LDV_i64_v2_areg; 1239249423Sdim break; 1240249423Sdim case MVT::f32: 1241249423Sdim Opcode = NVPTX::LDV_f32_v2_areg; 1242249423Sdim break; 1243249423Sdim case MVT::f64: 1244249423Sdim Opcode = NVPTX::LDV_f64_v2_areg; 1245249423Sdim break; 1246249423Sdim } 1247249423Sdim break; 1248249423Sdim case NVPTXISD::LoadV4: 1249249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1250249423Sdim default: 1251276479Sdim return nullptr; 1252249423Sdim case MVT::i8: 1253249423Sdim Opcode = NVPTX::LDV_i8_v4_areg; 1254249423Sdim break; 1255249423Sdim case MVT::i16: 1256249423Sdim Opcode = NVPTX::LDV_i16_v4_areg; 1257249423Sdim break; 1258249423Sdim case MVT::i32: 1259249423Sdim Opcode = NVPTX::LDV_i32_v4_areg; 1260249423Sdim break; 1261249423Sdim case MVT::f32: 1262249423Sdim Opcode = NVPTX::LDV_f32_v4_areg; 1263249423Sdim break; 1264249423Sdim } 1265249423Sdim break; 1266249423Sdim } 1267249423Sdim } 1268249423Sdim 1269288943Sdim SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL), 1270288943Sdim getI32Imm(VecType, DL), getI32Imm(FromType, DL), 1271288943Sdim getI32Imm(FromTypeWidth, DL), Op1, Chain }; 1272251662Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 1273249423Sdim } 1274249423Sdim 1275249423Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 1276249423Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 1277249423Sdim cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); 1278249423Sdim 1279249423Sdim return LD; 1280249423Sdim} 1281249423Sdim 1282276479SdimSDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) { 1283249423Sdim 1284249423Sdim SDValue Chain = N->getOperand(0); 1285276479Sdim SDValue Op1; 1286276479Sdim MemSDNode *Mem; 1287276479Sdim bool IsLDG = true; 1288276479Sdim 1289276479Sdim // If this is an LDG intrinsic, the address is the third operand. Its its an 1290276479Sdim // LDG/LDU SD node (from custom vector handling), then its the second operand 1291276479Sdim if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) { 1292276479Sdim Op1 = N->getOperand(2); 1293276479Sdim Mem = cast<MemIntrinsicSDNode>(N); 1294276479Sdim unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 1295276479Sdim switch (IID) { 1296276479Sdim default: 1297276479Sdim return NULL; 1298276479Sdim case Intrinsic::nvvm_ldg_global_f: 1299276479Sdim case Intrinsic::nvvm_ldg_global_i: 1300276479Sdim case Intrinsic::nvvm_ldg_global_p: 1301276479Sdim IsLDG = true; 1302276479Sdim break; 1303276479Sdim case Intrinsic::nvvm_ldu_global_f: 1304276479Sdim case Intrinsic::nvvm_ldu_global_i: 1305276479Sdim case Intrinsic::nvvm_ldu_global_p: 1306276479Sdim IsLDG = false; 1307276479Sdim break; 1308276479Sdim } 1309276479Sdim } else { 1310276479Sdim Op1 = N->getOperand(1); 1311276479Sdim Mem = cast<MemSDNode>(N); 1312276479Sdim } 1313276479Sdim 1314249423Sdim unsigned Opcode; 1315261991Sdim SDLoc DL(N); 1316249423Sdim SDNode *LD; 1317261991Sdim SDValue Base, Offset, Addr; 1318249423Sdim 1319276479Sdim EVT EltVT = Mem->getMemoryVT(); 1320276479Sdim if (EltVT.isVector()) { 1321276479Sdim EltVT = EltVT.getVectorElementType(); 1322276479Sdim } 1323249423Sdim 1324261991Sdim if (SelectDirectAddr(Op1, Addr)) { 1325249423Sdim switch (N->getOpcode()) { 1326249423Sdim default: 1327276479Sdim return nullptr; 1328276479Sdim case ISD::INTRINSIC_W_CHAIN: 1329276479Sdim if (IsLDG) { 1330276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1331276479Sdim default: 1332276479Sdim return nullptr; 1333276479Sdim case MVT::i8: 1334276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8avar; 1335276479Sdim break; 1336276479Sdim case MVT::i16: 1337276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16avar; 1338276479Sdim break; 1339276479Sdim case MVT::i32: 1340276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32avar; 1341276479Sdim break; 1342276479Sdim case MVT::i64: 1343276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64avar; 1344276479Sdim break; 1345276479Sdim case MVT::f32: 1346276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32avar; 1347276479Sdim break; 1348276479Sdim case MVT::f64: 1349276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64avar; 1350276479Sdim break; 1351276479Sdim } 1352276479Sdim } else { 1353276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1354276479Sdim default: 1355276479Sdim return nullptr; 1356276479Sdim case MVT::i8: 1357276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8avar; 1358276479Sdim break; 1359276479Sdim case MVT::i16: 1360276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16avar; 1361276479Sdim break; 1362276479Sdim case MVT::i32: 1363276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32avar; 1364276479Sdim break; 1365276479Sdim case MVT::i64: 1366276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64avar; 1367276479Sdim break; 1368276479Sdim case MVT::f32: 1369276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32avar; 1370276479Sdim break; 1371276479Sdim case MVT::f64: 1372276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64avar; 1373276479Sdim break; 1374276479Sdim } 1375276479Sdim } 1376276479Sdim break; 1377249423Sdim case NVPTXISD::LDGV2: 1378261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1379249423Sdim default: 1380276479Sdim return nullptr; 1381249423Sdim case MVT::i8: 1382261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar; 1383249423Sdim break; 1384249423Sdim case MVT::i16: 1385261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar; 1386249423Sdim break; 1387249423Sdim case MVT::i32: 1388261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar; 1389249423Sdim break; 1390249423Sdim case MVT::i64: 1391261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar; 1392249423Sdim break; 1393249423Sdim case MVT::f32: 1394261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar; 1395249423Sdim break; 1396249423Sdim case MVT::f64: 1397261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar; 1398249423Sdim break; 1399249423Sdim } 1400249423Sdim break; 1401261991Sdim case NVPTXISD::LDUV2: 1402261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1403249423Sdim default: 1404276479Sdim return nullptr; 1405249423Sdim case MVT::i8: 1406261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar; 1407249423Sdim break; 1408249423Sdim case MVT::i16: 1409261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar; 1410249423Sdim break; 1411249423Sdim case MVT::i32: 1412261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar; 1413249423Sdim break; 1414261991Sdim case MVT::i64: 1415261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar; 1416261991Sdim break; 1417249423Sdim case MVT::f32: 1418261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar; 1419249423Sdim break; 1420261991Sdim case MVT::f64: 1421261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar; 1422261991Sdim break; 1423249423Sdim } 1424249423Sdim break; 1425261991Sdim case NVPTXISD::LDGV4: 1426261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1427249423Sdim default: 1428276479Sdim return nullptr; 1429249423Sdim case MVT::i8: 1430261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar; 1431249423Sdim break; 1432249423Sdim case MVT::i16: 1433261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar; 1434249423Sdim break; 1435249423Sdim case MVT::i32: 1436261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar; 1437249423Sdim break; 1438249423Sdim case MVT::f32: 1439261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar; 1440249423Sdim break; 1441249423Sdim } 1442249423Sdim break; 1443249423Sdim case NVPTXISD::LDUV4: 1444261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1445249423Sdim default: 1446276479Sdim return nullptr; 1447249423Sdim case MVT::i8: 1448261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar; 1449249423Sdim break; 1450249423Sdim case MVT::i16: 1451261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar; 1452249423Sdim break; 1453249423Sdim case MVT::i32: 1454261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar; 1455249423Sdim break; 1456249423Sdim case MVT::f32: 1457261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar; 1458249423Sdim break; 1459249423Sdim } 1460249423Sdim break; 1461249423Sdim } 1462261991Sdim 1463261991Sdim SDValue Ops[] = { Addr, Chain }; 1464276479Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 1465288943Sdim } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) 1466288943Sdim : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { 1467288943Sdim if (TM.is64Bit()) { 1468261991Sdim switch (N->getOpcode()) { 1469249423Sdim default: 1470276479Sdim return nullptr; 1471296417Sdim case ISD::LOAD: 1472276479Sdim case ISD::INTRINSIC_W_CHAIN: 1473276479Sdim if (IsLDG) { 1474276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1475276479Sdim default: 1476276479Sdim return nullptr; 1477276479Sdim case MVT::i8: 1478276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari64; 1479276479Sdim break; 1480276479Sdim case MVT::i16: 1481276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari64; 1482276479Sdim break; 1483276479Sdim case MVT::i32: 1484276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari64; 1485276479Sdim break; 1486276479Sdim case MVT::i64: 1487276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari64; 1488276479Sdim break; 1489276479Sdim case MVT::f32: 1490276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari64; 1491276479Sdim break; 1492276479Sdim case MVT::f64: 1493276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari64; 1494276479Sdim break; 1495276479Sdim } 1496276479Sdim } else { 1497276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1498276479Sdim default: 1499276479Sdim return nullptr; 1500276479Sdim case MVT::i8: 1501276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari64; 1502276479Sdim break; 1503276479Sdim case MVT::i16: 1504276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari64; 1505276479Sdim break; 1506276479Sdim case MVT::i32: 1507276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari64; 1508276479Sdim break; 1509276479Sdim case MVT::i64: 1510276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari64; 1511276479Sdim break; 1512276479Sdim case MVT::f32: 1513276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari64; 1514276479Sdim break; 1515276479Sdim case MVT::f64: 1516276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari64; 1517276479Sdim break; 1518276479Sdim } 1519276479Sdim } 1520276479Sdim break; 1521296417Sdim case NVPTXISD::LoadV2: 1522261991Sdim case NVPTXISD::LDGV2: 1523261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1524261991Sdim default: 1525276479Sdim return nullptr; 1526261991Sdim case MVT::i8: 1527261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64; 1528261991Sdim break; 1529261991Sdim case MVT::i16: 1530261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64; 1531261991Sdim break; 1532261991Sdim case MVT::i32: 1533261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64; 1534261991Sdim break; 1535261991Sdim case MVT::i64: 1536261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64; 1537261991Sdim break; 1538261991Sdim case MVT::f32: 1539261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64; 1540261991Sdim break; 1541261991Sdim case MVT::f64: 1542261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64; 1543261991Sdim break; 1544261991Sdim } 1545249423Sdim break; 1546261991Sdim case NVPTXISD::LDUV2: 1547261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1548261991Sdim default: 1549276479Sdim return nullptr; 1550261991Sdim case MVT::i8: 1551261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64; 1552261991Sdim break; 1553261991Sdim case MVT::i16: 1554261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64; 1555261991Sdim break; 1556261991Sdim case MVT::i32: 1557261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64; 1558261991Sdim break; 1559261991Sdim case MVT::i64: 1560261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64; 1561261991Sdim break; 1562261991Sdim case MVT::f32: 1563261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64; 1564261991Sdim break; 1565261991Sdim case MVT::f64: 1566261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64; 1567261991Sdim break; 1568261991Sdim } 1569249423Sdim break; 1570296417Sdim case NVPTXISD::LoadV4: 1571261991Sdim case NVPTXISD::LDGV4: 1572261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1573261991Sdim default: 1574276479Sdim return nullptr; 1575261991Sdim case MVT::i8: 1576261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64; 1577261991Sdim break; 1578261991Sdim case MVT::i16: 1579261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64; 1580261991Sdim break; 1581261991Sdim case MVT::i32: 1582261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64; 1583261991Sdim break; 1584261991Sdim case MVT::f32: 1585261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64; 1586261991Sdim break; 1587261991Sdim } 1588249423Sdim break; 1589261991Sdim case NVPTXISD::LDUV4: 1590261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1591261991Sdim default: 1592276479Sdim return nullptr; 1593261991Sdim case MVT::i8: 1594261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64; 1595261991Sdim break; 1596261991Sdim case MVT::i16: 1597261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64; 1598261991Sdim break; 1599261991Sdim case MVT::i32: 1600261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64; 1601261991Sdim break; 1602261991Sdim case MVT::f32: 1603261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64; 1604261991Sdim break; 1605261991Sdim } 1606249423Sdim break; 1607249423Sdim } 1608261991Sdim } else { 1609261991Sdim switch (N->getOpcode()) { 1610249423Sdim default: 1611276479Sdim return nullptr; 1612296417Sdim case ISD::LOAD: 1613276479Sdim case ISD::INTRINSIC_W_CHAIN: 1614276479Sdim if (IsLDG) { 1615276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1616276479Sdim default: 1617276479Sdim return nullptr; 1618276479Sdim case MVT::i8: 1619276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari; 1620276479Sdim break; 1621276479Sdim case MVT::i16: 1622276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari; 1623276479Sdim break; 1624276479Sdim case MVT::i32: 1625276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari; 1626276479Sdim break; 1627276479Sdim case MVT::i64: 1628276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari; 1629276479Sdim break; 1630276479Sdim case MVT::f32: 1631276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari; 1632276479Sdim break; 1633276479Sdim case MVT::f64: 1634276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari; 1635276479Sdim break; 1636276479Sdim } 1637276479Sdim } else { 1638276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1639276479Sdim default: 1640276479Sdim return nullptr; 1641276479Sdim case MVT::i8: 1642276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari; 1643276479Sdim break; 1644276479Sdim case MVT::i16: 1645276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari; 1646276479Sdim break; 1647276479Sdim case MVT::i32: 1648276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari; 1649276479Sdim break; 1650276479Sdim case MVT::i64: 1651276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari; 1652276479Sdim break; 1653276479Sdim case MVT::f32: 1654276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari; 1655276479Sdim break; 1656276479Sdim case MVT::f64: 1657276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari; 1658276479Sdim break; 1659276479Sdim } 1660276479Sdim } 1661276479Sdim break; 1662296417Sdim case NVPTXISD::LoadV2: 1663261991Sdim case NVPTXISD::LDGV2: 1664261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1665261991Sdim default: 1666276479Sdim return nullptr; 1667261991Sdim case MVT::i8: 1668261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32; 1669261991Sdim break; 1670261991Sdim case MVT::i16: 1671261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32; 1672261991Sdim break; 1673261991Sdim case MVT::i32: 1674261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32; 1675261991Sdim break; 1676261991Sdim case MVT::i64: 1677261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32; 1678261991Sdim break; 1679261991Sdim case MVT::f32: 1680261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32; 1681261991Sdim break; 1682261991Sdim case MVT::f64: 1683261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32; 1684261991Sdim break; 1685261991Sdim } 1686249423Sdim break; 1687261991Sdim case NVPTXISD::LDUV2: 1688261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1689261991Sdim default: 1690276479Sdim return nullptr; 1691261991Sdim case MVT::i8: 1692261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32; 1693261991Sdim break; 1694261991Sdim case MVT::i16: 1695261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32; 1696261991Sdim break; 1697261991Sdim case MVT::i32: 1698261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32; 1699261991Sdim break; 1700261991Sdim case MVT::i64: 1701261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32; 1702261991Sdim break; 1703261991Sdim case MVT::f32: 1704261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32; 1705261991Sdim break; 1706261991Sdim case MVT::f64: 1707261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32; 1708261991Sdim break; 1709261991Sdim } 1710249423Sdim break; 1711296417Sdim case NVPTXISD::LoadV4: 1712261991Sdim case NVPTXISD::LDGV4: 1713261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1714261991Sdim default: 1715276479Sdim return nullptr; 1716261991Sdim case MVT::i8: 1717261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32; 1718261991Sdim break; 1719261991Sdim case MVT::i16: 1720261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32; 1721261991Sdim break; 1722261991Sdim case MVT::i32: 1723261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32; 1724261991Sdim break; 1725261991Sdim case MVT::f32: 1726261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32; 1727261991Sdim break; 1728261991Sdim } 1729249423Sdim break; 1730261991Sdim case NVPTXISD::LDUV4: 1731261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1732261991Sdim default: 1733276479Sdim return nullptr; 1734261991Sdim case MVT::i8: 1735261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32; 1736261991Sdim break; 1737261991Sdim case MVT::i16: 1738261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32; 1739261991Sdim break; 1740261991Sdim case MVT::i32: 1741261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32; 1742261991Sdim break; 1743261991Sdim case MVT::f32: 1744261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32; 1745261991Sdim break; 1746261991Sdim } 1747249423Sdim break; 1748249423Sdim } 1749261991Sdim } 1750261991Sdim 1751261991Sdim SDValue Ops[] = { Base, Offset, Chain }; 1752261991Sdim 1753276479Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 1754261991Sdim } else { 1755288943Sdim if (TM.is64Bit()) { 1756261991Sdim switch (N->getOpcode()) { 1757249423Sdim default: 1758276479Sdim return nullptr; 1759296417Sdim case ISD::LOAD: 1760276479Sdim case ISD::INTRINSIC_W_CHAIN: 1761276479Sdim if (IsLDG) { 1762276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1763276479Sdim default: 1764276479Sdim return nullptr; 1765276479Sdim case MVT::i8: 1766276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg64; 1767276479Sdim break; 1768276479Sdim case MVT::i16: 1769276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg64; 1770276479Sdim break; 1771276479Sdim case MVT::i32: 1772276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg64; 1773276479Sdim break; 1774276479Sdim case MVT::i64: 1775276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg64; 1776276479Sdim break; 1777276479Sdim case MVT::f32: 1778276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg64; 1779276479Sdim break; 1780276479Sdim case MVT::f64: 1781276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg64; 1782276479Sdim break; 1783276479Sdim } 1784276479Sdim } else { 1785276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1786276479Sdim default: 1787276479Sdim return nullptr; 1788276479Sdim case MVT::i8: 1789276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg64; 1790276479Sdim break; 1791276479Sdim case MVT::i16: 1792276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg64; 1793276479Sdim break; 1794276479Sdim case MVT::i32: 1795276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg64; 1796276479Sdim break; 1797276479Sdim case MVT::i64: 1798276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg64; 1799276479Sdim break; 1800276479Sdim case MVT::f32: 1801276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg64; 1802276479Sdim break; 1803276479Sdim case MVT::f64: 1804276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg64; 1805276479Sdim break; 1806276479Sdim } 1807276479Sdim } 1808276479Sdim break; 1809296417Sdim case NVPTXISD::LoadV2: 1810261991Sdim case NVPTXISD::LDGV2: 1811261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1812261991Sdim default: 1813276479Sdim return nullptr; 1814261991Sdim case MVT::i8: 1815261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64; 1816261991Sdim break; 1817261991Sdim case MVT::i16: 1818261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64; 1819261991Sdim break; 1820261991Sdim case MVT::i32: 1821261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64; 1822261991Sdim break; 1823261991Sdim case MVT::i64: 1824261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64; 1825261991Sdim break; 1826261991Sdim case MVT::f32: 1827261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64; 1828261991Sdim break; 1829261991Sdim case MVT::f64: 1830261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64; 1831261991Sdim break; 1832261991Sdim } 1833249423Sdim break; 1834261991Sdim case NVPTXISD::LDUV2: 1835261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1836261991Sdim default: 1837276479Sdim return nullptr; 1838261991Sdim case MVT::i8: 1839261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64; 1840261991Sdim break; 1841261991Sdim case MVT::i16: 1842261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64; 1843261991Sdim break; 1844261991Sdim case MVT::i32: 1845261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64; 1846261991Sdim break; 1847261991Sdim case MVT::i64: 1848261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64; 1849261991Sdim break; 1850261991Sdim case MVT::f32: 1851261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64; 1852261991Sdim break; 1853261991Sdim case MVT::f64: 1854261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64; 1855261991Sdim break; 1856261991Sdim } 1857249423Sdim break; 1858296417Sdim case NVPTXISD::LoadV4: 1859261991Sdim case NVPTXISD::LDGV4: 1860261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1861261991Sdim default: 1862276479Sdim return nullptr; 1863261991Sdim case MVT::i8: 1864261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64; 1865261991Sdim break; 1866261991Sdim case MVT::i16: 1867261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64; 1868261991Sdim break; 1869261991Sdim case MVT::i32: 1870261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64; 1871261991Sdim break; 1872261991Sdim case MVT::f32: 1873261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64; 1874261991Sdim break; 1875261991Sdim } 1876249423Sdim break; 1877261991Sdim case NVPTXISD::LDUV4: 1878261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1879261991Sdim default: 1880276479Sdim return nullptr; 1881261991Sdim case MVT::i8: 1882261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64; 1883261991Sdim break; 1884261991Sdim case MVT::i16: 1885261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64; 1886261991Sdim break; 1887261991Sdim case MVT::i32: 1888261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64; 1889261991Sdim break; 1890261991Sdim case MVT::f32: 1891261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64; 1892261991Sdim break; 1893261991Sdim } 1894249423Sdim break; 1895249423Sdim } 1896261991Sdim } else { 1897261991Sdim switch (N->getOpcode()) { 1898249423Sdim default: 1899276479Sdim return nullptr; 1900296417Sdim case ISD::LOAD: 1901276479Sdim case ISD::INTRINSIC_W_CHAIN: 1902276479Sdim if (IsLDG) { 1903276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1904276479Sdim default: 1905276479Sdim return nullptr; 1906276479Sdim case MVT::i8: 1907276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg; 1908276479Sdim break; 1909276479Sdim case MVT::i16: 1910276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg; 1911276479Sdim break; 1912276479Sdim case MVT::i32: 1913276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg; 1914276479Sdim break; 1915276479Sdim case MVT::i64: 1916276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg; 1917276479Sdim break; 1918276479Sdim case MVT::f32: 1919276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg; 1920276479Sdim break; 1921276479Sdim case MVT::f64: 1922276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg; 1923276479Sdim break; 1924276479Sdim } 1925276479Sdim } else { 1926276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1927276479Sdim default: 1928276479Sdim return nullptr; 1929276479Sdim case MVT::i8: 1930276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg; 1931276479Sdim break; 1932276479Sdim case MVT::i16: 1933276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg; 1934276479Sdim break; 1935276479Sdim case MVT::i32: 1936276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg; 1937276479Sdim break; 1938276479Sdim case MVT::i64: 1939276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg; 1940276479Sdim break; 1941276479Sdim case MVT::f32: 1942276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg; 1943276479Sdim break; 1944276479Sdim case MVT::f64: 1945276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg; 1946276479Sdim break; 1947276479Sdim } 1948276479Sdim } 1949276479Sdim break; 1950296417Sdim case NVPTXISD::LoadV2: 1951261991Sdim case NVPTXISD::LDGV2: 1952261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1953261991Sdim default: 1954276479Sdim return nullptr; 1955261991Sdim case MVT::i8: 1956261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32; 1957261991Sdim break; 1958261991Sdim case MVT::i16: 1959261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32; 1960261991Sdim break; 1961261991Sdim case MVT::i32: 1962261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32; 1963261991Sdim break; 1964261991Sdim case MVT::i64: 1965261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32; 1966261991Sdim break; 1967261991Sdim case MVT::f32: 1968261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32; 1969261991Sdim break; 1970261991Sdim case MVT::f64: 1971261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32; 1972261991Sdim break; 1973261991Sdim } 1974249423Sdim break; 1975261991Sdim case NVPTXISD::LDUV2: 1976261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1977261991Sdim default: 1978276479Sdim return nullptr; 1979261991Sdim case MVT::i8: 1980261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32; 1981261991Sdim break; 1982261991Sdim case MVT::i16: 1983261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32; 1984261991Sdim break; 1985261991Sdim case MVT::i32: 1986261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32; 1987261991Sdim break; 1988261991Sdim case MVT::i64: 1989261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32; 1990261991Sdim break; 1991261991Sdim case MVT::f32: 1992261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32; 1993261991Sdim break; 1994261991Sdim case MVT::f64: 1995261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32; 1996261991Sdim break; 1997261991Sdim } 1998249423Sdim break; 1999296417Sdim case NVPTXISD::LoadV4: 2000261991Sdim case NVPTXISD::LDGV4: 2001261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2002261991Sdim default: 2003276479Sdim return nullptr; 2004261991Sdim case MVT::i8: 2005261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32; 2006261991Sdim break; 2007261991Sdim case MVT::i16: 2008261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32; 2009261991Sdim break; 2010261991Sdim case MVT::i32: 2011261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32; 2012261991Sdim break; 2013261991Sdim case MVT::f32: 2014261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32; 2015261991Sdim break; 2016261991Sdim } 2017249423Sdim break; 2018261991Sdim case NVPTXISD::LDUV4: 2019261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2020261991Sdim default: 2021276479Sdim return nullptr; 2022261991Sdim case MVT::i8: 2023261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32; 2024261991Sdim break; 2025261991Sdim case MVT::i16: 2026261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32; 2027261991Sdim break; 2028261991Sdim case MVT::i32: 2029261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32; 2030261991Sdim break; 2031261991Sdim case MVT::f32: 2032261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32; 2033261991Sdim break; 2034261991Sdim } 2035249423Sdim break; 2036249423Sdim } 2037249423Sdim } 2038261991Sdim 2039261991Sdim SDValue Ops[] = { Op1, Chain }; 2040276479Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 2041249423Sdim } 2042249423Sdim 2043249423Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 2044276479Sdim MemRefs0[0] = Mem->getMemOperand(); 2045249423Sdim cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); 2046249423Sdim 2047249423Sdim return LD; 2048249423Sdim} 2049249423Sdim 2050249423SdimSDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) { 2051261991Sdim SDLoc dl(N); 2052239310Sdim StoreSDNode *ST = cast<StoreSDNode>(N); 2053239310Sdim EVT StoreVT = ST->getMemoryVT(); 2054276479Sdim SDNode *NVPTXST = nullptr; 2055239310Sdim 2056239310Sdim // do not support pre/post inc/dec 2057239310Sdim if (ST->isIndexed()) 2058276479Sdim return nullptr; 2059239310Sdim 2060239310Sdim if (!StoreVT.isSimple()) 2061276479Sdim return nullptr; 2062239310Sdim 2063239310Sdim // Address Space Setting 2064288943Sdim unsigned int codeAddrSpace = getCodeAddrSpace(ST); 2065239310Sdim 2066239310Sdim // Volatile Setting 2067239310Sdim // - .volatile is only availalble for .global and .shared 2068239310Sdim bool isVolatile = ST->isVolatile(); 2069239310Sdim if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 2070239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 2071239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 2072239310Sdim isVolatile = false; 2073239310Sdim 2074239310Sdim // Vector Setting 2075239310Sdim MVT SimpleVT = StoreVT.getSimpleVT(); 2076239310Sdim unsigned vecType = NVPTX::PTXLdStInstCode::Scalar; 2077239310Sdim if (SimpleVT.isVector()) { 2078239310Sdim unsigned num = SimpleVT.getVectorNumElements(); 2079239310Sdim if (num == 2) 2080239310Sdim vecType = NVPTX::PTXLdStInstCode::V2; 2081239310Sdim else if (num == 4) 2082239310Sdim vecType = NVPTX::PTXLdStInstCode::V4; 2083239310Sdim else 2084276479Sdim return nullptr; 2085239310Sdim } 2086239310Sdim 2087239310Sdim // Type Setting: toType + toTypeWidth 2088239310Sdim // - for integer type, always use 'u' 2089239310Sdim // 2090239310Sdim MVT ScalarVT = SimpleVT.getScalarType(); 2091249423Sdim unsigned toTypeWidth = ScalarVT.getSizeInBits(); 2092239310Sdim unsigned int toType; 2093239310Sdim if (ScalarVT.isFloatingPoint()) 2094239310Sdim toType = NVPTX::PTXLdStInstCode::Float; 2095239310Sdim else 2096239310Sdim toType = NVPTX::PTXLdStInstCode::Unsigned; 2097239310Sdim 2098239310Sdim // Create the machine instruction DAG 2099239310Sdim SDValue Chain = N->getOperand(0); 2100239310Sdim SDValue N1 = N->getOperand(1); 2101239310Sdim SDValue N2 = N->getOperand(2); 2102239310Sdim SDValue Addr; 2103239310Sdim SDValue Offset, Base; 2104239310Sdim unsigned Opcode; 2105261991Sdim MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy; 2106239310Sdim 2107239310Sdim if (SelectDirectAddr(N2, Addr)) { 2108239310Sdim switch (SourceVT) { 2109249423Sdim case MVT::i8: 2110249423Sdim Opcode = NVPTX::ST_i8_avar; 2111249423Sdim break; 2112249423Sdim case MVT::i16: 2113249423Sdim Opcode = NVPTX::ST_i16_avar; 2114249423Sdim break; 2115249423Sdim case MVT::i32: 2116249423Sdim Opcode = NVPTX::ST_i32_avar; 2117249423Sdim break; 2118249423Sdim case MVT::i64: 2119249423Sdim Opcode = NVPTX::ST_i64_avar; 2120249423Sdim break; 2121249423Sdim case MVT::f32: 2122249423Sdim Opcode = NVPTX::ST_f32_avar; 2123249423Sdim break; 2124249423Sdim case MVT::f64: 2125249423Sdim Opcode = NVPTX::ST_f64_avar; 2126249423Sdim break; 2127249423Sdim default: 2128276479Sdim return nullptr; 2129239310Sdim } 2130288943Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile, dl), 2131288943Sdim getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl), 2132288943Sdim getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Addr, 2133288943Sdim Chain }; 2134251662Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 2135288943Sdim } else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) 2136288943Sdim : SelectADDRsi(N2.getNode(), N2, Base, Offset)) { 2137239310Sdim switch (SourceVT) { 2138249423Sdim case MVT::i8: 2139249423Sdim Opcode = NVPTX::ST_i8_asi; 2140249423Sdim break; 2141249423Sdim case MVT::i16: 2142249423Sdim Opcode = NVPTX::ST_i16_asi; 2143249423Sdim break; 2144249423Sdim case MVT::i32: 2145249423Sdim Opcode = NVPTX::ST_i32_asi; 2146249423Sdim break; 2147249423Sdim case MVT::i64: 2148249423Sdim Opcode = NVPTX::ST_i64_asi; 2149249423Sdim break; 2150249423Sdim case MVT::f32: 2151249423Sdim Opcode = NVPTX::ST_f32_asi; 2152249423Sdim break; 2153249423Sdim case MVT::f64: 2154249423Sdim Opcode = NVPTX::ST_f64_asi; 2155249423Sdim break; 2156249423Sdim default: 2157276479Sdim return nullptr; 2158239310Sdim } 2159288943Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile, dl), 2160288943Sdim getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl), 2161288943Sdim getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Base, 2162288943Sdim Offset, Chain }; 2163251662Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 2164288943Sdim } else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset) 2165288943Sdim : SelectADDRri(N2.getNode(), N2, Base, Offset)) { 2166288943Sdim if (TM.is64Bit()) { 2167249423Sdim switch (SourceVT) { 2168249423Sdim case MVT::i8: 2169249423Sdim Opcode = NVPTX::ST_i8_ari_64; 2170249423Sdim break; 2171249423Sdim case MVT::i16: 2172249423Sdim Opcode = NVPTX::ST_i16_ari_64; 2173249423Sdim break; 2174249423Sdim case MVT::i32: 2175249423Sdim Opcode = NVPTX::ST_i32_ari_64; 2176249423Sdim break; 2177249423Sdim case MVT::i64: 2178249423Sdim Opcode = NVPTX::ST_i64_ari_64; 2179249423Sdim break; 2180249423Sdim case MVT::f32: 2181249423Sdim Opcode = NVPTX::ST_f32_ari_64; 2182249423Sdim break; 2183249423Sdim case MVT::f64: 2184249423Sdim Opcode = NVPTX::ST_f64_ari_64; 2185249423Sdim break; 2186249423Sdim default: 2187276479Sdim return nullptr; 2188249423Sdim } 2189249423Sdim } else { 2190249423Sdim switch (SourceVT) { 2191249423Sdim case MVT::i8: 2192249423Sdim Opcode = NVPTX::ST_i8_ari; 2193249423Sdim break; 2194249423Sdim case MVT::i16: 2195249423Sdim Opcode = NVPTX::ST_i16_ari; 2196249423Sdim break; 2197249423Sdim case MVT::i32: 2198249423Sdim Opcode = NVPTX::ST_i32_ari; 2199249423Sdim break; 2200249423Sdim case MVT::i64: 2201249423Sdim Opcode = NVPTX::ST_i64_ari; 2202249423Sdim break; 2203249423Sdim case MVT::f32: 2204249423Sdim Opcode = NVPTX::ST_f32_ari; 2205249423Sdim break; 2206249423Sdim case MVT::f64: 2207249423Sdim Opcode = NVPTX::ST_f64_ari; 2208249423Sdim break; 2209249423Sdim default: 2210276479Sdim return nullptr; 2211249423Sdim } 2212239310Sdim } 2213288943Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile, dl), 2214288943Sdim getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl), 2215288943Sdim getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Base, 2216288943Sdim Offset, Chain }; 2217251662Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 2218239310Sdim } else { 2219288943Sdim if (TM.is64Bit()) { 2220249423Sdim switch (SourceVT) { 2221249423Sdim case MVT::i8: 2222249423Sdim Opcode = NVPTX::ST_i8_areg_64; 2223249423Sdim break; 2224249423Sdim case MVT::i16: 2225249423Sdim Opcode = NVPTX::ST_i16_areg_64; 2226249423Sdim break; 2227249423Sdim case MVT::i32: 2228249423Sdim Opcode = NVPTX::ST_i32_areg_64; 2229249423Sdim break; 2230249423Sdim case MVT::i64: 2231249423Sdim Opcode = NVPTX::ST_i64_areg_64; 2232249423Sdim break; 2233249423Sdim case MVT::f32: 2234249423Sdim Opcode = NVPTX::ST_f32_areg_64; 2235249423Sdim break; 2236249423Sdim case MVT::f64: 2237249423Sdim Opcode = NVPTX::ST_f64_areg_64; 2238249423Sdim break; 2239249423Sdim default: 2240276479Sdim return nullptr; 2241249423Sdim } 2242249423Sdim } else { 2243249423Sdim switch (SourceVT) { 2244249423Sdim case MVT::i8: 2245249423Sdim Opcode = NVPTX::ST_i8_areg; 2246249423Sdim break; 2247249423Sdim case MVT::i16: 2248249423Sdim Opcode = NVPTX::ST_i16_areg; 2249249423Sdim break; 2250249423Sdim case MVT::i32: 2251249423Sdim Opcode = NVPTX::ST_i32_areg; 2252249423Sdim break; 2253249423Sdim case MVT::i64: 2254249423Sdim Opcode = NVPTX::ST_i64_areg; 2255249423Sdim break; 2256249423Sdim case MVT::f32: 2257249423Sdim Opcode = NVPTX::ST_f32_areg; 2258249423Sdim break; 2259249423Sdim case MVT::f64: 2260249423Sdim Opcode = NVPTX::ST_f64_areg; 2261249423Sdim break; 2262249423Sdim default: 2263276479Sdim return nullptr; 2264249423Sdim } 2265239310Sdim } 2266288943Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile, dl), 2267288943Sdim getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl), 2268288943Sdim getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), N2, 2269288943Sdim Chain }; 2270251662Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 2271239310Sdim } 2272239310Sdim 2273276479Sdim if (NVPTXST) { 2274239310Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 2275239310Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 2276239310Sdim cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1); 2277239310Sdim } 2278239310Sdim 2279239310Sdim return NVPTXST; 2280239310Sdim} 2281239310Sdim 2282249423SdimSDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) { 2283249423Sdim SDValue Chain = N->getOperand(0); 2284249423Sdim SDValue Op1 = N->getOperand(1); 2285249423Sdim SDValue Addr, Offset, Base; 2286249423Sdim unsigned Opcode; 2287261991Sdim SDLoc DL(N); 2288249423Sdim SDNode *ST; 2289249423Sdim EVT EltVT = Op1.getValueType(); 2290249423Sdim MemSDNode *MemSD = cast<MemSDNode>(N); 2291249423Sdim EVT StoreVT = MemSD->getMemoryVT(); 2292249423Sdim 2293249423Sdim // Address Space Setting 2294288943Sdim unsigned CodeAddrSpace = getCodeAddrSpace(MemSD); 2295249423Sdim 2296249423Sdim if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) { 2297249423Sdim report_fatal_error("Cannot store to pointer that points to constant " 2298249423Sdim "memory space"); 2299249423Sdim } 2300249423Sdim 2301249423Sdim // Volatile Setting 2302249423Sdim // - .volatile is only availalble for .global and .shared 2303249423Sdim bool IsVolatile = MemSD->isVolatile(); 2304249423Sdim if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 2305249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 2306249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 2307249423Sdim IsVolatile = false; 2308249423Sdim 2309249423Sdim // Type Setting: toType + toTypeWidth 2310249423Sdim // - for integer type, always use 'u' 2311249423Sdim assert(StoreVT.isSimple() && "Store value is not simple"); 2312249423Sdim MVT ScalarVT = StoreVT.getSimpleVT().getScalarType(); 2313249423Sdim unsigned ToTypeWidth = ScalarVT.getSizeInBits(); 2314249423Sdim unsigned ToType; 2315249423Sdim if (ScalarVT.isFloatingPoint()) 2316249423Sdim ToType = NVPTX::PTXLdStInstCode::Float; 2317249423Sdim else 2318249423Sdim ToType = NVPTX::PTXLdStInstCode::Unsigned; 2319249423Sdim 2320249423Sdim SmallVector<SDValue, 12> StOps; 2321249423Sdim SDValue N2; 2322249423Sdim unsigned VecType; 2323249423Sdim 2324249423Sdim switch (N->getOpcode()) { 2325249423Sdim case NVPTXISD::StoreV2: 2326249423Sdim VecType = NVPTX::PTXLdStInstCode::V2; 2327249423Sdim StOps.push_back(N->getOperand(1)); 2328249423Sdim StOps.push_back(N->getOperand(2)); 2329249423Sdim N2 = N->getOperand(3); 2330249423Sdim break; 2331249423Sdim case NVPTXISD::StoreV4: 2332249423Sdim VecType = NVPTX::PTXLdStInstCode::V4; 2333249423Sdim StOps.push_back(N->getOperand(1)); 2334249423Sdim StOps.push_back(N->getOperand(2)); 2335249423Sdim StOps.push_back(N->getOperand(3)); 2336249423Sdim StOps.push_back(N->getOperand(4)); 2337249423Sdim N2 = N->getOperand(5); 2338249423Sdim break; 2339249423Sdim default: 2340276479Sdim return nullptr; 2341249423Sdim } 2342249423Sdim 2343288943Sdim StOps.push_back(getI32Imm(IsVolatile, DL)); 2344288943Sdim StOps.push_back(getI32Imm(CodeAddrSpace, DL)); 2345288943Sdim StOps.push_back(getI32Imm(VecType, DL)); 2346288943Sdim StOps.push_back(getI32Imm(ToType, DL)); 2347288943Sdim StOps.push_back(getI32Imm(ToTypeWidth, DL)); 2348249423Sdim 2349249423Sdim if (SelectDirectAddr(N2, Addr)) { 2350249423Sdim switch (N->getOpcode()) { 2351249423Sdim default: 2352276479Sdim return nullptr; 2353249423Sdim case NVPTXISD::StoreV2: 2354249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2355249423Sdim default: 2356276479Sdim return nullptr; 2357249423Sdim case MVT::i8: 2358249423Sdim Opcode = NVPTX::STV_i8_v2_avar; 2359249423Sdim break; 2360249423Sdim case MVT::i16: 2361249423Sdim Opcode = NVPTX::STV_i16_v2_avar; 2362249423Sdim break; 2363249423Sdim case MVT::i32: 2364249423Sdim Opcode = NVPTX::STV_i32_v2_avar; 2365249423Sdim break; 2366249423Sdim case MVT::i64: 2367249423Sdim Opcode = NVPTX::STV_i64_v2_avar; 2368249423Sdim break; 2369249423Sdim case MVT::f32: 2370249423Sdim Opcode = NVPTX::STV_f32_v2_avar; 2371249423Sdim break; 2372249423Sdim case MVT::f64: 2373249423Sdim Opcode = NVPTX::STV_f64_v2_avar; 2374249423Sdim break; 2375249423Sdim } 2376249423Sdim break; 2377249423Sdim case NVPTXISD::StoreV4: 2378249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2379249423Sdim default: 2380276479Sdim return nullptr; 2381249423Sdim case MVT::i8: 2382249423Sdim Opcode = NVPTX::STV_i8_v4_avar; 2383249423Sdim break; 2384249423Sdim case MVT::i16: 2385249423Sdim Opcode = NVPTX::STV_i16_v4_avar; 2386249423Sdim break; 2387249423Sdim case MVT::i32: 2388249423Sdim Opcode = NVPTX::STV_i32_v4_avar; 2389249423Sdim break; 2390249423Sdim case MVT::f32: 2391249423Sdim Opcode = NVPTX::STV_f32_v4_avar; 2392249423Sdim break; 2393249423Sdim } 2394249423Sdim break; 2395249423Sdim } 2396249423Sdim StOps.push_back(Addr); 2397288943Sdim } else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) 2398288943Sdim : SelectADDRsi(N2.getNode(), N2, Base, Offset)) { 2399249423Sdim switch (N->getOpcode()) { 2400249423Sdim default: 2401276479Sdim return nullptr; 2402249423Sdim case NVPTXISD::StoreV2: 2403249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2404249423Sdim default: 2405276479Sdim return nullptr; 2406249423Sdim case MVT::i8: 2407249423Sdim Opcode = NVPTX::STV_i8_v2_asi; 2408249423Sdim break; 2409249423Sdim case MVT::i16: 2410249423Sdim Opcode = NVPTX::STV_i16_v2_asi; 2411249423Sdim break; 2412249423Sdim case MVT::i32: 2413249423Sdim Opcode = NVPTX::STV_i32_v2_asi; 2414249423Sdim break; 2415249423Sdim case MVT::i64: 2416249423Sdim Opcode = NVPTX::STV_i64_v2_asi; 2417249423Sdim break; 2418249423Sdim case MVT::f32: 2419249423Sdim Opcode = NVPTX::STV_f32_v2_asi; 2420249423Sdim break; 2421249423Sdim case MVT::f64: 2422249423Sdim Opcode = NVPTX::STV_f64_v2_asi; 2423249423Sdim break; 2424249423Sdim } 2425249423Sdim break; 2426249423Sdim case NVPTXISD::StoreV4: 2427249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2428249423Sdim default: 2429276479Sdim return nullptr; 2430249423Sdim case MVT::i8: 2431249423Sdim Opcode = NVPTX::STV_i8_v4_asi; 2432249423Sdim break; 2433249423Sdim case MVT::i16: 2434249423Sdim Opcode = NVPTX::STV_i16_v4_asi; 2435249423Sdim break; 2436249423Sdim case MVT::i32: 2437249423Sdim Opcode = NVPTX::STV_i32_v4_asi; 2438249423Sdim break; 2439249423Sdim case MVT::f32: 2440249423Sdim Opcode = NVPTX::STV_f32_v4_asi; 2441249423Sdim break; 2442249423Sdim } 2443249423Sdim break; 2444249423Sdim } 2445249423Sdim StOps.push_back(Base); 2446249423Sdim StOps.push_back(Offset); 2447288943Sdim } else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset) 2448288943Sdim : SelectADDRri(N2.getNode(), N2, Base, Offset)) { 2449288943Sdim if (TM.is64Bit()) { 2450249423Sdim switch (N->getOpcode()) { 2451249423Sdim default: 2452276479Sdim return nullptr; 2453249423Sdim case NVPTXISD::StoreV2: 2454249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2455249423Sdim default: 2456276479Sdim return nullptr; 2457249423Sdim case MVT::i8: 2458249423Sdim Opcode = NVPTX::STV_i8_v2_ari_64; 2459249423Sdim break; 2460249423Sdim case MVT::i16: 2461249423Sdim Opcode = NVPTX::STV_i16_v2_ari_64; 2462249423Sdim break; 2463249423Sdim case MVT::i32: 2464249423Sdim Opcode = NVPTX::STV_i32_v2_ari_64; 2465249423Sdim break; 2466249423Sdim case MVT::i64: 2467249423Sdim Opcode = NVPTX::STV_i64_v2_ari_64; 2468249423Sdim break; 2469249423Sdim case MVT::f32: 2470249423Sdim Opcode = NVPTX::STV_f32_v2_ari_64; 2471249423Sdim break; 2472249423Sdim case MVT::f64: 2473249423Sdim Opcode = NVPTX::STV_f64_v2_ari_64; 2474249423Sdim break; 2475249423Sdim } 2476249423Sdim break; 2477249423Sdim case NVPTXISD::StoreV4: 2478249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2479249423Sdim default: 2480276479Sdim return nullptr; 2481249423Sdim case MVT::i8: 2482249423Sdim Opcode = NVPTX::STV_i8_v4_ari_64; 2483249423Sdim break; 2484249423Sdim case MVT::i16: 2485249423Sdim Opcode = NVPTX::STV_i16_v4_ari_64; 2486249423Sdim break; 2487249423Sdim case MVT::i32: 2488249423Sdim Opcode = NVPTX::STV_i32_v4_ari_64; 2489249423Sdim break; 2490249423Sdim case MVT::f32: 2491249423Sdim Opcode = NVPTX::STV_f32_v4_ari_64; 2492249423Sdim break; 2493249423Sdim } 2494249423Sdim break; 2495249423Sdim } 2496249423Sdim } else { 2497249423Sdim switch (N->getOpcode()) { 2498249423Sdim default: 2499276479Sdim return nullptr; 2500249423Sdim case NVPTXISD::StoreV2: 2501249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2502249423Sdim default: 2503276479Sdim return nullptr; 2504249423Sdim case MVT::i8: 2505249423Sdim Opcode = NVPTX::STV_i8_v2_ari; 2506249423Sdim break; 2507249423Sdim case MVT::i16: 2508249423Sdim Opcode = NVPTX::STV_i16_v2_ari; 2509249423Sdim break; 2510249423Sdim case MVT::i32: 2511249423Sdim Opcode = NVPTX::STV_i32_v2_ari; 2512249423Sdim break; 2513249423Sdim case MVT::i64: 2514249423Sdim Opcode = NVPTX::STV_i64_v2_ari; 2515249423Sdim break; 2516249423Sdim case MVT::f32: 2517249423Sdim Opcode = NVPTX::STV_f32_v2_ari; 2518249423Sdim break; 2519249423Sdim case MVT::f64: 2520249423Sdim Opcode = NVPTX::STV_f64_v2_ari; 2521249423Sdim break; 2522249423Sdim } 2523249423Sdim break; 2524249423Sdim case NVPTXISD::StoreV4: 2525249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2526249423Sdim default: 2527276479Sdim return nullptr; 2528249423Sdim case MVT::i8: 2529249423Sdim Opcode = NVPTX::STV_i8_v4_ari; 2530249423Sdim break; 2531249423Sdim case MVT::i16: 2532249423Sdim Opcode = NVPTX::STV_i16_v4_ari; 2533249423Sdim break; 2534249423Sdim case MVT::i32: 2535249423Sdim Opcode = NVPTX::STV_i32_v4_ari; 2536249423Sdim break; 2537249423Sdim case MVT::f32: 2538249423Sdim Opcode = NVPTX::STV_f32_v4_ari; 2539249423Sdim break; 2540249423Sdim } 2541249423Sdim break; 2542249423Sdim } 2543249423Sdim } 2544249423Sdim StOps.push_back(Base); 2545249423Sdim StOps.push_back(Offset); 2546249423Sdim } else { 2547288943Sdim if (TM.is64Bit()) { 2548249423Sdim switch (N->getOpcode()) { 2549249423Sdim default: 2550276479Sdim return nullptr; 2551249423Sdim case NVPTXISD::StoreV2: 2552249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2553249423Sdim default: 2554276479Sdim return nullptr; 2555249423Sdim case MVT::i8: 2556249423Sdim Opcode = NVPTX::STV_i8_v2_areg_64; 2557249423Sdim break; 2558249423Sdim case MVT::i16: 2559249423Sdim Opcode = NVPTX::STV_i16_v2_areg_64; 2560249423Sdim break; 2561249423Sdim case MVT::i32: 2562249423Sdim Opcode = NVPTX::STV_i32_v2_areg_64; 2563249423Sdim break; 2564249423Sdim case MVT::i64: 2565249423Sdim Opcode = NVPTX::STV_i64_v2_areg_64; 2566249423Sdim break; 2567249423Sdim case MVT::f32: 2568249423Sdim Opcode = NVPTX::STV_f32_v2_areg_64; 2569249423Sdim break; 2570249423Sdim case MVT::f64: 2571249423Sdim Opcode = NVPTX::STV_f64_v2_areg_64; 2572249423Sdim break; 2573249423Sdim } 2574249423Sdim break; 2575249423Sdim case NVPTXISD::StoreV4: 2576249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2577249423Sdim default: 2578276479Sdim return nullptr; 2579249423Sdim case MVT::i8: 2580249423Sdim Opcode = NVPTX::STV_i8_v4_areg_64; 2581249423Sdim break; 2582249423Sdim case MVT::i16: 2583249423Sdim Opcode = NVPTX::STV_i16_v4_areg_64; 2584249423Sdim break; 2585249423Sdim case MVT::i32: 2586249423Sdim Opcode = NVPTX::STV_i32_v4_areg_64; 2587249423Sdim break; 2588249423Sdim case MVT::f32: 2589249423Sdim Opcode = NVPTX::STV_f32_v4_areg_64; 2590249423Sdim break; 2591249423Sdim } 2592249423Sdim break; 2593249423Sdim } 2594249423Sdim } else { 2595249423Sdim switch (N->getOpcode()) { 2596249423Sdim default: 2597276479Sdim return nullptr; 2598249423Sdim case NVPTXISD::StoreV2: 2599249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2600249423Sdim default: 2601276479Sdim return nullptr; 2602249423Sdim case MVT::i8: 2603249423Sdim Opcode = NVPTX::STV_i8_v2_areg; 2604249423Sdim break; 2605249423Sdim case MVT::i16: 2606249423Sdim Opcode = NVPTX::STV_i16_v2_areg; 2607249423Sdim break; 2608249423Sdim case MVT::i32: 2609249423Sdim Opcode = NVPTX::STV_i32_v2_areg; 2610249423Sdim break; 2611249423Sdim case MVT::i64: 2612249423Sdim Opcode = NVPTX::STV_i64_v2_areg; 2613249423Sdim break; 2614249423Sdim case MVT::f32: 2615249423Sdim Opcode = NVPTX::STV_f32_v2_areg; 2616249423Sdim break; 2617249423Sdim case MVT::f64: 2618249423Sdim Opcode = NVPTX::STV_f64_v2_areg; 2619249423Sdim break; 2620249423Sdim } 2621249423Sdim break; 2622249423Sdim case NVPTXISD::StoreV4: 2623249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2624249423Sdim default: 2625276479Sdim return nullptr; 2626249423Sdim case MVT::i8: 2627249423Sdim Opcode = NVPTX::STV_i8_v4_areg; 2628249423Sdim break; 2629249423Sdim case MVT::i16: 2630249423Sdim Opcode = NVPTX::STV_i16_v4_areg; 2631249423Sdim break; 2632249423Sdim case MVT::i32: 2633249423Sdim Opcode = NVPTX::STV_i32_v4_areg; 2634249423Sdim break; 2635249423Sdim case MVT::f32: 2636249423Sdim Opcode = NVPTX::STV_f32_v4_areg; 2637249423Sdim break; 2638249423Sdim } 2639249423Sdim break; 2640249423Sdim } 2641249423Sdim } 2642249423Sdim StOps.push_back(N2); 2643249423Sdim } 2644249423Sdim 2645249423Sdim StOps.push_back(Chain); 2646249423Sdim 2647251662Sdim ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps); 2648249423Sdim 2649249423Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 2650249423Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 2651249423Sdim cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1); 2652249423Sdim 2653249423Sdim return ST; 2654249423Sdim} 2655249423Sdim 2656261991SdimSDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) { 2657261991Sdim SDValue Chain = Node->getOperand(0); 2658261991Sdim SDValue Offset = Node->getOperand(2); 2659261991Sdim SDValue Flag = Node->getOperand(3); 2660261991Sdim SDLoc DL(Node); 2661261991Sdim MemSDNode *Mem = cast<MemSDNode>(Node); 2662261991Sdim 2663261991Sdim unsigned VecSize; 2664261991Sdim switch (Node->getOpcode()) { 2665261991Sdim default: 2666276479Sdim return nullptr; 2667261991Sdim case NVPTXISD::LoadParam: 2668261991Sdim VecSize = 1; 2669261991Sdim break; 2670261991Sdim case NVPTXISD::LoadParamV2: 2671261991Sdim VecSize = 2; 2672261991Sdim break; 2673261991Sdim case NVPTXISD::LoadParamV4: 2674261991Sdim VecSize = 4; 2675261991Sdim break; 2676261991Sdim } 2677261991Sdim 2678261991Sdim EVT EltVT = Node->getValueType(0); 2679261991Sdim EVT MemVT = Mem->getMemoryVT(); 2680261991Sdim 2681261991Sdim unsigned Opc = 0; 2682261991Sdim 2683261991Sdim switch (VecSize) { 2684261991Sdim default: 2685276479Sdim return nullptr; 2686261991Sdim case 1: 2687261991Sdim switch (MemVT.getSimpleVT().SimpleTy) { 2688261991Sdim default: 2689276479Sdim return nullptr; 2690261991Sdim case MVT::i1: 2691261991Sdim Opc = NVPTX::LoadParamMemI8; 2692261991Sdim break; 2693261991Sdim case MVT::i8: 2694261991Sdim Opc = NVPTX::LoadParamMemI8; 2695261991Sdim break; 2696261991Sdim case MVT::i16: 2697261991Sdim Opc = NVPTX::LoadParamMemI16; 2698261991Sdim break; 2699261991Sdim case MVT::i32: 2700261991Sdim Opc = NVPTX::LoadParamMemI32; 2701261991Sdim break; 2702261991Sdim case MVT::i64: 2703261991Sdim Opc = NVPTX::LoadParamMemI64; 2704261991Sdim break; 2705261991Sdim case MVT::f32: 2706261991Sdim Opc = NVPTX::LoadParamMemF32; 2707261991Sdim break; 2708261991Sdim case MVT::f64: 2709261991Sdim Opc = NVPTX::LoadParamMemF64; 2710261991Sdim break; 2711261991Sdim } 2712261991Sdim break; 2713261991Sdim case 2: 2714261991Sdim switch (MemVT.getSimpleVT().SimpleTy) { 2715261991Sdim default: 2716276479Sdim return nullptr; 2717261991Sdim case MVT::i1: 2718261991Sdim Opc = NVPTX::LoadParamMemV2I8; 2719261991Sdim break; 2720261991Sdim case MVT::i8: 2721261991Sdim Opc = NVPTX::LoadParamMemV2I8; 2722261991Sdim break; 2723261991Sdim case MVT::i16: 2724261991Sdim Opc = NVPTX::LoadParamMemV2I16; 2725261991Sdim break; 2726261991Sdim case MVT::i32: 2727261991Sdim Opc = NVPTX::LoadParamMemV2I32; 2728261991Sdim break; 2729261991Sdim case MVT::i64: 2730261991Sdim Opc = NVPTX::LoadParamMemV2I64; 2731261991Sdim break; 2732261991Sdim case MVT::f32: 2733261991Sdim Opc = NVPTX::LoadParamMemV2F32; 2734261991Sdim break; 2735261991Sdim case MVT::f64: 2736261991Sdim Opc = NVPTX::LoadParamMemV2F64; 2737261991Sdim break; 2738261991Sdim } 2739261991Sdim break; 2740261991Sdim case 4: 2741261991Sdim switch (MemVT.getSimpleVT().SimpleTy) { 2742261991Sdim default: 2743276479Sdim return nullptr; 2744261991Sdim case MVT::i1: 2745261991Sdim Opc = NVPTX::LoadParamMemV4I8; 2746261991Sdim break; 2747261991Sdim case MVT::i8: 2748261991Sdim Opc = NVPTX::LoadParamMemV4I8; 2749261991Sdim break; 2750261991Sdim case MVT::i16: 2751261991Sdim Opc = NVPTX::LoadParamMemV4I16; 2752261991Sdim break; 2753261991Sdim case MVT::i32: 2754261991Sdim Opc = NVPTX::LoadParamMemV4I32; 2755261991Sdim break; 2756261991Sdim case MVT::f32: 2757261991Sdim Opc = NVPTX::LoadParamMemV4F32; 2758261991Sdim break; 2759261991Sdim } 2760261991Sdim break; 2761261991Sdim } 2762261991Sdim 2763261991Sdim SDVTList VTs; 2764261991Sdim if (VecSize == 1) { 2765261991Sdim VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue); 2766261991Sdim } else if (VecSize == 2) { 2767261991Sdim VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue); 2768261991Sdim } else { 2769261991Sdim EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue }; 2770276479Sdim VTs = CurDAG->getVTList(EVTs); 2771261991Sdim } 2772261991Sdim 2773261991Sdim unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); 2774261991Sdim 2775261991Sdim SmallVector<SDValue, 2> Ops; 2776288943Sdim Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32)); 2777261991Sdim Ops.push_back(Chain); 2778261991Sdim Ops.push_back(Flag); 2779261991Sdim 2780288943Sdim return CurDAG->getMachineNode(Opc, DL, VTs, Ops); 2781261991Sdim} 2782261991Sdim 2783261991SdimSDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) { 2784261991Sdim SDLoc DL(N); 2785261991Sdim SDValue Chain = N->getOperand(0); 2786261991Sdim SDValue Offset = N->getOperand(1); 2787261991Sdim unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); 2788261991Sdim MemSDNode *Mem = cast<MemSDNode>(N); 2789261991Sdim 2790261991Sdim // How many elements do we have? 2791261991Sdim unsigned NumElts = 1; 2792261991Sdim switch (N->getOpcode()) { 2793261991Sdim default: 2794276479Sdim return nullptr; 2795261991Sdim case NVPTXISD::StoreRetval: 2796261991Sdim NumElts = 1; 2797261991Sdim break; 2798261991Sdim case NVPTXISD::StoreRetvalV2: 2799261991Sdim NumElts = 2; 2800261991Sdim break; 2801261991Sdim case NVPTXISD::StoreRetvalV4: 2802261991Sdim NumElts = 4; 2803261991Sdim break; 2804261991Sdim } 2805261991Sdim 2806261991Sdim // Build vector of operands 2807261991Sdim SmallVector<SDValue, 6> Ops; 2808261991Sdim for (unsigned i = 0; i < NumElts; ++i) 2809261991Sdim Ops.push_back(N->getOperand(i + 2)); 2810288943Sdim Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32)); 2811261991Sdim Ops.push_back(Chain); 2812261991Sdim 2813261991Sdim // Determine target opcode 2814261991Sdim // If we have an i1, use an 8-bit store. The lowering code in 2815261991Sdim // NVPTXISelLowering will have already emitted an upcast. 2816261991Sdim unsigned Opcode = 0; 2817261991Sdim switch (NumElts) { 2818261991Sdim default: 2819276479Sdim return nullptr; 2820261991Sdim case 1: 2821261991Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2822261991Sdim default: 2823276479Sdim return nullptr; 2824261991Sdim case MVT::i1: 2825261991Sdim Opcode = NVPTX::StoreRetvalI8; 2826261991Sdim break; 2827261991Sdim case MVT::i8: 2828261991Sdim Opcode = NVPTX::StoreRetvalI8; 2829261991Sdim break; 2830261991Sdim case MVT::i16: 2831261991Sdim Opcode = NVPTX::StoreRetvalI16; 2832261991Sdim break; 2833261991Sdim case MVT::i32: 2834261991Sdim Opcode = NVPTX::StoreRetvalI32; 2835261991Sdim break; 2836261991Sdim case MVT::i64: 2837261991Sdim Opcode = NVPTX::StoreRetvalI64; 2838261991Sdim break; 2839261991Sdim case MVT::f32: 2840261991Sdim Opcode = NVPTX::StoreRetvalF32; 2841261991Sdim break; 2842261991Sdim case MVT::f64: 2843261991Sdim Opcode = NVPTX::StoreRetvalF64; 2844261991Sdim break; 2845261991Sdim } 2846261991Sdim break; 2847261991Sdim case 2: 2848261991Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2849261991Sdim default: 2850276479Sdim return nullptr; 2851261991Sdim case MVT::i1: 2852261991Sdim Opcode = NVPTX::StoreRetvalV2I8; 2853261991Sdim break; 2854261991Sdim case MVT::i8: 2855261991Sdim Opcode = NVPTX::StoreRetvalV2I8; 2856261991Sdim break; 2857261991Sdim case MVT::i16: 2858261991Sdim Opcode = NVPTX::StoreRetvalV2I16; 2859261991Sdim break; 2860261991Sdim case MVT::i32: 2861261991Sdim Opcode = NVPTX::StoreRetvalV2I32; 2862261991Sdim break; 2863261991Sdim case MVT::i64: 2864261991Sdim Opcode = NVPTX::StoreRetvalV2I64; 2865261991Sdim break; 2866261991Sdim case MVT::f32: 2867261991Sdim Opcode = NVPTX::StoreRetvalV2F32; 2868261991Sdim break; 2869261991Sdim case MVT::f64: 2870261991Sdim Opcode = NVPTX::StoreRetvalV2F64; 2871261991Sdim break; 2872261991Sdim } 2873261991Sdim break; 2874261991Sdim case 4: 2875261991Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2876261991Sdim default: 2877276479Sdim return nullptr; 2878261991Sdim case MVT::i1: 2879261991Sdim Opcode = NVPTX::StoreRetvalV4I8; 2880261991Sdim break; 2881261991Sdim case MVT::i8: 2882261991Sdim Opcode = NVPTX::StoreRetvalV4I8; 2883261991Sdim break; 2884261991Sdim case MVT::i16: 2885261991Sdim Opcode = NVPTX::StoreRetvalV4I16; 2886261991Sdim break; 2887261991Sdim case MVT::i32: 2888261991Sdim Opcode = NVPTX::StoreRetvalV4I32; 2889261991Sdim break; 2890261991Sdim case MVT::f32: 2891261991Sdim Opcode = NVPTX::StoreRetvalV4F32; 2892261991Sdim break; 2893261991Sdim } 2894261991Sdim break; 2895261991Sdim } 2896261991Sdim 2897261991Sdim SDNode *Ret = 2898261991Sdim CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops); 2899261991Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 2900261991Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 2901261991Sdim cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1); 2902261991Sdim 2903261991Sdim return Ret; 2904261991Sdim} 2905261991Sdim 2906261991SdimSDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) { 2907261991Sdim SDLoc DL(N); 2908261991Sdim SDValue Chain = N->getOperand(0); 2909261991Sdim SDValue Param = N->getOperand(1); 2910261991Sdim unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue(); 2911261991Sdim SDValue Offset = N->getOperand(2); 2912261991Sdim unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); 2913261991Sdim MemSDNode *Mem = cast<MemSDNode>(N); 2914261991Sdim SDValue Flag = N->getOperand(N->getNumOperands() - 1); 2915261991Sdim 2916261991Sdim // How many elements do we have? 2917261991Sdim unsigned NumElts = 1; 2918261991Sdim switch (N->getOpcode()) { 2919261991Sdim default: 2920276479Sdim return nullptr; 2921261991Sdim case NVPTXISD::StoreParamU32: 2922261991Sdim case NVPTXISD::StoreParamS32: 2923261991Sdim case NVPTXISD::StoreParam: 2924261991Sdim NumElts = 1; 2925261991Sdim break; 2926261991Sdim case NVPTXISD::StoreParamV2: 2927261991Sdim NumElts = 2; 2928261991Sdim break; 2929261991Sdim case NVPTXISD::StoreParamV4: 2930261991Sdim NumElts = 4; 2931261991Sdim break; 2932261991Sdim } 2933261991Sdim 2934261991Sdim // Build vector of operands 2935261991Sdim SmallVector<SDValue, 8> Ops; 2936261991Sdim for (unsigned i = 0; i < NumElts; ++i) 2937261991Sdim Ops.push_back(N->getOperand(i + 3)); 2938288943Sdim Ops.push_back(CurDAG->getTargetConstant(ParamVal, DL, MVT::i32)); 2939288943Sdim Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32)); 2940261991Sdim Ops.push_back(Chain); 2941261991Sdim Ops.push_back(Flag); 2942261991Sdim 2943261991Sdim // Determine target opcode 2944261991Sdim // If we have an i1, use an 8-bit store. The lowering code in 2945261991Sdim // NVPTXISelLowering will have already emitted an upcast. 2946261991Sdim unsigned Opcode = 0; 2947261991Sdim switch (N->getOpcode()) { 2948261991Sdim default: 2949261991Sdim switch (NumElts) { 2950261991Sdim default: 2951276479Sdim return nullptr; 2952261991Sdim case 1: 2953261991Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2954261991Sdim default: 2955276479Sdim return nullptr; 2956261991Sdim case MVT::i1: 2957261991Sdim Opcode = NVPTX::StoreParamI8; 2958261991Sdim break; 2959261991Sdim case MVT::i8: 2960261991Sdim Opcode = NVPTX::StoreParamI8; 2961261991Sdim break; 2962261991Sdim case MVT::i16: 2963261991Sdim Opcode = NVPTX::StoreParamI16; 2964261991Sdim break; 2965261991Sdim case MVT::i32: 2966261991Sdim Opcode = NVPTX::StoreParamI32; 2967261991Sdim break; 2968261991Sdim case MVT::i64: 2969261991Sdim Opcode = NVPTX::StoreParamI64; 2970261991Sdim break; 2971261991Sdim case MVT::f32: 2972261991Sdim Opcode = NVPTX::StoreParamF32; 2973261991Sdim break; 2974261991Sdim case MVT::f64: 2975261991Sdim Opcode = NVPTX::StoreParamF64; 2976261991Sdim break; 2977261991Sdim } 2978261991Sdim break; 2979261991Sdim case 2: 2980261991Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2981261991Sdim default: 2982276479Sdim return nullptr; 2983261991Sdim case MVT::i1: 2984261991Sdim Opcode = NVPTX::StoreParamV2I8; 2985261991Sdim break; 2986261991Sdim case MVT::i8: 2987261991Sdim Opcode = NVPTX::StoreParamV2I8; 2988261991Sdim break; 2989261991Sdim case MVT::i16: 2990261991Sdim Opcode = NVPTX::StoreParamV2I16; 2991261991Sdim break; 2992261991Sdim case MVT::i32: 2993261991Sdim Opcode = NVPTX::StoreParamV2I32; 2994261991Sdim break; 2995261991Sdim case MVT::i64: 2996261991Sdim Opcode = NVPTX::StoreParamV2I64; 2997261991Sdim break; 2998261991Sdim case MVT::f32: 2999261991Sdim Opcode = NVPTX::StoreParamV2F32; 3000261991Sdim break; 3001261991Sdim case MVT::f64: 3002261991Sdim Opcode = NVPTX::StoreParamV2F64; 3003261991Sdim break; 3004261991Sdim } 3005261991Sdim break; 3006261991Sdim case 4: 3007261991Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 3008261991Sdim default: 3009276479Sdim return nullptr; 3010261991Sdim case MVT::i1: 3011261991Sdim Opcode = NVPTX::StoreParamV4I8; 3012261991Sdim break; 3013261991Sdim case MVT::i8: 3014261991Sdim Opcode = NVPTX::StoreParamV4I8; 3015261991Sdim break; 3016261991Sdim case MVT::i16: 3017261991Sdim Opcode = NVPTX::StoreParamV4I16; 3018261991Sdim break; 3019261991Sdim case MVT::i32: 3020261991Sdim Opcode = NVPTX::StoreParamV4I32; 3021261991Sdim break; 3022261991Sdim case MVT::f32: 3023261991Sdim Opcode = NVPTX::StoreParamV4F32; 3024261991Sdim break; 3025261991Sdim } 3026261991Sdim break; 3027261991Sdim } 3028261991Sdim break; 3029261991Sdim // Special case: if we have a sign-extend/zero-extend node, insert the 3030261991Sdim // conversion instruction first, and use that as the value operand to 3031261991Sdim // the selected StoreParam node. 3032261991Sdim case NVPTXISD::StoreParamU32: { 3033261991Sdim Opcode = NVPTX::StoreParamI32; 3034288943Sdim SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL, 3035261991Sdim MVT::i32); 3036261991Sdim SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL, 3037261991Sdim MVT::i32, Ops[0], CvtNone); 3038261991Sdim Ops[0] = SDValue(Cvt, 0); 3039261991Sdim break; 3040261991Sdim } 3041261991Sdim case NVPTXISD::StoreParamS32: { 3042261991Sdim Opcode = NVPTX::StoreParamI32; 3043288943Sdim SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL, 3044261991Sdim MVT::i32); 3045261991Sdim SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL, 3046261991Sdim MVT::i32, Ops[0], CvtNone); 3047261991Sdim Ops[0] = SDValue(Cvt, 0); 3048261991Sdim break; 3049261991Sdim } 3050261991Sdim } 3051261991Sdim 3052261991Sdim SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue); 3053261991Sdim SDNode *Ret = 3054261991Sdim CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops); 3055261991Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 3056261991Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 3057261991Sdim cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1); 3058261991Sdim 3059261991Sdim return Ret; 3060261991Sdim} 3061261991Sdim 3062276479SdimSDNode *NVPTXDAGToDAGISel::SelectTextureIntrinsic(SDNode *N) { 3063276479Sdim SDValue Chain = N->getOperand(0); 3064276479Sdim SDNode *Ret = nullptr; 3065276479Sdim unsigned Opc = 0; 3066276479Sdim SmallVector<SDValue, 8> Ops; 3067276479Sdim 3068276479Sdim switch (N->getOpcode()) { 3069276479Sdim default: return nullptr; 3070276479Sdim case NVPTXISD::Tex1DFloatS32: 3071276479Sdim Opc = NVPTX::TEX_1D_F32_S32; 3072276479Sdim break; 3073276479Sdim case NVPTXISD::Tex1DFloatFloat: 3074276479Sdim Opc = NVPTX::TEX_1D_F32_F32; 3075276479Sdim break; 3076276479Sdim case NVPTXISD::Tex1DFloatFloatLevel: 3077276479Sdim Opc = NVPTX::TEX_1D_F32_F32_LEVEL; 3078276479Sdim break; 3079276479Sdim case NVPTXISD::Tex1DFloatFloatGrad: 3080276479Sdim Opc = NVPTX::TEX_1D_F32_F32_GRAD; 3081276479Sdim break; 3082276479Sdim case NVPTXISD::Tex1DS32S32: 3083276479Sdim Opc = NVPTX::TEX_1D_S32_S32; 3084276479Sdim break; 3085276479Sdim case NVPTXISD::Tex1DS32Float: 3086276479Sdim Opc = NVPTX::TEX_1D_S32_F32; 3087276479Sdim break; 3088276479Sdim case NVPTXISD::Tex1DS32FloatLevel: 3089276479Sdim Opc = NVPTX::TEX_1D_S32_F32_LEVEL; 3090276479Sdim break; 3091276479Sdim case NVPTXISD::Tex1DS32FloatGrad: 3092276479Sdim Opc = NVPTX::TEX_1D_S32_F32_GRAD; 3093276479Sdim break; 3094276479Sdim case NVPTXISD::Tex1DU32S32: 3095276479Sdim Opc = NVPTX::TEX_1D_U32_S32; 3096276479Sdim break; 3097276479Sdim case NVPTXISD::Tex1DU32Float: 3098276479Sdim Opc = NVPTX::TEX_1D_U32_F32; 3099276479Sdim break; 3100276479Sdim case NVPTXISD::Tex1DU32FloatLevel: 3101276479Sdim Opc = NVPTX::TEX_1D_U32_F32_LEVEL; 3102276479Sdim break; 3103276479Sdim case NVPTXISD::Tex1DU32FloatGrad: 3104276479Sdim Opc = NVPTX::TEX_1D_U32_F32_GRAD; 3105276479Sdim break; 3106276479Sdim case NVPTXISD::Tex1DArrayFloatS32: 3107276479Sdim Opc = NVPTX::TEX_1D_ARRAY_F32_S32; 3108276479Sdim break; 3109276479Sdim case NVPTXISD::Tex1DArrayFloatFloat: 3110276479Sdim Opc = NVPTX::TEX_1D_ARRAY_F32_F32; 3111276479Sdim break; 3112276479Sdim case NVPTXISD::Tex1DArrayFloatFloatLevel: 3113276479Sdim Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL; 3114276479Sdim break; 3115276479Sdim case NVPTXISD::Tex1DArrayFloatFloatGrad: 3116276479Sdim Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD; 3117276479Sdim break; 3118276479Sdim case NVPTXISD::Tex1DArrayS32S32: 3119276479Sdim Opc = NVPTX::TEX_1D_ARRAY_S32_S32; 3120276479Sdim break; 3121276479Sdim case NVPTXISD::Tex1DArrayS32Float: 3122276479Sdim Opc = NVPTX::TEX_1D_ARRAY_S32_F32; 3123276479Sdim break; 3124276479Sdim case NVPTXISD::Tex1DArrayS32FloatLevel: 3125276479Sdim Opc = NVPTX::TEX_1D_ARRAY_S32_F32_LEVEL; 3126276479Sdim break; 3127276479Sdim case NVPTXISD::Tex1DArrayS32FloatGrad: 3128276479Sdim Opc = NVPTX::TEX_1D_ARRAY_S32_F32_GRAD; 3129276479Sdim break; 3130276479Sdim case NVPTXISD::Tex1DArrayU32S32: 3131276479Sdim Opc = NVPTX::TEX_1D_ARRAY_U32_S32; 3132276479Sdim break; 3133276479Sdim case NVPTXISD::Tex1DArrayU32Float: 3134276479Sdim Opc = NVPTX::TEX_1D_ARRAY_U32_F32; 3135276479Sdim break; 3136276479Sdim case NVPTXISD::Tex1DArrayU32FloatLevel: 3137276479Sdim Opc = NVPTX::TEX_1D_ARRAY_U32_F32_LEVEL; 3138276479Sdim break; 3139276479Sdim case NVPTXISD::Tex1DArrayU32FloatGrad: 3140276479Sdim Opc = NVPTX::TEX_1D_ARRAY_U32_F32_GRAD; 3141276479Sdim break; 3142276479Sdim case NVPTXISD::Tex2DFloatS32: 3143276479Sdim Opc = NVPTX::TEX_2D_F32_S32; 3144276479Sdim break; 3145276479Sdim case NVPTXISD::Tex2DFloatFloat: 3146276479Sdim Opc = NVPTX::TEX_2D_F32_F32; 3147276479Sdim break; 3148276479Sdim case NVPTXISD::Tex2DFloatFloatLevel: 3149276479Sdim Opc = NVPTX::TEX_2D_F32_F32_LEVEL; 3150276479Sdim break; 3151276479Sdim case NVPTXISD::Tex2DFloatFloatGrad: 3152276479Sdim Opc = NVPTX::TEX_2D_F32_F32_GRAD; 3153276479Sdim break; 3154276479Sdim case NVPTXISD::Tex2DS32S32: 3155276479Sdim Opc = NVPTX::TEX_2D_S32_S32; 3156276479Sdim break; 3157276479Sdim case NVPTXISD::Tex2DS32Float: 3158276479Sdim Opc = NVPTX::TEX_2D_S32_F32; 3159276479Sdim break; 3160276479Sdim case NVPTXISD::Tex2DS32FloatLevel: 3161276479Sdim Opc = NVPTX::TEX_2D_S32_F32_LEVEL; 3162276479Sdim break; 3163276479Sdim case NVPTXISD::Tex2DS32FloatGrad: 3164276479Sdim Opc = NVPTX::TEX_2D_S32_F32_GRAD; 3165276479Sdim break; 3166276479Sdim case NVPTXISD::Tex2DU32S32: 3167276479Sdim Opc = NVPTX::TEX_2D_U32_S32; 3168276479Sdim break; 3169276479Sdim case NVPTXISD::Tex2DU32Float: 3170276479Sdim Opc = NVPTX::TEX_2D_U32_F32; 3171276479Sdim break; 3172276479Sdim case NVPTXISD::Tex2DU32FloatLevel: 3173276479Sdim Opc = NVPTX::TEX_2D_U32_F32_LEVEL; 3174276479Sdim break; 3175276479Sdim case NVPTXISD::Tex2DU32FloatGrad: 3176276479Sdim Opc = NVPTX::TEX_2D_U32_F32_GRAD; 3177276479Sdim break; 3178276479Sdim case NVPTXISD::Tex2DArrayFloatS32: 3179276479Sdim Opc = NVPTX::TEX_2D_ARRAY_F32_S32; 3180276479Sdim break; 3181276479Sdim case NVPTXISD::Tex2DArrayFloatFloat: 3182276479Sdim Opc = NVPTX::TEX_2D_ARRAY_F32_F32; 3183276479Sdim break; 3184276479Sdim case NVPTXISD::Tex2DArrayFloatFloatLevel: 3185276479Sdim Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL; 3186276479Sdim break; 3187276479Sdim case NVPTXISD::Tex2DArrayFloatFloatGrad: 3188276479Sdim Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD; 3189276479Sdim break; 3190276479Sdim case NVPTXISD::Tex2DArrayS32S32: 3191276479Sdim Opc = NVPTX::TEX_2D_ARRAY_S32_S32; 3192276479Sdim break; 3193276479Sdim case NVPTXISD::Tex2DArrayS32Float: 3194276479Sdim Opc = NVPTX::TEX_2D_ARRAY_S32_F32; 3195276479Sdim break; 3196276479Sdim case NVPTXISD::Tex2DArrayS32FloatLevel: 3197276479Sdim Opc = NVPTX::TEX_2D_ARRAY_S32_F32_LEVEL; 3198276479Sdim break; 3199276479Sdim case NVPTXISD::Tex2DArrayS32FloatGrad: 3200276479Sdim Opc = NVPTX::TEX_2D_ARRAY_S32_F32_GRAD; 3201276479Sdim break; 3202276479Sdim case NVPTXISD::Tex2DArrayU32S32: 3203276479Sdim Opc = NVPTX::TEX_2D_ARRAY_U32_S32; 3204276479Sdim break; 3205276479Sdim case NVPTXISD::Tex2DArrayU32Float: 3206276479Sdim Opc = NVPTX::TEX_2D_ARRAY_U32_F32; 3207276479Sdim break; 3208276479Sdim case NVPTXISD::Tex2DArrayU32FloatLevel: 3209276479Sdim Opc = NVPTX::TEX_2D_ARRAY_U32_F32_LEVEL; 3210276479Sdim break; 3211276479Sdim case NVPTXISD::Tex2DArrayU32FloatGrad: 3212276479Sdim Opc = NVPTX::TEX_2D_ARRAY_U32_F32_GRAD; 3213276479Sdim break; 3214276479Sdim case NVPTXISD::Tex3DFloatS32: 3215276479Sdim Opc = NVPTX::TEX_3D_F32_S32; 3216276479Sdim break; 3217276479Sdim case NVPTXISD::Tex3DFloatFloat: 3218276479Sdim Opc = NVPTX::TEX_3D_F32_F32; 3219276479Sdim break; 3220276479Sdim case NVPTXISD::Tex3DFloatFloatLevel: 3221276479Sdim Opc = NVPTX::TEX_3D_F32_F32_LEVEL; 3222276479Sdim break; 3223276479Sdim case NVPTXISD::Tex3DFloatFloatGrad: 3224276479Sdim Opc = NVPTX::TEX_3D_F32_F32_GRAD; 3225276479Sdim break; 3226276479Sdim case NVPTXISD::Tex3DS32S32: 3227276479Sdim Opc = NVPTX::TEX_3D_S32_S32; 3228276479Sdim break; 3229276479Sdim case NVPTXISD::Tex3DS32Float: 3230276479Sdim Opc = NVPTX::TEX_3D_S32_F32; 3231276479Sdim break; 3232276479Sdim case NVPTXISD::Tex3DS32FloatLevel: 3233276479Sdim Opc = NVPTX::TEX_3D_S32_F32_LEVEL; 3234276479Sdim break; 3235276479Sdim case NVPTXISD::Tex3DS32FloatGrad: 3236276479Sdim Opc = NVPTX::TEX_3D_S32_F32_GRAD; 3237276479Sdim break; 3238276479Sdim case NVPTXISD::Tex3DU32S32: 3239276479Sdim Opc = NVPTX::TEX_3D_U32_S32; 3240276479Sdim break; 3241276479Sdim case NVPTXISD::Tex3DU32Float: 3242276479Sdim Opc = NVPTX::TEX_3D_U32_F32; 3243276479Sdim break; 3244276479Sdim case NVPTXISD::Tex3DU32FloatLevel: 3245276479Sdim Opc = NVPTX::TEX_3D_U32_F32_LEVEL; 3246276479Sdim break; 3247276479Sdim case NVPTXISD::Tex3DU32FloatGrad: 3248276479Sdim Opc = NVPTX::TEX_3D_U32_F32_GRAD; 3249276479Sdim break; 3250276479Sdim case NVPTXISD::TexCubeFloatFloat: 3251276479Sdim Opc = NVPTX::TEX_CUBE_F32_F32; 3252276479Sdim break; 3253276479Sdim case NVPTXISD::TexCubeFloatFloatLevel: 3254276479Sdim Opc = NVPTX::TEX_CUBE_F32_F32_LEVEL; 3255276479Sdim break; 3256276479Sdim case NVPTXISD::TexCubeS32Float: 3257276479Sdim Opc = NVPTX::TEX_CUBE_S32_F32; 3258276479Sdim break; 3259276479Sdim case NVPTXISD::TexCubeS32FloatLevel: 3260276479Sdim Opc = NVPTX::TEX_CUBE_S32_F32_LEVEL; 3261276479Sdim break; 3262276479Sdim case NVPTXISD::TexCubeU32Float: 3263276479Sdim Opc = NVPTX::TEX_CUBE_U32_F32; 3264276479Sdim break; 3265276479Sdim case NVPTXISD::TexCubeU32FloatLevel: 3266276479Sdim Opc = NVPTX::TEX_CUBE_U32_F32_LEVEL; 3267276479Sdim break; 3268276479Sdim case NVPTXISD::TexCubeArrayFloatFloat: 3269276479Sdim Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32; 3270276479Sdim break; 3271276479Sdim case NVPTXISD::TexCubeArrayFloatFloatLevel: 3272276479Sdim Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_LEVEL; 3273276479Sdim break; 3274276479Sdim case NVPTXISD::TexCubeArrayS32Float: 3275276479Sdim Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32; 3276276479Sdim break; 3277276479Sdim case NVPTXISD::TexCubeArrayS32FloatLevel: 3278276479Sdim Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_LEVEL; 3279276479Sdim break; 3280276479Sdim case NVPTXISD::TexCubeArrayU32Float: 3281276479Sdim Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32; 3282276479Sdim break; 3283276479Sdim case NVPTXISD::TexCubeArrayU32FloatLevel: 3284276479Sdim Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_LEVEL; 3285276479Sdim break; 3286276479Sdim case NVPTXISD::Tld4R2DFloatFloat: 3287276479Sdim Opc = NVPTX::TLD4_R_2D_F32_F32; 3288276479Sdim break; 3289276479Sdim case NVPTXISD::Tld4G2DFloatFloat: 3290276479Sdim Opc = NVPTX::TLD4_G_2D_F32_F32; 3291276479Sdim break; 3292276479Sdim case NVPTXISD::Tld4B2DFloatFloat: 3293276479Sdim Opc = NVPTX::TLD4_B_2D_F32_F32; 3294276479Sdim break; 3295276479Sdim case NVPTXISD::Tld4A2DFloatFloat: 3296276479Sdim Opc = NVPTX::TLD4_A_2D_F32_F32; 3297276479Sdim break; 3298276479Sdim case NVPTXISD::Tld4R2DS64Float: 3299276479Sdim Opc = NVPTX::TLD4_R_2D_S32_F32; 3300276479Sdim break; 3301276479Sdim case NVPTXISD::Tld4G2DS64Float: 3302276479Sdim Opc = NVPTX::TLD4_G_2D_S32_F32; 3303276479Sdim break; 3304276479Sdim case NVPTXISD::Tld4B2DS64Float: 3305276479Sdim Opc = NVPTX::TLD4_B_2D_S32_F32; 3306276479Sdim break; 3307276479Sdim case NVPTXISD::Tld4A2DS64Float: 3308276479Sdim Opc = NVPTX::TLD4_A_2D_S32_F32; 3309276479Sdim break; 3310276479Sdim case NVPTXISD::Tld4R2DU64Float: 3311276479Sdim Opc = NVPTX::TLD4_R_2D_U32_F32; 3312276479Sdim break; 3313276479Sdim case NVPTXISD::Tld4G2DU64Float: 3314276479Sdim Opc = NVPTX::TLD4_G_2D_U32_F32; 3315276479Sdim break; 3316276479Sdim case NVPTXISD::Tld4B2DU64Float: 3317276479Sdim Opc = NVPTX::TLD4_B_2D_U32_F32; 3318276479Sdim break; 3319276479Sdim case NVPTXISD::Tld4A2DU64Float: 3320276479Sdim Opc = NVPTX::TLD4_A_2D_U32_F32; 3321276479Sdim break; 3322276479Sdim case NVPTXISD::TexUnified1DFloatS32: 3323276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_F32_S32; 3324276479Sdim break; 3325276479Sdim case NVPTXISD::TexUnified1DFloatFloat: 3326276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_F32_F32; 3327276479Sdim break; 3328276479Sdim case NVPTXISD::TexUnified1DFloatFloatLevel: 3329276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_LEVEL; 3330276479Sdim break; 3331276479Sdim case NVPTXISD::TexUnified1DFloatFloatGrad: 3332276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_GRAD; 3333276479Sdim break; 3334276479Sdim case NVPTXISD::TexUnified1DS32S32: 3335276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_S32_S32; 3336276479Sdim break; 3337276479Sdim case NVPTXISD::TexUnified1DS32Float: 3338276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_S32_F32; 3339276479Sdim break; 3340276479Sdim case NVPTXISD::TexUnified1DS32FloatLevel: 3341276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_LEVEL; 3342276479Sdim break; 3343276479Sdim case NVPTXISD::TexUnified1DS32FloatGrad: 3344276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_GRAD; 3345276479Sdim break; 3346276479Sdim case NVPTXISD::TexUnified1DU32S32: 3347276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_U32_S32; 3348276479Sdim break; 3349276479Sdim case NVPTXISD::TexUnified1DU32Float: 3350276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_U32_F32; 3351276479Sdim break; 3352276479Sdim case NVPTXISD::TexUnified1DU32FloatLevel: 3353276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_LEVEL; 3354276479Sdim break; 3355276479Sdim case NVPTXISD::TexUnified1DU32FloatGrad: 3356276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_GRAD; 3357276479Sdim break; 3358276479Sdim case NVPTXISD::TexUnified1DArrayFloatS32: 3359276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_S32; 3360276479Sdim break; 3361276479Sdim case NVPTXISD::TexUnified1DArrayFloatFloat: 3362276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32; 3363276479Sdim break; 3364276479Sdim case NVPTXISD::TexUnified1DArrayFloatFloatLevel: 3365276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL; 3366276479Sdim break; 3367276479Sdim case NVPTXISD::TexUnified1DArrayFloatFloatGrad: 3368276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD; 3369276479Sdim break; 3370276479Sdim case NVPTXISD::TexUnified1DArrayS32S32: 3371276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_S32; 3372276479Sdim break; 3373276479Sdim case NVPTXISD::TexUnified1DArrayS32Float: 3374276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32; 3375276479Sdim break; 3376276479Sdim case NVPTXISD::TexUnified1DArrayS32FloatLevel: 3377276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL; 3378276479Sdim break; 3379276479Sdim case NVPTXISD::TexUnified1DArrayS32FloatGrad: 3380276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD; 3381276479Sdim break; 3382276479Sdim case NVPTXISD::TexUnified1DArrayU32S32: 3383276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_S32; 3384276479Sdim break; 3385276479Sdim case NVPTXISD::TexUnified1DArrayU32Float: 3386276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32; 3387276479Sdim break; 3388276479Sdim case NVPTXISD::TexUnified1DArrayU32FloatLevel: 3389276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL; 3390276479Sdim break; 3391276479Sdim case NVPTXISD::TexUnified1DArrayU32FloatGrad: 3392276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD; 3393276479Sdim break; 3394276479Sdim case NVPTXISD::TexUnified2DFloatS32: 3395276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_F32_S32; 3396276479Sdim break; 3397276479Sdim case NVPTXISD::TexUnified2DFloatFloat: 3398276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_F32_F32; 3399276479Sdim break; 3400276479Sdim case NVPTXISD::TexUnified2DFloatFloatLevel: 3401276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_LEVEL; 3402276479Sdim break; 3403276479Sdim case NVPTXISD::TexUnified2DFloatFloatGrad: 3404276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_GRAD; 3405276479Sdim break; 3406276479Sdim case NVPTXISD::TexUnified2DS32S32: 3407276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_S32_S32; 3408276479Sdim break; 3409276479Sdim case NVPTXISD::TexUnified2DS32Float: 3410276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_S32_F32; 3411276479Sdim break; 3412276479Sdim case NVPTXISD::TexUnified2DS32FloatLevel: 3413276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_LEVEL; 3414276479Sdim break; 3415276479Sdim case NVPTXISD::TexUnified2DS32FloatGrad: 3416276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_GRAD; 3417276479Sdim break; 3418276479Sdim case NVPTXISD::TexUnified2DU32S32: 3419276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_U32_S32; 3420276479Sdim break; 3421276479Sdim case NVPTXISD::TexUnified2DU32Float: 3422276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_U32_F32; 3423276479Sdim break; 3424276479Sdim case NVPTXISD::TexUnified2DU32FloatLevel: 3425276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_LEVEL; 3426276479Sdim break; 3427276479Sdim case NVPTXISD::TexUnified2DU32FloatGrad: 3428276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_GRAD; 3429276479Sdim break; 3430276479Sdim case NVPTXISD::TexUnified2DArrayFloatS32: 3431276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_S32; 3432276479Sdim break; 3433276479Sdim case NVPTXISD::TexUnified2DArrayFloatFloat: 3434276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32; 3435276479Sdim break; 3436276479Sdim case NVPTXISD::TexUnified2DArrayFloatFloatLevel: 3437276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL; 3438276479Sdim break; 3439276479Sdim case NVPTXISD::TexUnified2DArrayFloatFloatGrad: 3440276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD; 3441276479Sdim break; 3442276479Sdim case NVPTXISD::TexUnified2DArrayS32S32: 3443276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_S32; 3444276479Sdim break; 3445276479Sdim case NVPTXISD::TexUnified2DArrayS32Float: 3446276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32; 3447276479Sdim break; 3448276479Sdim case NVPTXISD::TexUnified2DArrayS32FloatLevel: 3449276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL; 3450276479Sdim break; 3451276479Sdim case NVPTXISD::TexUnified2DArrayS32FloatGrad: 3452276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD; 3453276479Sdim break; 3454276479Sdim case NVPTXISD::TexUnified2DArrayU32S32: 3455276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_S32; 3456276479Sdim break; 3457276479Sdim case NVPTXISD::TexUnified2DArrayU32Float: 3458276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32; 3459276479Sdim break; 3460276479Sdim case NVPTXISD::TexUnified2DArrayU32FloatLevel: 3461276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL; 3462276479Sdim break; 3463276479Sdim case NVPTXISD::TexUnified2DArrayU32FloatGrad: 3464276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD; 3465276479Sdim break; 3466276479Sdim case NVPTXISD::TexUnified3DFloatS32: 3467276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_F32_S32; 3468276479Sdim break; 3469276479Sdim case NVPTXISD::TexUnified3DFloatFloat: 3470276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_F32_F32; 3471276479Sdim break; 3472276479Sdim case NVPTXISD::TexUnified3DFloatFloatLevel: 3473276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_LEVEL; 3474276479Sdim break; 3475276479Sdim case NVPTXISD::TexUnified3DFloatFloatGrad: 3476276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_GRAD; 3477276479Sdim break; 3478276479Sdim case NVPTXISD::TexUnified3DS32S32: 3479276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_S32_S32; 3480276479Sdim break; 3481276479Sdim case NVPTXISD::TexUnified3DS32Float: 3482276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_S32_F32; 3483276479Sdim break; 3484276479Sdim case NVPTXISD::TexUnified3DS32FloatLevel: 3485276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_LEVEL; 3486276479Sdim break; 3487276479Sdim case NVPTXISD::TexUnified3DS32FloatGrad: 3488276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_GRAD; 3489276479Sdim break; 3490276479Sdim case NVPTXISD::TexUnified3DU32S32: 3491276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_U32_S32; 3492276479Sdim break; 3493276479Sdim case NVPTXISD::TexUnified3DU32Float: 3494276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_U32_F32; 3495276479Sdim break; 3496276479Sdim case NVPTXISD::TexUnified3DU32FloatLevel: 3497276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_LEVEL; 3498276479Sdim break; 3499276479Sdim case NVPTXISD::TexUnified3DU32FloatGrad: 3500276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_GRAD; 3501276479Sdim break; 3502276479Sdim case NVPTXISD::TexUnifiedCubeFloatFloat: 3503276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32; 3504276479Sdim break; 3505276479Sdim case NVPTXISD::TexUnifiedCubeFloatFloatLevel: 3506276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_LEVEL; 3507276479Sdim break; 3508276479Sdim case NVPTXISD::TexUnifiedCubeS32Float: 3509276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32; 3510276479Sdim break; 3511276479Sdim case NVPTXISD::TexUnifiedCubeS32FloatLevel: 3512276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_LEVEL; 3513276479Sdim break; 3514276479Sdim case NVPTXISD::TexUnifiedCubeU32Float: 3515276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32; 3516276479Sdim break; 3517276479Sdim case NVPTXISD::TexUnifiedCubeU32FloatLevel: 3518276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_LEVEL; 3519276479Sdim break; 3520276479Sdim case NVPTXISD::TexUnifiedCubeArrayFloatFloat: 3521276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32; 3522276479Sdim break; 3523276479Sdim case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel: 3524276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL; 3525276479Sdim break; 3526276479Sdim case NVPTXISD::TexUnifiedCubeArrayS32Float: 3527276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32; 3528276479Sdim break; 3529276479Sdim case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel: 3530276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL; 3531276479Sdim break; 3532276479Sdim case NVPTXISD::TexUnifiedCubeArrayU32Float: 3533276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32; 3534276479Sdim break; 3535276479Sdim case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel: 3536276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL; 3537276479Sdim break; 3538276479Sdim case NVPTXISD::Tld4UnifiedR2DFloatFloat: 3539276479Sdim Opc = NVPTX::TLD4_UNIFIED_R_2D_F32_F32; 3540276479Sdim break; 3541276479Sdim case NVPTXISD::Tld4UnifiedG2DFloatFloat: 3542276479Sdim Opc = NVPTX::TLD4_UNIFIED_G_2D_F32_F32; 3543276479Sdim break; 3544276479Sdim case NVPTXISD::Tld4UnifiedB2DFloatFloat: 3545276479Sdim Opc = NVPTX::TLD4_UNIFIED_B_2D_F32_F32; 3546276479Sdim break; 3547276479Sdim case NVPTXISD::Tld4UnifiedA2DFloatFloat: 3548276479Sdim Opc = NVPTX::TLD4_UNIFIED_A_2D_F32_F32; 3549276479Sdim break; 3550276479Sdim case NVPTXISD::Tld4UnifiedR2DS64Float: 3551276479Sdim Opc = NVPTX::TLD4_UNIFIED_R_2D_S32_F32; 3552276479Sdim break; 3553276479Sdim case NVPTXISD::Tld4UnifiedG2DS64Float: 3554276479Sdim Opc = NVPTX::TLD4_UNIFIED_G_2D_S32_F32; 3555276479Sdim break; 3556276479Sdim case NVPTXISD::Tld4UnifiedB2DS64Float: 3557276479Sdim Opc = NVPTX::TLD4_UNIFIED_B_2D_S32_F32; 3558276479Sdim break; 3559276479Sdim case NVPTXISD::Tld4UnifiedA2DS64Float: 3560276479Sdim Opc = NVPTX::TLD4_UNIFIED_A_2D_S32_F32; 3561276479Sdim break; 3562276479Sdim case NVPTXISD::Tld4UnifiedR2DU64Float: 3563276479Sdim Opc = NVPTX::TLD4_UNIFIED_R_2D_U32_F32; 3564276479Sdim break; 3565276479Sdim case NVPTXISD::Tld4UnifiedG2DU64Float: 3566276479Sdim Opc = NVPTX::TLD4_UNIFIED_G_2D_U32_F32; 3567276479Sdim break; 3568276479Sdim case NVPTXISD::Tld4UnifiedB2DU64Float: 3569276479Sdim Opc = NVPTX::TLD4_UNIFIED_B_2D_U32_F32; 3570276479Sdim break; 3571276479Sdim case NVPTXISD::Tld4UnifiedA2DU64Float: 3572276479Sdim Opc = NVPTX::TLD4_UNIFIED_A_2D_U32_F32; 3573276479Sdim break; 3574276479Sdim } 3575276479Sdim 3576276479Sdim // Copy over operands 3577276479Sdim for (unsigned i = 1; i < N->getNumOperands(); ++i) { 3578276479Sdim Ops.push_back(N->getOperand(i)); 3579276479Sdim } 3580276479Sdim 3581276479Sdim Ops.push_back(Chain); 3582276479Sdim Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops); 3583276479Sdim return Ret; 3584276479Sdim} 3585276479Sdim 3586276479SdimSDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) { 3587276479Sdim SDValue Chain = N->getOperand(0); 3588276479Sdim SDValue TexHandle = N->getOperand(1); 3589276479Sdim SDNode *Ret = nullptr; 3590276479Sdim unsigned Opc = 0; 3591276479Sdim SmallVector<SDValue, 8> Ops; 3592276479Sdim switch (N->getOpcode()) { 3593276479Sdim default: return nullptr; 3594276479Sdim case NVPTXISD::Suld1DI8Clamp: 3595276479Sdim Opc = NVPTX::SULD_1D_I8_CLAMP; 3596276479Sdim Ops.push_back(TexHandle); 3597276479Sdim Ops.push_back(N->getOperand(2)); 3598276479Sdim Ops.push_back(Chain); 3599276479Sdim break; 3600276479Sdim case NVPTXISD::Suld1DI16Clamp: 3601276479Sdim Opc = NVPTX::SULD_1D_I16_CLAMP; 3602276479Sdim Ops.push_back(TexHandle); 3603276479Sdim Ops.push_back(N->getOperand(2)); 3604276479Sdim Ops.push_back(Chain); 3605276479Sdim break; 3606276479Sdim case NVPTXISD::Suld1DI32Clamp: 3607276479Sdim Opc = NVPTX::SULD_1D_I32_CLAMP; 3608276479Sdim Ops.push_back(TexHandle); 3609276479Sdim Ops.push_back(N->getOperand(2)); 3610276479Sdim Ops.push_back(Chain); 3611276479Sdim break; 3612276479Sdim case NVPTXISD::Suld1DI64Clamp: 3613276479Sdim Opc = NVPTX::SULD_1D_I64_CLAMP; 3614276479Sdim Ops.push_back(TexHandle); 3615276479Sdim Ops.push_back(N->getOperand(2)); 3616276479Sdim Ops.push_back(Chain); 3617276479Sdim break; 3618276479Sdim case NVPTXISD::Suld1DV2I8Clamp: 3619276479Sdim Opc = NVPTX::SULD_1D_V2I8_CLAMP; 3620276479Sdim Ops.push_back(TexHandle); 3621276479Sdim Ops.push_back(N->getOperand(2)); 3622276479Sdim Ops.push_back(Chain); 3623276479Sdim break; 3624276479Sdim case NVPTXISD::Suld1DV2I16Clamp: 3625276479Sdim Opc = NVPTX::SULD_1D_V2I16_CLAMP; 3626276479Sdim Ops.push_back(TexHandle); 3627276479Sdim Ops.push_back(N->getOperand(2)); 3628276479Sdim Ops.push_back(Chain); 3629276479Sdim break; 3630276479Sdim case NVPTXISD::Suld1DV2I32Clamp: 3631276479Sdim Opc = NVPTX::SULD_1D_V2I32_CLAMP; 3632276479Sdim Ops.push_back(TexHandle); 3633276479Sdim Ops.push_back(N->getOperand(2)); 3634276479Sdim Ops.push_back(Chain); 3635276479Sdim break; 3636276479Sdim case NVPTXISD::Suld1DV2I64Clamp: 3637276479Sdim Opc = NVPTX::SULD_1D_V2I64_CLAMP; 3638276479Sdim Ops.push_back(TexHandle); 3639276479Sdim Ops.push_back(N->getOperand(2)); 3640276479Sdim Ops.push_back(Chain); 3641276479Sdim break; 3642276479Sdim case NVPTXISD::Suld1DV4I8Clamp: 3643276479Sdim Opc = NVPTX::SULD_1D_V4I8_CLAMP; 3644276479Sdim Ops.push_back(TexHandle); 3645276479Sdim Ops.push_back(N->getOperand(2)); 3646276479Sdim Ops.push_back(Chain); 3647276479Sdim break; 3648276479Sdim case NVPTXISD::Suld1DV4I16Clamp: 3649276479Sdim Opc = NVPTX::SULD_1D_V4I16_CLAMP; 3650276479Sdim Ops.push_back(TexHandle); 3651276479Sdim Ops.push_back(N->getOperand(2)); 3652276479Sdim Ops.push_back(Chain); 3653276479Sdim break; 3654276479Sdim case NVPTXISD::Suld1DV4I32Clamp: 3655276479Sdim Opc = NVPTX::SULD_1D_V4I32_CLAMP; 3656276479Sdim Ops.push_back(TexHandle); 3657276479Sdim Ops.push_back(N->getOperand(2)); 3658276479Sdim Ops.push_back(Chain); 3659276479Sdim break; 3660276479Sdim case NVPTXISD::Suld1DArrayI8Clamp: 3661276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I8_CLAMP; 3662276479Sdim Ops.push_back(TexHandle); 3663276479Sdim Ops.push_back(N->getOperand(2)); 3664276479Sdim Ops.push_back(N->getOperand(3)); 3665276479Sdim Ops.push_back(Chain); 3666276479Sdim break; 3667276479Sdim case NVPTXISD::Suld1DArrayI16Clamp: 3668276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I16_CLAMP; 3669276479Sdim Ops.push_back(TexHandle); 3670276479Sdim Ops.push_back(N->getOperand(2)); 3671276479Sdim Ops.push_back(N->getOperand(3)); 3672276479Sdim Ops.push_back(Chain); 3673276479Sdim break; 3674276479Sdim case NVPTXISD::Suld1DArrayI32Clamp: 3675276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I32_CLAMP; 3676276479Sdim Ops.push_back(TexHandle); 3677276479Sdim Ops.push_back(N->getOperand(2)); 3678276479Sdim Ops.push_back(N->getOperand(3)); 3679276479Sdim Ops.push_back(Chain); 3680276479Sdim break; 3681276479Sdim case NVPTXISD::Suld1DArrayI64Clamp: 3682276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I64_CLAMP; 3683276479Sdim Ops.push_back(TexHandle); 3684276479Sdim Ops.push_back(N->getOperand(2)); 3685276479Sdim Ops.push_back(N->getOperand(3)); 3686276479Sdim Ops.push_back(Chain); 3687276479Sdim break; 3688276479Sdim case NVPTXISD::Suld1DArrayV2I8Clamp: 3689276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I8_CLAMP; 3690276479Sdim Ops.push_back(TexHandle); 3691276479Sdim Ops.push_back(N->getOperand(2)); 3692276479Sdim Ops.push_back(N->getOperand(3)); 3693276479Sdim Ops.push_back(Chain); 3694276479Sdim break; 3695276479Sdim case NVPTXISD::Suld1DArrayV2I16Clamp: 3696276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I16_CLAMP; 3697276479Sdim Ops.push_back(TexHandle); 3698276479Sdim Ops.push_back(N->getOperand(2)); 3699276479Sdim Ops.push_back(N->getOperand(3)); 3700276479Sdim Ops.push_back(Chain); 3701276479Sdim break; 3702276479Sdim case NVPTXISD::Suld1DArrayV2I32Clamp: 3703276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I32_CLAMP; 3704276479Sdim Ops.push_back(TexHandle); 3705276479Sdim Ops.push_back(N->getOperand(2)); 3706276479Sdim Ops.push_back(N->getOperand(3)); 3707276479Sdim Ops.push_back(Chain); 3708276479Sdim break; 3709276479Sdim case NVPTXISD::Suld1DArrayV2I64Clamp: 3710276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I64_CLAMP; 3711276479Sdim Ops.push_back(TexHandle); 3712276479Sdim Ops.push_back(N->getOperand(2)); 3713276479Sdim Ops.push_back(N->getOperand(3)); 3714276479Sdim Ops.push_back(Chain); 3715276479Sdim break; 3716276479Sdim case NVPTXISD::Suld1DArrayV4I8Clamp: 3717276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I8_CLAMP; 3718276479Sdim Ops.push_back(TexHandle); 3719276479Sdim Ops.push_back(N->getOperand(2)); 3720276479Sdim Ops.push_back(N->getOperand(3)); 3721276479Sdim Ops.push_back(Chain); 3722276479Sdim break; 3723276479Sdim case NVPTXISD::Suld1DArrayV4I16Clamp: 3724276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I16_CLAMP; 3725276479Sdim Ops.push_back(TexHandle); 3726276479Sdim Ops.push_back(N->getOperand(2)); 3727276479Sdim Ops.push_back(N->getOperand(3)); 3728276479Sdim Ops.push_back(Chain); 3729276479Sdim break; 3730276479Sdim case NVPTXISD::Suld1DArrayV4I32Clamp: 3731276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I32_CLAMP; 3732276479Sdim Ops.push_back(TexHandle); 3733276479Sdim Ops.push_back(N->getOperand(2)); 3734276479Sdim Ops.push_back(N->getOperand(3)); 3735276479Sdim Ops.push_back(Chain); 3736276479Sdim break; 3737276479Sdim case NVPTXISD::Suld2DI8Clamp: 3738276479Sdim Opc = NVPTX::SULD_2D_I8_CLAMP; 3739276479Sdim Ops.push_back(TexHandle); 3740276479Sdim Ops.push_back(N->getOperand(2)); 3741276479Sdim Ops.push_back(N->getOperand(3)); 3742276479Sdim Ops.push_back(Chain); 3743276479Sdim break; 3744276479Sdim case NVPTXISD::Suld2DI16Clamp: 3745276479Sdim Opc = NVPTX::SULD_2D_I16_CLAMP; 3746276479Sdim Ops.push_back(TexHandle); 3747276479Sdim Ops.push_back(N->getOperand(2)); 3748276479Sdim Ops.push_back(N->getOperand(3)); 3749276479Sdim Ops.push_back(Chain); 3750276479Sdim break; 3751276479Sdim case NVPTXISD::Suld2DI32Clamp: 3752276479Sdim Opc = NVPTX::SULD_2D_I32_CLAMP; 3753276479Sdim Ops.push_back(TexHandle); 3754276479Sdim Ops.push_back(N->getOperand(2)); 3755276479Sdim Ops.push_back(N->getOperand(3)); 3756276479Sdim Ops.push_back(Chain); 3757276479Sdim break; 3758276479Sdim case NVPTXISD::Suld2DI64Clamp: 3759276479Sdim Opc = NVPTX::SULD_2D_I64_CLAMP; 3760276479Sdim Ops.push_back(TexHandle); 3761276479Sdim Ops.push_back(N->getOperand(2)); 3762276479Sdim Ops.push_back(N->getOperand(3)); 3763276479Sdim Ops.push_back(Chain); 3764276479Sdim break; 3765276479Sdim case NVPTXISD::Suld2DV2I8Clamp: 3766276479Sdim Opc = NVPTX::SULD_2D_V2I8_CLAMP; 3767276479Sdim Ops.push_back(TexHandle); 3768276479Sdim Ops.push_back(N->getOperand(2)); 3769276479Sdim Ops.push_back(N->getOperand(3)); 3770276479Sdim Ops.push_back(Chain); 3771276479Sdim break; 3772276479Sdim case NVPTXISD::Suld2DV2I16Clamp: 3773276479Sdim Opc = NVPTX::SULD_2D_V2I16_CLAMP; 3774276479Sdim Ops.push_back(TexHandle); 3775276479Sdim Ops.push_back(N->getOperand(2)); 3776276479Sdim Ops.push_back(N->getOperand(3)); 3777276479Sdim Ops.push_back(Chain); 3778276479Sdim break; 3779276479Sdim case NVPTXISD::Suld2DV2I32Clamp: 3780276479Sdim Opc = NVPTX::SULD_2D_V2I32_CLAMP; 3781276479Sdim Ops.push_back(TexHandle); 3782276479Sdim Ops.push_back(N->getOperand(2)); 3783276479Sdim Ops.push_back(N->getOperand(3)); 3784276479Sdim Ops.push_back(Chain); 3785276479Sdim break; 3786276479Sdim case NVPTXISD::Suld2DV2I64Clamp: 3787276479Sdim Opc = NVPTX::SULD_2D_V2I64_CLAMP; 3788276479Sdim Ops.push_back(TexHandle); 3789276479Sdim Ops.push_back(N->getOperand(2)); 3790276479Sdim Ops.push_back(N->getOperand(3)); 3791276479Sdim Ops.push_back(Chain); 3792276479Sdim break; 3793276479Sdim case NVPTXISD::Suld2DV4I8Clamp: 3794276479Sdim Opc = NVPTX::SULD_2D_V4I8_CLAMP; 3795276479Sdim Ops.push_back(TexHandle); 3796276479Sdim Ops.push_back(N->getOperand(2)); 3797276479Sdim Ops.push_back(N->getOperand(3)); 3798276479Sdim Ops.push_back(Chain); 3799276479Sdim break; 3800276479Sdim case NVPTXISD::Suld2DV4I16Clamp: 3801276479Sdim Opc = NVPTX::SULD_2D_V4I16_CLAMP; 3802276479Sdim Ops.push_back(TexHandle); 3803276479Sdim Ops.push_back(N->getOperand(2)); 3804276479Sdim Ops.push_back(N->getOperand(3)); 3805276479Sdim Ops.push_back(Chain); 3806276479Sdim break; 3807276479Sdim case NVPTXISD::Suld2DV4I32Clamp: 3808276479Sdim Opc = NVPTX::SULD_2D_V4I32_CLAMP; 3809276479Sdim Ops.push_back(TexHandle); 3810276479Sdim Ops.push_back(N->getOperand(2)); 3811276479Sdim Ops.push_back(N->getOperand(3)); 3812276479Sdim Ops.push_back(Chain); 3813276479Sdim break; 3814276479Sdim case NVPTXISD::Suld2DArrayI8Clamp: 3815276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I8_CLAMP; 3816276479Sdim Ops.push_back(TexHandle); 3817276479Sdim Ops.push_back(N->getOperand(2)); 3818276479Sdim Ops.push_back(N->getOperand(3)); 3819276479Sdim Ops.push_back(N->getOperand(4)); 3820276479Sdim Ops.push_back(Chain); 3821276479Sdim break; 3822276479Sdim case NVPTXISD::Suld2DArrayI16Clamp: 3823276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I16_CLAMP; 3824276479Sdim Ops.push_back(TexHandle); 3825276479Sdim Ops.push_back(N->getOperand(2)); 3826276479Sdim Ops.push_back(N->getOperand(3)); 3827276479Sdim Ops.push_back(N->getOperand(4)); 3828276479Sdim Ops.push_back(Chain); 3829276479Sdim break; 3830276479Sdim case NVPTXISD::Suld2DArrayI32Clamp: 3831276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I32_CLAMP; 3832276479Sdim Ops.push_back(TexHandle); 3833276479Sdim Ops.push_back(N->getOperand(2)); 3834276479Sdim Ops.push_back(N->getOperand(3)); 3835276479Sdim Ops.push_back(N->getOperand(4)); 3836276479Sdim Ops.push_back(Chain); 3837276479Sdim break; 3838276479Sdim case NVPTXISD::Suld2DArrayI64Clamp: 3839276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I64_CLAMP; 3840276479Sdim Ops.push_back(TexHandle); 3841276479Sdim Ops.push_back(N->getOperand(2)); 3842276479Sdim Ops.push_back(N->getOperand(3)); 3843276479Sdim Ops.push_back(N->getOperand(4)); 3844276479Sdim Ops.push_back(Chain); 3845276479Sdim break; 3846276479Sdim case NVPTXISD::Suld2DArrayV2I8Clamp: 3847276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I8_CLAMP; 3848276479Sdim Ops.push_back(TexHandle); 3849276479Sdim Ops.push_back(N->getOperand(2)); 3850276479Sdim Ops.push_back(N->getOperand(3)); 3851276479Sdim Ops.push_back(N->getOperand(4)); 3852276479Sdim Ops.push_back(Chain); 3853276479Sdim break; 3854276479Sdim case NVPTXISD::Suld2DArrayV2I16Clamp: 3855276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I16_CLAMP; 3856276479Sdim Ops.push_back(TexHandle); 3857276479Sdim Ops.push_back(N->getOperand(2)); 3858276479Sdim Ops.push_back(N->getOperand(3)); 3859276479Sdim Ops.push_back(N->getOperand(4)); 3860276479Sdim Ops.push_back(Chain); 3861276479Sdim break; 3862276479Sdim case NVPTXISD::Suld2DArrayV2I32Clamp: 3863276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I32_CLAMP; 3864276479Sdim Ops.push_back(TexHandle); 3865276479Sdim Ops.push_back(N->getOperand(2)); 3866276479Sdim Ops.push_back(N->getOperand(3)); 3867276479Sdim Ops.push_back(N->getOperand(4)); 3868276479Sdim Ops.push_back(Chain); 3869276479Sdim break; 3870276479Sdim case NVPTXISD::Suld2DArrayV2I64Clamp: 3871276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I64_CLAMP; 3872276479Sdim Ops.push_back(TexHandle); 3873276479Sdim Ops.push_back(N->getOperand(2)); 3874276479Sdim Ops.push_back(N->getOperand(3)); 3875276479Sdim Ops.push_back(N->getOperand(4)); 3876276479Sdim Ops.push_back(Chain); 3877276479Sdim break; 3878276479Sdim case NVPTXISD::Suld2DArrayV4I8Clamp: 3879276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I8_CLAMP; 3880276479Sdim Ops.push_back(TexHandle); 3881276479Sdim Ops.push_back(N->getOperand(2)); 3882276479Sdim Ops.push_back(N->getOperand(3)); 3883276479Sdim Ops.push_back(N->getOperand(4)); 3884276479Sdim Ops.push_back(Chain); 3885276479Sdim break; 3886276479Sdim case NVPTXISD::Suld2DArrayV4I16Clamp: 3887276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I16_CLAMP; 3888276479Sdim Ops.push_back(TexHandle); 3889276479Sdim Ops.push_back(N->getOperand(2)); 3890276479Sdim Ops.push_back(N->getOperand(3)); 3891276479Sdim Ops.push_back(N->getOperand(4)); 3892276479Sdim Ops.push_back(Chain); 3893276479Sdim break; 3894276479Sdim case NVPTXISD::Suld2DArrayV4I32Clamp: 3895276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I32_CLAMP; 3896276479Sdim Ops.push_back(TexHandle); 3897276479Sdim Ops.push_back(N->getOperand(2)); 3898276479Sdim Ops.push_back(N->getOperand(3)); 3899276479Sdim Ops.push_back(N->getOperand(4)); 3900276479Sdim Ops.push_back(Chain); 3901276479Sdim break; 3902276479Sdim case NVPTXISD::Suld3DI8Clamp: 3903276479Sdim Opc = NVPTX::SULD_3D_I8_CLAMP; 3904276479Sdim Ops.push_back(TexHandle); 3905276479Sdim Ops.push_back(N->getOperand(2)); 3906276479Sdim Ops.push_back(N->getOperand(3)); 3907276479Sdim Ops.push_back(N->getOperand(4)); 3908276479Sdim Ops.push_back(Chain); 3909276479Sdim break; 3910276479Sdim case NVPTXISD::Suld3DI16Clamp: 3911276479Sdim Opc = NVPTX::SULD_3D_I16_CLAMP; 3912276479Sdim Ops.push_back(TexHandle); 3913276479Sdim Ops.push_back(N->getOperand(2)); 3914276479Sdim Ops.push_back(N->getOperand(3)); 3915276479Sdim Ops.push_back(N->getOperand(4)); 3916276479Sdim Ops.push_back(Chain); 3917276479Sdim break; 3918276479Sdim case NVPTXISD::Suld3DI32Clamp: 3919276479Sdim Opc = NVPTX::SULD_3D_I32_CLAMP; 3920276479Sdim Ops.push_back(TexHandle); 3921276479Sdim Ops.push_back(N->getOperand(2)); 3922276479Sdim Ops.push_back(N->getOperand(3)); 3923276479Sdim Ops.push_back(N->getOperand(4)); 3924276479Sdim Ops.push_back(Chain); 3925276479Sdim break; 3926276479Sdim case NVPTXISD::Suld3DI64Clamp: 3927276479Sdim Opc = NVPTX::SULD_3D_I64_CLAMP; 3928276479Sdim Ops.push_back(TexHandle); 3929276479Sdim Ops.push_back(N->getOperand(2)); 3930276479Sdim Ops.push_back(N->getOperand(3)); 3931276479Sdim Ops.push_back(N->getOperand(4)); 3932276479Sdim Ops.push_back(Chain); 3933276479Sdim break; 3934276479Sdim case NVPTXISD::Suld3DV2I8Clamp: 3935276479Sdim Opc = NVPTX::SULD_3D_V2I8_CLAMP; 3936276479Sdim Ops.push_back(TexHandle); 3937276479Sdim Ops.push_back(N->getOperand(2)); 3938276479Sdim Ops.push_back(N->getOperand(3)); 3939276479Sdim Ops.push_back(N->getOperand(4)); 3940276479Sdim Ops.push_back(Chain); 3941276479Sdim break; 3942276479Sdim case NVPTXISD::Suld3DV2I16Clamp: 3943276479Sdim Opc = NVPTX::SULD_3D_V2I16_CLAMP; 3944276479Sdim Ops.push_back(TexHandle); 3945276479Sdim Ops.push_back(N->getOperand(2)); 3946276479Sdim Ops.push_back(N->getOperand(3)); 3947276479Sdim Ops.push_back(N->getOperand(4)); 3948276479Sdim Ops.push_back(Chain); 3949276479Sdim break; 3950276479Sdim case NVPTXISD::Suld3DV2I32Clamp: 3951276479Sdim Opc = NVPTX::SULD_3D_V2I32_CLAMP; 3952276479Sdim Ops.push_back(TexHandle); 3953276479Sdim Ops.push_back(N->getOperand(2)); 3954276479Sdim Ops.push_back(N->getOperand(3)); 3955276479Sdim Ops.push_back(N->getOperand(4)); 3956276479Sdim Ops.push_back(Chain); 3957276479Sdim break; 3958276479Sdim case NVPTXISD::Suld3DV2I64Clamp: 3959276479Sdim Opc = NVPTX::SULD_3D_V2I64_CLAMP; 3960276479Sdim Ops.push_back(TexHandle); 3961276479Sdim Ops.push_back(N->getOperand(2)); 3962276479Sdim Ops.push_back(N->getOperand(3)); 3963276479Sdim Ops.push_back(N->getOperand(4)); 3964276479Sdim Ops.push_back(Chain); 3965276479Sdim break; 3966276479Sdim case NVPTXISD::Suld3DV4I8Clamp: 3967276479Sdim Opc = NVPTX::SULD_3D_V4I8_CLAMP; 3968276479Sdim Ops.push_back(TexHandle); 3969276479Sdim Ops.push_back(N->getOperand(2)); 3970276479Sdim Ops.push_back(N->getOperand(3)); 3971276479Sdim Ops.push_back(N->getOperand(4)); 3972276479Sdim Ops.push_back(Chain); 3973276479Sdim break; 3974276479Sdim case NVPTXISD::Suld3DV4I16Clamp: 3975276479Sdim Opc = NVPTX::SULD_3D_V4I16_CLAMP; 3976276479Sdim Ops.push_back(TexHandle); 3977276479Sdim Ops.push_back(N->getOperand(2)); 3978276479Sdim Ops.push_back(N->getOperand(3)); 3979276479Sdim Ops.push_back(N->getOperand(4)); 3980276479Sdim Ops.push_back(Chain); 3981276479Sdim break; 3982276479Sdim case NVPTXISD::Suld3DV4I32Clamp: 3983276479Sdim Opc = NVPTX::SULD_3D_V4I32_CLAMP; 3984276479Sdim Ops.push_back(TexHandle); 3985276479Sdim Ops.push_back(N->getOperand(2)); 3986276479Sdim Ops.push_back(N->getOperand(3)); 3987276479Sdim Ops.push_back(N->getOperand(4)); 3988276479Sdim Ops.push_back(Chain); 3989276479Sdim break; 3990276479Sdim case NVPTXISD::Suld1DI8Trap: 3991276479Sdim Opc = NVPTX::SULD_1D_I8_TRAP; 3992276479Sdim Ops.push_back(TexHandle); 3993276479Sdim Ops.push_back(N->getOperand(2)); 3994276479Sdim Ops.push_back(Chain); 3995276479Sdim break; 3996276479Sdim case NVPTXISD::Suld1DI16Trap: 3997276479Sdim Opc = NVPTX::SULD_1D_I16_TRAP; 3998276479Sdim Ops.push_back(TexHandle); 3999276479Sdim Ops.push_back(N->getOperand(2)); 4000276479Sdim Ops.push_back(Chain); 4001276479Sdim break; 4002276479Sdim case NVPTXISD::Suld1DI32Trap: 4003276479Sdim Opc = NVPTX::SULD_1D_I32_TRAP; 4004276479Sdim Ops.push_back(TexHandle); 4005276479Sdim Ops.push_back(N->getOperand(2)); 4006276479Sdim Ops.push_back(Chain); 4007276479Sdim break; 4008276479Sdim case NVPTXISD::Suld1DI64Trap: 4009276479Sdim Opc = NVPTX::SULD_1D_I64_TRAP; 4010276479Sdim Ops.push_back(TexHandle); 4011276479Sdim Ops.push_back(N->getOperand(2)); 4012276479Sdim Ops.push_back(Chain); 4013276479Sdim break; 4014276479Sdim case NVPTXISD::Suld1DV2I8Trap: 4015276479Sdim Opc = NVPTX::SULD_1D_V2I8_TRAP; 4016276479Sdim Ops.push_back(TexHandle); 4017276479Sdim Ops.push_back(N->getOperand(2)); 4018276479Sdim Ops.push_back(Chain); 4019276479Sdim break; 4020276479Sdim case NVPTXISD::Suld1DV2I16Trap: 4021276479Sdim Opc = NVPTX::SULD_1D_V2I16_TRAP; 4022276479Sdim Ops.push_back(TexHandle); 4023276479Sdim Ops.push_back(N->getOperand(2)); 4024276479Sdim Ops.push_back(Chain); 4025276479Sdim break; 4026276479Sdim case NVPTXISD::Suld1DV2I32Trap: 4027276479Sdim Opc = NVPTX::SULD_1D_V2I32_TRAP; 4028276479Sdim Ops.push_back(TexHandle); 4029276479Sdim Ops.push_back(N->getOperand(2)); 4030276479Sdim Ops.push_back(Chain); 4031276479Sdim break; 4032276479Sdim case NVPTXISD::Suld1DV2I64Trap: 4033276479Sdim Opc = NVPTX::SULD_1D_V2I64_TRAP; 4034276479Sdim Ops.push_back(TexHandle); 4035276479Sdim Ops.push_back(N->getOperand(2)); 4036276479Sdim Ops.push_back(Chain); 4037276479Sdim break; 4038276479Sdim case NVPTXISD::Suld1DV4I8Trap: 4039276479Sdim Opc = NVPTX::SULD_1D_V4I8_TRAP; 4040276479Sdim Ops.push_back(TexHandle); 4041276479Sdim Ops.push_back(N->getOperand(2)); 4042276479Sdim Ops.push_back(Chain); 4043276479Sdim break; 4044276479Sdim case NVPTXISD::Suld1DV4I16Trap: 4045276479Sdim Opc = NVPTX::SULD_1D_V4I16_TRAP; 4046276479Sdim Ops.push_back(TexHandle); 4047276479Sdim Ops.push_back(N->getOperand(2)); 4048276479Sdim Ops.push_back(Chain); 4049276479Sdim break; 4050276479Sdim case NVPTXISD::Suld1DV4I32Trap: 4051276479Sdim Opc = NVPTX::SULD_1D_V4I32_TRAP; 4052276479Sdim Ops.push_back(TexHandle); 4053276479Sdim Ops.push_back(N->getOperand(2)); 4054276479Sdim Ops.push_back(Chain); 4055276479Sdim break; 4056276479Sdim case NVPTXISD::Suld1DArrayI8Trap: 4057276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP; 4058276479Sdim Ops.push_back(TexHandle); 4059276479Sdim Ops.push_back(N->getOperand(2)); 4060276479Sdim Ops.push_back(N->getOperand(3)); 4061276479Sdim Ops.push_back(Chain); 4062276479Sdim break; 4063276479Sdim case NVPTXISD::Suld1DArrayI16Trap: 4064276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP; 4065276479Sdim Ops.push_back(TexHandle); 4066276479Sdim Ops.push_back(N->getOperand(2)); 4067276479Sdim Ops.push_back(N->getOperand(3)); 4068276479Sdim Ops.push_back(Chain); 4069276479Sdim break; 4070276479Sdim case NVPTXISD::Suld1DArrayI32Trap: 4071276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP; 4072276479Sdim Ops.push_back(TexHandle); 4073276479Sdim Ops.push_back(N->getOperand(2)); 4074276479Sdim Ops.push_back(N->getOperand(3)); 4075276479Sdim Ops.push_back(Chain); 4076276479Sdim break; 4077276479Sdim case NVPTXISD::Suld1DArrayI64Trap: 4078276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I64_TRAP; 4079276479Sdim Ops.push_back(TexHandle); 4080276479Sdim Ops.push_back(N->getOperand(2)); 4081276479Sdim Ops.push_back(N->getOperand(3)); 4082276479Sdim Ops.push_back(Chain); 4083276479Sdim break; 4084276479Sdim case NVPTXISD::Suld1DArrayV2I8Trap: 4085276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP; 4086276479Sdim Ops.push_back(TexHandle); 4087276479Sdim Ops.push_back(N->getOperand(2)); 4088276479Sdim Ops.push_back(N->getOperand(3)); 4089276479Sdim Ops.push_back(Chain); 4090276479Sdim break; 4091276479Sdim case NVPTXISD::Suld1DArrayV2I16Trap: 4092276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP; 4093276479Sdim Ops.push_back(TexHandle); 4094276479Sdim Ops.push_back(N->getOperand(2)); 4095276479Sdim Ops.push_back(N->getOperand(3)); 4096276479Sdim Ops.push_back(Chain); 4097276479Sdim break; 4098276479Sdim case NVPTXISD::Suld1DArrayV2I32Trap: 4099276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP; 4100276479Sdim Ops.push_back(TexHandle); 4101276479Sdim Ops.push_back(N->getOperand(2)); 4102276479Sdim Ops.push_back(N->getOperand(3)); 4103276479Sdim Ops.push_back(Chain); 4104276479Sdim break; 4105276479Sdim case NVPTXISD::Suld1DArrayV2I64Trap: 4106276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I64_TRAP; 4107276479Sdim Ops.push_back(TexHandle); 4108276479Sdim Ops.push_back(N->getOperand(2)); 4109276479Sdim Ops.push_back(N->getOperand(3)); 4110276479Sdim Ops.push_back(Chain); 4111276479Sdim break; 4112276479Sdim case NVPTXISD::Suld1DArrayV4I8Trap: 4113276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP; 4114276479Sdim Ops.push_back(TexHandle); 4115276479Sdim Ops.push_back(N->getOperand(2)); 4116276479Sdim Ops.push_back(N->getOperand(3)); 4117276479Sdim Ops.push_back(Chain); 4118276479Sdim break; 4119276479Sdim case NVPTXISD::Suld1DArrayV4I16Trap: 4120276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP; 4121276479Sdim Ops.push_back(TexHandle); 4122276479Sdim Ops.push_back(N->getOperand(2)); 4123276479Sdim Ops.push_back(N->getOperand(3)); 4124276479Sdim Ops.push_back(Chain); 4125276479Sdim break; 4126276479Sdim case NVPTXISD::Suld1DArrayV4I32Trap: 4127276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP; 4128276479Sdim Ops.push_back(TexHandle); 4129276479Sdim Ops.push_back(N->getOperand(2)); 4130276479Sdim Ops.push_back(N->getOperand(3)); 4131276479Sdim Ops.push_back(Chain); 4132276479Sdim break; 4133276479Sdim case NVPTXISD::Suld2DI8Trap: 4134276479Sdim Opc = NVPTX::SULD_2D_I8_TRAP; 4135276479Sdim Ops.push_back(TexHandle); 4136276479Sdim Ops.push_back(N->getOperand(2)); 4137276479Sdim Ops.push_back(N->getOperand(3)); 4138276479Sdim Ops.push_back(Chain); 4139276479Sdim break; 4140276479Sdim case NVPTXISD::Suld2DI16Trap: 4141276479Sdim Opc = NVPTX::SULD_2D_I16_TRAP; 4142276479Sdim Ops.push_back(TexHandle); 4143276479Sdim Ops.push_back(N->getOperand(2)); 4144276479Sdim Ops.push_back(N->getOperand(3)); 4145276479Sdim Ops.push_back(Chain); 4146276479Sdim break; 4147276479Sdim case NVPTXISD::Suld2DI32Trap: 4148276479Sdim Opc = NVPTX::SULD_2D_I32_TRAP; 4149276479Sdim Ops.push_back(TexHandle); 4150276479Sdim Ops.push_back(N->getOperand(2)); 4151276479Sdim Ops.push_back(N->getOperand(3)); 4152276479Sdim Ops.push_back(Chain); 4153276479Sdim break; 4154276479Sdim case NVPTXISD::Suld2DI64Trap: 4155276479Sdim Opc = NVPTX::SULD_2D_I64_TRAP; 4156276479Sdim Ops.push_back(TexHandle); 4157276479Sdim Ops.push_back(N->getOperand(2)); 4158276479Sdim Ops.push_back(N->getOperand(3)); 4159276479Sdim Ops.push_back(Chain); 4160276479Sdim break; 4161276479Sdim case NVPTXISD::Suld2DV2I8Trap: 4162276479Sdim Opc = NVPTX::SULD_2D_V2I8_TRAP; 4163276479Sdim Ops.push_back(TexHandle); 4164276479Sdim Ops.push_back(N->getOperand(2)); 4165276479Sdim Ops.push_back(N->getOperand(3)); 4166276479Sdim Ops.push_back(Chain); 4167276479Sdim break; 4168276479Sdim case NVPTXISD::Suld2DV2I16Trap: 4169276479Sdim Opc = NVPTX::SULD_2D_V2I16_TRAP; 4170276479Sdim Ops.push_back(TexHandle); 4171276479Sdim Ops.push_back(N->getOperand(2)); 4172276479Sdim Ops.push_back(N->getOperand(3)); 4173276479Sdim Ops.push_back(Chain); 4174276479Sdim break; 4175276479Sdim case NVPTXISD::Suld2DV2I32Trap: 4176276479Sdim Opc = NVPTX::SULD_2D_V2I32_TRAP; 4177276479Sdim Ops.push_back(TexHandle); 4178276479Sdim Ops.push_back(N->getOperand(2)); 4179276479Sdim Ops.push_back(N->getOperand(3)); 4180276479Sdim Ops.push_back(Chain); 4181276479Sdim break; 4182276479Sdim case NVPTXISD::Suld2DV2I64Trap: 4183276479Sdim Opc = NVPTX::SULD_2D_V2I64_TRAP; 4184276479Sdim Ops.push_back(TexHandle); 4185276479Sdim Ops.push_back(N->getOperand(2)); 4186276479Sdim Ops.push_back(N->getOperand(3)); 4187276479Sdim Ops.push_back(Chain); 4188276479Sdim break; 4189276479Sdim case NVPTXISD::Suld2DV4I8Trap: 4190276479Sdim Opc = NVPTX::SULD_2D_V4I8_TRAP; 4191276479Sdim Ops.push_back(TexHandle); 4192276479Sdim Ops.push_back(N->getOperand(2)); 4193276479Sdim Ops.push_back(N->getOperand(3)); 4194276479Sdim Ops.push_back(Chain); 4195276479Sdim break; 4196276479Sdim case NVPTXISD::Suld2DV4I16Trap: 4197276479Sdim Opc = NVPTX::SULD_2D_V4I16_TRAP; 4198276479Sdim Ops.push_back(TexHandle); 4199276479Sdim Ops.push_back(N->getOperand(2)); 4200276479Sdim Ops.push_back(N->getOperand(3)); 4201276479Sdim Ops.push_back(Chain); 4202276479Sdim break; 4203276479Sdim case NVPTXISD::Suld2DV4I32Trap: 4204276479Sdim Opc = NVPTX::SULD_2D_V4I32_TRAP; 4205276479Sdim Ops.push_back(TexHandle); 4206276479Sdim Ops.push_back(N->getOperand(2)); 4207276479Sdim Ops.push_back(N->getOperand(3)); 4208276479Sdim Ops.push_back(Chain); 4209276479Sdim break; 4210276479Sdim case NVPTXISD::Suld2DArrayI8Trap: 4211276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP; 4212276479Sdim Ops.push_back(TexHandle); 4213276479Sdim Ops.push_back(N->getOperand(2)); 4214276479Sdim Ops.push_back(N->getOperand(3)); 4215276479Sdim Ops.push_back(N->getOperand(4)); 4216276479Sdim Ops.push_back(Chain); 4217276479Sdim break; 4218276479Sdim case NVPTXISD::Suld2DArrayI16Trap: 4219276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP; 4220276479Sdim Ops.push_back(TexHandle); 4221276479Sdim Ops.push_back(N->getOperand(2)); 4222276479Sdim Ops.push_back(N->getOperand(3)); 4223276479Sdim Ops.push_back(N->getOperand(4)); 4224276479Sdim Ops.push_back(Chain); 4225276479Sdim break; 4226276479Sdim case NVPTXISD::Suld2DArrayI32Trap: 4227276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP; 4228276479Sdim Ops.push_back(TexHandle); 4229276479Sdim Ops.push_back(N->getOperand(2)); 4230276479Sdim Ops.push_back(N->getOperand(3)); 4231276479Sdim Ops.push_back(N->getOperand(4)); 4232276479Sdim Ops.push_back(Chain); 4233276479Sdim break; 4234276479Sdim case NVPTXISD::Suld2DArrayI64Trap: 4235276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I64_TRAP; 4236276479Sdim Ops.push_back(TexHandle); 4237276479Sdim Ops.push_back(N->getOperand(2)); 4238276479Sdim Ops.push_back(N->getOperand(3)); 4239276479Sdim Ops.push_back(N->getOperand(4)); 4240276479Sdim Ops.push_back(Chain); 4241276479Sdim break; 4242276479Sdim case NVPTXISD::Suld2DArrayV2I8Trap: 4243276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP; 4244276479Sdim Ops.push_back(TexHandle); 4245276479Sdim Ops.push_back(N->getOperand(2)); 4246276479Sdim Ops.push_back(N->getOperand(3)); 4247276479Sdim Ops.push_back(N->getOperand(4)); 4248276479Sdim Ops.push_back(Chain); 4249276479Sdim break; 4250276479Sdim case NVPTXISD::Suld2DArrayV2I16Trap: 4251276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP; 4252276479Sdim Ops.push_back(TexHandle); 4253276479Sdim Ops.push_back(N->getOperand(2)); 4254276479Sdim Ops.push_back(N->getOperand(3)); 4255276479Sdim Ops.push_back(N->getOperand(4)); 4256276479Sdim Ops.push_back(Chain); 4257276479Sdim break; 4258276479Sdim case NVPTXISD::Suld2DArrayV2I32Trap: 4259276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP; 4260276479Sdim Ops.push_back(TexHandle); 4261276479Sdim Ops.push_back(N->getOperand(2)); 4262276479Sdim Ops.push_back(N->getOperand(3)); 4263276479Sdim Ops.push_back(N->getOperand(4)); 4264276479Sdim Ops.push_back(Chain); 4265276479Sdim break; 4266276479Sdim case NVPTXISD::Suld2DArrayV2I64Trap: 4267276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I64_TRAP; 4268276479Sdim Ops.push_back(TexHandle); 4269276479Sdim Ops.push_back(N->getOperand(2)); 4270276479Sdim Ops.push_back(N->getOperand(3)); 4271276479Sdim Ops.push_back(N->getOperand(4)); 4272276479Sdim Ops.push_back(Chain); 4273276479Sdim break; 4274276479Sdim case NVPTXISD::Suld2DArrayV4I8Trap: 4275276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP; 4276276479Sdim Ops.push_back(TexHandle); 4277276479Sdim Ops.push_back(N->getOperand(2)); 4278276479Sdim Ops.push_back(N->getOperand(3)); 4279276479Sdim Ops.push_back(N->getOperand(4)); 4280276479Sdim Ops.push_back(Chain); 4281276479Sdim break; 4282276479Sdim case NVPTXISD::Suld2DArrayV4I16Trap: 4283276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP; 4284276479Sdim Ops.push_back(TexHandle); 4285276479Sdim Ops.push_back(N->getOperand(2)); 4286276479Sdim Ops.push_back(N->getOperand(3)); 4287276479Sdim Ops.push_back(N->getOperand(4)); 4288276479Sdim Ops.push_back(Chain); 4289276479Sdim break; 4290276479Sdim case NVPTXISD::Suld2DArrayV4I32Trap: 4291276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP; 4292276479Sdim Ops.push_back(TexHandle); 4293276479Sdim Ops.push_back(N->getOperand(2)); 4294276479Sdim Ops.push_back(N->getOperand(3)); 4295276479Sdim Ops.push_back(N->getOperand(4)); 4296276479Sdim Ops.push_back(Chain); 4297276479Sdim break; 4298276479Sdim case NVPTXISD::Suld3DI8Trap: 4299276479Sdim Opc = NVPTX::SULD_3D_I8_TRAP; 4300276479Sdim Ops.push_back(TexHandle); 4301276479Sdim Ops.push_back(N->getOperand(2)); 4302276479Sdim Ops.push_back(N->getOperand(3)); 4303276479Sdim Ops.push_back(N->getOperand(4)); 4304276479Sdim Ops.push_back(Chain); 4305276479Sdim break; 4306276479Sdim case NVPTXISD::Suld3DI16Trap: 4307276479Sdim Opc = NVPTX::SULD_3D_I16_TRAP; 4308276479Sdim Ops.push_back(TexHandle); 4309276479Sdim Ops.push_back(N->getOperand(2)); 4310276479Sdim Ops.push_back(N->getOperand(3)); 4311276479Sdim Ops.push_back(N->getOperand(4)); 4312276479Sdim Ops.push_back(Chain); 4313276479Sdim break; 4314276479Sdim case NVPTXISD::Suld3DI32Trap: 4315276479Sdim Opc = NVPTX::SULD_3D_I32_TRAP; 4316276479Sdim Ops.push_back(TexHandle); 4317276479Sdim Ops.push_back(N->getOperand(2)); 4318276479Sdim Ops.push_back(N->getOperand(3)); 4319276479Sdim Ops.push_back(N->getOperand(4)); 4320276479Sdim Ops.push_back(Chain); 4321276479Sdim break; 4322276479Sdim case NVPTXISD::Suld3DI64Trap: 4323276479Sdim Opc = NVPTX::SULD_3D_I64_TRAP; 4324276479Sdim Ops.push_back(TexHandle); 4325276479Sdim Ops.push_back(N->getOperand(2)); 4326276479Sdim Ops.push_back(N->getOperand(3)); 4327276479Sdim Ops.push_back(N->getOperand(4)); 4328276479Sdim Ops.push_back(Chain); 4329276479Sdim break; 4330276479Sdim case NVPTXISD::Suld3DV2I8Trap: 4331276479Sdim Opc = NVPTX::SULD_3D_V2I8_TRAP; 4332276479Sdim Ops.push_back(TexHandle); 4333276479Sdim Ops.push_back(N->getOperand(2)); 4334276479Sdim Ops.push_back(N->getOperand(3)); 4335276479Sdim Ops.push_back(N->getOperand(4)); 4336276479Sdim Ops.push_back(Chain); 4337276479Sdim break; 4338276479Sdim case NVPTXISD::Suld3DV2I16Trap: 4339276479Sdim Opc = NVPTX::SULD_3D_V2I16_TRAP; 4340276479Sdim Ops.push_back(TexHandle); 4341276479Sdim Ops.push_back(N->getOperand(2)); 4342276479Sdim Ops.push_back(N->getOperand(3)); 4343276479Sdim Ops.push_back(N->getOperand(4)); 4344276479Sdim Ops.push_back(Chain); 4345276479Sdim break; 4346276479Sdim case NVPTXISD::Suld3DV2I32Trap: 4347276479Sdim Opc = NVPTX::SULD_3D_V2I32_TRAP; 4348276479Sdim Ops.push_back(TexHandle); 4349276479Sdim Ops.push_back(N->getOperand(2)); 4350276479Sdim Ops.push_back(N->getOperand(3)); 4351276479Sdim Ops.push_back(N->getOperand(4)); 4352276479Sdim Ops.push_back(Chain); 4353276479Sdim break; 4354276479Sdim case NVPTXISD::Suld3DV2I64Trap: 4355276479Sdim Opc = NVPTX::SULD_3D_V2I64_TRAP; 4356276479Sdim Ops.push_back(TexHandle); 4357276479Sdim Ops.push_back(N->getOperand(2)); 4358276479Sdim Ops.push_back(N->getOperand(3)); 4359276479Sdim Ops.push_back(N->getOperand(4)); 4360276479Sdim Ops.push_back(Chain); 4361276479Sdim break; 4362276479Sdim case NVPTXISD::Suld3DV4I8Trap: 4363276479Sdim Opc = NVPTX::SULD_3D_V4I8_TRAP; 4364276479Sdim Ops.push_back(TexHandle); 4365276479Sdim Ops.push_back(N->getOperand(2)); 4366276479Sdim Ops.push_back(N->getOperand(3)); 4367276479Sdim Ops.push_back(N->getOperand(4)); 4368276479Sdim Ops.push_back(Chain); 4369276479Sdim break; 4370276479Sdim case NVPTXISD::Suld3DV4I16Trap: 4371276479Sdim Opc = NVPTX::SULD_3D_V4I16_TRAP; 4372276479Sdim Ops.push_back(TexHandle); 4373276479Sdim Ops.push_back(N->getOperand(2)); 4374276479Sdim Ops.push_back(N->getOperand(3)); 4375276479Sdim Ops.push_back(N->getOperand(4)); 4376276479Sdim Ops.push_back(Chain); 4377276479Sdim break; 4378276479Sdim case NVPTXISD::Suld3DV4I32Trap: 4379276479Sdim Opc = NVPTX::SULD_3D_V4I32_TRAP; 4380276479Sdim Ops.push_back(TexHandle); 4381276479Sdim Ops.push_back(N->getOperand(2)); 4382276479Sdim Ops.push_back(N->getOperand(3)); 4383276479Sdim Ops.push_back(N->getOperand(4)); 4384276479Sdim Ops.push_back(Chain); 4385276479Sdim break; 4386276479Sdim case NVPTXISD::Suld1DI8Zero: 4387276479Sdim Opc = NVPTX::SULD_1D_I8_ZERO; 4388276479Sdim Ops.push_back(TexHandle); 4389276479Sdim Ops.push_back(N->getOperand(2)); 4390276479Sdim Ops.push_back(Chain); 4391276479Sdim break; 4392276479Sdim case NVPTXISD::Suld1DI16Zero: 4393276479Sdim Opc = NVPTX::SULD_1D_I16_ZERO; 4394276479Sdim Ops.push_back(TexHandle); 4395276479Sdim Ops.push_back(N->getOperand(2)); 4396276479Sdim Ops.push_back(Chain); 4397276479Sdim break; 4398276479Sdim case NVPTXISD::Suld1DI32Zero: 4399276479Sdim Opc = NVPTX::SULD_1D_I32_ZERO; 4400276479Sdim Ops.push_back(TexHandle); 4401276479Sdim Ops.push_back(N->getOperand(2)); 4402276479Sdim Ops.push_back(Chain); 4403276479Sdim break; 4404276479Sdim case NVPTXISD::Suld1DI64Zero: 4405276479Sdim Opc = NVPTX::SULD_1D_I64_ZERO; 4406276479Sdim Ops.push_back(TexHandle); 4407276479Sdim Ops.push_back(N->getOperand(2)); 4408276479Sdim Ops.push_back(Chain); 4409276479Sdim break; 4410276479Sdim case NVPTXISD::Suld1DV2I8Zero: 4411276479Sdim Opc = NVPTX::SULD_1D_V2I8_ZERO; 4412276479Sdim Ops.push_back(TexHandle); 4413276479Sdim Ops.push_back(N->getOperand(2)); 4414276479Sdim Ops.push_back(Chain); 4415276479Sdim break; 4416276479Sdim case NVPTXISD::Suld1DV2I16Zero: 4417276479Sdim Opc = NVPTX::SULD_1D_V2I16_ZERO; 4418276479Sdim Ops.push_back(TexHandle); 4419276479Sdim Ops.push_back(N->getOperand(2)); 4420276479Sdim Ops.push_back(Chain); 4421276479Sdim break; 4422276479Sdim case NVPTXISD::Suld1DV2I32Zero: 4423276479Sdim Opc = NVPTX::SULD_1D_V2I32_ZERO; 4424276479Sdim Ops.push_back(TexHandle); 4425276479Sdim Ops.push_back(N->getOperand(2)); 4426276479Sdim Ops.push_back(Chain); 4427276479Sdim break; 4428276479Sdim case NVPTXISD::Suld1DV2I64Zero: 4429276479Sdim Opc = NVPTX::SULD_1D_V2I64_ZERO; 4430276479Sdim Ops.push_back(TexHandle); 4431276479Sdim Ops.push_back(N->getOperand(2)); 4432276479Sdim Ops.push_back(Chain); 4433276479Sdim break; 4434276479Sdim case NVPTXISD::Suld1DV4I8Zero: 4435276479Sdim Opc = NVPTX::SULD_1D_V4I8_ZERO; 4436276479Sdim Ops.push_back(TexHandle); 4437276479Sdim Ops.push_back(N->getOperand(2)); 4438276479Sdim Ops.push_back(Chain); 4439276479Sdim break; 4440276479Sdim case NVPTXISD::Suld1DV4I16Zero: 4441276479Sdim Opc = NVPTX::SULD_1D_V4I16_ZERO; 4442276479Sdim Ops.push_back(TexHandle); 4443276479Sdim Ops.push_back(N->getOperand(2)); 4444276479Sdim Ops.push_back(Chain); 4445276479Sdim break; 4446276479Sdim case NVPTXISD::Suld1DV4I32Zero: 4447276479Sdim Opc = NVPTX::SULD_1D_V4I32_ZERO; 4448276479Sdim Ops.push_back(TexHandle); 4449276479Sdim Ops.push_back(N->getOperand(2)); 4450276479Sdim Ops.push_back(Chain); 4451276479Sdim break; 4452276479Sdim case NVPTXISD::Suld1DArrayI8Zero: 4453276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I8_ZERO; 4454276479Sdim Ops.push_back(TexHandle); 4455276479Sdim Ops.push_back(N->getOperand(2)); 4456276479Sdim Ops.push_back(N->getOperand(3)); 4457276479Sdim Ops.push_back(Chain); 4458276479Sdim break; 4459276479Sdim case NVPTXISD::Suld1DArrayI16Zero: 4460276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I16_ZERO; 4461276479Sdim Ops.push_back(TexHandle); 4462276479Sdim Ops.push_back(N->getOperand(2)); 4463276479Sdim Ops.push_back(N->getOperand(3)); 4464276479Sdim Ops.push_back(Chain); 4465276479Sdim break; 4466276479Sdim case NVPTXISD::Suld1DArrayI32Zero: 4467276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I32_ZERO; 4468276479Sdim Ops.push_back(TexHandle); 4469276479Sdim Ops.push_back(N->getOperand(2)); 4470276479Sdim Ops.push_back(N->getOperand(3)); 4471276479Sdim Ops.push_back(Chain); 4472276479Sdim break; 4473276479Sdim case NVPTXISD::Suld1DArrayI64Zero: 4474276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I64_ZERO; 4475276479Sdim Ops.push_back(TexHandle); 4476276479Sdim Ops.push_back(N->getOperand(2)); 4477276479Sdim Ops.push_back(N->getOperand(3)); 4478276479Sdim Ops.push_back(Chain); 4479276479Sdim break; 4480276479Sdim case NVPTXISD::Suld1DArrayV2I8Zero: 4481276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I8_ZERO; 4482276479Sdim Ops.push_back(TexHandle); 4483276479Sdim Ops.push_back(N->getOperand(2)); 4484276479Sdim Ops.push_back(N->getOperand(3)); 4485276479Sdim Ops.push_back(Chain); 4486276479Sdim break; 4487276479Sdim case NVPTXISD::Suld1DArrayV2I16Zero: 4488276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I16_ZERO; 4489276479Sdim Ops.push_back(TexHandle); 4490276479Sdim Ops.push_back(N->getOperand(2)); 4491276479Sdim Ops.push_back(N->getOperand(3)); 4492276479Sdim Ops.push_back(Chain); 4493276479Sdim break; 4494276479Sdim case NVPTXISD::Suld1DArrayV2I32Zero: 4495276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I32_ZERO; 4496276479Sdim Ops.push_back(TexHandle); 4497276479Sdim Ops.push_back(N->getOperand(2)); 4498276479Sdim Ops.push_back(N->getOperand(3)); 4499276479Sdim Ops.push_back(Chain); 4500276479Sdim break; 4501276479Sdim case NVPTXISD::Suld1DArrayV2I64Zero: 4502276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I64_ZERO; 4503276479Sdim Ops.push_back(TexHandle); 4504276479Sdim Ops.push_back(N->getOperand(2)); 4505276479Sdim Ops.push_back(N->getOperand(3)); 4506276479Sdim Ops.push_back(Chain); 4507276479Sdim break; 4508276479Sdim case NVPTXISD::Suld1DArrayV4I8Zero: 4509276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I8_ZERO; 4510276479Sdim Ops.push_back(TexHandle); 4511276479Sdim Ops.push_back(N->getOperand(2)); 4512276479Sdim Ops.push_back(N->getOperand(3)); 4513276479Sdim Ops.push_back(Chain); 4514276479Sdim break; 4515276479Sdim case NVPTXISD::Suld1DArrayV4I16Zero: 4516276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I16_ZERO; 4517276479Sdim Ops.push_back(TexHandle); 4518276479Sdim Ops.push_back(N->getOperand(2)); 4519276479Sdim Ops.push_back(N->getOperand(3)); 4520276479Sdim Ops.push_back(Chain); 4521276479Sdim break; 4522276479Sdim case NVPTXISD::Suld1DArrayV4I32Zero: 4523276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I32_ZERO; 4524276479Sdim Ops.push_back(TexHandle); 4525276479Sdim Ops.push_back(N->getOperand(2)); 4526276479Sdim Ops.push_back(N->getOperand(3)); 4527276479Sdim Ops.push_back(Chain); 4528276479Sdim break; 4529276479Sdim case NVPTXISD::Suld2DI8Zero: 4530276479Sdim Opc = NVPTX::SULD_2D_I8_ZERO; 4531276479Sdim Ops.push_back(TexHandle); 4532276479Sdim Ops.push_back(N->getOperand(2)); 4533276479Sdim Ops.push_back(N->getOperand(3)); 4534276479Sdim Ops.push_back(Chain); 4535276479Sdim break; 4536276479Sdim case NVPTXISD::Suld2DI16Zero: 4537276479Sdim Opc = NVPTX::SULD_2D_I16_ZERO; 4538276479Sdim Ops.push_back(TexHandle); 4539276479Sdim Ops.push_back(N->getOperand(2)); 4540276479Sdim Ops.push_back(N->getOperand(3)); 4541276479Sdim Ops.push_back(Chain); 4542276479Sdim break; 4543276479Sdim case NVPTXISD::Suld2DI32Zero: 4544276479Sdim Opc = NVPTX::SULD_2D_I32_ZERO; 4545276479Sdim Ops.push_back(TexHandle); 4546276479Sdim Ops.push_back(N->getOperand(2)); 4547276479Sdim Ops.push_back(N->getOperand(3)); 4548276479Sdim Ops.push_back(Chain); 4549276479Sdim break; 4550276479Sdim case NVPTXISD::Suld2DI64Zero: 4551276479Sdim Opc = NVPTX::SULD_2D_I64_ZERO; 4552276479Sdim Ops.push_back(TexHandle); 4553276479Sdim Ops.push_back(N->getOperand(2)); 4554276479Sdim Ops.push_back(N->getOperand(3)); 4555276479Sdim Ops.push_back(Chain); 4556276479Sdim break; 4557276479Sdim case NVPTXISD::Suld2DV2I8Zero: 4558276479Sdim Opc = NVPTX::SULD_2D_V2I8_ZERO; 4559276479Sdim Ops.push_back(TexHandle); 4560276479Sdim Ops.push_back(N->getOperand(2)); 4561276479Sdim Ops.push_back(N->getOperand(3)); 4562276479Sdim Ops.push_back(Chain); 4563276479Sdim break; 4564276479Sdim case NVPTXISD::Suld2DV2I16Zero: 4565276479Sdim Opc = NVPTX::SULD_2D_V2I16_ZERO; 4566276479Sdim Ops.push_back(TexHandle); 4567276479Sdim Ops.push_back(N->getOperand(2)); 4568276479Sdim Ops.push_back(N->getOperand(3)); 4569276479Sdim Ops.push_back(Chain); 4570276479Sdim break; 4571276479Sdim case NVPTXISD::Suld2DV2I32Zero: 4572276479Sdim Opc = NVPTX::SULD_2D_V2I32_ZERO; 4573276479Sdim Ops.push_back(TexHandle); 4574276479Sdim Ops.push_back(N->getOperand(2)); 4575276479Sdim Ops.push_back(N->getOperand(3)); 4576276479Sdim Ops.push_back(Chain); 4577276479Sdim break; 4578276479Sdim case NVPTXISD::Suld2DV2I64Zero: 4579276479Sdim Opc = NVPTX::SULD_2D_V2I64_ZERO; 4580276479Sdim Ops.push_back(TexHandle); 4581276479Sdim Ops.push_back(N->getOperand(2)); 4582276479Sdim Ops.push_back(N->getOperand(3)); 4583276479Sdim Ops.push_back(Chain); 4584276479Sdim break; 4585276479Sdim case NVPTXISD::Suld2DV4I8Zero: 4586276479Sdim Opc = NVPTX::SULD_2D_V4I8_ZERO; 4587276479Sdim Ops.push_back(TexHandle); 4588276479Sdim Ops.push_back(N->getOperand(2)); 4589276479Sdim Ops.push_back(N->getOperand(3)); 4590276479Sdim Ops.push_back(Chain); 4591276479Sdim break; 4592276479Sdim case NVPTXISD::Suld2DV4I16Zero: 4593276479Sdim Opc = NVPTX::SULD_2D_V4I16_ZERO; 4594276479Sdim Ops.push_back(TexHandle); 4595276479Sdim Ops.push_back(N->getOperand(2)); 4596276479Sdim Ops.push_back(N->getOperand(3)); 4597276479Sdim Ops.push_back(Chain); 4598276479Sdim break; 4599276479Sdim case NVPTXISD::Suld2DV4I32Zero: 4600276479Sdim Opc = NVPTX::SULD_2D_V4I32_ZERO; 4601276479Sdim Ops.push_back(TexHandle); 4602276479Sdim Ops.push_back(N->getOperand(2)); 4603276479Sdim Ops.push_back(N->getOperand(3)); 4604276479Sdim Ops.push_back(Chain); 4605276479Sdim break; 4606276479Sdim case NVPTXISD::Suld2DArrayI8Zero: 4607276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I8_ZERO; 4608276479Sdim Ops.push_back(TexHandle); 4609276479Sdim Ops.push_back(N->getOperand(2)); 4610276479Sdim Ops.push_back(N->getOperand(3)); 4611276479Sdim Ops.push_back(N->getOperand(4)); 4612276479Sdim Ops.push_back(Chain); 4613276479Sdim break; 4614276479Sdim case NVPTXISD::Suld2DArrayI16Zero: 4615276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I16_ZERO; 4616276479Sdim Ops.push_back(TexHandle); 4617276479Sdim Ops.push_back(N->getOperand(2)); 4618276479Sdim Ops.push_back(N->getOperand(3)); 4619276479Sdim Ops.push_back(N->getOperand(4)); 4620276479Sdim Ops.push_back(Chain); 4621276479Sdim break; 4622276479Sdim case NVPTXISD::Suld2DArrayI32Zero: 4623276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I32_ZERO; 4624276479Sdim Ops.push_back(TexHandle); 4625276479Sdim Ops.push_back(N->getOperand(2)); 4626276479Sdim Ops.push_back(N->getOperand(3)); 4627276479Sdim Ops.push_back(N->getOperand(4)); 4628276479Sdim Ops.push_back(Chain); 4629276479Sdim break; 4630276479Sdim case NVPTXISD::Suld2DArrayI64Zero: 4631276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I64_ZERO; 4632276479Sdim Ops.push_back(TexHandle); 4633276479Sdim Ops.push_back(N->getOperand(2)); 4634276479Sdim Ops.push_back(N->getOperand(3)); 4635276479Sdim Ops.push_back(N->getOperand(4)); 4636276479Sdim Ops.push_back(Chain); 4637276479Sdim break; 4638276479Sdim case NVPTXISD::Suld2DArrayV2I8Zero: 4639276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I8_ZERO; 4640276479Sdim Ops.push_back(TexHandle); 4641276479Sdim Ops.push_back(N->getOperand(2)); 4642276479Sdim Ops.push_back(N->getOperand(3)); 4643276479Sdim Ops.push_back(N->getOperand(4)); 4644276479Sdim Ops.push_back(Chain); 4645276479Sdim break; 4646276479Sdim case NVPTXISD::Suld2DArrayV2I16Zero: 4647276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I16_ZERO; 4648276479Sdim Ops.push_back(TexHandle); 4649276479Sdim Ops.push_back(N->getOperand(2)); 4650276479Sdim Ops.push_back(N->getOperand(3)); 4651276479Sdim Ops.push_back(N->getOperand(4)); 4652276479Sdim Ops.push_back(Chain); 4653276479Sdim break; 4654276479Sdim case NVPTXISD::Suld2DArrayV2I32Zero: 4655276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I32_ZERO; 4656276479Sdim Ops.push_back(TexHandle); 4657276479Sdim Ops.push_back(N->getOperand(2)); 4658276479Sdim Ops.push_back(N->getOperand(3)); 4659276479Sdim Ops.push_back(N->getOperand(4)); 4660276479Sdim Ops.push_back(Chain); 4661276479Sdim break; 4662276479Sdim case NVPTXISD::Suld2DArrayV2I64Zero: 4663276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I64_ZERO; 4664276479Sdim Ops.push_back(TexHandle); 4665276479Sdim Ops.push_back(N->getOperand(2)); 4666276479Sdim Ops.push_back(N->getOperand(3)); 4667276479Sdim Ops.push_back(N->getOperand(4)); 4668276479Sdim Ops.push_back(Chain); 4669276479Sdim break; 4670276479Sdim case NVPTXISD::Suld2DArrayV4I8Zero: 4671276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I8_ZERO; 4672276479Sdim Ops.push_back(TexHandle); 4673276479Sdim Ops.push_back(N->getOperand(2)); 4674276479Sdim Ops.push_back(N->getOperand(3)); 4675276479Sdim Ops.push_back(N->getOperand(4)); 4676276479Sdim Ops.push_back(Chain); 4677276479Sdim break; 4678276479Sdim case NVPTXISD::Suld2DArrayV4I16Zero: 4679276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I16_ZERO; 4680276479Sdim Ops.push_back(TexHandle); 4681276479Sdim Ops.push_back(N->getOperand(2)); 4682276479Sdim Ops.push_back(N->getOperand(3)); 4683276479Sdim Ops.push_back(N->getOperand(4)); 4684276479Sdim Ops.push_back(Chain); 4685276479Sdim break; 4686276479Sdim case NVPTXISD::Suld2DArrayV4I32Zero: 4687276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I32_ZERO; 4688276479Sdim Ops.push_back(TexHandle); 4689276479Sdim Ops.push_back(N->getOperand(2)); 4690276479Sdim Ops.push_back(N->getOperand(3)); 4691276479Sdim Ops.push_back(N->getOperand(4)); 4692276479Sdim Ops.push_back(Chain); 4693276479Sdim break; 4694276479Sdim case NVPTXISD::Suld3DI8Zero: 4695276479Sdim Opc = NVPTX::SULD_3D_I8_ZERO; 4696276479Sdim Ops.push_back(TexHandle); 4697276479Sdim Ops.push_back(N->getOperand(2)); 4698276479Sdim Ops.push_back(N->getOperand(3)); 4699276479Sdim Ops.push_back(N->getOperand(4)); 4700276479Sdim Ops.push_back(Chain); 4701276479Sdim break; 4702276479Sdim case NVPTXISD::Suld3DI16Zero: 4703276479Sdim Opc = NVPTX::SULD_3D_I16_ZERO; 4704276479Sdim Ops.push_back(TexHandle); 4705276479Sdim Ops.push_back(N->getOperand(2)); 4706276479Sdim Ops.push_back(N->getOperand(3)); 4707276479Sdim Ops.push_back(N->getOperand(4)); 4708276479Sdim Ops.push_back(Chain); 4709276479Sdim break; 4710276479Sdim case NVPTXISD::Suld3DI32Zero: 4711276479Sdim Opc = NVPTX::SULD_3D_I32_ZERO; 4712276479Sdim Ops.push_back(TexHandle); 4713276479Sdim Ops.push_back(N->getOperand(2)); 4714276479Sdim Ops.push_back(N->getOperand(3)); 4715276479Sdim Ops.push_back(N->getOperand(4)); 4716276479Sdim Ops.push_back(Chain); 4717276479Sdim break; 4718276479Sdim case NVPTXISD::Suld3DI64Zero: 4719276479Sdim Opc = NVPTX::SULD_3D_I64_ZERO; 4720276479Sdim Ops.push_back(TexHandle); 4721276479Sdim Ops.push_back(N->getOperand(2)); 4722276479Sdim Ops.push_back(N->getOperand(3)); 4723276479Sdim Ops.push_back(N->getOperand(4)); 4724276479Sdim Ops.push_back(Chain); 4725276479Sdim break; 4726276479Sdim case NVPTXISD::Suld3DV2I8Zero: 4727276479Sdim Opc = NVPTX::SULD_3D_V2I8_ZERO; 4728276479Sdim Ops.push_back(TexHandle); 4729276479Sdim Ops.push_back(N->getOperand(2)); 4730276479Sdim Ops.push_back(N->getOperand(3)); 4731276479Sdim Ops.push_back(N->getOperand(4)); 4732276479Sdim Ops.push_back(Chain); 4733276479Sdim break; 4734276479Sdim case NVPTXISD::Suld3DV2I16Zero: 4735276479Sdim Opc = NVPTX::SULD_3D_V2I16_ZERO; 4736276479Sdim Ops.push_back(TexHandle); 4737276479Sdim Ops.push_back(N->getOperand(2)); 4738276479Sdim Ops.push_back(N->getOperand(3)); 4739276479Sdim Ops.push_back(N->getOperand(4)); 4740276479Sdim Ops.push_back(Chain); 4741276479Sdim break; 4742276479Sdim case NVPTXISD::Suld3DV2I32Zero: 4743276479Sdim Opc = NVPTX::SULD_3D_V2I32_ZERO; 4744276479Sdim Ops.push_back(TexHandle); 4745276479Sdim Ops.push_back(N->getOperand(2)); 4746276479Sdim Ops.push_back(N->getOperand(3)); 4747276479Sdim Ops.push_back(N->getOperand(4)); 4748276479Sdim Ops.push_back(Chain); 4749276479Sdim break; 4750276479Sdim case NVPTXISD::Suld3DV2I64Zero: 4751276479Sdim Opc = NVPTX::SULD_3D_V2I64_ZERO; 4752276479Sdim Ops.push_back(TexHandle); 4753276479Sdim Ops.push_back(N->getOperand(2)); 4754276479Sdim Ops.push_back(N->getOperand(3)); 4755276479Sdim Ops.push_back(N->getOperand(4)); 4756276479Sdim Ops.push_back(Chain); 4757276479Sdim break; 4758276479Sdim case NVPTXISD::Suld3DV4I8Zero: 4759276479Sdim Opc = NVPTX::SULD_3D_V4I8_ZERO; 4760276479Sdim Ops.push_back(TexHandle); 4761276479Sdim Ops.push_back(N->getOperand(2)); 4762276479Sdim Ops.push_back(N->getOperand(3)); 4763276479Sdim Ops.push_back(N->getOperand(4)); 4764276479Sdim Ops.push_back(Chain); 4765276479Sdim break; 4766276479Sdim case NVPTXISD::Suld3DV4I16Zero: 4767276479Sdim Opc = NVPTX::SULD_3D_V4I16_ZERO; 4768276479Sdim Ops.push_back(TexHandle); 4769276479Sdim Ops.push_back(N->getOperand(2)); 4770276479Sdim Ops.push_back(N->getOperand(3)); 4771276479Sdim Ops.push_back(N->getOperand(4)); 4772276479Sdim Ops.push_back(Chain); 4773276479Sdim break; 4774276479Sdim case NVPTXISD::Suld3DV4I32Zero: 4775276479Sdim Opc = NVPTX::SULD_3D_V4I32_ZERO; 4776276479Sdim Ops.push_back(TexHandle); 4777276479Sdim Ops.push_back(N->getOperand(2)); 4778276479Sdim Ops.push_back(N->getOperand(3)); 4779276479Sdim Ops.push_back(N->getOperand(4)); 4780276479Sdim Ops.push_back(Chain); 4781276479Sdim break; 4782276479Sdim } 4783276479Sdim Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops); 4784276479Sdim return Ret; 4785276479Sdim} 4786276479Sdim 4787276479Sdim 4788276479Sdim/// SelectBFE - Look for instruction sequences that can be made more efficient 4789276479Sdim/// by using the 'bfe' (bit-field extract) PTX instruction 4790276479SdimSDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) { 4791288943Sdim SDLoc DL(N); 4792276479Sdim SDValue LHS = N->getOperand(0); 4793276479Sdim SDValue RHS = N->getOperand(1); 4794276479Sdim SDValue Len; 4795276479Sdim SDValue Start; 4796276479Sdim SDValue Val; 4797276479Sdim bool IsSigned = false; 4798276479Sdim 4799276479Sdim if (N->getOpcode() == ISD::AND) { 4800276479Sdim // Canonicalize the operands 4801276479Sdim // We want 'and %val, %mask' 4802276479Sdim if (isa<ConstantSDNode>(LHS) && !isa<ConstantSDNode>(RHS)) { 4803276479Sdim std::swap(LHS, RHS); 4804276479Sdim } 4805276479Sdim 4806276479Sdim ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(RHS); 4807276479Sdim if (!Mask) { 4808276479Sdim // We need a constant mask on the RHS of the AND 4809276479Sdim return NULL; 4810276479Sdim } 4811276479Sdim 4812276479Sdim // Extract the mask bits 4813276479Sdim uint64_t MaskVal = Mask->getZExtValue(); 4814276479Sdim if (!isMask_64(MaskVal)) { 4815276479Sdim // We *could* handle shifted masks here, but doing so would require an 4816276479Sdim // 'and' operation to fix up the low-order bits so we would trade 4817276479Sdim // shr+and for bfe+and, which has the same throughput 4818276479Sdim return NULL; 4819276479Sdim } 4820276479Sdim 4821276479Sdim // How many bits are in our mask? 4822288943Sdim uint64_t NumBits = countTrailingOnes(MaskVal); 4823288943Sdim Len = CurDAG->getTargetConstant(NumBits, DL, MVT::i32); 4824276479Sdim 4825276479Sdim if (LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SRA) { 4826276479Sdim // We have a 'srl/and' pair, extract the effective start bit and length 4827276479Sdim Val = LHS.getNode()->getOperand(0); 4828276479Sdim Start = LHS.getNode()->getOperand(1); 4829276479Sdim ConstantSDNode *StartConst = dyn_cast<ConstantSDNode>(Start); 4830276479Sdim if (StartConst) { 4831276479Sdim uint64_t StartVal = StartConst->getZExtValue(); 4832276479Sdim // How many "good" bits do we have left? "good" is defined here as bits 4833276479Sdim // that exist in the original value, not shifted in. 4834276479Sdim uint64_t GoodBits = Start.getValueType().getSizeInBits() - StartVal; 4835276479Sdim if (NumBits > GoodBits) { 4836276479Sdim // Do not handle the case where bits have been shifted in. In theory 4837276479Sdim // we could handle this, but the cost is likely higher than just 4838276479Sdim // emitting the srl/and pair. 4839276479Sdim return NULL; 4840276479Sdim } 4841288943Sdim Start = CurDAG->getTargetConstant(StartVal, DL, MVT::i32); 4842276479Sdim } else { 4843276479Sdim // Do not handle the case where the shift amount (can be zero if no srl 4844276479Sdim // was found) is not constant. We could handle this case, but it would 4845276479Sdim // require run-time logic that would be more expensive than just 4846276479Sdim // emitting the srl/and pair. 4847276479Sdim return NULL; 4848276479Sdim } 4849276479Sdim } else { 4850276479Sdim // Do not handle the case where the LHS of the and is not a shift. While 4851276479Sdim // it would be trivial to handle this case, it would just transform 4852276479Sdim // 'and' -> 'bfe', but 'and' has higher-throughput. 4853276479Sdim return NULL; 4854276479Sdim } 4855276479Sdim } else if (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) { 4856276479Sdim if (LHS->getOpcode() == ISD::AND) { 4857276479Sdim ConstantSDNode *ShiftCnst = dyn_cast<ConstantSDNode>(RHS); 4858276479Sdim if (!ShiftCnst) { 4859276479Sdim // Shift amount must be constant 4860276479Sdim return NULL; 4861276479Sdim } 4862276479Sdim 4863276479Sdim uint64_t ShiftAmt = ShiftCnst->getZExtValue(); 4864276479Sdim 4865276479Sdim SDValue AndLHS = LHS->getOperand(0); 4866276479Sdim SDValue AndRHS = LHS->getOperand(1); 4867276479Sdim 4868276479Sdim // Canonicalize the AND to have the mask on the RHS 4869276479Sdim if (isa<ConstantSDNode>(AndLHS)) { 4870276479Sdim std::swap(AndLHS, AndRHS); 4871276479Sdim } 4872276479Sdim 4873276479Sdim ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(AndRHS); 4874276479Sdim if (!MaskCnst) { 4875276479Sdim // Mask must be constant 4876276479Sdim return NULL; 4877276479Sdim } 4878276479Sdim 4879276479Sdim uint64_t MaskVal = MaskCnst->getZExtValue(); 4880276479Sdim uint64_t NumZeros; 4881276479Sdim uint64_t NumBits; 4882276479Sdim if (isMask_64(MaskVal)) { 4883276479Sdim NumZeros = 0; 4884276479Sdim // The number of bits in the result bitfield will be the number of 4885276479Sdim // trailing ones (the AND) minus the number of bits we shift off 4886288943Sdim NumBits = countTrailingOnes(MaskVal) - ShiftAmt; 4887276479Sdim } else if (isShiftedMask_64(MaskVal)) { 4888276479Sdim NumZeros = countTrailingZeros(MaskVal); 4889288943Sdim unsigned NumOnes = countTrailingOnes(MaskVal >> NumZeros); 4890276479Sdim // The number of bits in the result bitfield will be the number of 4891276479Sdim // trailing zeros plus the number of set bits in the mask minus the 4892276479Sdim // number of bits we shift off 4893276479Sdim NumBits = NumZeros + NumOnes - ShiftAmt; 4894276479Sdim } else { 4895276479Sdim // This is not a mask we can handle 4896276479Sdim return NULL; 4897276479Sdim } 4898276479Sdim 4899276479Sdim if (ShiftAmt < NumZeros) { 4900276479Sdim // Handling this case would require extra logic that would make this 4901276479Sdim // transformation non-profitable 4902276479Sdim return NULL; 4903276479Sdim } 4904276479Sdim 4905276479Sdim Val = AndLHS; 4906288943Sdim Start = CurDAG->getTargetConstant(ShiftAmt, DL, MVT::i32); 4907288943Sdim Len = CurDAG->getTargetConstant(NumBits, DL, MVT::i32); 4908276479Sdim } else if (LHS->getOpcode() == ISD::SHL) { 4909276479Sdim // Here, we have a pattern like: 4910276479Sdim // 4911276479Sdim // (sra (shl val, NN), MM) 4912276479Sdim // or 4913276479Sdim // (srl (shl val, NN), MM) 4914276479Sdim // 4915276479Sdim // If MM >= NN, we can efficiently optimize this with bfe 4916276479Sdim Val = LHS->getOperand(0); 4917276479Sdim 4918276479Sdim SDValue ShlRHS = LHS->getOperand(1); 4919276479Sdim ConstantSDNode *ShlCnst = dyn_cast<ConstantSDNode>(ShlRHS); 4920276479Sdim if (!ShlCnst) { 4921276479Sdim // Shift amount must be constant 4922276479Sdim return NULL; 4923276479Sdim } 4924276479Sdim uint64_t InnerShiftAmt = ShlCnst->getZExtValue(); 4925276479Sdim 4926276479Sdim SDValue ShrRHS = RHS; 4927276479Sdim ConstantSDNode *ShrCnst = dyn_cast<ConstantSDNode>(ShrRHS); 4928276479Sdim if (!ShrCnst) { 4929276479Sdim // Shift amount must be constant 4930276479Sdim return NULL; 4931276479Sdim } 4932276479Sdim uint64_t OuterShiftAmt = ShrCnst->getZExtValue(); 4933276479Sdim 4934276479Sdim // To avoid extra codegen and be profitable, we need Outer >= Inner 4935276479Sdim if (OuterShiftAmt < InnerShiftAmt) { 4936276479Sdim return NULL; 4937276479Sdim } 4938276479Sdim 4939276479Sdim // If the outer shift is more than the type size, we have no bitfield to 4940276479Sdim // extract (since we also check that the inner shift is <= the outer shift 4941276479Sdim // then this also implies that the inner shift is < the type size) 4942276479Sdim if (OuterShiftAmt >= Val.getValueType().getSizeInBits()) { 4943276479Sdim return NULL; 4944276479Sdim } 4945276479Sdim 4946276479Sdim Start = 4947288943Sdim CurDAG->getTargetConstant(OuterShiftAmt - InnerShiftAmt, DL, MVT::i32); 4948276479Sdim Len = 4949276479Sdim CurDAG->getTargetConstant(Val.getValueType().getSizeInBits() - 4950288943Sdim OuterShiftAmt, DL, MVT::i32); 4951276479Sdim 4952276479Sdim if (N->getOpcode() == ISD::SRA) { 4953276479Sdim // If we have a arithmetic right shift, we need to use the signed bfe 4954276479Sdim // variant 4955276479Sdim IsSigned = true; 4956276479Sdim } 4957276479Sdim } else { 4958276479Sdim // No can do... 4959276479Sdim return NULL; 4960276479Sdim } 4961276479Sdim } else { 4962276479Sdim // No can do... 4963276479Sdim return NULL; 4964276479Sdim } 4965276479Sdim 4966276479Sdim 4967276479Sdim unsigned Opc; 4968276479Sdim // For the BFE operations we form here from "and" and "srl", always use the 4969276479Sdim // unsigned variants. 4970276479Sdim if (Val.getValueType() == MVT::i32) { 4971276479Sdim if (IsSigned) { 4972276479Sdim Opc = NVPTX::BFE_S32rii; 4973276479Sdim } else { 4974276479Sdim Opc = NVPTX::BFE_U32rii; 4975276479Sdim } 4976276479Sdim } else if (Val.getValueType() == MVT::i64) { 4977276479Sdim if (IsSigned) { 4978276479Sdim Opc = NVPTX::BFE_S64rii; 4979276479Sdim } else { 4980276479Sdim Opc = NVPTX::BFE_U64rii; 4981276479Sdim } 4982276479Sdim } else { 4983276479Sdim // We cannot handle this type 4984276479Sdim return NULL; 4985276479Sdim } 4986276479Sdim 4987276479Sdim SDValue Ops[] = { 4988276479Sdim Val, Start, Len 4989276479Sdim }; 4990276479Sdim 4991288943Sdim return CurDAG->getMachineNode(Opc, DL, N->getVTList(), Ops); 4992276479Sdim} 4993276479Sdim 4994239310Sdim// SelectDirectAddr - Match a direct address for DAG. 4995239310Sdim// A direct address could be a globaladdress or externalsymbol. 4996239310Sdimbool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) { 4997239310Sdim // Return true if TGA or ES. 4998249423Sdim if (N.getOpcode() == ISD::TargetGlobalAddress || 4999249423Sdim N.getOpcode() == ISD::TargetExternalSymbol) { 5000239310Sdim Address = N; 5001239310Sdim return true; 5002239310Sdim } 5003239310Sdim if (N.getOpcode() == NVPTXISD::Wrapper) { 5004239310Sdim Address = N.getOperand(0); 5005239310Sdim return true; 5006239310Sdim } 5007239310Sdim if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) { 5008239310Sdim unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue(); 5009239310Sdim if (IID == Intrinsic::nvvm_ptr_gen_to_param) 5010239310Sdim if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam) 5011239310Sdim return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address)); 5012239310Sdim } 5013239310Sdim return false; 5014239310Sdim} 5015239310Sdim 5016239310Sdim// symbol+offset 5017249423Sdimbool NVPTXDAGToDAGISel::SelectADDRsi_imp( 5018249423Sdim SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { 5019239310Sdim if (Addr.getOpcode() == ISD::ADD) { 5020239310Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) { 5021249423Sdim SDValue base = Addr.getOperand(0); 5022239310Sdim if (SelectDirectAddr(base, Base)) { 5023288943Sdim Offset = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(OpNode), 5024288943Sdim mvt); 5025239310Sdim return true; 5026239310Sdim } 5027239310Sdim } 5028239310Sdim } 5029239310Sdim return false; 5030239310Sdim} 5031239310Sdim 5032239310Sdim// symbol+offset 5033239310Sdimbool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr, 5034239310Sdim SDValue &Base, SDValue &Offset) { 5035239310Sdim return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32); 5036239310Sdim} 5037239310Sdim 5038239310Sdim// symbol+offset 5039239310Sdimbool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr, 5040239310Sdim SDValue &Base, SDValue &Offset) { 5041239310Sdim return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64); 5042239310Sdim} 5043239310Sdim 5044239310Sdim// register+offset 5045249423Sdimbool NVPTXDAGToDAGISel::SelectADDRri_imp( 5046249423Sdim SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { 5047239310Sdim if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) { 5048239310Sdim Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt); 5049288943Sdim Offset = CurDAG->getTargetConstant(0, SDLoc(OpNode), mvt); 5050239310Sdim return true; 5051239310Sdim } 5052239310Sdim if (Addr.getOpcode() == ISD::TargetExternalSymbol || 5053239310Sdim Addr.getOpcode() == ISD::TargetGlobalAddress) 5054249423Sdim return false; // direct calls. 5055239310Sdim 5056239310Sdim if (Addr.getOpcode() == ISD::ADD) { 5057239310Sdim if (SelectDirectAddr(Addr.getOperand(0), Addr)) { 5058239310Sdim return false; 5059239310Sdim } 5060239310Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) { 5061239310Sdim if (FrameIndexSDNode *FIN = 5062249423Sdim dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) 5063239310Sdim // Constant offset from frame ref. 5064239310Sdim Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt); 5065239310Sdim else 5066239310Sdim Base = Addr.getOperand(0); 5067288943Sdim Offset = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(OpNode), 5068288943Sdim mvt); 5069239310Sdim return true; 5070239310Sdim } 5071239310Sdim } 5072239310Sdim return false; 5073239310Sdim} 5074239310Sdim 5075239310Sdim// register+offset 5076239310Sdimbool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr, 5077239310Sdim SDValue &Base, SDValue &Offset) { 5078239310Sdim return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32); 5079239310Sdim} 5080239310Sdim 5081239310Sdim// register+offset 5082239310Sdimbool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr, 5083239310Sdim SDValue &Base, SDValue &Offset) { 5084239310Sdim return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64); 5085239310Sdim} 5086239310Sdim 5087239310Sdimbool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N, 5088239310Sdim unsigned int spN) const { 5089276479Sdim const Value *Src = nullptr; 5090239310Sdim if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) { 5091276479Sdim if (spN == 0 && mN->getMemOperand()->getPseudoValue()) 5092276479Sdim return true; 5093276479Sdim Src = mN->getMemOperand()->getValue(); 5094239310Sdim } 5095239310Sdim if (!Src) 5096239310Sdim return false; 5097296417Sdim if (auto *PT = dyn_cast<PointerType>(Src->getType())) 5098239310Sdim return (PT->getAddressSpace() == spN); 5099239310Sdim return false; 5100239310Sdim} 5101239310Sdim 5102239310Sdim/// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 5103239310Sdim/// inline asm expressions. 5104249423Sdimbool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand( 5105288943Sdim const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) { 5106239310Sdim SDValue Op0, Op1; 5107288943Sdim switch (ConstraintID) { 5108249423Sdim default: 5109249423Sdim return true; 5110288943Sdim case InlineAsm::Constraint_m: // memory 5111239310Sdim if (SelectDirectAddr(Op, Op0)) { 5112239310Sdim OutOps.push_back(Op0); 5113288943Sdim OutOps.push_back(CurDAG->getTargetConstant(0, SDLoc(Op), MVT::i32)); 5114239310Sdim return false; 5115239310Sdim } 5116239310Sdim if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) { 5117239310Sdim OutOps.push_back(Op0); 5118239310Sdim OutOps.push_back(Op1); 5119239310Sdim return false; 5120239310Sdim } 5121239310Sdim break; 5122239310Sdim } 5123239310Sdim return true; 5124239310Sdim} 5125