NVPTXISelDAGToDAG.cpp revision 276479
1239310Sdim//===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===// 2239310Sdim// 3239310Sdim// The LLVM Compiler Infrastructure 4239310Sdim// 5239310Sdim// This file is distributed under the University of Illinois Open Source 6239310Sdim// License. See LICENSE.TXT for details. 7239310Sdim// 8239310Sdim//===----------------------------------------------------------------------===// 9239310Sdim// 10239310Sdim// This file defines an instruction selector for the NVPTX target. 11239310Sdim// 12239310Sdim//===----------------------------------------------------------------------===// 13239310Sdim 14239310Sdim#include "NVPTXISelDAGToDAG.h" 15249423Sdim#include "llvm/IR/GlobalValue.h" 16249423Sdim#include "llvm/IR/Instructions.h" 17249423Sdim#include "llvm/Support/CommandLine.h" 18239310Sdim#include "llvm/Support/Debug.h" 19239310Sdim#include "llvm/Support/ErrorHandling.h" 20249423Sdim#include "llvm/Support/raw_ostream.h" 21239310Sdim#include "llvm/Target/TargetIntrinsicInfo.h" 22239310Sdim 23239310Sdimusing namespace llvm; 24239310Sdim 25276479Sdim#define DEBUG_TYPE "nvptx-isel" 26239310Sdim 27249423Sdimstatic cl::opt<int> UsePrecDivF32( 28261991Sdim "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden, 29249423Sdim cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use" 30249423Sdim " IEEE Compliant F32 div.rnd if avaiable."), 31249423Sdim cl::init(2)); 32239310Sdim 33251662Sdimstatic cl::opt<bool> 34261991SdimUsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden, 35251662Sdim cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."), 36251662Sdim cl::init(true)); 37251662Sdim 38261991Sdimstatic cl::opt<bool> 39261991SdimFtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden, 40261991Sdim cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."), 41261991Sdim cl::init(false)); 42261991Sdim 43261991Sdim 44239310Sdim/// createNVPTXISelDag - This pass converts a legalized DAG into a 45239310Sdim/// NVPTX-specific DAG, ready for instruction scheduling. 46239310SdimFunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM, 47239310Sdim llvm::CodeGenOpt::Level OptLevel) { 48239310Sdim return new NVPTXDAGToDAGISel(TM, OptLevel); 49239310Sdim} 50239310Sdim 51239310SdimNVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm, 52239310Sdim CodeGenOpt::Level OptLevel) 53249423Sdim : SelectionDAGISel(tm, OptLevel), 54249423Sdim Subtarget(tm.getSubtarget<NVPTXSubtarget>()) { 55239310Sdim doMulWide = (OptLevel > 0); 56261991Sdim} 57239310Sdim 58261991Sdimint NVPTXDAGToDAGISel::getDivF32Level() const { 59261991Sdim if (UsePrecDivF32.getNumOccurrences() > 0) { 60261991Sdim // If nvptx-prec-div32=N is used on the command-line, always honor it 61261991Sdim return UsePrecDivF32; 62261991Sdim } else { 63261991Sdim // Otherwise, use div.approx if fast math is enabled 64261991Sdim if (TM.Options.UnsafeFPMath) 65261991Sdim return 0; 66261991Sdim else 67261991Sdim return 2; 68261991Sdim } 69261991Sdim} 70239310Sdim 71261991Sdimbool NVPTXDAGToDAGISel::usePrecSqrtF32() const { 72261991Sdim if (UsePrecSqrtF32.getNumOccurrences() > 0) { 73261991Sdim // If nvptx-prec-sqrtf32 is used on the command-line, always honor it 74261991Sdim return UsePrecSqrtF32; 75261991Sdim } else { 76261991Sdim // Otherwise, use sqrt.approx if fast math is enabled 77261991Sdim if (TM.Options.UnsafeFPMath) 78261991Sdim return false; 79261991Sdim else 80261991Sdim return true; 81261991Sdim } 82239310Sdim} 83239310Sdim 84261991Sdimbool NVPTXDAGToDAGISel::useF32FTZ() const { 85261991Sdim if (FtzEnabled.getNumOccurrences() > 0) { 86261991Sdim // If nvptx-f32ftz is used on the command-line, always honor it 87261991Sdim return FtzEnabled; 88261991Sdim } else { 89261991Sdim const Function *F = MF->getFunction(); 90261991Sdim // Otherwise, check for an nvptx-f32ftz attribute on the function 91261991Sdim if (F->hasFnAttribute("nvptx-f32ftz")) 92261991Sdim return (F->getAttributes().getAttribute(AttributeSet::FunctionIndex, 93261991Sdim "nvptx-f32ftz") 94261991Sdim .getValueAsString() == "true"); 95261991Sdim else 96261991Sdim return false; 97261991Sdim } 98261991Sdim} 99261991Sdim 100276479Sdimbool NVPTXDAGToDAGISel::allowFMA() const { 101276479Sdim const NVPTXTargetLowering *TL = Subtarget.getTargetLowering(); 102276479Sdim return TL->allowFMA(*MF, OptLevel); 103276479Sdim} 104276479Sdim 105239310Sdim/// Select - Select instructions not customized! Used for 106239310Sdim/// expanded, promoted and normal instructions. 107249423SdimSDNode *NVPTXDAGToDAGISel::Select(SDNode *N) { 108239310Sdim 109255804Sdim if (N->isMachineOpcode()) { 110255804Sdim N->setNodeId(-1); 111276479Sdim return nullptr; // Already selected. 112255804Sdim } 113239310Sdim 114276479Sdim SDNode *ResNode = nullptr; 115239310Sdim switch (N->getOpcode()) { 116239310Sdim case ISD::LOAD: 117239310Sdim ResNode = SelectLoad(N); 118239310Sdim break; 119239310Sdim case ISD::STORE: 120239310Sdim ResNode = SelectStore(N); 121239310Sdim break; 122249423Sdim case NVPTXISD::LoadV2: 123249423Sdim case NVPTXISD::LoadV4: 124249423Sdim ResNode = SelectLoadVector(N); 125249423Sdim break; 126249423Sdim case NVPTXISD::LDGV2: 127249423Sdim case NVPTXISD::LDGV4: 128249423Sdim case NVPTXISD::LDUV2: 129249423Sdim case NVPTXISD::LDUV4: 130276479Sdim ResNode = SelectLDGLDU(N); 131249423Sdim break; 132249423Sdim case NVPTXISD::StoreV2: 133249423Sdim case NVPTXISD::StoreV4: 134249423Sdim ResNode = SelectStoreVector(N); 135249423Sdim break; 136261991Sdim case NVPTXISD::LoadParam: 137261991Sdim case NVPTXISD::LoadParamV2: 138261991Sdim case NVPTXISD::LoadParamV4: 139261991Sdim ResNode = SelectLoadParam(N); 140261991Sdim break; 141261991Sdim case NVPTXISD::StoreRetval: 142261991Sdim case NVPTXISD::StoreRetvalV2: 143261991Sdim case NVPTXISD::StoreRetvalV4: 144261991Sdim ResNode = SelectStoreRetval(N); 145261991Sdim break; 146261991Sdim case NVPTXISD::StoreParam: 147261991Sdim case NVPTXISD::StoreParamV2: 148261991Sdim case NVPTXISD::StoreParamV4: 149261991Sdim case NVPTXISD::StoreParamS32: 150261991Sdim case NVPTXISD::StoreParamU32: 151261991Sdim ResNode = SelectStoreParam(N); 152261991Sdim break; 153276479Sdim case ISD::INTRINSIC_WO_CHAIN: 154276479Sdim ResNode = SelectIntrinsicNoChain(N); 155276479Sdim break; 156276479Sdim case ISD::INTRINSIC_W_CHAIN: 157276479Sdim ResNode = SelectIntrinsicChain(N); 158276479Sdim break; 159276479Sdim case NVPTXISD::Tex1DFloatS32: 160276479Sdim case NVPTXISD::Tex1DFloatFloat: 161276479Sdim case NVPTXISD::Tex1DFloatFloatLevel: 162276479Sdim case NVPTXISD::Tex1DFloatFloatGrad: 163276479Sdim case NVPTXISD::Tex1DS32S32: 164276479Sdim case NVPTXISD::Tex1DS32Float: 165276479Sdim case NVPTXISD::Tex1DS32FloatLevel: 166276479Sdim case NVPTXISD::Tex1DS32FloatGrad: 167276479Sdim case NVPTXISD::Tex1DU32S32: 168276479Sdim case NVPTXISD::Tex1DU32Float: 169276479Sdim case NVPTXISD::Tex1DU32FloatLevel: 170276479Sdim case NVPTXISD::Tex1DU32FloatGrad: 171276479Sdim case NVPTXISD::Tex1DArrayFloatS32: 172276479Sdim case NVPTXISD::Tex1DArrayFloatFloat: 173276479Sdim case NVPTXISD::Tex1DArrayFloatFloatLevel: 174276479Sdim case NVPTXISD::Tex1DArrayFloatFloatGrad: 175276479Sdim case NVPTXISD::Tex1DArrayS32S32: 176276479Sdim case NVPTXISD::Tex1DArrayS32Float: 177276479Sdim case NVPTXISD::Tex1DArrayS32FloatLevel: 178276479Sdim case NVPTXISD::Tex1DArrayS32FloatGrad: 179276479Sdim case NVPTXISD::Tex1DArrayU32S32: 180276479Sdim case NVPTXISD::Tex1DArrayU32Float: 181276479Sdim case NVPTXISD::Tex1DArrayU32FloatLevel: 182276479Sdim case NVPTXISD::Tex1DArrayU32FloatGrad: 183276479Sdim case NVPTXISD::Tex2DFloatS32: 184276479Sdim case NVPTXISD::Tex2DFloatFloat: 185276479Sdim case NVPTXISD::Tex2DFloatFloatLevel: 186276479Sdim case NVPTXISD::Tex2DFloatFloatGrad: 187276479Sdim case NVPTXISD::Tex2DS32S32: 188276479Sdim case NVPTXISD::Tex2DS32Float: 189276479Sdim case NVPTXISD::Tex2DS32FloatLevel: 190276479Sdim case NVPTXISD::Tex2DS32FloatGrad: 191276479Sdim case NVPTXISD::Tex2DU32S32: 192276479Sdim case NVPTXISD::Tex2DU32Float: 193276479Sdim case NVPTXISD::Tex2DU32FloatLevel: 194276479Sdim case NVPTXISD::Tex2DU32FloatGrad: 195276479Sdim case NVPTXISD::Tex2DArrayFloatS32: 196276479Sdim case NVPTXISD::Tex2DArrayFloatFloat: 197276479Sdim case NVPTXISD::Tex2DArrayFloatFloatLevel: 198276479Sdim case NVPTXISD::Tex2DArrayFloatFloatGrad: 199276479Sdim case NVPTXISD::Tex2DArrayS32S32: 200276479Sdim case NVPTXISD::Tex2DArrayS32Float: 201276479Sdim case NVPTXISD::Tex2DArrayS32FloatLevel: 202276479Sdim case NVPTXISD::Tex2DArrayS32FloatGrad: 203276479Sdim case NVPTXISD::Tex2DArrayU32S32: 204276479Sdim case NVPTXISD::Tex2DArrayU32Float: 205276479Sdim case NVPTXISD::Tex2DArrayU32FloatLevel: 206276479Sdim case NVPTXISD::Tex2DArrayU32FloatGrad: 207276479Sdim case NVPTXISD::Tex3DFloatS32: 208276479Sdim case NVPTXISD::Tex3DFloatFloat: 209276479Sdim case NVPTXISD::Tex3DFloatFloatLevel: 210276479Sdim case NVPTXISD::Tex3DFloatFloatGrad: 211276479Sdim case NVPTXISD::Tex3DS32S32: 212276479Sdim case NVPTXISD::Tex3DS32Float: 213276479Sdim case NVPTXISD::Tex3DS32FloatLevel: 214276479Sdim case NVPTXISD::Tex3DS32FloatGrad: 215276479Sdim case NVPTXISD::Tex3DU32S32: 216276479Sdim case NVPTXISD::Tex3DU32Float: 217276479Sdim case NVPTXISD::Tex3DU32FloatLevel: 218276479Sdim case NVPTXISD::Tex3DU32FloatGrad: 219276479Sdim case NVPTXISD::TexCubeFloatFloat: 220276479Sdim case NVPTXISD::TexCubeFloatFloatLevel: 221276479Sdim case NVPTXISD::TexCubeS32Float: 222276479Sdim case NVPTXISD::TexCubeS32FloatLevel: 223276479Sdim case NVPTXISD::TexCubeU32Float: 224276479Sdim case NVPTXISD::TexCubeU32FloatLevel: 225276479Sdim case NVPTXISD::TexCubeArrayFloatFloat: 226276479Sdim case NVPTXISD::TexCubeArrayFloatFloatLevel: 227276479Sdim case NVPTXISD::TexCubeArrayS32Float: 228276479Sdim case NVPTXISD::TexCubeArrayS32FloatLevel: 229276479Sdim case NVPTXISD::TexCubeArrayU32Float: 230276479Sdim case NVPTXISD::TexCubeArrayU32FloatLevel: 231276479Sdim case NVPTXISD::Tld4R2DFloatFloat: 232276479Sdim case NVPTXISD::Tld4G2DFloatFloat: 233276479Sdim case NVPTXISD::Tld4B2DFloatFloat: 234276479Sdim case NVPTXISD::Tld4A2DFloatFloat: 235276479Sdim case NVPTXISD::Tld4R2DS64Float: 236276479Sdim case NVPTXISD::Tld4G2DS64Float: 237276479Sdim case NVPTXISD::Tld4B2DS64Float: 238276479Sdim case NVPTXISD::Tld4A2DS64Float: 239276479Sdim case NVPTXISD::Tld4R2DU64Float: 240276479Sdim case NVPTXISD::Tld4G2DU64Float: 241276479Sdim case NVPTXISD::Tld4B2DU64Float: 242276479Sdim case NVPTXISD::Tld4A2DU64Float: 243276479Sdim case NVPTXISD::TexUnified1DFloatS32: 244276479Sdim case NVPTXISD::TexUnified1DFloatFloat: 245276479Sdim case NVPTXISD::TexUnified1DFloatFloatLevel: 246276479Sdim case NVPTXISD::TexUnified1DFloatFloatGrad: 247276479Sdim case NVPTXISD::TexUnified1DS32S32: 248276479Sdim case NVPTXISD::TexUnified1DS32Float: 249276479Sdim case NVPTXISD::TexUnified1DS32FloatLevel: 250276479Sdim case NVPTXISD::TexUnified1DS32FloatGrad: 251276479Sdim case NVPTXISD::TexUnified1DU32S32: 252276479Sdim case NVPTXISD::TexUnified1DU32Float: 253276479Sdim case NVPTXISD::TexUnified1DU32FloatLevel: 254276479Sdim case NVPTXISD::TexUnified1DU32FloatGrad: 255276479Sdim case NVPTXISD::TexUnified1DArrayFloatS32: 256276479Sdim case NVPTXISD::TexUnified1DArrayFloatFloat: 257276479Sdim case NVPTXISD::TexUnified1DArrayFloatFloatLevel: 258276479Sdim case NVPTXISD::TexUnified1DArrayFloatFloatGrad: 259276479Sdim case NVPTXISD::TexUnified1DArrayS32S32: 260276479Sdim case NVPTXISD::TexUnified1DArrayS32Float: 261276479Sdim case NVPTXISD::TexUnified1DArrayS32FloatLevel: 262276479Sdim case NVPTXISD::TexUnified1DArrayS32FloatGrad: 263276479Sdim case NVPTXISD::TexUnified1DArrayU32S32: 264276479Sdim case NVPTXISD::TexUnified1DArrayU32Float: 265276479Sdim case NVPTXISD::TexUnified1DArrayU32FloatLevel: 266276479Sdim case NVPTXISD::TexUnified1DArrayU32FloatGrad: 267276479Sdim case NVPTXISD::TexUnified2DFloatS32: 268276479Sdim case NVPTXISD::TexUnified2DFloatFloat: 269276479Sdim case NVPTXISD::TexUnified2DFloatFloatLevel: 270276479Sdim case NVPTXISD::TexUnified2DFloatFloatGrad: 271276479Sdim case NVPTXISD::TexUnified2DS32S32: 272276479Sdim case NVPTXISD::TexUnified2DS32Float: 273276479Sdim case NVPTXISD::TexUnified2DS32FloatLevel: 274276479Sdim case NVPTXISD::TexUnified2DS32FloatGrad: 275276479Sdim case NVPTXISD::TexUnified2DU32S32: 276276479Sdim case NVPTXISD::TexUnified2DU32Float: 277276479Sdim case NVPTXISD::TexUnified2DU32FloatLevel: 278276479Sdim case NVPTXISD::TexUnified2DU32FloatGrad: 279276479Sdim case NVPTXISD::TexUnified2DArrayFloatS32: 280276479Sdim case NVPTXISD::TexUnified2DArrayFloatFloat: 281276479Sdim case NVPTXISD::TexUnified2DArrayFloatFloatLevel: 282276479Sdim case NVPTXISD::TexUnified2DArrayFloatFloatGrad: 283276479Sdim case NVPTXISD::TexUnified2DArrayS32S32: 284276479Sdim case NVPTXISD::TexUnified2DArrayS32Float: 285276479Sdim case NVPTXISD::TexUnified2DArrayS32FloatLevel: 286276479Sdim case NVPTXISD::TexUnified2DArrayS32FloatGrad: 287276479Sdim case NVPTXISD::TexUnified2DArrayU32S32: 288276479Sdim case NVPTXISD::TexUnified2DArrayU32Float: 289276479Sdim case NVPTXISD::TexUnified2DArrayU32FloatLevel: 290276479Sdim case NVPTXISD::TexUnified2DArrayU32FloatGrad: 291276479Sdim case NVPTXISD::TexUnified3DFloatS32: 292276479Sdim case NVPTXISD::TexUnified3DFloatFloat: 293276479Sdim case NVPTXISD::TexUnified3DFloatFloatLevel: 294276479Sdim case NVPTXISD::TexUnified3DFloatFloatGrad: 295276479Sdim case NVPTXISD::TexUnified3DS32S32: 296276479Sdim case NVPTXISD::TexUnified3DS32Float: 297276479Sdim case NVPTXISD::TexUnified3DS32FloatLevel: 298276479Sdim case NVPTXISD::TexUnified3DS32FloatGrad: 299276479Sdim case NVPTXISD::TexUnified3DU32S32: 300276479Sdim case NVPTXISD::TexUnified3DU32Float: 301276479Sdim case NVPTXISD::TexUnified3DU32FloatLevel: 302276479Sdim case NVPTXISD::TexUnified3DU32FloatGrad: 303276479Sdim case NVPTXISD::TexUnifiedCubeFloatFloat: 304276479Sdim case NVPTXISD::TexUnifiedCubeFloatFloatLevel: 305276479Sdim case NVPTXISD::TexUnifiedCubeS32Float: 306276479Sdim case NVPTXISD::TexUnifiedCubeS32FloatLevel: 307276479Sdim case NVPTXISD::TexUnifiedCubeU32Float: 308276479Sdim case NVPTXISD::TexUnifiedCubeU32FloatLevel: 309276479Sdim case NVPTXISD::TexUnifiedCubeArrayFloatFloat: 310276479Sdim case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel: 311276479Sdim case NVPTXISD::TexUnifiedCubeArrayS32Float: 312276479Sdim case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel: 313276479Sdim case NVPTXISD::TexUnifiedCubeArrayU32Float: 314276479Sdim case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel: 315276479Sdim case NVPTXISD::Tld4UnifiedR2DFloatFloat: 316276479Sdim case NVPTXISD::Tld4UnifiedG2DFloatFloat: 317276479Sdim case NVPTXISD::Tld4UnifiedB2DFloatFloat: 318276479Sdim case NVPTXISD::Tld4UnifiedA2DFloatFloat: 319276479Sdim case NVPTXISD::Tld4UnifiedR2DS64Float: 320276479Sdim case NVPTXISD::Tld4UnifiedG2DS64Float: 321276479Sdim case NVPTXISD::Tld4UnifiedB2DS64Float: 322276479Sdim case NVPTXISD::Tld4UnifiedA2DS64Float: 323276479Sdim case NVPTXISD::Tld4UnifiedR2DU64Float: 324276479Sdim case NVPTXISD::Tld4UnifiedG2DU64Float: 325276479Sdim case NVPTXISD::Tld4UnifiedB2DU64Float: 326276479Sdim case NVPTXISD::Tld4UnifiedA2DU64Float: 327276479Sdim ResNode = SelectTextureIntrinsic(N); 328276479Sdim break; 329276479Sdim case NVPTXISD::Suld1DI8Clamp: 330276479Sdim case NVPTXISD::Suld1DI16Clamp: 331276479Sdim case NVPTXISD::Suld1DI32Clamp: 332276479Sdim case NVPTXISD::Suld1DI64Clamp: 333276479Sdim case NVPTXISD::Suld1DV2I8Clamp: 334276479Sdim case NVPTXISD::Suld1DV2I16Clamp: 335276479Sdim case NVPTXISD::Suld1DV2I32Clamp: 336276479Sdim case NVPTXISD::Suld1DV2I64Clamp: 337276479Sdim case NVPTXISD::Suld1DV4I8Clamp: 338276479Sdim case NVPTXISD::Suld1DV4I16Clamp: 339276479Sdim case NVPTXISD::Suld1DV4I32Clamp: 340276479Sdim case NVPTXISD::Suld1DArrayI8Clamp: 341276479Sdim case NVPTXISD::Suld1DArrayI16Clamp: 342276479Sdim case NVPTXISD::Suld1DArrayI32Clamp: 343276479Sdim case NVPTXISD::Suld1DArrayI64Clamp: 344276479Sdim case NVPTXISD::Suld1DArrayV2I8Clamp: 345276479Sdim case NVPTXISD::Suld1DArrayV2I16Clamp: 346276479Sdim case NVPTXISD::Suld1DArrayV2I32Clamp: 347276479Sdim case NVPTXISD::Suld1DArrayV2I64Clamp: 348276479Sdim case NVPTXISD::Suld1DArrayV4I8Clamp: 349276479Sdim case NVPTXISD::Suld1DArrayV4I16Clamp: 350276479Sdim case NVPTXISD::Suld1DArrayV4I32Clamp: 351276479Sdim case NVPTXISD::Suld2DI8Clamp: 352276479Sdim case NVPTXISD::Suld2DI16Clamp: 353276479Sdim case NVPTXISD::Suld2DI32Clamp: 354276479Sdim case NVPTXISD::Suld2DI64Clamp: 355276479Sdim case NVPTXISD::Suld2DV2I8Clamp: 356276479Sdim case NVPTXISD::Suld2DV2I16Clamp: 357276479Sdim case NVPTXISD::Suld2DV2I32Clamp: 358276479Sdim case NVPTXISD::Suld2DV2I64Clamp: 359276479Sdim case NVPTXISD::Suld2DV4I8Clamp: 360276479Sdim case NVPTXISD::Suld2DV4I16Clamp: 361276479Sdim case NVPTXISD::Suld2DV4I32Clamp: 362276479Sdim case NVPTXISD::Suld2DArrayI8Clamp: 363276479Sdim case NVPTXISD::Suld2DArrayI16Clamp: 364276479Sdim case NVPTXISD::Suld2DArrayI32Clamp: 365276479Sdim case NVPTXISD::Suld2DArrayI64Clamp: 366276479Sdim case NVPTXISD::Suld2DArrayV2I8Clamp: 367276479Sdim case NVPTXISD::Suld2DArrayV2I16Clamp: 368276479Sdim case NVPTXISD::Suld2DArrayV2I32Clamp: 369276479Sdim case NVPTXISD::Suld2DArrayV2I64Clamp: 370276479Sdim case NVPTXISD::Suld2DArrayV4I8Clamp: 371276479Sdim case NVPTXISD::Suld2DArrayV4I16Clamp: 372276479Sdim case NVPTXISD::Suld2DArrayV4I32Clamp: 373276479Sdim case NVPTXISD::Suld3DI8Clamp: 374276479Sdim case NVPTXISD::Suld3DI16Clamp: 375276479Sdim case NVPTXISD::Suld3DI32Clamp: 376276479Sdim case NVPTXISD::Suld3DI64Clamp: 377276479Sdim case NVPTXISD::Suld3DV2I8Clamp: 378276479Sdim case NVPTXISD::Suld3DV2I16Clamp: 379276479Sdim case NVPTXISD::Suld3DV2I32Clamp: 380276479Sdim case NVPTXISD::Suld3DV2I64Clamp: 381276479Sdim case NVPTXISD::Suld3DV4I8Clamp: 382276479Sdim case NVPTXISD::Suld3DV4I16Clamp: 383276479Sdim case NVPTXISD::Suld3DV4I32Clamp: 384276479Sdim case NVPTXISD::Suld1DI8Trap: 385276479Sdim case NVPTXISD::Suld1DI16Trap: 386276479Sdim case NVPTXISD::Suld1DI32Trap: 387276479Sdim case NVPTXISD::Suld1DI64Trap: 388276479Sdim case NVPTXISD::Suld1DV2I8Trap: 389276479Sdim case NVPTXISD::Suld1DV2I16Trap: 390276479Sdim case NVPTXISD::Suld1DV2I32Trap: 391276479Sdim case NVPTXISD::Suld1DV2I64Trap: 392276479Sdim case NVPTXISD::Suld1DV4I8Trap: 393276479Sdim case NVPTXISD::Suld1DV4I16Trap: 394276479Sdim case NVPTXISD::Suld1DV4I32Trap: 395276479Sdim case NVPTXISD::Suld1DArrayI8Trap: 396276479Sdim case NVPTXISD::Suld1DArrayI16Trap: 397276479Sdim case NVPTXISD::Suld1DArrayI32Trap: 398276479Sdim case NVPTXISD::Suld1DArrayI64Trap: 399276479Sdim case NVPTXISD::Suld1DArrayV2I8Trap: 400276479Sdim case NVPTXISD::Suld1DArrayV2I16Trap: 401276479Sdim case NVPTXISD::Suld1DArrayV2I32Trap: 402276479Sdim case NVPTXISD::Suld1DArrayV2I64Trap: 403276479Sdim case NVPTXISD::Suld1DArrayV4I8Trap: 404276479Sdim case NVPTXISD::Suld1DArrayV4I16Trap: 405276479Sdim case NVPTXISD::Suld1DArrayV4I32Trap: 406276479Sdim case NVPTXISD::Suld2DI8Trap: 407276479Sdim case NVPTXISD::Suld2DI16Trap: 408276479Sdim case NVPTXISD::Suld2DI32Trap: 409276479Sdim case NVPTXISD::Suld2DI64Trap: 410276479Sdim case NVPTXISD::Suld2DV2I8Trap: 411276479Sdim case NVPTXISD::Suld2DV2I16Trap: 412276479Sdim case NVPTXISD::Suld2DV2I32Trap: 413276479Sdim case NVPTXISD::Suld2DV2I64Trap: 414276479Sdim case NVPTXISD::Suld2DV4I8Trap: 415276479Sdim case NVPTXISD::Suld2DV4I16Trap: 416276479Sdim case NVPTXISD::Suld2DV4I32Trap: 417276479Sdim case NVPTXISD::Suld2DArrayI8Trap: 418276479Sdim case NVPTXISD::Suld2DArrayI16Trap: 419276479Sdim case NVPTXISD::Suld2DArrayI32Trap: 420276479Sdim case NVPTXISD::Suld2DArrayI64Trap: 421276479Sdim case NVPTXISD::Suld2DArrayV2I8Trap: 422276479Sdim case NVPTXISD::Suld2DArrayV2I16Trap: 423276479Sdim case NVPTXISD::Suld2DArrayV2I32Trap: 424276479Sdim case NVPTXISD::Suld2DArrayV2I64Trap: 425276479Sdim case NVPTXISD::Suld2DArrayV4I8Trap: 426276479Sdim case NVPTXISD::Suld2DArrayV4I16Trap: 427276479Sdim case NVPTXISD::Suld2DArrayV4I32Trap: 428276479Sdim case NVPTXISD::Suld3DI8Trap: 429276479Sdim case NVPTXISD::Suld3DI16Trap: 430276479Sdim case NVPTXISD::Suld3DI32Trap: 431276479Sdim case NVPTXISD::Suld3DI64Trap: 432276479Sdim case NVPTXISD::Suld3DV2I8Trap: 433276479Sdim case NVPTXISD::Suld3DV2I16Trap: 434276479Sdim case NVPTXISD::Suld3DV2I32Trap: 435276479Sdim case NVPTXISD::Suld3DV2I64Trap: 436276479Sdim case NVPTXISD::Suld3DV4I8Trap: 437276479Sdim case NVPTXISD::Suld3DV4I16Trap: 438276479Sdim case NVPTXISD::Suld3DV4I32Trap: 439276479Sdim case NVPTXISD::Suld1DI8Zero: 440276479Sdim case NVPTXISD::Suld1DI16Zero: 441276479Sdim case NVPTXISD::Suld1DI32Zero: 442276479Sdim case NVPTXISD::Suld1DI64Zero: 443276479Sdim case NVPTXISD::Suld1DV2I8Zero: 444276479Sdim case NVPTXISD::Suld1DV2I16Zero: 445276479Sdim case NVPTXISD::Suld1DV2I32Zero: 446276479Sdim case NVPTXISD::Suld1DV2I64Zero: 447276479Sdim case NVPTXISD::Suld1DV4I8Zero: 448276479Sdim case NVPTXISD::Suld1DV4I16Zero: 449276479Sdim case NVPTXISD::Suld1DV4I32Zero: 450276479Sdim case NVPTXISD::Suld1DArrayI8Zero: 451276479Sdim case NVPTXISD::Suld1DArrayI16Zero: 452276479Sdim case NVPTXISD::Suld1DArrayI32Zero: 453276479Sdim case NVPTXISD::Suld1DArrayI64Zero: 454276479Sdim case NVPTXISD::Suld1DArrayV2I8Zero: 455276479Sdim case NVPTXISD::Suld1DArrayV2I16Zero: 456276479Sdim case NVPTXISD::Suld1DArrayV2I32Zero: 457276479Sdim case NVPTXISD::Suld1DArrayV2I64Zero: 458276479Sdim case NVPTXISD::Suld1DArrayV4I8Zero: 459276479Sdim case NVPTXISD::Suld1DArrayV4I16Zero: 460276479Sdim case NVPTXISD::Suld1DArrayV4I32Zero: 461276479Sdim case NVPTXISD::Suld2DI8Zero: 462276479Sdim case NVPTXISD::Suld2DI16Zero: 463276479Sdim case NVPTXISD::Suld2DI32Zero: 464276479Sdim case NVPTXISD::Suld2DI64Zero: 465276479Sdim case NVPTXISD::Suld2DV2I8Zero: 466276479Sdim case NVPTXISD::Suld2DV2I16Zero: 467276479Sdim case NVPTXISD::Suld2DV2I32Zero: 468276479Sdim case NVPTXISD::Suld2DV2I64Zero: 469276479Sdim case NVPTXISD::Suld2DV4I8Zero: 470276479Sdim case NVPTXISD::Suld2DV4I16Zero: 471276479Sdim case NVPTXISD::Suld2DV4I32Zero: 472276479Sdim case NVPTXISD::Suld2DArrayI8Zero: 473276479Sdim case NVPTXISD::Suld2DArrayI16Zero: 474276479Sdim case NVPTXISD::Suld2DArrayI32Zero: 475276479Sdim case NVPTXISD::Suld2DArrayI64Zero: 476276479Sdim case NVPTXISD::Suld2DArrayV2I8Zero: 477276479Sdim case NVPTXISD::Suld2DArrayV2I16Zero: 478276479Sdim case NVPTXISD::Suld2DArrayV2I32Zero: 479276479Sdim case NVPTXISD::Suld2DArrayV2I64Zero: 480276479Sdim case NVPTXISD::Suld2DArrayV4I8Zero: 481276479Sdim case NVPTXISD::Suld2DArrayV4I16Zero: 482276479Sdim case NVPTXISD::Suld2DArrayV4I32Zero: 483276479Sdim case NVPTXISD::Suld3DI8Zero: 484276479Sdim case NVPTXISD::Suld3DI16Zero: 485276479Sdim case NVPTXISD::Suld3DI32Zero: 486276479Sdim case NVPTXISD::Suld3DI64Zero: 487276479Sdim case NVPTXISD::Suld3DV2I8Zero: 488276479Sdim case NVPTXISD::Suld3DV2I16Zero: 489276479Sdim case NVPTXISD::Suld3DV2I32Zero: 490276479Sdim case NVPTXISD::Suld3DV2I64Zero: 491276479Sdim case NVPTXISD::Suld3DV4I8Zero: 492276479Sdim case NVPTXISD::Suld3DV4I16Zero: 493276479Sdim case NVPTXISD::Suld3DV4I32Zero: 494276479Sdim ResNode = SelectSurfaceIntrinsic(N); 495276479Sdim break; 496276479Sdim case ISD::AND: 497276479Sdim case ISD::SRA: 498276479Sdim case ISD::SRL: 499276479Sdim // Try to select BFE 500276479Sdim ResNode = SelectBFE(N); 501276479Sdim break; 502276479Sdim case ISD::ADDRSPACECAST: 503276479Sdim ResNode = SelectAddrSpaceCast(N); 504276479Sdim break; 505249423Sdim default: 506249423Sdim break; 507239310Sdim } 508239310Sdim if (ResNode) 509239310Sdim return ResNode; 510239310Sdim return SelectCode(N); 511239310Sdim} 512239310Sdim 513276479SdimSDNode *NVPTXDAGToDAGISel::SelectIntrinsicChain(SDNode *N) { 514276479Sdim unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 515276479Sdim switch (IID) { 516276479Sdim default: 517276479Sdim return NULL; 518276479Sdim case Intrinsic::nvvm_ldg_global_f: 519276479Sdim case Intrinsic::nvvm_ldg_global_i: 520276479Sdim case Intrinsic::nvvm_ldg_global_p: 521276479Sdim case Intrinsic::nvvm_ldu_global_f: 522276479Sdim case Intrinsic::nvvm_ldu_global_i: 523276479Sdim case Intrinsic::nvvm_ldu_global_p: 524276479Sdim return SelectLDGLDU(N); 525276479Sdim } 526276479Sdim} 527276479Sdim 528249423Sdimstatic unsigned int getCodeAddrSpace(MemSDNode *N, 529249423Sdim const NVPTXSubtarget &Subtarget) { 530276479Sdim const Value *Src = N->getMemOperand()->getValue(); 531261991Sdim 532239310Sdim if (!Src) 533261991Sdim return NVPTX::PTXLdStInstCode::GENERIC; 534239310Sdim 535239310Sdim if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) { 536239310Sdim switch (PT->getAddressSpace()) { 537261991Sdim case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL; 538261991Sdim case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL; 539261991Sdim case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED; 540261991Sdim case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC; 541261991Sdim case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM; 542261991Sdim case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT; 543261991Sdim default: break; 544239310Sdim } 545239310Sdim } 546261991Sdim return NVPTX::PTXLdStInstCode::GENERIC; 547239310Sdim} 548239310Sdim 549276479SdimSDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) { 550276479Sdim unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 551276479Sdim switch (IID) { 552276479Sdim default: 553276479Sdim return nullptr; 554276479Sdim case Intrinsic::nvvm_texsurf_handle_internal: 555276479Sdim return SelectTexSurfHandle(N); 556276479Sdim } 557276479Sdim} 558276479Sdim 559276479SdimSDNode *NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) { 560276479Sdim // Op 0 is the intrinsic ID 561276479Sdim SDValue Wrapper = N->getOperand(1); 562276479Sdim SDValue GlobalVal = Wrapper.getOperand(0); 563276479Sdim return CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N), MVT::i64, 564276479Sdim GlobalVal); 565276479Sdim} 566276479Sdim 567276479SdimSDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) { 568276479Sdim SDValue Src = N->getOperand(0); 569276479Sdim AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N); 570276479Sdim unsigned SrcAddrSpace = CastN->getSrcAddressSpace(); 571276479Sdim unsigned DstAddrSpace = CastN->getDestAddressSpace(); 572276479Sdim 573276479Sdim assert(SrcAddrSpace != DstAddrSpace && 574276479Sdim "addrspacecast must be between different address spaces"); 575276479Sdim 576276479Sdim if (DstAddrSpace == ADDRESS_SPACE_GENERIC) { 577276479Sdim // Specific to generic 578276479Sdim unsigned Opc; 579276479Sdim switch (SrcAddrSpace) { 580276479Sdim default: report_fatal_error("Bad address space in addrspacecast"); 581276479Sdim case ADDRESS_SPACE_GLOBAL: 582276479Sdim Opc = Subtarget.is64Bit() ? NVPTX::cvta_global_yes_64 583276479Sdim : NVPTX::cvta_global_yes; 584276479Sdim break; 585276479Sdim case ADDRESS_SPACE_SHARED: 586276479Sdim Opc = Subtarget.is64Bit() ? NVPTX::cvta_shared_yes_64 587276479Sdim : NVPTX::cvta_shared_yes; 588276479Sdim break; 589276479Sdim case ADDRESS_SPACE_CONST: 590276479Sdim Opc = Subtarget.is64Bit() ? NVPTX::cvta_const_yes_64 591276479Sdim : NVPTX::cvta_const_yes; 592276479Sdim break; 593276479Sdim case ADDRESS_SPACE_LOCAL: 594276479Sdim Opc = Subtarget.is64Bit() ? NVPTX::cvta_local_yes_64 595276479Sdim : NVPTX::cvta_local_yes; 596276479Sdim break; 597276479Sdim } 598276479Sdim return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src); 599276479Sdim } else { 600276479Sdim // Generic to specific 601276479Sdim if (SrcAddrSpace != 0) 602276479Sdim report_fatal_error("Cannot cast between two non-generic address spaces"); 603276479Sdim unsigned Opc; 604276479Sdim switch (DstAddrSpace) { 605276479Sdim default: report_fatal_error("Bad address space in addrspacecast"); 606276479Sdim case ADDRESS_SPACE_GLOBAL: 607276479Sdim Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_global_yes_64 608276479Sdim : NVPTX::cvta_to_global_yes; 609276479Sdim break; 610276479Sdim case ADDRESS_SPACE_SHARED: 611276479Sdim Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_shared_yes_64 612276479Sdim : NVPTX::cvta_to_shared_yes; 613276479Sdim break; 614276479Sdim case ADDRESS_SPACE_CONST: 615276479Sdim Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_const_yes_64 616276479Sdim : NVPTX::cvta_to_const_yes; 617276479Sdim break; 618276479Sdim case ADDRESS_SPACE_LOCAL: 619276479Sdim Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_local_yes_64 620276479Sdim : NVPTX::cvta_to_local_yes; 621276479Sdim break; 622276479Sdim } 623276479Sdim return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src); 624276479Sdim } 625276479Sdim} 626276479Sdim 627249423SdimSDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) { 628261991Sdim SDLoc dl(N); 629239310Sdim LoadSDNode *LD = cast<LoadSDNode>(N); 630239310Sdim EVT LoadedVT = LD->getMemoryVT(); 631276479Sdim SDNode *NVPTXLD = nullptr; 632239310Sdim 633239310Sdim // do not support pre/post inc/dec 634239310Sdim if (LD->isIndexed()) 635276479Sdim return nullptr; 636239310Sdim 637239310Sdim if (!LoadedVT.isSimple()) 638276479Sdim return nullptr; 639239310Sdim 640239310Sdim // Address Space Setting 641239310Sdim unsigned int codeAddrSpace = getCodeAddrSpace(LD, Subtarget); 642239310Sdim 643239310Sdim // Volatile Setting 644239310Sdim // - .volatile is only availalble for .global and .shared 645239310Sdim bool isVolatile = LD->isVolatile(); 646239310Sdim if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 647239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 648239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 649239310Sdim isVolatile = false; 650239310Sdim 651239310Sdim // Vector Setting 652239310Sdim MVT SimpleVT = LoadedVT.getSimpleVT(); 653239310Sdim unsigned vecType = NVPTX::PTXLdStInstCode::Scalar; 654239310Sdim if (SimpleVT.isVector()) { 655239310Sdim unsigned num = SimpleVT.getVectorNumElements(); 656239310Sdim if (num == 2) 657239310Sdim vecType = NVPTX::PTXLdStInstCode::V2; 658239310Sdim else if (num == 4) 659239310Sdim vecType = NVPTX::PTXLdStInstCode::V4; 660239310Sdim else 661276479Sdim return nullptr; 662239310Sdim } 663239310Sdim 664239310Sdim // Type Setting: fromType + fromTypeWidth 665239310Sdim // 666239310Sdim // Sign : ISD::SEXTLOAD 667239310Sdim // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the 668239310Sdim // type is integer 669239310Sdim // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float 670239310Sdim MVT ScalarVT = SimpleVT.getScalarType(); 671261991Sdim // Read at least 8 bits (predicates are stored as 8-bit values) 672261991Sdim unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits()); 673239310Sdim unsigned int fromType; 674239310Sdim if ((LD->getExtensionType() == ISD::SEXTLOAD)) 675239310Sdim fromType = NVPTX::PTXLdStInstCode::Signed; 676239310Sdim else if (ScalarVT.isFloatingPoint()) 677239310Sdim fromType = NVPTX::PTXLdStInstCode::Float; 678239310Sdim else 679239310Sdim fromType = NVPTX::PTXLdStInstCode::Unsigned; 680239310Sdim 681239310Sdim // Create the machine instruction DAG 682239310Sdim SDValue Chain = N->getOperand(0); 683239310Sdim SDValue N1 = N->getOperand(1); 684239310Sdim SDValue Addr; 685239310Sdim SDValue Offset, Base; 686239310Sdim unsigned Opcode; 687261991Sdim MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy; 688239310Sdim 689239310Sdim if (SelectDirectAddr(N1, Addr)) { 690239310Sdim switch (TargetVT) { 691249423Sdim case MVT::i8: 692249423Sdim Opcode = NVPTX::LD_i8_avar; 693249423Sdim break; 694249423Sdim case MVT::i16: 695249423Sdim Opcode = NVPTX::LD_i16_avar; 696249423Sdim break; 697249423Sdim case MVT::i32: 698249423Sdim Opcode = NVPTX::LD_i32_avar; 699249423Sdim break; 700249423Sdim case MVT::i64: 701249423Sdim Opcode = NVPTX::LD_i64_avar; 702249423Sdim break; 703249423Sdim case MVT::f32: 704249423Sdim Opcode = NVPTX::LD_f32_avar; 705249423Sdim break; 706249423Sdim case MVT::f64: 707249423Sdim Opcode = NVPTX::LD_f64_avar; 708249423Sdim break; 709249423Sdim default: 710276479Sdim return nullptr; 711239310Sdim } 712249423Sdim SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 713249423Sdim getI32Imm(vecType), getI32Imm(fromType), 714249423Sdim getI32Imm(fromTypeWidth), Addr, Chain }; 715251662Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 716249423Sdim } else if (Subtarget.is64Bit() 717249423Sdim ? SelectADDRsi64(N1.getNode(), N1, Base, Offset) 718249423Sdim : SelectADDRsi(N1.getNode(), N1, Base, Offset)) { 719239310Sdim switch (TargetVT) { 720249423Sdim case MVT::i8: 721249423Sdim Opcode = NVPTX::LD_i8_asi; 722249423Sdim break; 723249423Sdim case MVT::i16: 724249423Sdim Opcode = NVPTX::LD_i16_asi; 725249423Sdim break; 726249423Sdim case MVT::i32: 727249423Sdim Opcode = NVPTX::LD_i32_asi; 728249423Sdim break; 729249423Sdim case MVT::i64: 730249423Sdim Opcode = NVPTX::LD_i64_asi; 731249423Sdim break; 732249423Sdim case MVT::f32: 733249423Sdim Opcode = NVPTX::LD_f32_asi; 734249423Sdim break; 735249423Sdim case MVT::f64: 736249423Sdim Opcode = NVPTX::LD_f64_asi; 737249423Sdim break; 738249423Sdim default: 739276479Sdim return nullptr; 740239310Sdim } 741249423Sdim SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 742249423Sdim getI32Imm(vecType), getI32Imm(fromType), 743249423Sdim getI32Imm(fromTypeWidth), Base, Offset, Chain }; 744251662Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 745249423Sdim } else if (Subtarget.is64Bit() 746249423Sdim ? SelectADDRri64(N1.getNode(), N1, Base, Offset) 747249423Sdim : SelectADDRri(N1.getNode(), N1, Base, Offset)) { 748249423Sdim if (Subtarget.is64Bit()) { 749249423Sdim switch (TargetVT) { 750249423Sdim case MVT::i8: 751249423Sdim Opcode = NVPTX::LD_i8_ari_64; 752249423Sdim break; 753249423Sdim case MVT::i16: 754249423Sdim Opcode = NVPTX::LD_i16_ari_64; 755249423Sdim break; 756249423Sdim case MVT::i32: 757249423Sdim Opcode = NVPTX::LD_i32_ari_64; 758249423Sdim break; 759249423Sdim case MVT::i64: 760249423Sdim Opcode = NVPTX::LD_i64_ari_64; 761249423Sdim break; 762249423Sdim case MVT::f32: 763249423Sdim Opcode = NVPTX::LD_f32_ari_64; 764249423Sdim break; 765249423Sdim case MVT::f64: 766249423Sdim Opcode = NVPTX::LD_f64_ari_64; 767249423Sdim break; 768249423Sdim default: 769276479Sdim return nullptr; 770249423Sdim } 771249423Sdim } else { 772249423Sdim switch (TargetVT) { 773249423Sdim case MVT::i8: 774249423Sdim Opcode = NVPTX::LD_i8_ari; 775249423Sdim break; 776249423Sdim case MVT::i16: 777249423Sdim Opcode = NVPTX::LD_i16_ari; 778249423Sdim break; 779249423Sdim case MVT::i32: 780249423Sdim Opcode = NVPTX::LD_i32_ari; 781249423Sdim break; 782249423Sdim case MVT::i64: 783249423Sdim Opcode = NVPTX::LD_i64_ari; 784249423Sdim break; 785249423Sdim case MVT::f32: 786249423Sdim Opcode = NVPTX::LD_f32_ari; 787249423Sdim break; 788249423Sdim case MVT::f64: 789249423Sdim Opcode = NVPTX::LD_f64_ari; 790249423Sdim break; 791249423Sdim default: 792276479Sdim return nullptr; 793249423Sdim } 794239310Sdim } 795249423Sdim SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 796249423Sdim getI32Imm(vecType), getI32Imm(fromType), 797249423Sdim getI32Imm(fromTypeWidth), Base, Offset, Chain }; 798251662Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 799249423Sdim } else { 800249423Sdim if (Subtarget.is64Bit()) { 801249423Sdim switch (TargetVT) { 802249423Sdim case MVT::i8: 803249423Sdim Opcode = NVPTX::LD_i8_areg_64; 804249423Sdim break; 805249423Sdim case MVT::i16: 806249423Sdim Opcode = NVPTX::LD_i16_areg_64; 807249423Sdim break; 808249423Sdim case MVT::i32: 809249423Sdim Opcode = NVPTX::LD_i32_areg_64; 810249423Sdim break; 811249423Sdim case MVT::i64: 812249423Sdim Opcode = NVPTX::LD_i64_areg_64; 813249423Sdim break; 814249423Sdim case MVT::f32: 815249423Sdim Opcode = NVPTX::LD_f32_areg_64; 816249423Sdim break; 817249423Sdim case MVT::f64: 818249423Sdim Opcode = NVPTX::LD_f64_areg_64; 819249423Sdim break; 820249423Sdim default: 821276479Sdim return nullptr; 822249423Sdim } 823249423Sdim } else { 824249423Sdim switch (TargetVT) { 825249423Sdim case MVT::i8: 826249423Sdim Opcode = NVPTX::LD_i8_areg; 827249423Sdim break; 828249423Sdim case MVT::i16: 829249423Sdim Opcode = NVPTX::LD_i16_areg; 830249423Sdim break; 831249423Sdim case MVT::i32: 832249423Sdim Opcode = NVPTX::LD_i32_areg; 833249423Sdim break; 834249423Sdim case MVT::i64: 835249423Sdim Opcode = NVPTX::LD_i64_areg; 836249423Sdim break; 837249423Sdim case MVT::f32: 838249423Sdim Opcode = NVPTX::LD_f32_areg; 839249423Sdim break; 840249423Sdim case MVT::f64: 841249423Sdim Opcode = NVPTX::LD_f64_areg; 842249423Sdim break; 843249423Sdim default: 844276479Sdim return nullptr; 845249423Sdim } 846239310Sdim } 847249423Sdim SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 848249423Sdim getI32Imm(vecType), getI32Imm(fromType), 849249423Sdim getI32Imm(fromTypeWidth), N1, Chain }; 850251662Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 851239310Sdim } 852239310Sdim 853276479Sdim if (NVPTXLD) { 854239310Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 855239310Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 856239310Sdim cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1); 857239310Sdim } 858239310Sdim 859239310Sdim return NVPTXLD; 860239310Sdim} 861239310Sdim 862249423SdimSDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) { 863249423Sdim 864249423Sdim SDValue Chain = N->getOperand(0); 865249423Sdim SDValue Op1 = N->getOperand(1); 866249423Sdim SDValue Addr, Offset, Base; 867249423Sdim unsigned Opcode; 868261991Sdim SDLoc DL(N); 869249423Sdim SDNode *LD; 870249423Sdim MemSDNode *MemSD = cast<MemSDNode>(N); 871249423Sdim EVT LoadedVT = MemSD->getMemoryVT(); 872249423Sdim 873249423Sdim if (!LoadedVT.isSimple()) 874276479Sdim return nullptr; 875249423Sdim 876249423Sdim // Address Space Setting 877249423Sdim unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget); 878249423Sdim 879249423Sdim // Volatile Setting 880249423Sdim // - .volatile is only availalble for .global and .shared 881249423Sdim bool IsVolatile = MemSD->isVolatile(); 882249423Sdim if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 883249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 884249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 885249423Sdim IsVolatile = false; 886249423Sdim 887249423Sdim // Vector Setting 888249423Sdim MVT SimpleVT = LoadedVT.getSimpleVT(); 889249423Sdim 890249423Sdim // Type Setting: fromType + fromTypeWidth 891249423Sdim // 892249423Sdim // Sign : ISD::SEXTLOAD 893249423Sdim // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the 894249423Sdim // type is integer 895249423Sdim // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float 896249423Sdim MVT ScalarVT = SimpleVT.getScalarType(); 897261991Sdim // Read at least 8 bits (predicates are stored as 8-bit values) 898261991Sdim unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits()); 899249423Sdim unsigned int FromType; 900249423Sdim // The last operand holds the original LoadSDNode::getExtensionType() value 901249423Sdim unsigned ExtensionType = cast<ConstantSDNode>( 902249423Sdim N->getOperand(N->getNumOperands() - 1))->getZExtValue(); 903249423Sdim if (ExtensionType == ISD::SEXTLOAD) 904249423Sdim FromType = NVPTX::PTXLdStInstCode::Signed; 905249423Sdim else if (ScalarVT.isFloatingPoint()) 906249423Sdim FromType = NVPTX::PTXLdStInstCode::Float; 907249423Sdim else 908249423Sdim FromType = NVPTX::PTXLdStInstCode::Unsigned; 909249423Sdim 910249423Sdim unsigned VecType; 911249423Sdim 912249423Sdim switch (N->getOpcode()) { 913249423Sdim case NVPTXISD::LoadV2: 914249423Sdim VecType = NVPTX::PTXLdStInstCode::V2; 915249423Sdim break; 916249423Sdim case NVPTXISD::LoadV4: 917249423Sdim VecType = NVPTX::PTXLdStInstCode::V4; 918249423Sdim break; 919249423Sdim default: 920276479Sdim return nullptr; 921249423Sdim } 922249423Sdim 923249423Sdim EVT EltVT = N->getValueType(0); 924249423Sdim 925249423Sdim if (SelectDirectAddr(Op1, Addr)) { 926249423Sdim switch (N->getOpcode()) { 927249423Sdim default: 928276479Sdim return nullptr; 929249423Sdim case NVPTXISD::LoadV2: 930249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 931249423Sdim default: 932276479Sdim return nullptr; 933249423Sdim case MVT::i8: 934249423Sdim Opcode = NVPTX::LDV_i8_v2_avar; 935249423Sdim break; 936249423Sdim case MVT::i16: 937249423Sdim Opcode = NVPTX::LDV_i16_v2_avar; 938249423Sdim break; 939249423Sdim case MVT::i32: 940249423Sdim Opcode = NVPTX::LDV_i32_v2_avar; 941249423Sdim break; 942249423Sdim case MVT::i64: 943249423Sdim Opcode = NVPTX::LDV_i64_v2_avar; 944249423Sdim break; 945249423Sdim case MVT::f32: 946249423Sdim Opcode = NVPTX::LDV_f32_v2_avar; 947249423Sdim break; 948249423Sdim case MVT::f64: 949249423Sdim Opcode = NVPTX::LDV_f64_v2_avar; 950249423Sdim break; 951249423Sdim } 952249423Sdim break; 953249423Sdim case NVPTXISD::LoadV4: 954249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 955249423Sdim default: 956276479Sdim return nullptr; 957249423Sdim case MVT::i8: 958249423Sdim Opcode = NVPTX::LDV_i8_v4_avar; 959249423Sdim break; 960249423Sdim case MVT::i16: 961249423Sdim Opcode = NVPTX::LDV_i16_v4_avar; 962249423Sdim break; 963249423Sdim case MVT::i32: 964249423Sdim Opcode = NVPTX::LDV_i32_v4_avar; 965249423Sdim break; 966249423Sdim case MVT::f32: 967249423Sdim Opcode = NVPTX::LDV_f32_v4_avar; 968249423Sdim break; 969249423Sdim } 970249423Sdim break; 971249423Sdim } 972249423Sdim 973249423Sdim SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 974249423Sdim getI32Imm(VecType), getI32Imm(FromType), 975249423Sdim getI32Imm(FromTypeWidth), Addr, Chain }; 976251662Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 977249423Sdim } else if (Subtarget.is64Bit() 978249423Sdim ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset) 979249423Sdim : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) { 980249423Sdim switch (N->getOpcode()) { 981249423Sdim default: 982276479Sdim return nullptr; 983249423Sdim case NVPTXISD::LoadV2: 984249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 985249423Sdim default: 986276479Sdim return nullptr; 987249423Sdim case MVT::i8: 988249423Sdim Opcode = NVPTX::LDV_i8_v2_asi; 989249423Sdim break; 990249423Sdim case MVT::i16: 991249423Sdim Opcode = NVPTX::LDV_i16_v2_asi; 992249423Sdim break; 993249423Sdim case MVT::i32: 994249423Sdim Opcode = NVPTX::LDV_i32_v2_asi; 995249423Sdim break; 996249423Sdim case MVT::i64: 997249423Sdim Opcode = NVPTX::LDV_i64_v2_asi; 998249423Sdim break; 999249423Sdim case MVT::f32: 1000249423Sdim Opcode = NVPTX::LDV_f32_v2_asi; 1001249423Sdim break; 1002249423Sdim case MVT::f64: 1003249423Sdim Opcode = NVPTX::LDV_f64_v2_asi; 1004249423Sdim break; 1005249423Sdim } 1006249423Sdim break; 1007249423Sdim case NVPTXISD::LoadV4: 1008249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1009249423Sdim default: 1010276479Sdim return nullptr; 1011249423Sdim case MVT::i8: 1012249423Sdim Opcode = NVPTX::LDV_i8_v4_asi; 1013249423Sdim break; 1014249423Sdim case MVT::i16: 1015249423Sdim Opcode = NVPTX::LDV_i16_v4_asi; 1016249423Sdim break; 1017249423Sdim case MVT::i32: 1018249423Sdim Opcode = NVPTX::LDV_i32_v4_asi; 1019249423Sdim break; 1020249423Sdim case MVT::f32: 1021249423Sdim Opcode = NVPTX::LDV_f32_v4_asi; 1022249423Sdim break; 1023249423Sdim } 1024249423Sdim break; 1025249423Sdim } 1026249423Sdim 1027249423Sdim SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 1028249423Sdim getI32Imm(VecType), getI32Imm(FromType), 1029249423Sdim getI32Imm(FromTypeWidth), Base, Offset, Chain }; 1030251662Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 1031249423Sdim } else if (Subtarget.is64Bit() 1032249423Sdim ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) 1033249423Sdim : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { 1034249423Sdim if (Subtarget.is64Bit()) { 1035249423Sdim switch (N->getOpcode()) { 1036249423Sdim default: 1037276479Sdim return nullptr; 1038249423Sdim case NVPTXISD::LoadV2: 1039249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1040249423Sdim default: 1041276479Sdim return nullptr; 1042249423Sdim case MVT::i8: 1043249423Sdim Opcode = NVPTX::LDV_i8_v2_ari_64; 1044249423Sdim break; 1045249423Sdim case MVT::i16: 1046249423Sdim Opcode = NVPTX::LDV_i16_v2_ari_64; 1047249423Sdim break; 1048249423Sdim case MVT::i32: 1049249423Sdim Opcode = NVPTX::LDV_i32_v2_ari_64; 1050249423Sdim break; 1051249423Sdim case MVT::i64: 1052249423Sdim Opcode = NVPTX::LDV_i64_v2_ari_64; 1053249423Sdim break; 1054249423Sdim case MVT::f32: 1055249423Sdim Opcode = NVPTX::LDV_f32_v2_ari_64; 1056249423Sdim break; 1057249423Sdim case MVT::f64: 1058249423Sdim Opcode = NVPTX::LDV_f64_v2_ari_64; 1059249423Sdim break; 1060249423Sdim } 1061249423Sdim break; 1062249423Sdim case NVPTXISD::LoadV4: 1063249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1064249423Sdim default: 1065276479Sdim return nullptr; 1066249423Sdim case MVT::i8: 1067249423Sdim Opcode = NVPTX::LDV_i8_v4_ari_64; 1068249423Sdim break; 1069249423Sdim case MVT::i16: 1070249423Sdim Opcode = NVPTX::LDV_i16_v4_ari_64; 1071249423Sdim break; 1072249423Sdim case MVT::i32: 1073249423Sdim Opcode = NVPTX::LDV_i32_v4_ari_64; 1074249423Sdim break; 1075249423Sdim case MVT::f32: 1076249423Sdim Opcode = NVPTX::LDV_f32_v4_ari_64; 1077249423Sdim break; 1078249423Sdim } 1079249423Sdim break; 1080249423Sdim } 1081249423Sdim } else { 1082249423Sdim switch (N->getOpcode()) { 1083249423Sdim default: 1084276479Sdim return nullptr; 1085249423Sdim case NVPTXISD::LoadV2: 1086249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1087249423Sdim default: 1088276479Sdim return nullptr; 1089249423Sdim case MVT::i8: 1090249423Sdim Opcode = NVPTX::LDV_i8_v2_ari; 1091249423Sdim break; 1092249423Sdim case MVT::i16: 1093249423Sdim Opcode = NVPTX::LDV_i16_v2_ari; 1094249423Sdim break; 1095249423Sdim case MVT::i32: 1096249423Sdim Opcode = NVPTX::LDV_i32_v2_ari; 1097249423Sdim break; 1098249423Sdim case MVT::i64: 1099249423Sdim Opcode = NVPTX::LDV_i64_v2_ari; 1100249423Sdim break; 1101249423Sdim case MVT::f32: 1102249423Sdim Opcode = NVPTX::LDV_f32_v2_ari; 1103249423Sdim break; 1104249423Sdim case MVT::f64: 1105249423Sdim Opcode = NVPTX::LDV_f64_v2_ari; 1106249423Sdim break; 1107249423Sdim } 1108249423Sdim break; 1109249423Sdim case NVPTXISD::LoadV4: 1110249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1111249423Sdim default: 1112276479Sdim return nullptr; 1113249423Sdim case MVT::i8: 1114249423Sdim Opcode = NVPTX::LDV_i8_v4_ari; 1115249423Sdim break; 1116249423Sdim case MVT::i16: 1117249423Sdim Opcode = NVPTX::LDV_i16_v4_ari; 1118249423Sdim break; 1119249423Sdim case MVT::i32: 1120249423Sdim Opcode = NVPTX::LDV_i32_v4_ari; 1121249423Sdim break; 1122249423Sdim case MVT::f32: 1123249423Sdim Opcode = NVPTX::LDV_f32_v4_ari; 1124249423Sdim break; 1125249423Sdim } 1126249423Sdim break; 1127249423Sdim } 1128249423Sdim } 1129249423Sdim 1130249423Sdim SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 1131249423Sdim getI32Imm(VecType), getI32Imm(FromType), 1132249423Sdim getI32Imm(FromTypeWidth), Base, Offset, Chain }; 1133249423Sdim 1134251662Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 1135249423Sdim } else { 1136249423Sdim if (Subtarget.is64Bit()) { 1137249423Sdim switch (N->getOpcode()) { 1138249423Sdim default: 1139276479Sdim return nullptr; 1140249423Sdim case NVPTXISD::LoadV2: 1141249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1142249423Sdim default: 1143276479Sdim return nullptr; 1144249423Sdim case MVT::i8: 1145249423Sdim Opcode = NVPTX::LDV_i8_v2_areg_64; 1146249423Sdim break; 1147249423Sdim case MVT::i16: 1148249423Sdim Opcode = NVPTX::LDV_i16_v2_areg_64; 1149249423Sdim break; 1150249423Sdim case MVT::i32: 1151249423Sdim Opcode = NVPTX::LDV_i32_v2_areg_64; 1152249423Sdim break; 1153249423Sdim case MVT::i64: 1154249423Sdim Opcode = NVPTX::LDV_i64_v2_areg_64; 1155249423Sdim break; 1156249423Sdim case MVT::f32: 1157249423Sdim Opcode = NVPTX::LDV_f32_v2_areg_64; 1158249423Sdim break; 1159249423Sdim case MVT::f64: 1160249423Sdim Opcode = NVPTX::LDV_f64_v2_areg_64; 1161249423Sdim break; 1162249423Sdim } 1163249423Sdim break; 1164249423Sdim case NVPTXISD::LoadV4: 1165249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1166249423Sdim default: 1167276479Sdim return nullptr; 1168249423Sdim case MVT::i8: 1169249423Sdim Opcode = NVPTX::LDV_i8_v4_areg_64; 1170249423Sdim break; 1171249423Sdim case MVT::i16: 1172249423Sdim Opcode = NVPTX::LDV_i16_v4_areg_64; 1173249423Sdim break; 1174249423Sdim case MVT::i32: 1175249423Sdim Opcode = NVPTX::LDV_i32_v4_areg_64; 1176249423Sdim break; 1177249423Sdim case MVT::f32: 1178249423Sdim Opcode = NVPTX::LDV_f32_v4_areg_64; 1179249423Sdim break; 1180249423Sdim } 1181249423Sdim break; 1182249423Sdim } 1183249423Sdim } else { 1184249423Sdim switch (N->getOpcode()) { 1185249423Sdim default: 1186276479Sdim return nullptr; 1187249423Sdim case NVPTXISD::LoadV2: 1188249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1189249423Sdim default: 1190276479Sdim return nullptr; 1191249423Sdim case MVT::i8: 1192249423Sdim Opcode = NVPTX::LDV_i8_v2_areg; 1193249423Sdim break; 1194249423Sdim case MVT::i16: 1195249423Sdim Opcode = NVPTX::LDV_i16_v2_areg; 1196249423Sdim break; 1197249423Sdim case MVT::i32: 1198249423Sdim Opcode = NVPTX::LDV_i32_v2_areg; 1199249423Sdim break; 1200249423Sdim case MVT::i64: 1201249423Sdim Opcode = NVPTX::LDV_i64_v2_areg; 1202249423Sdim break; 1203249423Sdim case MVT::f32: 1204249423Sdim Opcode = NVPTX::LDV_f32_v2_areg; 1205249423Sdim break; 1206249423Sdim case MVT::f64: 1207249423Sdim Opcode = NVPTX::LDV_f64_v2_areg; 1208249423Sdim break; 1209249423Sdim } 1210249423Sdim break; 1211249423Sdim case NVPTXISD::LoadV4: 1212249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1213249423Sdim default: 1214276479Sdim return nullptr; 1215249423Sdim case MVT::i8: 1216249423Sdim Opcode = NVPTX::LDV_i8_v4_areg; 1217249423Sdim break; 1218249423Sdim case MVT::i16: 1219249423Sdim Opcode = NVPTX::LDV_i16_v4_areg; 1220249423Sdim break; 1221249423Sdim case MVT::i32: 1222249423Sdim Opcode = NVPTX::LDV_i32_v4_areg; 1223249423Sdim break; 1224249423Sdim case MVT::f32: 1225249423Sdim Opcode = NVPTX::LDV_f32_v4_areg; 1226249423Sdim break; 1227249423Sdim } 1228249423Sdim break; 1229249423Sdim } 1230249423Sdim } 1231249423Sdim 1232249423Sdim SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 1233249423Sdim getI32Imm(VecType), getI32Imm(FromType), 1234249423Sdim getI32Imm(FromTypeWidth), Op1, Chain }; 1235251662Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 1236249423Sdim } 1237249423Sdim 1238249423Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 1239249423Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 1240249423Sdim cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); 1241249423Sdim 1242249423Sdim return LD; 1243249423Sdim} 1244249423Sdim 1245276479SdimSDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) { 1246249423Sdim 1247249423Sdim SDValue Chain = N->getOperand(0); 1248276479Sdim SDValue Op1; 1249276479Sdim MemSDNode *Mem; 1250276479Sdim bool IsLDG = true; 1251276479Sdim 1252276479Sdim // If this is an LDG intrinsic, the address is the third operand. Its its an 1253276479Sdim // LDG/LDU SD node (from custom vector handling), then its the second operand 1254276479Sdim if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) { 1255276479Sdim Op1 = N->getOperand(2); 1256276479Sdim Mem = cast<MemIntrinsicSDNode>(N); 1257276479Sdim unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 1258276479Sdim switch (IID) { 1259276479Sdim default: 1260276479Sdim return NULL; 1261276479Sdim case Intrinsic::nvvm_ldg_global_f: 1262276479Sdim case Intrinsic::nvvm_ldg_global_i: 1263276479Sdim case Intrinsic::nvvm_ldg_global_p: 1264276479Sdim IsLDG = true; 1265276479Sdim break; 1266276479Sdim case Intrinsic::nvvm_ldu_global_f: 1267276479Sdim case Intrinsic::nvvm_ldu_global_i: 1268276479Sdim case Intrinsic::nvvm_ldu_global_p: 1269276479Sdim IsLDG = false; 1270276479Sdim break; 1271276479Sdim } 1272276479Sdim } else { 1273276479Sdim Op1 = N->getOperand(1); 1274276479Sdim Mem = cast<MemSDNode>(N); 1275276479Sdim } 1276276479Sdim 1277249423Sdim unsigned Opcode; 1278261991Sdim SDLoc DL(N); 1279249423Sdim SDNode *LD; 1280261991Sdim SDValue Base, Offset, Addr; 1281249423Sdim 1282276479Sdim EVT EltVT = Mem->getMemoryVT(); 1283276479Sdim if (EltVT.isVector()) { 1284276479Sdim EltVT = EltVT.getVectorElementType(); 1285276479Sdim } 1286249423Sdim 1287261991Sdim if (SelectDirectAddr(Op1, Addr)) { 1288249423Sdim switch (N->getOpcode()) { 1289249423Sdim default: 1290276479Sdim return nullptr; 1291276479Sdim case ISD::INTRINSIC_W_CHAIN: 1292276479Sdim if (IsLDG) { 1293276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1294276479Sdim default: 1295276479Sdim return nullptr; 1296276479Sdim case MVT::i8: 1297276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8avar; 1298276479Sdim break; 1299276479Sdim case MVT::i16: 1300276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16avar; 1301276479Sdim break; 1302276479Sdim case MVT::i32: 1303276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32avar; 1304276479Sdim break; 1305276479Sdim case MVT::i64: 1306276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64avar; 1307276479Sdim break; 1308276479Sdim case MVT::f32: 1309276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32avar; 1310276479Sdim break; 1311276479Sdim case MVT::f64: 1312276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64avar; 1313276479Sdim break; 1314276479Sdim } 1315276479Sdim } else { 1316276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1317276479Sdim default: 1318276479Sdim return nullptr; 1319276479Sdim case MVT::i8: 1320276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8avar; 1321276479Sdim break; 1322276479Sdim case MVT::i16: 1323276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16avar; 1324276479Sdim break; 1325276479Sdim case MVT::i32: 1326276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32avar; 1327276479Sdim break; 1328276479Sdim case MVT::i64: 1329276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64avar; 1330276479Sdim break; 1331276479Sdim case MVT::f32: 1332276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32avar; 1333276479Sdim break; 1334276479Sdim case MVT::f64: 1335276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64avar; 1336276479Sdim break; 1337276479Sdim } 1338276479Sdim } 1339276479Sdim break; 1340249423Sdim case NVPTXISD::LDGV2: 1341261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1342249423Sdim default: 1343276479Sdim return nullptr; 1344249423Sdim case MVT::i8: 1345261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar; 1346249423Sdim break; 1347249423Sdim case MVT::i16: 1348261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar; 1349249423Sdim break; 1350249423Sdim case MVT::i32: 1351261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar; 1352249423Sdim break; 1353249423Sdim case MVT::i64: 1354261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar; 1355249423Sdim break; 1356249423Sdim case MVT::f32: 1357261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar; 1358249423Sdim break; 1359249423Sdim case MVT::f64: 1360261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar; 1361249423Sdim break; 1362249423Sdim } 1363249423Sdim break; 1364261991Sdim case NVPTXISD::LDUV2: 1365261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1366249423Sdim default: 1367276479Sdim return nullptr; 1368249423Sdim case MVT::i8: 1369261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar; 1370249423Sdim break; 1371249423Sdim case MVT::i16: 1372261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar; 1373249423Sdim break; 1374249423Sdim case MVT::i32: 1375261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar; 1376249423Sdim break; 1377261991Sdim case MVT::i64: 1378261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar; 1379261991Sdim break; 1380249423Sdim case MVT::f32: 1381261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar; 1382249423Sdim break; 1383261991Sdim case MVT::f64: 1384261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar; 1385261991Sdim break; 1386249423Sdim } 1387249423Sdim break; 1388261991Sdim case NVPTXISD::LDGV4: 1389261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1390249423Sdim default: 1391276479Sdim return nullptr; 1392249423Sdim case MVT::i8: 1393261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar; 1394249423Sdim break; 1395249423Sdim case MVT::i16: 1396261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar; 1397249423Sdim break; 1398249423Sdim case MVT::i32: 1399261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar; 1400249423Sdim break; 1401249423Sdim case MVT::f32: 1402261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar; 1403249423Sdim break; 1404249423Sdim } 1405249423Sdim break; 1406249423Sdim case NVPTXISD::LDUV4: 1407261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1408249423Sdim default: 1409276479Sdim return nullptr; 1410249423Sdim case MVT::i8: 1411261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar; 1412249423Sdim break; 1413249423Sdim case MVT::i16: 1414261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar; 1415249423Sdim break; 1416249423Sdim case MVT::i32: 1417261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar; 1418249423Sdim break; 1419249423Sdim case MVT::f32: 1420261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar; 1421249423Sdim break; 1422249423Sdim } 1423249423Sdim break; 1424249423Sdim } 1425261991Sdim 1426261991Sdim SDValue Ops[] = { Addr, Chain }; 1427276479Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 1428261991Sdim } else if (Subtarget.is64Bit() 1429261991Sdim ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) 1430261991Sdim : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { 1431261991Sdim if (Subtarget.is64Bit()) { 1432261991Sdim switch (N->getOpcode()) { 1433249423Sdim default: 1434276479Sdim return nullptr; 1435276479Sdim case ISD::INTRINSIC_W_CHAIN: 1436276479Sdim if (IsLDG) { 1437276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1438276479Sdim default: 1439276479Sdim return nullptr; 1440276479Sdim case MVT::i8: 1441276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari64; 1442276479Sdim break; 1443276479Sdim case MVT::i16: 1444276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari64; 1445276479Sdim break; 1446276479Sdim case MVT::i32: 1447276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari64; 1448276479Sdim break; 1449276479Sdim case MVT::i64: 1450276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari64; 1451276479Sdim break; 1452276479Sdim case MVT::f32: 1453276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari64; 1454276479Sdim break; 1455276479Sdim case MVT::f64: 1456276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari64; 1457276479Sdim break; 1458276479Sdim } 1459276479Sdim } else { 1460276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1461276479Sdim default: 1462276479Sdim return nullptr; 1463276479Sdim case MVT::i8: 1464276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari64; 1465276479Sdim break; 1466276479Sdim case MVT::i16: 1467276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari64; 1468276479Sdim break; 1469276479Sdim case MVT::i32: 1470276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari64; 1471276479Sdim break; 1472276479Sdim case MVT::i64: 1473276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari64; 1474276479Sdim break; 1475276479Sdim case MVT::f32: 1476276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari64; 1477276479Sdim break; 1478276479Sdim case MVT::f64: 1479276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari64; 1480276479Sdim break; 1481276479Sdim } 1482276479Sdim } 1483276479Sdim break; 1484261991Sdim case NVPTXISD::LDGV2: 1485261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1486261991Sdim default: 1487276479Sdim return nullptr; 1488261991Sdim case MVT::i8: 1489261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64; 1490261991Sdim break; 1491261991Sdim case MVT::i16: 1492261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64; 1493261991Sdim break; 1494261991Sdim case MVT::i32: 1495261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64; 1496261991Sdim break; 1497261991Sdim case MVT::i64: 1498261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64; 1499261991Sdim break; 1500261991Sdim case MVT::f32: 1501261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64; 1502261991Sdim break; 1503261991Sdim case MVT::f64: 1504261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64; 1505261991Sdim break; 1506261991Sdim } 1507249423Sdim break; 1508261991Sdim case NVPTXISD::LDUV2: 1509261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1510261991Sdim default: 1511276479Sdim return nullptr; 1512261991Sdim case MVT::i8: 1513261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64; 1514261991Sdim break; 1515261991Sdim case MVT::i16: 1516261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64; 1517261991Sdim break; 1518261991Sdim case MVT::i32: 1519261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64; 1520261991Sdim break; 1521261991Sdim case MVT::i64: 1522261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64; 1523261991Sdim break; 1524261991Sdim case MVT::f32: 1525261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64; 1526261991Sdim break; 1527261991Sdim case MVT::f64: 1528261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64; 1529261991Sdim break; 1530261991Sdim } 1531249423Sdim break; 1532261991Sdim case NVPTXISD::LDGV4: 1533261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1534261991Sdim default: 1535276479Sdim return nullptr; 1536261991Sdim case MVT::i8: 1537261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64; 1538261991Sdim break; 1539261991Sdim case MVT::i16: 1540261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64; 1541261991Sdim break; 1542261991Sdim case MVT::i32: 1543261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64; 1544261991Sdim break; 1545261991Sdim case MVT::f32: 1546261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64; 1547261991Sdim break; 1548261991Sdim } 1549249423Sdim break; 1550261991Sdim case NVPTXISD::LDUV4: 1551261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1552261991Sdim default: 1553276479Sdim return nullptr; 1554261991Sdim case MVT::i8: 1555261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64; 1556261991Sdim break; 1557261991Sdim case MVT::i16: 1558261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64; 1559261991Sdim break; 1560261991Sdim case MVT::i32: 1561261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64; 1562261991Sdim break; 1563261991Sdim case MVT::f32: 1564261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64; 1565261991Sdim break; 1566261991Sdim } 1567249423Sdim break; 1568249423Sdim } 1569261991Sdim } else { 1570261991Sdim switch (N->getOpcode()) { 1571249423Sdim default: 1572276479Sdim return nullptr; 1573276479Sdim case ISD::INTRINSIC_W_CHAIN: 1574276479Sdim if (IsLDG) { 1575276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1576276479Sdim default: 1577276479Sdim return nullptr; 1578276479Sdim case MVT::i8: 1579276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari; 1580276479Sdim break; 1581276479Sdim case MVT::i16: 1582276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari; 1583276479Sdim break; 1584276479Sdim case MVT::i32: 1585276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari; 1586276479Sdim break; 1587276479Sdim case MVT::i64: 1588276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari; 1589276479Sdim break; 1590276479Sdim case MVT::f32: 1591276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari; 1592276479Sdim break; 1593276479Sdim case MVT::f64: 1594276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari; 1595276479Sdim break; 1596276479Sdim } 1597276479Sdim } else { 1598276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1599276479Sdim default: 1600276479Sdim return nullptr; 1601276479Sdim case MVT::i8: 1602276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari; 1603276479Sdim break; 1604276479Sdim case MVT::i16: 1605276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari; 1606276479Sdim break; 1607276479Sdim case MVT::i32: 1608276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari; 1609276479Sdim break; 1610276479Sdim case MVT::i64: 1611276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari; 1612276479Sdim break; 1613276479Sdim case MVT::f32: 1614276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari; 1615276479Sdim break; 1616276479Sdim case MVT::f64: 1617276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari; 1618276479Sdim break; 1619276479Sdim } 1620276479Sdim } 1621276479Sdim break; 1622261991Sdim case NVPTXISD::LDGV2: 1623261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1624261991Sdim default: 1625276479Sdim return nullptr; 1626261991Sdim case MVT::i8: 1627261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32; 1628261991Sdim break; 1629261991Sdim case MVT::i16: 1630261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32; 1631261991Sdim break; 1632261991Sdim case MVT::i32: 1633261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32; 1634261991Sdim break; 1635261991Sdim case MVT::i64: 1636261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32; 1637261991Sdim break; 1638261991Sdim case MVT::f32: 1639261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32; 1640261991Sdim break; 1641261991Sdim case MVT::f64: 1642261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32; 1643261991Sdim break; 1644261991Sdim } 1645249423Sdim break; 1646261991Sdim case NVPTXISD::LDUV2: 1647261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1648261991Sdim default: 1649276479Sdim return nullptr; 1650261991Sdim case MVT::i8: 1651261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32; 1652261991Sdim break; 1653261991Sdim case MVT::i16: 1654261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32; 1655261991Sdim break; 1656261991Sdim case MVT::i32: 1657261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32; 1658261991Sdim break; 1659261991Sdim case MVT::i64: 1660261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32; 1661261991Sdim break; 1662261991Sdim case MVT::f32: 1663261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32; 1664261991Sdim break; 1665261991Sdim case MVT::f64: 1666261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32; 1667261991Sdim break; 1668261991Sdim } 1669249423Sdim break; 1670261991Sdim case NVPTXISD::LDGV4: 1671261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1672261991Sdim default: 1673276479Sdim return nullptr; 1674261991Sdim case MVT::i8: 1675261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32; 1676261991Sdim break; 1677261991Sdim case MVT::i16: 1678261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32; 1679261991Sdim break; 1680261991Sdim case MVT::i32: 1681261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32; 1682261991Sdim break; 1683261991Sdim case MVT::f32: 1684261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32; 1685261991Sdim break; 1686261991Sdim } 1687249423Sdim break; 1688261991Sdim case NVPTXISD::LDUV4: 1689261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1690261991Sdim default: 1691276479Sdim return nullptr; 1692261991Sdim case MVT::i8: 1693261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32; 1694261991Sdim break; 1695261991Sdim case MVT::i16: 1696261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32; 1697261991Sdim break; 1698261991Sdim case MVT::i32: 1699261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32; 1700261991Sdim break; 1701261991Sdim case MVT::f32: 1702261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32; 1703261991Sdim break; 1704261991Sdim } 1705249423Sdim break; 1706249423Sdim } 1707261991Sdim } 1708261991Sdim 1709261991Sdim SDValue Ops[] = { Base, Offset, Chain }; 1710261991Sdim 1711276479Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 1712261991Sdim } else { 1713261991Sdim if (Subtarget.is64Bit()) { 1714261991Sdim switch (N->getOpcode()) { 1715249423Sdim default: 1716276479Sdim return nullptr; 1717276479Sdim case ISD::INTRINSIC_W_CHAIN: 1718276479Sdim if (IsLDG) { 1719276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1720276479Sdim default: 1721276479Sdim return nullptr; 1722276479Sdim case MVT::i8: 1723276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg64; 1724276479Sdim break; 1725276479Sdim case MVT::i16: 1726276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg64; 1727276479Sdim break; 1728276479Sdim case MVT::i32: 1729276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg64; 1730276479Sdim break; 1731276479Sdim case MVT::i64: 1732276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg64; 1733276479Sdim break; 1734276479Sdim case MVT::f32: 1735276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg64; 1736276479Sdim break; 1737276479Sdim case MVT::f64: 1738276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg64; 1739276479Sdim break; 1740276479Sdim } 1741276479Sdim } else { 1742276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1743276479Sdim default: 1744276479Sdim return nullptr; 1745276479Sdim case MVT::i8: 1746276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg64; 1747276479Sdim break; 1748276479Sdim case MVT::i16: 1749276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg64; 1750276479Sdim break; 1751276479Sdim case MVT::i32: 1752276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg64; 1753276479Sdim break; 1754276479Sdim case MVT::i64: 1755276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg64; 1756276479Sdim break; 1757276479Sdim case MVT::f32: 1758276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg64; 1759276479Sdim break; 1760276479Sdim case MVT::f64: 1761276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg64; 1762276479Sdim break; 1763276479Sdim } 1764276479Sdim } 1765276479Sdim break; 1766261991Sdim case NVPTXISD::LDGV2: 1767261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1768261991Sdim default: 1769276479Sdim return nullptr; 1770261991Sdim case MVT::i8: 1771261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64; 1772261991Sdim break; 1773261991Sdim case MVT::i16: 1774261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64; 1775261991Sdim break; 1776261991Sdim case MVT::i32: 1777261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64; 1778261991Sdim break; 1779261991Sdim case MVT::i64: 1780261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64; 1781261991Sdim break; 1782261991Sdim case MVT::f32: 1783261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64; 1784261991Sdim break; 1785261991Sdim case MVT::f64: 1786261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64; 1787261991Sdim break; 1788261991Sdim } 1789249423Sdim break; 1790261991Sdim case NVPTXISD::LDUV2: 1791261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1792261991Sdim default: 1793276479Sdim return nullptr; 1794261991Sdim case MVT::i8: 1795261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64; 1796261991Sdim break; 1797261991Sdim case MVT::i16: 1798261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64; 1799261991Sdim break; 1800261991Sdim case MVT::i32: 1801261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64; 1802261991Sdim break; 1803261991Sdim case MVT::i64: 1804261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64; 1805261991Sdim break; 1806261991Sdim case MVT::f32: 1807261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64; 1808261991Sdim break; 1809261991Sdim case MVT::f64: 1810261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64; 1811261991Sdim break; 1812261991Sdim } 1813249423Sdim break; 1814261991Sdim case NVPTXISD::LDGV4: 1815261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1816261991Sdim default: 1817276479Sdim return nullptr; 1818261991Sdim case MVT::i8: 1819261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64; 1820261991Sdim break; 1821261991Sdim case MVT::i16: 1822261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64; 1823261991Sdim break; 1824261991Sdim case MVT::i32: 1825261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64; 1826261991Sdim break; 1827261991Sdim case MVT::f32: 1828261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64; 1829261991Sdim break; 1830261991Sdim } 1831249423Sdim break; 1832261991Sdim case NVPTXISD::LDUV4: 1833261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1834261991Sdim default: 1835276479Sdim return nullptr; 1836261991Sdim case MVT::i8: 1837261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64; 1838261991Sdim break; 1839261991Sdim case MVT::i16: 1840261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64; 1841261991Sdim break; 1842261991Sdim case MVT::i32: 1843261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64; 1844261991Sdim break; 1845261991Sdim case MVT::f32: 1846261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64; 1847261991Sdim break; 1848261991Sdim } 1849249423Sdim break; 1850249423Sdim } 1851261991Sdim } else { 1852261991Sdim switch (N->getOpcode()) { 1853249423Sdim default: 1854276479Sdim return nullptr; 1855276479Sdim case ISD::INTRINSIC_W_CHAIN: 1856276479Sdim if (IsLDG) { 1857276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1858276479Sdim default: 1859276479Sdim return nullptr; 1860276479Sdim case MVT::i8: 1861276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg; 1862276479Sdim break; 1863276479Sdim case MVT::i16: 1864276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg; 1865276479Sdim break; 1866276479Sdim case MVT::i32: 1867276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg; 1868276479Sdim break; 1869276479Sdim case MVT::i64: 1870276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg; 1871276479Sdim break; 1872276479Sdim case MVT::f32: 1873276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg; 1874276479Sdim break; 1875276479Sdim case MVT::f64: 1876276479Sdim Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg; 1877276479Sdim break; 1878276479Sdim } 1879276479Sdim } else { 1880276479Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1881276479Sdim default: 1882276479Sdim return nullptr; 1883276479Sdim case MVT::i8: 1884276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg; 1885276479Sdim break; 1886276479Sdim case MVT::i16: 1887276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg; 1888276479Sdim break; 1889276479Sdim case MVT::i32: 1890276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg; 1891276479Sdim break; 1892276479Sdim case MVT::i64: 1893276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg; 1894276479Sdim break; 1895276479Sdim case MVT::f32: 1896276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg; 1897276479Sdim break; 1898276479Sdim case MVT::f64: 1899276479Sdim Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg; 1900276479Sdim break; 1901276479Sdim } 1902276479Sdim } 1903276479Sdim break; 1904261991Sdim case NVPTXISD::LDGV2: 1905261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1906261991Sdim default: 1907276479Sdim return nullptr; 1908261991Sdim case MVT::i8: 1909261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32; 1910261991Sdim break; 1911261991Sdim case MVT::i16: 1912261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32; 1913261991Sdim break; 1914261991Sdim case MVT::i32: 1915261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32; 1916261991Sdim break; 1917261991Sdim case MVT::i64: 1918261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32; 1919261991Sdim break; 1920261991Sdim case MVT::f32: 1921261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32; 1922261991Sdim break; 1923261991Sdim case MVT::f64: 1924261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32; 1925261991Sdim break; 1926261991Sdim } 1927249423Sdim break; 1928261991Sdim case NVPTXISD::LDUV2: 1929261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1930261991Sdim default: 1931276479Sdim return nullptr; 1932261991Sdim case MVT::i8: 1933261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32; 1934261991Sdim break; 1935261991Sdim case MVT::i16: 1936261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32; 1937261991Sdim break; 1938261991Sdim case MVT::i32: 1939261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32; 1940261991Sdim break; 1941261991Sdim case MVT::i64: 1942261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32; 1943261991Sdim break; 1944261991Sdim case MVT::f32: 1945261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32; 1946261991Sdim break; 1947261991Sdim case MVT::f64: 1948261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32; 1949261991Sdim break; 1950261991Sdim } 1951249423Sdim break; 1952261991Sdim case NVPTXISD::LDGV4: 1953261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1954261991Sdim default: 1955276479Sdim return nullptr; 1956261991Sdim case MVT::i8: 1957261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32; 1958261991Sdim break; 1959261991Sdim case MVT::i16: 1960261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32; 1961261991Sdim break; 1962261991Sdim case MVT::i32: 1963261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32; 1964261991Sdim break; 1965261991Sdim case MVT::f32: 1966261991Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32; 1967261991Sdim break; 1968261991Sdim } 1969249423Sdim break; 1970261991Sdim case NVPTXISD::LDUV4: 1971261991Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1972261991Sdim default: 1973276479Sdim return nullptr; 1974261991Sdim case MVT::i8: 1975261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32; 1976261991Sdim break; 1977261991Sdim case MVT::i16: 1978261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32; 1979261991Sdim break; 1980261991Sdim case MVT::i32: 1981261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32; 1982261991Sdim break; 1983261991Sdim case MVT::f32: 1984261991Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32; 1985261991Sdim break; 1986261991Sdim } 1987249423Sdim break; 1988249423Sdim } 1989249423Sdim } 1990261991Sdim 1991261991Sdim SDValue Ops[] = { Op1, Chain }; 1992276479Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 1993249423Sdim } 1994249423Sdim 1995249423Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 1996276479Sdim MemRefs0[0] = Mem->getMemOperand(); 1997249423Sdim cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); 1998249423Sdim 1999249423Sdim return LD; 2000249423Sdim} 2001249423Sdim 2002249423SdimSDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) { 2003261991Sdim SDLoc dl(N); 2004239310Sdim StoreSDNode *ST = cast<StoreSDNode>(N); 2005239310Sdim EVT StoreVT = ST->getMemoryVT(); 2006276479Sdim SDNode *NVPTXST = nullptr; 2007239310Sdim 2008239310Sdim // do not support pre/post inc/dec 2009239310Sdim if (ST->isIndexed()) 2010276479Sdim return nullptr; 2011239310Sdim 2012239310Sdim if (!StoreVT.isSimple()) 2013276479Sdim return nullptr; 2014239310Sdim 2015239310Sdim // Address Space Setting 2016239310Sdim unsigned int codeAddrSpace = getCodeAddrSpace(ST, Subtarget); 2017239310Sdim 2018239310Sdim // Volatile Setting 2019239310Sdim // - .volatile is only availalble for .global and .shared 2020239310Sdim bool isVolatile = ST->isVolatile(); 2021239310Sdim if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 2022239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 2023239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 2024239310Sdim isVolatile = false; 2025239310Sdim 2026239310Sdim // Vector Setting 2027239310Sdim MVT SimpleVT = StoreVT.getSimpleVT(); 2028239310Sdim unsigned vecType = NVPTX::PTXLdStInstCode::Scalar; 2029239310Sdim if (SimpleVT.isVector()) { 2030239310Sdim unsigned num = SimpleVT.getVectorNumElements(); 2031239310Sdim if (num == 2) 2032239310Sdim vecType = NVPTX::PTXLdStInstCode::V2; 2033239310Sdim else if (num == 4) 2034239310Sdim vecType = NVPTX::PTXLdStInstCode::V4; 2035239310Sdim else 2036276479Sdim return nullptr; 2037239310Sdim } 2038239310Sdim 2039239310Sdim // Type Setting: toType + toTypeWidth 2040239310Sdim // - for integer type, always use 'u' 2041239310Sdim // 2042239310Sdim MVT ScalarVT = SimpleVT.getScalarType(); 2043249423Sdim unsigned toTypeWidth = ScalarVT.getSizeInBits(); 2044239310Sdim unsigned int toType; 2045239310Sdim if (ScalarVT.isFloatingPoint()) 2046239310Sdim toType = NVPTX::PTXLdStInstCode::Float; 2047239310Sdim else 2048239310Sdim toType = NVPTX::PTXLdStInstCode::Unsigned; 2049239310Sdim 2050239310Sdim // Create the machine instruction DAG 2051239310Sdim SDValue Chain = N->getOperand(0); 2052239310Sdim SDValue N1 = N->getOperand(1); 2053239310Sdim SDValue N2 = N->getOperand(2); 2054239310Sdim SDValue Addr; 2055239310Sdim SDValue Offset, Base; 2056239310Sdim unsigned Opcode; 2057261991Sdim MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy; 2058239310Sdim 2059239310Sdim if (SelectDirectAddr(N2, Addr)) { 2060239310Sdim switch (SourceVT) { 2061249423Sdim case MVT::i8: 2062249423Sdim Opcode = NVPTX::ST_i8_avar; 2063249423Sdim break; 2064249423Sdim case MVT::i16: 2065249423Sdim Opcode = NVPTX::ST_i16_avar; 2066249423Sdim break; 2067249423Sdim case MVT::i32: 2068249423Sdim Opcode = NVPTX::ST_i32_avar; 2069249423Sdim break; 2070249423Sdim case MVT::i64: 2071249423Sdim Opcode = NVPTX::ST_i64_avar; 2072249423Sdim break; 2073249423Sdim case MVT::f32: 2074249423Sdim Opcode = NVPTX::ST_f32_avar; 2075249423Sdim break; 2076249423Sdim case MVT::f64: 2077249423Sdim Opcode = NVPTX::ST_f64_avar; 2078249423Sdim break; 2079249423Sdim default: 2080276479Sdim return nullptr; 2081239310Sdim } 2082249423Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 2083249423Sdim getI32Imm(vecType), getI32Imm(toType), 2084249423Sdim getI32Imm(toTypeWidth), Addr, Chain }; 2085251662Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 2086249423Sdim } else if (Subtarget.is64Bit() 2087249423Sdim ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) 2088249423Sdim : SelectADDRsi(N2.getNode(), N2, Base, Offset)) { 2089239310Sdim switch (SourceVT) { 2090249423Sdim case MVT::i8: 2091249423Sdim Opcode = NVPTX::ST_i8_asi; 2092249423Sdim break; 2093249423Sdim case MVT::i16: 2094249423Sdim Opcode = NVPTX::ST_i16_asi; 2095249423Sdim break; 2096249423Sdim case MVT::i32: 2097249423Sdim Opcode = NVPTX::ST_i32_asi; 2098249423Sdim break; 2099249423Sdim case MVT::i64: 2100249423Sdim Opcode = NVPTX::ST_i64_asi; 2101249423Sdim break; 2102249423Sdim case MVT::f32: 2103249423Sdim Opcode = NVPTX::ST_f32_asi; 2104249423Sdim break; 2105249423Sdim case MVT::f64: 2106249423Sdim Opcode = NVPTX::ST_f64_asi; 2107249423Sdim break; 2108249423Sdim default: 2109276479Sdim return nullptr; 2110239310Sdim } 2111249423Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 2112249423Sdim getI32Imm(vecType), getI32Imm(toType), 2113249423Sdim getI32Imm(toTypeWidth), Base, Offset, Chain }; 2114251662Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 2115249423Sdim } else if (Subtarget.is64Bit() 2116249423Sdim ? SelectADDRri64(N2.getNode(), N2, Base, Offset) 2117249423Sdim : SelectADDRri(N2.getNode(), N2, Base, Offset)) { 2118249423Sdim if (Subtarget.is64Bit()) { 2119249423Sdim switch (SourceVT) { 2120249423Sdim case MVT::i8: 2121249423Sdim Opcode = NVPTX::ST_i8_ari_64; 2122249423Sdim break; 2123249423Sdim case MVT::i16: 2124249423Sdim Opcode = NVPTX::ST_i16_ari_64; 2125249423Sdim break; 2126249423Sdim case MVT::i32: 2127249423Sdim Opcode = NVPTX::ST_i32_ari_64; 2128249423Sdim break; 2129249423Sdim case MVT::i64: 2130249423Sdim Opcode = NVPTX::ST_i64_ari_64; 2131249423Sdim break; 2132249423Sdim case MVT::f32: 2133249423Sdim Opcode = NVPTX::ST_f32_ari_64; 2134249423Sdim break; 2135249423Sdim case MVT::f64: 2136249423Sdim Opcode = NVPTX::ST_f64_ari_64; 2137249423Sdim break; 2138249423Sdim default: 2139276479Sdim return nullptr; 2140249423Sdim } 2141249423Sdim } else { 2142249423Sdim switch (SourceVT) { 2143249423Sdim case MVT::i8: 2144249423Sdim Opcode = NVPTX::ST_i8_ari; 2145249423Sdim break; 2146249423Sdim case MVT::i16: 2147249423Sdim Opcode = NVPTX::ST_i16_ari; 2148249423Sdim break; 2149249423Sdim case MVT::i32: 2150249423Sdim Opcode = NVPTX::ST_i32_ari; 2151249423Sdim break; 2152249423Sdim case MVT::i64: 2153249423Sdim Opcode = NVPTX::ST_i64_ari; 2154249423Sdim break; 2155249423Sdim case MVT::f32: 2156249423Sdim Opcode = NVPTX::ST_f32_ari; 2157249423Sdim break; 2158249423Sdim case MVT::f64: 2159249423Sdim Opcode = NVPTX::ST_f64_ari; 2160249423Sdim break; 2161249423Sdim default: 2162276479Sdim return nullptr; 2163249423Sdim } 2164239310Sdim } 2165249423Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 2166249423Sdim getI32Imm(vecType), getI32Imm(toType), 2167249423Sdim getI32Imm(toTypeWidth), Base, Offset, Chain }; 2168251662Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 2169239310Sdim } else { 2170249423Sdim if (Subtarget.is64Bit()) { 2171249423Sdim switch (SourceVT) { 2172249423Sdim case MVT::i8: 2173249423Sdim Opcode = NVPTX::ST_i8_areg_64; 2174249423Sdim break; 2175249423Sdim case MVT::i16: 2176249423Sdim Opcode = NVPTX::ST_i16_areg_64; 2177249423Sdim break; 2178249423Sdim case MVT::i32: 2179249423Sdim Opcode = NVPTX::ST_i32_areg_64; 2180249423Sdim break; 2181249423Sdim case MVT::i64: 2182249423Sdim Opcode = NVPTX::ST_i64_areg_64; 2183249423Sdim break; 2184249423Sdim case MVT::f32: 2185249423Sdim Opcode = NVPTX::ST_f32_areg_64; 2186249423Sdim break; 2187249423Sdim case MVT::f64: 2188249423Sdim Opcode = NVPTX::ST_f64_areg_64; 2189249423Sdim break; 2190249423Sdim default: 2191276479Sdim return nullptr; 2192249423Sdim } 2193249423Sdim } else { 2194249423Sdim switch (SourceVT) { 2195249423Sdim case MVT::i8: 2196249423Sdim Opcode = NVPTX::ST_i8_areg; 2197249423Sdim break; 2198249423Sdim case MVT::i16: 2199249423Sdim Opcode = NVPTX::ST_i16_areg; 2200249423Sdim break; 2201249423Sdim case MVT::i32: 2202249423Sdim Opcode = NVPTX::ST_i32_areg; 2203249423Sdim break; 2204249423Sdim case MVT::i64: 2205249423Sdim Opcode = NVPTX::ST_i64_areg; 2206249423Sdim break; 2207249423Sdim case MVT::f32: 2208249423Sdim Opcode = NVPTX::ST_f32_areg; 2209249423Sdim break; 2210249423Sdim case MVT::f64: 2211249423Sdim Opcode = NVPTX::ST_f64_areg; 2212249423Sdim break; 2213249423Sdim default: 2214276479Sdim return nullptr; 2215249423Sdim } 2216239310Sdim } 2217249423Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 2218249423Sdim getI32Imm(vecType), getI32Imm(toType), 2219249423Sdim getI32Imm(toTypeWidth), N2, Chain }; 2220251662Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 2221239310Sdim } 2222239310Sdim 2223276479Sdim if (NVPTXST) { 2224239310Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 2225239310Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 2226239310Sdim cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1); 2227239310Sdim } 2228239310Sdim 2229239310Sdim return NVPTXST; 2230239310Sdim} 2231239310Sdim 2232249423SdimSDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) { 2233249423Sdim SDValue Chain = N->getOperand(0); 2234249423Sdim SDValue Op1 = N->getOperand(1); 2235249423Sdim SDValue Addr, Offset, Base; 2236249423Sdim unsigned Opcode; 2237261991Sdim SDLoc DL(N); 2238249423Sdim SDNode *ST; 2239249423Sdim EVT EltVT = Op1.getValueType(); 2240249423Sdim MemSDNode *MemSD = cast<MemSDNode>(N); 2241249423Sdim EVT StoreVT = MemSD->getMemoryVT(); 2242249423Sdim 2243249423Sdim // Address Space Setting 2244249423Sdim unsigned CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget); 2245249423Sdim 2246249423Sdim if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) { 2247249423Sdim report_fatal_error("Cannot store to pointer that points to constant " 2248249423Sdim "memory space"); 2249249423Sdim } 2250249423Sdim 2251249423Sdim // Volatile Setting 2252249423Sdim // - .volatile is only availalble for .global and .shared 2253249423Sdim bool IsVolatile = MemSD->isVolatile(); 2254249423Sdim if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 2255249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 2256249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 2257249423Sdim IsVolatile = false; 2258249423Sdim 2259249423Sdim // Type Setting: toType + toTypeWidth 2260249423Sdim // - for integer type, always use 'u' 2261249423Sdim assert(StoreVT.isSimple() && "Store value is not simple"); 2262249423Sdim MVT ScalarVT = StoreVT.getSimpleVT().getScalarType(); 2263249423Sdim unsigned ToTypeWidth = ScalarVT.getSizeInBits(); 2264249423Sdim unsigned ToType; 2265249423Sdim if (ScalarVT.isFloatingPoint()) 2266249423Sdim ToType = NVPTX::PTXLdStInstCode::Float; 2267249423Sdim else 2268249423Sdim ToType = NVPTX::PTXLdStInstCode::Unsigned; 2269249423Sdim 2270249423Sdim SmallVector<SDValue, 12> StOps; 2271249423Sdim SDValue N2; 2272249423Sdim unsigned VecType; 2273249423Sdim 2274249423Sdim switch (N->getOpcode()) { 2275249423Sdim case NVPTXISD::StoreV2: 2276249423Sdim VecType = NVPTX::PTXLdStInstCode::V2; 2277249423Sdim StOps.push_back(N->getOperand(1)); 2278249423Sdim StOps.push_back(N->getOperand(2)); 2279249423Sdim N2 = N->getOperand(3); 2280249423Sdim break; 2281249423Sdim case NVPTXISD::StoreV4: 2282249423Sdim VecType = NVPTX::PTXLdStInstCode::V4; 2283249423Sdim StOps.push_back(N->getOperand(1)); 2284249423Sdim StOps.push_back(N->getOperand(2)); 2285249423Sdim StOps.push_back(N->getOperand(3)); 2286249423Sdim StOps.push_back(N->getOperand(4)); 2287249423Sdim N2 = N->getOperand(5); 2288249423Sdim break; 2289249423Sdim default: 2290276479Sdim return nullptr; 2291249423Sdim } 2292249423Sdim 2293249423Sdim StOps.push_back(getI32Imm(IsVolatile)); 2294249423Sdim StOps.push_back(getI32Imm(CodeAddrSpace)); 2295249423Sdim StOps.push_back(getI32Imm(VecType)); 2296249423Sdim StOps.push_back(getI32Imm(ToType)); 2297249423Sdim StOps.push_back(getI32Imm(ToTypeWidth)); 2298249423Sdim 2299249423Sdim if (SelectDirectAddr(N2, Addr)) { 2300249423Sdim switch (N->getOpcode()) { 2301249423Sdim default: 2302276479Sdim return nullptr; 2303249423Sdim case NVPTXISD::StoreV2: 2304249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2305249423Sdim default: 2306276479Sdim return nullptr; 2307249423Sdim case MVT::i8: 2308249423Sdim Opcode = NVPTX::STV_i8_v2_avar; 2309249423Sdim break; 2310249423Sdim case MVT::i16: 2311249423Sdim Opcode = NVPTX::STV_i16_v2_avar; 2312249423Sdim break; 2313249423Sdim case MVT::i32: 2314249423Sdim Opcode = NVPTX::STV_i32_v2_avar; 2315249423Sdim break; 2316249423Sdim case MVT::i64: 2317249423Sdim Opcode = NVPTX::STV_i64_v2_avar; 2318249423Sdim break; 2319249423Sdim case MVT::f32: 2320249423Sdim Opcode = NVPTX::STV_f32_v2_avar; 2321249423Sdim break; 2322249423Sdim case MVT::f64: 2323249423Sdim Opcode = NVPTX::STV_f64_v2_avar; 2324249423Sdim break; 2325249423Sdim } 2326249423Sdim break; 2327249423Sdim case NVPTXISD::StoreV4: 2328249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2329249423Sdim default: 2330276479Sdim return nullptr; 2331249423Sdim case MVT::i8: 2332249423Sdim Opcode = NVPTX::STV_i8_v4_avar; 2333249423Sdim break; 2334249423Sdim case MVT::i16: 2335249423Sdim Opcode = NVPTX::STV_i16_v4_avar; 2336249423Sdim break; 2337249423Sdim case MVT::i32: 2338249423Sdim Opcode = NVPTX::STV_i32_v4_avar; 2339249423Sdim break; 2340249423Sdim case MVT::f32: 2341249423Sdim Opcode = NVPTX::STV_f32_v4_avar; 2342249423Sdim break; 2343249423Sdim } 2344249423Sdim break; 2345249423Sdim } 2346249423Sdim StOps.push_back(Addr); 2347249423Sdim } else if (Subtarget.is64Bit() 2348249423Sdim ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) 2349249423Sdim : SelectADDRsi(N2.getNode(), N2, Base, Offset)) { 2350249423Sdim switch (N->getOpcode()) { 2351249423Sdim default: 2352276479Sdim return nullptr; 2353249423Sdim case NVPTXISD::StoreV2: 2354249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2355249423Sdim default: 2356276479Sdim return nullptr; 2357249423Sdim case MVT::i8: 2358249423Sdim Opcode = NVPTX::STV_i8_v2_asi; 2359249423Sdim break; 2360249423Sdim case MVT::i16: 2361249423Sdim Opcode = NVPTX::STV_i16_v2_asi; 2362249423Sdim break; 2363249423Sdim case MVT::i32: 2364249423Sdim Opcode = NVPTX::STV_i32_v2_asi; 2365249423Sdim break; 2366249423Sdim case MVT::i64: 2367249423Sdim Opcode = NVPTX::STV_i64_v2_asi; 2368249423Sdim break; 2369249423Sdim case MVT::f32: 2370249423Sdim Opcode = NVPTX::STV_f32_v2_asi; 2371249423Sdim break; 2372249423Sdim case MVT::f64: 2373249423Sdim Opcode = NVPTX::STV_f64_v2_asi; 2374249423Sdim break; 2375249423Sdim } 2376249423Sdim break; 2377249423Sdim case NVPTXISD::StoreV4: 2378249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2379249423Sdim default: 2380276479Sdim return nullptr; 2381249423Sdim case MVT::i8: 2382249423Sdim Opcode = NVPTX::STV_i8_v4_asi; 2383249423Sdim break; 2384249423Sdim case MVT::i16: 2385249423Sdim Opcode = NVPTX::STV_i16_v4_asi; 2386249423Sdim break; 2387249423Sdim case MVT::i32: 2388249423Sdim Opcode = NVPTX::STV_i32_v4_asi; 2389249423Sdim break; 2390249423Sdim case MVT::f32: 2391249423Sdim Opcode = NVPTX::STV_f32_v4_asi; 2392249423Sdim break; 2393249423Sdim } 2394249423Sdim break; 2395249423Sdim } 2396249423Sdim StOps.push_back(Base); 2397249423Sdim StOps.push_back(Offset); 2398249423Sdim } else if (Subtarget.is64Bit() 2399249423Sdim ? SelectADDRri64(N2.getNode(), N2, Base, Offset) 2400249423Sdim : SelectADDRri(N2.getNode(), N2, Base, Offset)) { 2401249423Sdim if (Subtarget.is64Bit()) { 2402249423Sdim switch (N->getOpcode()) { 2403249423Sdim default: 2404276479Sdim return nullptr; 2405249423Sdim case NVPTXISD::StoreV2: 2406249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2407249423Sdim default: 2408276479Sdim return nullptr; 2409249423Sdim case MVT::i8: 2410249423Sdim Opcode = NVPTX::STV_i8_v2_ari_64; 2411249423Sdim break; 2412249423Sdim case MVT::i16: 2413249423Sdim Opcode = NVPTX::STV_i16_v2_ari_64; 2414249423Sdim break; 2415249423Sdim case MVT::i32: 2416249423Sdim Opcode = NVPTX::STV_i32_v2_ari_64; 2417249423Sdim break; 2418249423Sdim case MVT::i64: 2419249423Sdim Opcode = NVPTX::STV_i64_v2_ari_64; 2420249423Sdim break; 2421249423Sdim case MVT::f32: 2422249423Sdim Opcode = NVPTX::STV_f32_v2_ari_64; 2423249423Sdim break; 2424249423Sdim case MVT::f64: 2425249423Sdim Opcode = NVPTX::STV_f64_v2_ari_64; 2426249423Sdim break; 2427249423Sdim } 2428249423Sdim break; 2429249423Sdim case NVPTXISD::StoreV4: 2430249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2431249423Sdim default: 2432276479Sdim return nullptr; 2433249423Sdim case MVT::i8: 2434249423Sdim Opcode = NVPTX::STV_i8_v4_ari_64; 2435249423Sdim break; 2436249423Sdim case MVT::i16: 2437249423Sdim Opcode = NVPTX::STV_i16_v4_ari_64; 2438249423Sdim break; 2439249423Sdim case MVT::i32: 2440249423Sdim Opcode = NVPTX::STV_i32_v4_ari_64; 2441249423Sdim break; 2442249423Sdim case MVT::f32: 2443249423Sdim Opcode = NVPTX::STV_f32_v4_ari_64; 2444249423Sdim break; 2445249423Sdim } 2446249423Sdim break; 2447249423Sdim } 2448249423Sdim } else { 2449249423Sdim switch (N->getOpcode()) { 2450249423Sdim default: 2451276479Sdim return nullptr; 2452249423Sdim case NVPTXISD::StoreV2: 2453249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2454249423Sdim default: 2455276479Sdim return nullptr; 2456249423Sdim case MVT::i8: 2457249423Sdim Opcode = NVPTX::STV_i8_v2_ari; 2458249423Sdim break; 2459249423Sdim case MVT::i16: 2460249423Sdim Opcode = NVPTX::STV_i16_v2_ari; 2461249423Sdim break; 2462249423Sdim case MVT::i32: 2463249423Sdim Opcode = NVPTX::STV_i32_v2_ari; 2464249423Sdim break; 2465249423Sdim case MVT::i64: 2466249423Sdim Opcode = NVPTX::STV_i64_v2_ari; 2467249423Sdim break; 2468249423Sdim case MVT::f32: 2469249423Sdim Opcode = NVPTX::STV_f32_v2_ari; 2470249423Sdim break; 2471249423Sdim case MVT::f64: 2472249423Sdim Opcode = NVPTX::STV_f64_v2_ari; 2473249423Sdim break; 2474249423Sdim } 2475249423Sdim break; 2476249423Sdim case NVPTXISD::StoreV4: 2477249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2478249423Sdim default: 2479276479Sdim return nullptr; 2480249423Sdim case MVT::i8: 2481249423Sdim Opcode = NVPTX::STV_i8_v4_ari; 2482249423Sdim break; 2483249423Sdim case MVT::i16: 2484249423Sdim Opcode = NVPTX::STV_i16_v4_ari; 2485249423Sdim break; 2486249423Sdim case MVT::i32: 2487249423Sdim Opcode = NVPTX::STV_i32_v4_ari; 2488249423Sdim break; 2489249423Sdim case MVT::f32: 2490249423Sdim Opcode = NVPTX::STV_f32_v4_ari; 2491249423Sdim break; 2492249423Sdim } 2493249423Sdim break; 2494249423Sdim } 2495249423Sdim } 2496249423Sdim StOps.push_back(Base); 2497249423Sdim StOps.push_back(Offset); 2498249423Sdim } else { 2499249423Sdim if (Subtarget.is64Bit()) { 2500249423Sdim switch (N->getOpcode()) { 2501249423Sdim default: 2502276479Sdim return nullptr; 2503249423Sdim case NVPTXISD::StoreV2: 2504249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2505249423Sdim default: 2506276479Sdim return nullptr; 2507249423Sdim case MVT::i8: 2508249423Sdim Opcode = NVPTX::STV_i8_v2_areg_64; 2509249423Sdim break; 2510249423Sdim case MVT::i16: 2511249423Sdim Opcode = NVPTX::STV_i16_v2_areg_64; 2512249423Sdim break; 2513249423Sdim case MVT::i32: 2514249423Sdim Opcode = NVPTX::STV_i32_v2_areg_64; 2515249423Sdim break; 2516249423Sdim case MVT::i64: 2517249423Sdim Opcode = NVPTX::STV_i64_v2_areg_64; 2518249423Sdim break; 2519249423Sdim case MVT::f32: 2520249423Sdim Opcode = NVPTX::STV_f32_v2_areg_64; 2521249423Sdim break; 2522249423Sdim case MVT::f64: 2523249423Sdim Opcode = NVPTX::STV_f64_v2_areg_64; 2524249423Sdim break; 2525249423Sdim } 2526249423Sdim break; 2527249423Sdim case NVPTXISD::StoreV4: 2528249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2529249423Sdim default: 2530276479Sdim return nullptr; 2531249423Sdim case MVT::i8: 2532249423Sdim Opcode = NVPTX::STV_i8_v4_areg_64; 2533249423Sdim break; 2534249423Sdim case MVT::i16: 2535249423Sdim Opcode = NVPTX::STV_i16_v4_areg_64; 2536249423Sdim break; 2537249423Sdim case MVT::i32: 2538249423Sdim Opcode = NVPTX::STV_i32_v4_areg_64; 2539249423Sdim break; 2540249423Sdim case MVT::f32: 2541249423Sdim Opcode = NVPTX::STV_f32_v4_areg_64; 2542249423Sdim break; 2543249423Sdim } 2544249423Sdim break; 2545249423Sdim } 2546249423Sdim } else { 2547249423Sdim switch (N->getOpcode()) { 2548249423Sdim default: 2549276479Sdim return nullptr; 2550249423Sdim case NVPTXISD::StoreV2: 2551249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2552249423Sdim default: 2553276479Sdim return nullptr; 2554249423Sdim case MVT::i8: 2555249423Sdim Opcode = NVPTX::STV_i8_v2_areg; 2556249423Sdim break; 2557249423Sdim case MVT::i16: 2558249423Sdim Opcode = NVPTX::STV_i16_v2_areg; 2559249423Sdim break; 2560249423Sdim case MVT::i32: 2561249423Sdim Opcode = NVPTX::STV_i32_v2_areg; 2562249423Sdim break; 2563249423Sdim case MVT::i64: 2564249423Sdim Opcode = NVPTX::STV_i64_v2_areg; 2565249423Sdim break; 2566249423Sdim case MVT::f32: 2567249423Sdim Opcode = NVPTX::STV_f32_v2_areg; 2568249423Sdim break; 2569249423Sdim case MVT::f64: 2570249423Sdim Opcode = NVPTX::STV_f64_v2_areg; 2571249423Sdim break; 2572249423Sdim } 2573249423Sdim break; 2574249423Sdim case NVPTXISD::StoreV4: 2575249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 2576249423Sdim default: 2577276479Sdim return nullptr; 2578249423Sdim case MVT::i8: 2579249423Sdim Opcode = NVPTX::STV_i8_v4_areg; 2580249423Sdim break; 2581249423Sdim case MVT::i16: 2582249423Sdim Opcode = NVPTX::STV_i16_v4_areg; 2583249423Sdim break; 2584249423Sdim case MVT::i32: 2585249423Sdim Opcode = NVPTX::STV_i32_v4_areg; 2586249423Sdim break; 2587249423Sdim case MVT::f32: 2588249423Sdim Opcode = NVPTX::STV_f32_v4_areg; 2589249423Sdim break; 2590249423Sdim } 2591249423Sdim break; 2592249423Sdim } 2593249423Sdim } 2594249423Sdim StOps.push_back(N2); 2595249423Sdim } 2596249423Sdim 2597249423Sdim StOps.push_back(Chain); 2598249423Sdim 2599251662Sdim ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps); 2600249423Sdim 2601249423Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 2602249423Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 2603249423Sdim cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1); 2604249423Sdim 2605249423Sdim return ST; 2606249423Sdim} 2607249423Sdim 2608261991SdimSDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) { 2609261991Sdim SDValue Chain = Node->getOperand(0); 2610261991Sdim SDValue Offset = Node->getOperand(2); 2611261991Sdim SDValue Flag = Node->getOperand(3); 2612261991Sdim SDLoc DL(Node); 2613261991Sdim MemSDNode *Mem = cast<MemSDNode>(Node); 2614261991Sdim 2615261991Sdim unsigned VecSize; 2616261991Sdim switch (Node->getOpcode()) { 2617261991Sdim default: 2618276479Sdim return nullptr; 2619261991Sdim case NVPTXISD::LoadParam: 2620261991Sdim VecSize = 1; 2621261991Sdim break; 2622261991Sdim case NVPTXISD::LoadParamV2: 2623261991Sdim VecSize = 2; 2624261991Sdim break; 2625261991Sdim case NVPTXISD::LoadParamV4: 2626261991Sdim VecSize = 4; 2627261991Sdim break; 2628261991Sdim } 2629261991Sdim 2630261991Sdim EVT EltVT = Node->getValueType(0); 2631261991Sdim EVT MemVT = Mem->getMemoryVT(); 2632261991Sdim 2633261991Sdim unsigned Opc = 0; 2634261991Sdim 2635261991Sdim switch (VecSize) { 2636261991Sdim default: 2637276479Sdim return nullptr; 2638261991Sdim case 1: 2639261991Sdim switch (MemVT.getSimpleVT().SimpleTy) { 2640261991Sdim default: 2641276479Sdim return nullptr; 2642261991Sdim case MVT::i1: 2643261991Sdim Opc = NVPTX::LoadParamMemI8; 2644261991Sdim break; 2645261991Sdim case MVT::i8: 2646261991Sdim Opc = NVPTX::LoadParamMemI8; 2647261991Sdim break; 2648261991Sdim case MVT::i16: 2649261991Sdim Opc = NVPTX::LoadParamMemI16; 2650261991Sdim break; 2651261991Sdim case MVT::i32: 2652261991Sdim Opc = NVPTX::LoadParamMemI32; 2653261991Sdim break; 2654261991Sdim case MVT::i64: 2655261991Sdim Opc = NVPTX::LoadParamMemI64; 2656261991Sdim break; 2657261991Sdim case MVT::f32: 2658261991Sdim Opc = NVPTX::LoadParamMemF32; 2659261991Sdim break; 2660261991Sdim case MVT::f64: 2661261991Sdim Opc = NVPTX::LoadParamMemF64; 2662261991Sdim break; 2663261991Sdim } 2664261991Sdim break; 2665261991Sdim case 2: 2666261991Sdim switch (MemVT.getSimpleVT().SimpleTy) { 2667261991Sdim default: 2668276479Sdim return nullptr; 2669261991Sdim case MVT::i1: 2670261991Sdim Opc = NVPTX::LoadParamMemV2I8; 2671261991Sdim break; 2672261991Sdim case MVT::i8: 2673261991Sdim Opc = NVPTX::LoadParamMemV2I8; 2674261991Sdim break; 2675261991Sdim case MVT::i16: 2676261991Sdim Opc = NVPTX::LoadParamMemV2I16; 2677261991Sdim break; 2678261991Sdim case MVT::i32: 2679261991Sdim Opc = NVPTX::LoadParamMemV2I32; 2680261991Sdim break; 2681261991Sdim case MVT::i64: 2682261991Sdim Opc = NVPTX::LoadParamMemV2I64; 2683261991Sdim break; 2684261991Sdim case MVT::f32: 2685261991Sdim Opc = NVPTX::LoadParamMemV2F32; 2686261991Sdim break; 2687261991Sdim case MVT::f64: 2688261991Sdim Opc = NVPTX::LoadParamMemV2F64; 2689261991Sdim break; 2690261991Sdim } 2691261991Sdim break; 2692261991Sdim case 4: 2693261991Sdim switch (MemVT.getSimpleVT().SimpleTy) { 2694261991Sdim default: 2695276479Sdim return nullptr; 2696261991Sdim case MVT::i1: 2697261991Sdim Opc = NVPTX::LoadParamMemV4I8; 2698261991Sdim break; 2699261991Sdim case MVT::i8: 2700261991Sdim Opc = NVPTX::LoadParamMemV4I8; 2701261991Sdim break; 2702261991Sdim case MVT::i16: 2703261991Sdim Opc = NVPTX::LoadParamMemV4I16; 2704261991Sdim break; 2705261991Sdim case MVT::i32: 2706261991Sdim Opc = NVPTX::LoadParamMemV4I32; 2707261991Sdim break; 2708261991Sdim case MVT::f32: 2709261991Sdim Opc = NVPTX::LoadParamMemV4F32; 2710261991Sdim break; 2711261991Sdim } 2712261991Sdim break; 2713261991Sdim } 2714261991Sdim 2715261991Sdim SDVTList VTs; 2716261991Sdim if (VecSize == 1) { 2717261991Sdim VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue); 2718261991Sdim } else if (VecSize == 2) { 2719261991Sdim VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue); 2720261991Sdim } else { 2721261991Sdim EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue }; 2722276479Sdim VTs = CurDAG->getVTList(EVTs); 2723261991Sdim } 2724261991Sdim 2725261991Sdim unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); 2726261991Sdim 2727261991Sdim SmallVector<SDValue, 2> Ops; 2728261991Sdim Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32)); 2729261991Sdim Ops.push_back(Chain); 2730261991Sdim Ops.push_back(Flag); 2731261991Sdim 2732261991Sdim SDNode *Ret = 2733261991Sdim CurDAG->getMachineNode(Opc, DL, VTs, Ops); 2734261991Sdim return Ret; 2735261991Sdim} 2736261991Sdim 2737261991SdimSDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) { 2738261991Sdim SDLoc DL(N); 2739261991Sdim SDValue Chain = N->getOperand(0); 2740261991Sdim SDValue Offset = N->getOperand(1); 2741261991Sdim unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); 2742261991Sdim MemSDNode *Mem = cast<MemSDNode>(N); 2743261991Sdim 2744261991Sdim // How many elements do we have? 2745261991Sdim unsigned NumElts = 1; 2746261991Sdim switch (N->getOpcode()) { 2747261991Sdim default: 2748276479Sdim return nullptr; 2749261991Sdim case NVPTXISD::StoreRetval: 2750261991Sdim NumElts = 1; 2751261991Sdim break; 2752261991Sdim case NVPTXISD::StoreRetvalV2: 2753261991Sdim NumElts = 2; 2754261991Sdim break; 2755261991Sdim case NVPTXISD::StoreRetvalV4: 2756261991Sdim NumElts = 4; 2757261991Sdim break; 2758261991Sdim } 2759261991Sdim 2760261991Sdim // Build vector of operands 2761261991Sdim SmallVector<SDValue, 6> Ops; 2762261991Sdim for (unsigned i = 0; i < NumElts; ++i) 2763261991Sdim Ops.push_back(N->getOperand(i + 2)); 2764261991Sdim Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32)); 2765261991Sdim Ops.push_back(Chain); 2766261991Sdim 2767261991Sdim // Determine target opcode 2768261991Sdim // If we have an i1, use an 8-bit store. The lowering code in 2769261991Sdim // NVPTXISelLowering will have already emitted an upcast. 2770261991Sdim unsigned Opcode = 0; 2771261991Sdim switch (NumElts) { 2772261991Sdim default: 2773276479Sdim return nullptr; 2774261991Sdim case 1: 2775261991Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2776261991Sdim default: 2777276479Sdim return nullptr; 2778261991Sdim case MVT::i1: 2779261991Sdim Opcode = NVPTX::StoreRetvalI8; 2780261991Sdim break; 2781261991Sdim case MVT::i8: 2782261991Sdim Opcode = NVPTX::StoreRetvalI8; 2783261991Sdim break; 2784261991Sdim case MVT::i16: 2785261991Sdim Opcode = NVPTX::StoreRetvalI16; 2786261991Sdim break; 2787261991Sdim case MVT::i32: 2788261991Sdim Opcode = NVPTX::StoreRetvalI32; 2789261991Sdim break; 2790261991Sdim case MVT::i64: 2791261991Sdim Opcode = NVPTX::StoreRetvalI64; 2792261991Sdim break; 2793261991Sdim case MVT::f32: 2794261991Sdim Opcode = NVPTX::StoreRetvalF32; 2795261991Sdim break; 2796261991Sdim case MVT::f64: 2797261991Sdim Opcode = NVPTX::StoreRetvalF64; 2798261991Sdim break; 2799261991Sdim } 2800261991Sdim break; 2801261991Sdim case 2: 2802261991Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2803261991Sdim default: 2804276479Sdim return nullptr; 2805261991Sdim case MVT::i1: 2806261991Sdim Opcode = NVPTX::StoreRetvalV2I8; 2807261991Sdim break; 2808261991Sdim case MVT::i8: 2809261991Sdim Opcode = NVPTX::StoreRetvalV2I8; 2810261991Sdim break; 2811261991Sdim case MVT::i16: 2812261991Sdim Opcode = NVPTX::StoreRetvalV2I16; 2813261991Sdim break; 2814261991Sdim case MVT::i32: 2815261991Sdim Opcode = NVPTX::StoreRetvalV2I32; 2816261991Sdim break; 2817261991Sdim case MVT::i64: 2818261991Sdim Opcode = NVPTX::StoreRetvalV2I64; 2819261991Sdim break; 2820261991Sdim case MVT::f32: 2821261991Sdim Opcode = NVPTX::StoreRetvalV2F32; 2822261991Sdim break; 2823261991Sdim case MVT::f64: 2824261991Sdim Opcode = NVPTX::StoreRetvalV2F64; 2825261991Sdim break; 2826261991Sdim } 2827261991Sdim break; 2828261991Sdim case 4: 2829261991Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2830261991Sdim default: 2831276479Sdim return nullptr; 2832261991Sdim case MVT::i1: 2833261991Sdim Opcode = NVPTX::StoreRetvalV4I8; 2834261991Sdim break; 2835261991Sdim case MVT::i8: 2836261991Sdim Opcode = NVPTX::StoreRetvalV4I8; 2837261991Sdim break; 2838261991Sdim case MVT::i16: 2839261991Sdim Opcode = NVPTX::StoreRetvalV4I16; 2840261991Sdim break; 2841261991Sdim case MVT::i32: 2842261991Sdim Opcode = NVPTX::StoreRetvalV4I32; 2843261991Sdim break; 2844261991Sdim case MVT::f32: 2845261991Sdim Opcode = NVPTX::StoreRetvalV4F32; 2846261991Sdim break; 2847261991Sdim } 2848261991Sdim break; 2849261991Sdim } 2850261991Sdim 2851261991Sdim SDNode *Ret = 2852261991Sdim CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops); 2853261991Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 2854261991Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 2855261991Sdim cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1); 2856261991Sdim 2857261991Sdim return Ret; 2858261991Sdim} 2859261991Sdim 2860261991SdimSDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) { 2861261991Sdim SDLoc DL(N); 2862261991Sdim SDValue Chain = N->getOperand(0); 2863261991Sdim SDValue Param = N->getOperand(1); 2864261991Sdim unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue(); 2865261991Sdim SDValue Offset = N->getOperand(2); 2866261991Sdim unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); 2867261991Sdim MemSDNode *Mem = cast<MemSDNode>(N); 2868261991Sdim SDValue Flag = N->getOperand(N->getNumOperands() - 1); 2869261991Sdim 2870261991Sdim // How many elements do we have? 2871261991Sdim unsigned NumElts = 1; 2872261991Sdim switch (N->getOpcode()) { 2873261991Sdim default: 2874276479Sdim return nullptr; 2875261991Sdim case NVPTXISD::StoreParamU32: 2876261991Sdim case NVPTXISD::StoreParamS32: 2877261991Sdim case NVPTXISD::StoreParam: 2878261991Sdim NumElts = 1; 2879261991Sdim break; 2880261991Sdim case NVPTXISD::StoreParamV2: 2881261991Sdim NumElts = 2; 2882261991Sdim break; 2883261991Sdim case NVPTXISD::StoreParamV4: 2884261991Sdim NumElts = 4; 2885261991Sdim break; 2886261991Sdim } 2887261991Sdim 2888261991Sdim // Build vector of operands 2889261991Sdim SmallVector<SDValue, 8> Ops; 2890261991Sdim for (unsigned i = 0; i < NumElts; ++i) 2891261991Sdim Ops.push_back(N->getOperand(i + 3)); 2892261991Sdim Ops.push_back(CurDAG->getTargetConstant(ParamVal, MVT::i32)); 2893261991Sdim Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32)); 2894261991Sdim Ops.push_back(Chain); 2895261991Sdim Ops.push_back(Flag); 2896261991Sdim 2897261991Sdim // Determine target opcode 2898261991Sdim // If we have an i1, use an 8-bit store. The lowering code in 2899261991Sdim // NVPTXISelLowering will have already emitted an upcast. 2900261991Sdim unsigned Opcode = 0; 2901261991Sdim switch (N->getOpcode()) { 2902261991Sdim default: 2903261991Sdim switch (NumElts) { 2904261991Sdim default: 2905276479Sdim return nullptr; 2906261991Sdim case 1: 2907261991Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2908261991Sdim default: 2909276479Sdim return nullptr; 2910261991Sdim case MVT::i1: 2911261991Sdim Opcode = NVPTX::StoreParamI8; 2912261991Sdim break; 2913261991Sdim case MVT::i8: 2914261991Sdim Opcode = NVPTX::StoreParamI8; 2915261991Sdim break; 2916261991Sdim case MVT::i16: 2917261991Sdim Opcode = NVPTX::StoreParamI16; 2918261991Sdim break; 2919261991Sdim case MVT::i32: 2920261991Sdim Opcode = NVPTX::StoreParamI32; 2921261991Sdim break; 2922261991Sdim case MVT::i64: 2923261991Sdim Opcode = NVPTX::StoreParamI64; 2924261991Sdim break; 2925261991Sdim case MVT::f32: 2926261991Sdim Opcode = NVPTX::StoreParamF32; 2927261991Sdim break; 2928261991Sdim case MVT::f64: 2929261991Sdim Opcode = NVPTX::StoreParamF64; 2930261991Sdim break; 2931261991Sdim } 2932261991Sdim break; 2933261991Sdim case 2: 2934261991Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2935261991Sdim default: 2936276479Sdim return nullptr; 2937261991Sdim case MVT::i1: 2938261991Sdim Opcode = NVPTX::StoreParamV2I8; 2939261991Sdim break; 2940261991Sdim case MVT::i8: 2941261991Sdim Opcode = NVPTX::StoreParamV2I8; 2942261991Sdim break; 2943261991Sdim case MVT::i16: 2944261991Sdim Opcode = NVPTX::StoreParamV2I16; 2945261991Sdim break; 2946261991Sdim case MVT::i32: 2947261991Sdim Opcode = NVPTX::StoreParamV2I32; 2948261991Sdim break; 2949261991Sdim case MVT::i64: 2950261991Sdim Opcode = NVPTX::StoreParamV2I64; 2951261991Sdim break; 2952261991Sdim case MVT::f32: 2953261991Sdim Opcode = NVPTX::StoreParamV2F32; 2954261991Sdim break; 2955261991Sdim case MVT::f64: 2956261991Sdim Opcode = NVPTX::StoreParamV2F64; 2957261991Sdim break; 2958261991Sdim } 2959261991Sdim break; 2960261991Sdim case 4: 2961261991Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2962261991Sdim default: 2963276479Sdim return nullptr; 2964261991Sdim case MVT::i1: 2965261991Sdim Opcode = NVPTX::StoreParamV4I8; 2966261991Sdim break; 2967261991Sdim case MVT::i8: 2968261991Sdim Opcode = NVPTX::StoreParamV4I8; 2969261991Sdim break; 2970261991Sdim case MVT::i16: 2971261991Sdim Opcode = NVPTX::StoreParamV4I16; 2972261991Sdim break; 2973261991Sdim case MVT::i32: 2974261991Sdim Opcode = NVPTX::StoreParamV4I32; 2975261991Sdim break; 2976261991Sdim case MVT::f32: 2977261991Sdim Opcode = NVPTX::StoreParamV4F32; 2978261991Sdim break; 2979261991Sdim } 2980261991Sdim break; 2981261991Sdim } 2982261991Sdim break; 2983261991Sdim // Special case: if we have a sign-extend/zero-extend node, insert the 2984261991Sdim // conversion instruction first, and use that as the value operand to 2985261991Sdim // the selected StoreParam node. 2986261991Sdim case NVPTXISD::StoreParamU32: { 2987261991Sdim Opcode = NVPTX::StoreParamI32; 2988261991Sdim SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, 2989261991Sdim MVT::i32); 2990261991Sdim SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL, 2991261991Sdim MVT::i32, Ops[0], CvtNone); 2992261991Sdim Ops[0] = SDValue(Cvt, 0); 2993261991Sdim break; 2994261991Sdim } 2995261991Sdim case NVPTXISD::StoreParamS32: { 2996261991Sdim Opcode = NVPTX::StoreParamI32; 2997261991Sdim SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, 2998261991Sdim MVT::i32); 2999261991Sdim SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL, 3000261991Sdim MVT::i32, Ops[0], CvtNone); 3001261991Sdim Ops[0] = SDValue(Cvt, 0); 3002261991Sdim break; 3003261991Sdim } 3004261991Sdim } 3005261991Sdim 3006261991Sdim SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue); 3007261991Sdim SDNode *Ret = 3008261991Sdim CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops); 3009261991Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 3010261991Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 3011261991Sdim cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1); 3012261991Sdim 3013261991Sdim return Ret; 3014261991Sdim} 3015261991Sdim 3016276479SdimSDNode *NVPTXDAGToDAGISel::SelectTextureIntrinsic(SDNode *N) { 3017276479Sdim SDValue Chain = N->getOperand(0); 3018276479Sdim SDNode *Ret = nullptr; 3019276479Sdim unsigned Opc = 0; 3020276479Sdim SmallVector<SDValue, 8> Ops; 3021276479Sdim 3022276479Sdim switch (N->getOpcode()) { 3023276479Sdim default: return nullptr; 3024276479Sdim case NVPTXISD::Tex1DFloatS32: 3025276479Sdim Opc = NVPTX::TEX_1D_F32_S32; 3026276479Sdim break; 3027276479Sdim case NVPTXISD::Tex1DFloatFloat: 3028276479Sdim Opc = NVPTX::TEX_1D_F32_F32; 3029276479Sdim break; 3030276479Sdim case NVPTXISD::Tex1DFloatFloatLevel: 3031276479Sdim Opc = NVPTX::TEX_1D_F32_F32_LEVEL; 3032276479Sdim break; 3033276479Sdim case NVPTXISD::Tex1DFloatFloatGrad: 3034276479Sdim Opc = NVPTX::TEX_1D_F32_F32_GRAD; 3035276479Sdim break; 3036276479Sdim case NVPTXISD::Tex1DS32S32: 3037276479Sdim Opc = NVPTX::TEX_1D_S32_S32; 3038276479Sdim break; 3039276479Sdim case NVPTXISD::Tex1DS32Float: 3040276479Sdim Opc = NVPTX::TEX_1D_S32_F32; 3041276479Sdim break; 3042276479Sdim case NVPTXISD::Tex1DS32FloatLevel: 3043276479Sdim Opc = NVPTX::TEX_1D_S32_F32_LEVEL; 3044276479Sdim break; 3045276479Sdim case NVPTXISD::Tex1DS32FloatGrad: 3046276479Sdim Opc = NVPTX::TEX_1D_S32_F32_GRAD; 3047276479Sdim break; 3048276479Sdim case NVPTXISD::Tex1DU32S32: 3049276479Sdim Opc = NVPTX::TEX_1D_U32_S32; 3050276479Sdim break; 3051276479Sdim case NVPTXISD::Tex1DU32Float: 3052276479Sdim Opc = NVPTX::TEX_1D_U32_F32; 3053276479Sdim break; 3054276479Sdim case NVPTXISD::Tex1DU32FloatLevel: 3055276479Sdim Opc = NVPTX::TEX_1D_U32_F32_LEVEL; 3056276479Sdim break; 3057276479Sdim case NVPTXISD::Tex1DU32FloatGrad: 3058276479Sdim Opc = NVPTX::TEX_1D_U32_F32_GRAD; 3059276479Sdim break; 3060276479Sdim case NVPTXISD::Tex1DArrayFloatS32: 3061276479Sdim Opc = NVPTX::TEX_1D_ARRAY_F32_S32; 3062276479Sdim break; 3063276479Sdim case NVPTXISD::Tex1DArrayFloatFloat: 3064276479Sdim Opc = NVPTX::TEX_1D_ARRAY_F32_F32; 3065276479Sdim break; 3066276479Sdim case NVPTXISD::Tex1DArrayFloatFloatLevel: 3067276479Sdim Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL; 3068276479Sdim break; 3069276479Sdim case NVPTXISD::Tex1DArrayFloatFloatGrad: 3070276479Sdim Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD; 3071276479Sdim break; 3072276479Sdim case NVPTXISD::Tex1DArrayS32S32: 3073276479Sdim Opc = NVPTX::TEX_1D_ARRAY_S32_S32; 3074276479Sdim break; 3075276479Sdim case NVPTXISD::Tex1DArrayS32Float: 3076276479Sdim Opc = NVPTX::TEX_1D_ARRAY_S32_F32; 3077276479Sdim break; 3078276479Sdim case NVPTXISD::Tex1DArrayS32FloatLevel: 3079276479Sdim Opc = NVPTX::TEX_1D_ARRAY_S32_F32_LEVEL; 3080276479Sdim break; 3081276479Sdim case NVPTXISD::Tex1DArrayS32FloatGrad: 3082276479Sdim Opc = NVPTX::TEX_1D_ARRAY_S32_F32_GRAD; 3083276479Sdim break; 3084276479Sdim case NVPTXISD::Tex1DArrayU32S32: 3085276479Sdim Opc = NVPTX::TEX_1D_ARRAY_U32_S32; 3086276479Sdim break; 3087276479Sdim case NVPTXISD::Tex1DArrayU32Float: 3088276479Sdim Opc = NVPTX::TEX_1D_ARRAY_U32_F32; 3089276479Sdim break; 3090276479Sdim case NVPTXISD::Tex1DArrayU32FloatLevel: 3091276479Sdim Opc = NVPTX::TEX_1D_ARRAY_U32_F32_LEVEL; 3092276479Sdim break; 3093276479Sdim case NVPTXISD::Tex1DArrayU32FloatGrad: 3094276479Sdim Opc = NVPTX::TEX_1D_ARRAY_U32_F32_GRAD; 3095276479Sdim break; 3096276479Sdim case NVPTXISD::Tex2DFloatS32: 3097276479Sdim Opc = NVPTX::TEX_2D_F32_S32; 3098276479Sdim break; 3099276479Sdim case NVPTXISD::Tex2DFloatFloat: 3100276479Sdim Opc = NVPTX::TEX_2D_F32_F32; 3101276479Sdim break; 3102276479Sdim case NVPTXISD::Tex2DFloatFloatLevel: 3103276479Sdim Opc = NVPTX::TEX_2D_F32_F32_LEVEL; 3104276479Sdim break; 3105276479Sdim case NVPTXISD::Tex2DFloatFloatGrad: 3106276479Sdim Opc = NVPTX::TEX_2D_F32_F32_GRAD; 3107276479Sdim break; 3108276479Sdim case NVPTXISD::Tex2DS32S32: 3109276479Sdim Opc = NVPTX::TEX_2D_S32_S32; 3110276479Sdim break; 3111276479Sdim case NVPTXISD::Tex2DS32Float: 3112276479Sdim Opc = NVPTX::TEX_2D_S32_F32; 3113276479Sdim break; 3114276479Sdim case NVPTXISD::Tex2DS32FloatLevel: 3115276479Sdim Opc = NVPTX::TEX_2D_S32_F32_LEVEL; 3116276479Sdim break; 3117276479Sdim case NVPTXISD::Tex2DS32FloatGrad: 3118276479Sdim Opc = NVPTX::TEX_2D_S32_F32_GRAD; 3119276479Sdim break; 3120276479Sdim case NVPTXISD::Tex2DU32S32: 3121276479Sdim Opc = NVPTX::TEX_2D_U32_S32; 3122276479Sdim break; 3123276479Sdim case NVPTXISD::Tex2DU32Float: 3124276479Sdim Opc = NVPTX::TEX_2D_U32_F32; 3125276479Sdim break; 3126276479Sdim case NVPTXISD::Tex2DU32FloatLevel: 3127276479Sdim Opc = NVPTX::TEX_2D_U32_F32_LEVEL; 3128276479Sdim break; 3129276479Sdim case NVPTXISD::Tex2DU32FloatGrad: 3130276479Sdim Opc = NVPTX::TEX_2D_U32_F32_GRAD; 3131276479Sdim break; 3132276479Sdim case NVPTXISD::Tex2DArrayFloatS32: 3133276479Sdim Opc = NVPTX::TEX_2D_ARRAY_F32_S32; 3134276479Sdim break; 3135276479Sdim case NVPTXISD::Tex2DArrayFloatFloat: 3136276479Sdim Opc = NVPTX::TEX_2D_ARRAY_F32_F32; 3137276479Sdim break; 3138276479Sdim case NVPTXISD::Tex2DArrayFloatFloatLevel: 3139276479Sdim Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL; 3140276479Sdim break; 3141276479Sdim case NVPTXISD::Tex2DArrayFloatFloatGrad: 3142276479Sdim Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD; 3143276479Sdim break; 3144276479Sdim case NVPTXISD::Tex2DArrayS32S32: 3145276479Sdim Opc = NVPTX::TEX_2D_ARRAY_S32_S32; 3146276479Sdim break; 3147276479Sdim case NVPTXISD::Tex2DArrayS32Float: 3148276479Sdim Opc = NVPTX::TEX_2D_ARRAY_S32_F32; 3149276479Sdim break; 3150276479Sdim case NVPTXISD::Tex2DArrayS32FloatLevel: 3151276479Sdim Opc = NVPTX::TEX_2D_ARRAY_S32_F32_LEVEL; 3152276479Sdim break; 3153276479Sdim case NVPTXISD::Tex2DArrayS32FloatGrad: 3154276479Sdim Opc = NVPTX::TEX_2D_ARRAY_S32_F32_GRAD; 3155276479Sdim break; 3156276479Sdim case NVPTXISD::Tex2DArrayU32S32: 3157276479Sdim Opc = NVPTX::TEX_2D_ARRAY_U32_S32; 3158276479Sdim break; 3159276479Sdim case NVPTXISD::Tex2DArrayU32Float: 3160276479Sdim Opc = NVPTX::TEX_2D_ARRAY_U32_F32; 3161276479Sdim break; 3162276479Sdim case NVPTXISD::Tex2DArrayU32FloatLevel: 3163276479Sdim Opc = NVPTX::TEX_2D_ARRAY_U32_F32_LEVEL; 3164276479Sdim break; 3165276479Sdim case NVPTXISD::Tex2DArrayU32FloatGrad: 3166276479Sdim Opc = NVPTX::TEX_2D_ARRAY_U32_F32_GRAD; 3167276479Sdim break; 3168276479Sdim case NVPTXISD::Tex3DFloatS32: 3169276479Sdim Opc = NVPTX::TEX_3D_F32_S32; 3170276479Sdim break; 3171276479Sdim case NVPTXISD::Tex3DFloatFloat: 3172276479Sdim Opc = NVPTX::TEX_3D_F32_F32; 3173276479Sdim break; 3174276479Sdim case NVPTXISD::Tex3DFloatFloatLevel: 3175276479Sdim Opc = NVPTX::TEX_3D_F32_F32_LEVEL; 3176276479Sdim break; 3177276479Sdim case NVPTXISD::Tex3DFloatFloatGrad: 3178276479Sdim Opc = NVPTX::TEX_3D_F32_F32_GRAD; 3179276479Sdim break; 3180276479Sdim case NVPTXISD::Tex3DS32S32: 3181276479Sdim Opc = NVPTX::TEX_3D_S32_S32; 3182276479Sdim break; 3183276479Sdim case NVPTXISD::Tex3DS32Float: 3184276479Sdim Opc = NVPTX::TEX_3D_S32_F32; 3185276479Sdim break; 3186276479Sdim case NVPTXISD::Tex3DS32FloatLevel: 3187276479Sdim Opc = NVPTX::TEX_3D_S32_F32_LEVEL; 3188276479Sdim break; 3189276479Sdim case NVPTXISD::Tex3DS32FloatGrad: 3190276479Sdim Opc = NVPTX::TEX_3D_S32_F32_GRAD; 3191276479Sdim break; 3192276479Sdim case NVPTXISD::Tex3DU32S32: 3193276479Sdim Opc = NVPTX::TEX_3D_U32_S32; 3194276479Sdim break; 3195276479Sdim case NVPTXISD::Tex3DU32Float: 3196276479Sdim Opc = NVPTX::TEX_3D_U32_F32; 3197276479Sdim break; 3198276479Sdim case NVPTXISD::Tex3DU32FloatLevel: 3199276479Sdim Opc = NVPTX::TEX_3D_U32_F32_LEVEL; 3200276479Sdim break; 3201276479Sdim case NVPTXISD::Tex3DU32FloatGrad: 3202276479Sdim Opc = NVPTX::TEX_3D_U32_F32_GRAD; 3203276479Sdim break; 3204276479Sdim case NVPTXISD::TexCubeFloatFloat: 3205276479Sdim Opc = NVPTX::TEX_CUBE_F32_F32; 3206276479Sdim break; 3207276479Sdim case NVPTXISD::TexCubeFloatFloatLevel: 3208276479Sdim Opc = NVPTX::TEX_CUBE_F32_F32_LEVEL; 3209276479Sdim break; 3210276479Sdim case NVPTXISD::TexCubeS32Float: 3211276479Sdim Opc = NVPTX::TEX_CUBE_S32_F32; 3212276479Sdim break; 3213276479Sdim case NVPTXISD::TexCubeS32FloatLevel: 3214276479Sdim Opc = NVPTX::TEX_CUBE_S32_F32_LEVEL; 3215276479Sdim break; 3216276479Sdim case NVPTXISD::TexCubeU32Float: 3217276479Sdim Opc = NVPTX::TEX_CUBE_U32_F32; 3218276479Sdim break; 3219276479Sdim case NVPTXISD::TexCubeU32FloatLevel: 3220276479Sdim Opc = NVPTX::TEX_CUBE_U32_F32_LEVEL; 3221276479Sdim break; 3222276479Sdim case NVPTXISD::TexCubeArrayFloatFloat: 3223276479Sdim Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32; 3224276479Sdim break; 3225276479Sdim case NVPTXISD::TexCubeArrayFloatFloatLevel: 3226276479Sdim Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_LEVEL; 3227276479Sdim break; 3228276479Sdim case NVPTXISD::TexCubeArrayS32Float: 3229276479Sdim Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32; 3230276479Sdim break; 3231276479Sdim case NVPTXISD::TexCubeArrayS32FloatLevel: 3232276479Sdim Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_LEVEL; 3233276479Sdim break; 3234276479Sdim case NVPTXISD::TexCubeArrayU32Float: 3235276479Sdim Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32; 3236276479Sdim break; 3237276479Sdim case NVPTXISD::TexCubeArrayU32FloatLevel: 3238276479Sdim Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_LEVEL; 3239276479Sdim break; 3240276479Sdim case NVPTXISD::Tld4R2DFloatFloat: 3241276479Sdim Opc = NVPTX::TLD4_R_2D_F32_F32; 3242276479Sdim break; 3243276479Sdim case NVPTXISD::Tld4G2DFloatFloat: 3244276479Sdim Opc = NVPTX::TLD4_G_2D_F32_F32; 3245276479Sdim break; 3246276479Sdim case NVPTXISD::Tld4B2DFloatFloat: 3247276479Sdim Opc = NVPTX::TLD4_B_2D_F32_F32; 3248276479Sdim break; 3249276479Sdim case NVPTXISD::Tld4A2DFloatFloat: 3250276479Sdim Opc = NVPTX::TLD4_A_2D_F32_F32; 3251276479Sdim break; 3252276479Sdim case NVPTXISD::Tld4R2DS64Float: 3253276479Sdim Opc = NVPTX::TLD4_R_2D_S32_F32; 3254276479Sdim break; 3255276479Sdim case NVPTXISD::Tld4G2DS64Float: 3256276479Sdim Opc = NVPTX::TLD4_G_2D_S32_F32; 3257276479Sdim break; 3258276479Sdim case NVPTXISD::Tld4B2DS64Float: 3259276479Sdim Opc = NVPTX::TLD4_B_2D_S32_F32; 3260276479Sdim break; 3261276479Sdim case NVPTXISD::Tld4A2DS64Float: 3262276479Sdim Opc = NVPTX::TLD4_A_2D_S32_F32; 3263276479Sdim break; 3264276479Sdim case NVPTXISD::Tld4R2DU64Float: 3265276479Sdim Opc = NVPTX::TLD4_R_2D_U32_F32; 3266276479Sdim break; 3267276479Sdim case NVPTXISD::Tld4G2DU64Float: 3268276479Sdim Opc = NVPTX::TLD4_G_2D_U32_F32; 3269276479Sdim break; 3270276479Sdim case NVPTXISD::Tld4B2DU64Float: 3271276479Sdim Opc = NVPTX::TLD4_B_2D_U32_F32; 3272276479Sdim break; 3273276479Sdim case NVPTXISD::Tld4A2DU64Float: 3274276479Sdim Opc = NVPTX::TLD4_A_2D_U32_F32; 3275276479Sdim break; 3276276479Sdim case NVPTXISD::TexUnified1DFloatS32: 3277276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_F32_S32; 3278276479Sdim break; 3279276479Sdim case NVPTXISD::TexUnified1DFloatFloat: 3280276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_F32_F32; 3281276479Sdim break; 3282276479Sdim case NVPTXISD::TexUnified1DFloatFloatLevel: 3283276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_LEVEL; 3284276479Sdim break; 3285276479Sdim case NVPTXISD::TexUnified1DFloatFloatGrad: 3286276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_GRAD; 3287276479Sdim break; 3288276479Sdim case NVPTXISD::TexUnified1DS32S32: 3289276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_S32_S32; 3290276479Sdim break; 3291276479Sdim case NVPTXISD::TexUnified1DS32Float: 3292276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_S32_F32; 3293276479Sdim break; 3294276479Sdim case NVPTXISD::TexUnified1DS32FloatLevel: 3295276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_LEVEL; 3296276479Sdim break; 3297276479Sdim case NVPTXISD::TexUnified1DS32FloatGrad: 3298276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_GRAD; 3299276479Sdim break; 3300276479Sdim case NVPTXISD::TexUnified1DU32S32: 3301276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_U32_S32; 3302276479Sdim break; 3303276479Sdim case NVPTXISD::TexUnified1DU32Float: 3304276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_U32_F32; 3305276479Sdim break; 3306276479Sdim case NVPTXISD::TexUnified1DU32FloatLevel: 3307276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_LEVEL; 3308276479Sdim break; 3309276479Sdim case NVPTXISD::TexUnified1DU32FloatGrad: 3310276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_GRAD; 3311276479Sdim break; 3312276479Sdim case NVPTXISD::TexUnified1DArrayFloatS32: 3313276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_S32; 3314276479Sdim break; 3315276479Sdim case NVPTXISD::TexUnified1DArrayFloatFloat: 3316276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32; 3317276479Sdim break; 3318276479Sdim case NVPTXISD::TexUnified1DArrayFloatFloatLevel: 3319276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL; 3320276479Sdim break; 3321276479Sdim case NVPTXISD::TexUnified1DArrayFloatFloatGrad: 3322276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD; 3323276479Sdim break; 3324276479Sdim case NVPTXISD::TexUnified1DArrayS32S32: 3325276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_S32; 3326276479Sdim break; 3327276479Sdim case NVPTXISD::TexUnified1DArrayS32Float: 3328276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32; 3329276479Sdim break; 3330276479Sdim case NVPTXISD::TexUnified1DArrayS32FloatLevel: 3331276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL; 3332276479Sdim break; 3333276479Sdim case NVPTXISD::TexUnified1DArrayS32FloatGrad: 3334276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD; 3335276479Sdim break; 3336276479Sdim case NVPTXISD::TexUnified1DArrayU32S32: 3337276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_S32; 3338276479Sdim break; 3339276479Sdim case NVPTXISD::TexUnified1DArrayU32Float: 3340276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32; 3341276479Sdim break; 3342276479Sdim case NVPTXISD::TexUnified1DArrayU32FloatLevel: 3343276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL; 3344276479Sdim break; 3345276479Sdim case NVPTXISD::TexUnified1DArrayU32FloatGrad: 3346276479Sdim Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD; 3347276479Sdim break; 3348276479Sdim case NVPTXISD::TexUnified2DFloatS32: 3349276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_F32_S32; 3350276479Sdim break; 3351276479Sdim case NVPTXISD::TexUnified2DFloatFloat: 3352276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_F32_F32; 3353276479Sdim break; 3354276479Sdim case NVPTXISD::TexUnified2DFloatFloatLevel: 3355276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_LEVEL; 3356276479Sdim break; 3357276479Sdim case NVPTXISD::TexUnified2DFloatFloatGrad: 3358276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_GRAD; 3359276479Sdim break; 3360276479Sdim case NVPTXISD::TexUnified2DS32S32: 3361276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_S32_S32; 3362276479Sdim break; 3363276479Sdim case NVPTXISD::TexUnified2DS32Float: 3364276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_S32_F32; 3365276479Sdim break; 3366276479Sdim case NVPTXISD::TexUnified2DS32FloatLevel: 3367276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_LEVEL; 3368276479Sdim break; 3369276479Sdim case NVPTXISD::TexUnified2DS32FloatGrad: 3370276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_GRAD; 3371276479Sdim break; 3372276479Sdim case NVPTXISD::TexUnified2DU32S32: 3373276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_U32_S32; 3374276479Sdim break; 3375276479Sdim case NVPTXISD::TexUnified2DU32Float: 3376276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_U32_F32; 3377276479Sdim break; 3378276479Sdim case NVPTXISD::TexUnified2DU32FloatLevel: 3379276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_LEVEL; 3380276479Sdim break; 3381276479Sdim case NVPTXISD::TexUnified2DU32FloatGrad: 3382276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_GRAD; 3383276479Sdim break; 3384276479Sdim case NVPTXISD::TexUnified2DArrayFloatS32: 3385276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_S32; 3386276479Sdim break; 3387276479Sdim case NVPTXISD::TexUnified2DArrayFloatFloat: 3388276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32; 3389276479Sdim break; 3390276479Sdim case NVPTXISD::TexUnified2DArrayFloatFloatLevel: 3391276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL; 3392276479Sdim break; 3393276479Sdim case NVPTXISD::TexUnified2DArrayFloatFloatGrad: 3394276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD; 3395276479Sdim break; 3396276479Sdim case NVPTXISD::TexUnified2DArrayS32S32: 3397276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_S32; 3398276479Sdim break; 3399276479Sdim case NVPTXISD::TexUnified2DArrayS32Float: 3400276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32; 3401276479Sdim break; 3402276479Sdim case NVPTXISD::TexUnified2DArrayS32FloatLevel: 3403276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL; 3404276479Sdim break; 3405276479Sdim case NVPTXISD::TexUnified2DArrayS32FloatGrad: 3406276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD; 3407276479Sdim break; 3408276479Sdim case NVPTXISD::TexUnified2DArrayU32S32: 3409276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_S32; 3410276479Sdim break; 3411276479Sdim case NVPTXISD::TexUnified2DArrayU32Float: 3412276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32; 3413276479Sdim break; 3414276479Sdim case NVPTXISD::TexUnified2DArrayU32FloatLevel: 3415276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL; 3416276479Sdim break; 3417276479Sdim case NVPTXISD::TexUnified2DArrayU32FloatGrad: 3418276479Sdim Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD; 3419276479Sdim break; 3420276479Sdim case NVPTXISD::TexUnified3DFloatS32: 3421276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_F32_S32; 3422276479Sdim break; 3423276479Sdim case NVPTXISD::TexUnified3DFloatFloat: 3424276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_F32_F32; 3425276479Sdim break; 3426276479Sdim case NVPTXISD::TexUnified3DFloatFloatLevel: 3427276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_LEVEL; 3428276479Sdim break; 3429276479Sdim case NVPTXISD::TexUnified3DFloatFloatGrad: 3430276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_GRAD; 3431276479Sdim break; 3432276479Sdim case NVPTXISD::TexUnified3DS32S32: 3433276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_S32_S32; 3434276479Sdim break; 3435276479Sdim case NVPTXISD::TexUnified3DS32Float: 3436276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_S32_F32; 3437276479Sdim break; 3438276479Sdim case NVPTXISD::TexUnified3DS32FloatLevel: 3439276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_LEVEL; 3440276479Sdim break; 3441276479Sdim case NVPTXISD::TexUnified3DS32FloatGrad: 3442276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_GRAD; 3443276479Sdim break; 3444276479Sdim case NVPTXISD::TexUnified3DU32S32: 3445276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_U32_S32; 3446276479Sdim break; 3447276479Sdim case NVPTXISD::TexUnified3DU32Float: 3448276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_U32_F32; 3449276479Sdim break; 3450276479Sdim case NVPTXISD::TexUnified3DU32FloatLevel: 3451276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_LEVEL; 3452276479Sdim break; 3453276479Sdim case NVPTXISD::TexUnified3DU32FloatGrad: 3454276479Sdim Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_GRAD; 3455276479Sdim break; 3456276479Sdim case NVPTXISD::TexUnifiedCubeFloatFloat: 3457276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32; 3458276479Sdim break; 3459276479Sdim case NVPTXISD::TexUnifiedCubeFloatFloatLevel: 3460276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_LEVEL; 3461276479Sdim break; 3462276479Sdim case NVPTXISD::TexUnifiedCubeS32Float: 3463276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32; 3464276479Sdim break; 3465276479Sdim case NVPTXISD::TexUnifiedCubeS32FloatLevel: 3466276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_LEVEL; 3467276479Sdim break; 3468276479Sdim case NVPTXISD::TexUnifiedCubeU32Float: 3469276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32; 3470276479Sdim break; 3471276479Sdim case NVPTXISD::TexUnifiedCubeU32FloatLevel: 3472276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_LEVEL; 3473276479Sdim break; 3474276479Sdim case NVPTXISD::TexUnifiedCubeArrayFloatFloat: 3475276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32; 3476276479Sdim break; 3477276479Sdim case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel: 3478276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL; 3479276479Sdim break; 3480276479Sdim case NVPTXISD::TexUnifiedCubeArrayS32Float: 3481276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32; 3482276479Sdim break; 3483276479Sdim case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel: 3484276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL; 3485276479Sdim break; 3486276479Sdim case NVPTXISD::TexUnifiedCubeArrayU32Float: 3487276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32; 3488276479Sdim break; 3489276479Sdim case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel: 3490276479Sdim Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL; 3491276479Sdim break; 3492276479Sdim case NVPTXISD::Tld4UnifiedR2DFloatFloat: 3493276479Sdim Opc = NVPTX::TLD4_UNIFIED_R_2D_F32_F32; 3494276479Sdim break; 3495276479Sdim case NVPTXISD::Tld4UnifiedG2DFloatFloat: 3496276479Sdim Opc = NVPTX::TLD4_UNIFIED_G_2D_F32_F32; 3497276479Sdim break; 3498276479Sdim case NVPTXISD::Tld4UnifiedB2DFloatFloat: 3499276479Sdim Opc = NVPTX::TLD4_UNIFIED_B_2D_F32_F32; 3500276479Sdim break; 3501276479Sdim case NVPTXISD::Tld4UnifiedA2DFloatFloat: 3502276479Sdim Opc = NVPTX::TLD4_UNIFIED_A_2D_F32_F32; 3503276479Sdim break; 3504276479Sdim case NVPTXISD::Tld4UnifiedR2DS64Float: 3505276479Sdim Opc = NVPTX::TLD4_UNIFIED_R_2D_S32_F32; 3506276479Sdim break; 3507276479Sdim case NVPTXISD::Tld4UnifiedG2DS64Float: 3508276479Sdim Opc = NVPTX::TLD4_UNIFIED_G_2D_S32_F32; 3509276479Sdim break; 3510276479Sdim case NVPTXISD::Tld4UnifiedB2DS64Float: 3511276479Sdim Opc = NVPTX::TLD4_UNIFIED_B_2D_S32_F32; 3512276479Sdim break; 3513276479Sdim case NVPTXISD::Tld4UnifiedA2DS64Float: 3514276479Sdim Opc = NVPTX::TLD4_UNIFIED_A_2D_S32_F32; 3515276479Sdim break; 3516276479Sdim case NVPTXISD::Tld4UnifiedR2DU64Float: 3517276479Sdim Opc = NVPTX::TLD4_UNIFIED_R_2D_U32_F32; 3518276479Sdim break; 3519276479Sdim case NVPTXISD::Tld4UnifiedG2DU64Float: 3520276479Sdim Opc = NVPTX::TLD4_UNIFIED_G_2D_U32_F32; 3521276479Sdim break; 3522276479Sdim case NVPTXISD::Tld4UnifiedB2DU64Float: 3523276479Sdim Opc = NVPTX::TLD4_UNIFIED_B_2D_U32_F32; 3524276479Sdim break; 3525276479Sdim case NVPTXISD::Tld4UnifiedA2DU64Float: 3526276479Sdim Opc = NVPTX::TLD4_UNIFIED_A_2D_U32_F32; 3527276479Sdim break; 3528276479Sdim } 3529276479Sdim 3530276479Sdim // Copy over operands 3531276479Sdim for (unsigned i = 1; i < N->getNumOperands(); ++i) { 3532276479Sdim Ops.push_back(N->getOperand(i)); 3533276479Sdim } 3534276479Sdim 3535276479Sdim Ops.push_back(Chain); 3536276479Sdim Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops); 3537276479Sdim return Ret; 3538276479Sdim} 3539276479Sdim 3540276479SdimSDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) { 3541276479Sdim SDValue Chain = N->getOperand(0); 3542276479Sdim SDValue TexHandle = N->getOperand(1); 3543276479Sdim SDNode *Ret = nullptr; 3544276479Sdim unsigned Opc = 0; 3545276479Sdim SmallVector<SDValue, 8> Ops; 3546276479Sdim switch (N->getOpcode()) { 3547276479Sdim default: return nullptr; 3548276479Sdim case NVPTXISD::Suld1DI8Clamp: 3549276479Sdim Opc = NVPTX::SULD_1D_I8_CLAMP; 3550276479Sdim Ops.push_back(TexHandle); 3551276479Sdim Ops.push_back(N->getOperand(2)); 3552276479Sdim Ops.push_back(Chain); 3553276479Sdim break; 3554276479Sdim case NVPTXISD::Suld1DI16Clamp: 3555276479Sdim Opc = NVPTX::SULD_1D_I16_CLAMP; 3556276479Sdim Ops.push_back(TexHandle); 3557276479Sdim Ops.push_back(N->getOperand(2)); 3558276479Sdim Ops.push_back(Chain); 3559276479Sdim break; 3560276479Sdim case NVPTXISD::Suld1DI32Clamp: 3561276479Sdim Opc = NVPTX::SULD_1D_I32_CLAMP; 3562276479Sdim Ops.push_back(TexHandle); 3563276479Sdim Ops.push_back(N->getOperand(2)); 3564276479Sdim Ops.push_back(Chain); 3565276479Sdim break; 3566276479Sdim case NVPTXISD::Suld1DI64Clamp: 3567276479Sdim Opc = NVPTX::SULD_1D_I64_CLAMP; 3568276479Sdim Ops.push_back(TexHandle); 3569276479Sdim Ops.push_back(N->getOperand(2)); 3570276479Sdim Ops.push_back(Chain); 3571276479Sdim break; 3572276479Sdim case NVPTXISD::Suld1DV2I8Clamp: 3573276479Sdim Opc = NVPTX::SULD_1D_V2I8_CLAMP; 3574276479Sdim Ops.push_back(TexHandle); 3575276479Sdim Ops.push_back(N->getOperand(2)); 3576276479Sdim Ops.push_back(Chain); 3577276479Sdim break; 3578276479Sdim case NVPTXISD::Suld1DV2I16Clamp: 3579276479Sdim Opc = NVPTX::SULD_1D_V2I16_CLAMP; 3580276479Sdim Ops.push_back(TexHandle); 3581276479Sdim Ops.push_back(N->getOperand(2)); 3582276479Sdim Ops.push_back(Chain); 3583276479Sdim break; 3584276479Sdim case NVPTXISD::Suld1DV2I32Clamp: 3585276479Sdim Opc = NVPTX::SULD_1D_V2I32_CLAMP; 3586276479Sdim Ops.push_back(TexHandle); 3587276479Sdim Ops.push_back(N->getOperand(2)); 3588276479Sdim Ops.push_back(Chain); 3589276479Sdim break; 3590276479Sdim case NVPTXISD::Suld1DV2I64Clamp: 3591276479Sdim Opc = NVPTX::SULD_1D_V2I64_CLAMP; 3592276479Sdim Ops.push_back(TexHandle); 3593276479Sdim Ops.push_back(N->getOperand(2)); 3594276479Sdim Ops.push_back(Chain); 3595276479Sdim break; 3596276479Sdim case NVPTXISD::Suld1DV4I8Clamp: 3597276479Sdim Opc = NVPTX::SULD_1D_V4I8_CLAMP; 3598276479Sdim Ops.push_back(TexHandle); 3599276479Sdim Ops.push_back(N->getOperand(2)); 3600276479Sdim Ops.push_back(Chain); 3601276479Sdim break; 3602276479Sdim case NVPTXISD::Suld1DV4I16Clamp: 3603276479Sdim Opc = NVPTX::SULD_1D_V4I16_CLAMP; 3604276479Sdim Ops.push_back(TexHandle); 3605276479Sdim Ops.push_back(N->getOperand(2)); 3606276479Sdim Ops.push_back(Chain); 3607276479Sdim break; 3608276479Sdim case NVPTXISD::Suld1DV4I32Clamp: 3609276479Sdim Opc = NVPTX::SULD_1D_V4I32_CLAMP; 3610276479Sdim Ops.push_back(TexHandle); 3611276479Sdim Ops.push_back(N->getOperand(2)); 3612276479Sdim Ops.push_back(Chain); 3613276479Sdim break; 3614276479Sdim case NVPTXISD::Suld1DArrayI8Clamp: 3615276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I8_CLAMP; 3616276479Sdim Ops.push_back(TexHandle); 3617276479Sdim Ops.push_back(N->getOperand(2)); 3618276479Sdim Ops.push_back(N->getOperand(3)); 3619276479Sdim Ops.push_back(Chain); 3620276479Sdim break; 3621276479Sdim case NVPTXISD::Suld1DArrayI16Clamp: 3622276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I16_CLAMP; 3623276479Sdim Ops.push_back(TexHandle); 3624276479Sdim Ops.push_back(N->getOperand(2)); 3625276479Sdim Ops.push_back(N->getOperand(3)); 3626276479Sdim Ops.push_back(Chain); 3627276479Sdim break; 3628276479Sdim case NVPTXISD::Suld1DArrayI32Clamp: 3629276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I32_CLAMP; 3630276479Sdim Ops.push_back(TexHandle); 3631276479Sdim Ops.push_back(N->getOperand(2)); 3632276479Sdim Ops.push_back(N->getOperand(3)); 3633276479Sdim Ops.push_back(Chain); 3634276479Sdim break; 3635276479Sdim case NVPTXISD::Suld1DArrayI64Clamp: 3636276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I64_CLAMP; 3637276479Sdim Ops.push_back(TexHandle); 3638276479Sdim Ops.push_back(N->getOperand(2)); 3639276479Sdim Ops.push_back(N->getOperand(3)); 3640276479Sdim Ops.push_back(Chain); 3641276479Sdim break; 3642276479Sdim case NVPTXISD::Suld1DArrayV2I8Clamp: 3643276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I8_CLAMP; 3644276479Sdim Ops.push_back(TexHandle); 3645276479Sdim Ops.push_back(N->getOperand(2)); 3646276479Sdim Ops.push_back(N->getOperand(3)); 3647276479Sdim Ops.push_back(Chain); 3648276479Sdim break; 3649276479Sdim case NVPTXISD::Suld1DArrayV2I16Clamp: 3650276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I16_CLAMP; 3651276479Sdim Ops.push_back(TexHandle); 3652276479Sdim Ops.push_back(N->getOperand(2)); 3653276479Sdim Ops.push_back(N->getOperand(3)); 3654276479Sdim Ops.push_back(Chain); 3655276479Sdim break; 3656276479Sdim case NVPTXISD::Suld1DArrayV2I32Clamp: 3657276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I32_CLAMP; 3658276479Sdim Ops.push_back(TexHandle); 3659276479Sdim Ops.push_back(N->getOperand(2)); 3660276479Sdim Ops.push_back(N->getOperand(3)); 3661276479Sdim Ops.push_back(Chain); 3662276479Sdim break; 3663276479Sdim case NVPTXISD::Suld1DArrayV2I64Clamp: 3664276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I64_CLAMP; 3665276479Sdim Ops.push_back(TexHandle); 3666276479Sdim Ops.push_back(N->getOperand(2)); 3667276479Sdim Ops.push_back(N->getOperand(3)); 3668276479Sdim Ops.push_back(Chain); 3669276479Sdim break; 3670276479Sdim case NVPTXISD::Suld1DArrayV4I8Clamp: 3671276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I8_CLAMP; 3672276479Sdim Ops.push_back(TexHandle); 3673276479Sdim Ops.push_back(N->getOperand(2)); 3674276479Sdim Ops.push_back(N->getOperand(3)); 3675276479Sdim Ops.push_back(Chain); 3676276479Sdim break; 3677276479Sdim case NVPTXISD::Suld1DArrayV4I16Clamp: 3678276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I16_CLAMP; 3679276479Sdim Ops.push_back(TexHandle); 3680276479Sdim Ops.push_back(N->getOperand(2)); 3681276479Sdim Ops.push_back(N->getOperand(3)); 3682276479Sdim Ops.push_back(Chain); 3683276479Sdim break; 3684276479Sdim case NVPTXISD::Suld1DArrayV4I32Clamp: 3685276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I32_CLAMP; 3686276479Sdim Ops.push_back(TexHandle); 3687276479Sdim Ops.push_back(N->getOperand(2)); 3688276479Sdim Ops.push_back(N->getOperand(3)); 3689276479Sdim Ops.push_back(Chain); 3690276479Sdim break; 3691276479Sdim case NVPTXISD::Suld2DI8Clamp: 3692276479Sdim Opc = NVPTX::SULD_2D_I8_CLAMP; 3693276479Sdim Ops.push_back(TexHandle); 3694276479Sdim Ops.push_back(N->getOperand(2)); 3695276479Sdim Ops.push_back(N->getOperand(3)); 3696276479Sdim Ops.push_back(Chain); 3697276479Sdim break; 3698276479Sdim case NVPTXISD::Suld2DI16Clamp: 3699276479Sdim Opc = NVPTX::SULD_2D_I16_CLAMP; 3700276479Sdim Ops.push_back(TexHandle); 3701276479Sdim Ops.push_back(N->getOperand(2)); 3702276479Sdim Ops.push_back(N->getOperand(3)); 3703276479Sdim Ops.push_back(Chain); 3704276479Sdim break; 3705276479Sdim case NVPTXISD::Suld2DI32Clamp: 3706276479Sdim Opc = NVPTX::SULD_2D_I32_CLAMP; 3707276479Sdim Ops.push_back(TexHandle); 3708276479Sdim Ops.push_back(N->getOperand(2)); 3709276479Sdim Ops.push_back(N->getOperand(3)); 3710276479Sdim Ops.push_back(Chain); 3711276479Sdim break; 3712276479Sdim case NVPTXISD::Suld2DI64Clamp: 3713276479Sdim Opc = NVPTX::SULD_2D_I64_CLAMP; 3714276479Sdim Ops.push_back(TexHandle); 3715276479Sdim Ops.push_back(N->getOperand(2)); 3716276479Sdim Ops.push_back(N->getOperand(3)); 3717276479Sdim Ops.push_back(Chain); 3718276479Sdim break; 3719276479Sdim case NVPTXISD::Suld2DV2I8Clamp: 3720276479Sdim Opc = NVPTX::SULD_2D_V2I8_CLAMP; 3721276479Sdim Ops.push_back(TexHandle); 3722276479Sdim Ops.push_back(N->getOperand(2)); 3723276479Sdim Ops.push_back(N->getOperand(3)); 3724276479Sdim Ops.push_back(Chain); 3725276479Sdim break; 3726276479Sdim case NVPTXISD::Suld2DV2I16Clamp: 3727276479Sdim Opc = NVPTX::SULD_2D_V2I16_CLAMP; 3728276479Sdim Ops.push_back(TexHandle); 3729276479Sdim Ops.push_back(N->getOperand(2)); 3730276479Sdim Ops.push_back(N->getOperand(3)); 3731276479Sdim Ops.push_back(Chain); 3732276479Sdim break; 3733276479Sdim case NVPTXISD::Suld2DV2I32Clamp: 3734276479Sdim Opc = NVPTX::SULD_2D_V2I32_CLAMP; 3735276479Sdim Ops.push_back(TexHandle); 3736276479Sdim Ops.push_back(N->getOperand(2)); 3737276479Sdim Ops.push_back(N->getOperand(3)); 3738276479Sdim Ops.push_back(Chain); 3739276479Sdim break; 3740276479Sdim case NVPTXISD::Suld2DV2I64Clamp: 3741276479Sdim Opc = NVPTX::SULD_2D_V2I64_CLAMP; 3742276479Sdim Ops.push_back(TexHandle); 3743276479Sdim Ops.push_back(N->getOperand(2)); 3744276479Sdim Ops.push_back(N->getOperand(3)); 3745276479Sdim Ops.push_back(Chain); 3746276479Sdim break; 3747276479Sdim case NVPTXISD::Suld2DV4I8Clamp: 3748276479Sdim Opc = NVPTX::SULD_2D_V4I8_CLAMP; 3749276479Sdim Ops.push_back(TexHandle); 3750276479Sdim Ops.push_back(N->getOperand(2)); 3751276479Sdim Ops.push_back(N->getOperand(3)); 3752276479Sdim Ops.push_back(Chain); 3753276479Sdim break; 3754276479Sdim case NVPTXISD::Suld2DV4I16Clamp: 3755276479Sdim Opc = NVPTX::SULD_2D_V4I16_CLAMP; 3756276479Sdim Ops.push_back(TexHandle); 3757276479Sdim Ops.push_back(N->getOperand(2)); 3758276479Sdim Ops.push_back(N->getOperand(3)); 3759276479Sdim Ops.push_back(Chain); 3760276479Sdim break; 3761276479Sdim case NVPTXISD::Suld2DV4I32Clamp: 3762276479Sdim Opc = NVPTX::SULD_2D_V4I32_CLAMP; 3763276479Sdim Ops.push_back(TexHandle); 3764276479Sdim Ops.push_back(N->getOperand(2)); 3765276479Sdim Ops.push_back(N->getOperand(3)); 3766276479Sdim Ops.push_back(Chain); 3767276479Sdim break; 3768276479Sdim case NVPTXISD::Suld2DArrayI8Clamp: 3769276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I8_CLAMP; 3770276479Sdim Ops.push_back(TexHandle); 3771276479Sdim Ops.push_back(N->getOperand(2)); 3772276479Sdim Ops.push_back(N->getOperand(3)); 3773276479Sdim Ops.push_back(N->getOperand(4)); 3774276479Sdim Ops.push_back(Chain); 3775276479Sdim break; 3776276479Sdim case NVPTXISD::Suld2DArrayI16Clamp: 3777276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I16_CLAMP; 3778276479Sdim Ops.push_back(TexHandle); 3779276479Sdim Ops.push_back(N->getOperand(2)); 3780276479Sdim Ops.push_back(N->getOperand(3)); 3781276479Sdim Ops.push_back(N->getOperand(4)); 3782276479Sdim Ops.push_back(Chain); 3783276479Sdim break; 3784276479Sdim case NVPTXISD::Suld2DArrayI32Clamp: 3785276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I32_CLAMP; 3786276479Sdim Ops.push_back(TexHandle); 3787276479Sdim Ops.push_back(N->getOperand(2)); 3788276479Sdim Ops.push_back(N->getOperand(3)); 3789276479Sdim Ops.push_back(N->getOperand(4)); 3790276479Sdim Ops.push_back(Chain); 3791276479Sdim break; 3792276479Sdim case NVPTXISD::Suld2DArrayI64Clamp: 3793276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I64_CLAMP; 3794276479Sdim Ops.push_back(TexHandle); 3795276479Sdim Ops.push_back(N->getOperand(2)); 3796276479Sdim Ops.push_back(N->getOperand(3)); 3797276479Sdim Ops.push_back(N->getOperand(4)); 3798276479Sdim Ops.push_back(Chain); 3799276479Sdim break; 3800276479Sdim case NVPTXISD::Suld2DArrayV2I8Clamp: 3801276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I8_CLAMP; 3802276479Sdim Ops.push_back(TexHandle); 3803276479Sdim Ops.push_back(N->getOperand(2)); 3804276479Sdim Ops.push_back(N->getOperand(3)); 3805276479Sdim Ops.push_back(N->getOperand(4)); 3806276479Sdim Ops.push_back(Chain); 3807276479Sdim break; 3808276479Sdim case NVPTXISD::Suld2DArrayV2I16Clamp: 3809276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I16_CLAMP; 3810276479Sdim Ops.push_back(TexHandle); 3811276479Sdim Ops.push_back(N->getOperand(2)); 3812276479Sdim Ops.push_back(N->getOperand(3)); 3813276479Sdim Ops.push_back(N->getOperand(4)); 3814276479Sdim Ops.push_back(Chain); 3815276479Sdim break; 3816276479Sdim case NVPTXISD::Suld2DArrayV2I32Clamp: 3817276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I32_CLAMP; 3818276479Sdim Ops.push_back(TexHandle); 3819276479Sdim Ops.push_back(N->getOperand(2)); 3820276479Sdim Ops.push_back(N->getOperand(3)); 3821276479Sdim Ops.push_back(N->getOperand(4)); 3822276479Sdim Ops.push_back(Chain); 3823276479Sdim break; 3824276479Sdim case NVPTXISD::Suld2DArrayV2I64Clamp: 3825276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I64_CLAMP; 3826276479Sdim Ops.push_back(TexHandle); 3827276479Sdim Ops.push_back(N->getOperand(2)); 3828276479Sdim Ops.push_back(N->getOperand(3)); 3829276479Sdim Ops.push_back(N->getOperand(4)); 3830276479Sdim Ops.push_back(Chain); 3831276479Sdim break; 3832276479Sdim case NVPTXISD::Suld2DArrayV4I8Clamp: 3833276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I8_CLAMP; 3834276479Sdim Ops.push_back(TexHandle); 3835276479Sdim Ops.push_back(N->getOperand(2)); 3836276479Sdim Ops.push_back(N->getOperand(3)); 3837276479Sdim Ops.push_back(N->getOperand(4)); 3838276479Sdim Ops.push_back(Chain); 3839276479Sdim break; 3840276479Sdim case NVPTXISD::Suld2DArrayV4I16Clamp: 3841276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I16_CLAMP; 3842276479Sdim Ops.push_back(TexHandle); 3843276479Sdim Ops.push_back(N->getOperand(2)); 3844276479Sdim Ops.push_back(N->getOperand(3)); 3845276479Sdim Ops.push_back(N->getOperand(4)); 3846276479Sdim Ops.push_back(Chain); 3847276479Sdim break; 3848276479Sdim case NVPTXISD::Suld2DArrayV4I32Clamp: 3849276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I32_CLAMP; 3850276479Sdim Ops.push_back(TexHandle); 3851276479Sdim Ops.push_back(N->getOperand(2)); 3852276479Sdim Ops.push_back(N->getOperand(3)); 3853276479Sdim Ops.push_back(N->getOperand(4)); 3854276479Sdim Ops.push_back(Chain); 3855276479Sdim break; 3856276479Sdim case NVPTXISD::Suld3DI8Clamp: 3857276479Sdim Opc = NVPTX::SULD_3D_I8_CLAMP; 3858276479Sdim Ops.push_back(TexHandle); 3859276479Sdim Ops.push_back(N->getOperand(2)); 3860276479Sdim Ops.push_back(N->getOperand(3)); 3861276479Sdim Ops.push_back(N->getOperand(4)); 3862276479Sdim Ops.push_back(Chain); 3863276479Sdim break; 3864276479Sdim case NVPTXISD::Suld3DI16Clamp: 3865276479Sdim Opc = NVPTX::SULD_3D_I16_CLAMP; 3866276479Sdim Ops.push_back(TexHandle); 3867276479Sdim Ops.push_back(N->getOperand(2)); 3868276479Sdim Ops.push_back(N->getOperand(3)); 3869276479Sdim Ops.push_back(N->getOperand(4)); 3870276479Sdim Ops.push_back(Chain); 3871276479Sdim break; 3872276479Sdim case NVPTXISD::Suld3DI32Clamp: 3873276479Sdim Opc = NVPTX::SULD_3D_I32_CLAMP; 3874276479Sdim Ops.push_back(TexHandle); 3875276479Sdim Ops.push_back(N->getOperand(2)); 3876276479Sdim Ops.push_back(N->getOperand(3)); 3877276479Sdim Ops.push_back(N->getOperand(4)); 3878276479Sdim Ops.push_back(Chain); 3879276479Sdim break; 3880276479Sdim case NVPTXISD::Suld3DI64Clamp: 3881276479Sdim Opc = NVPTX::SULD_3D_I64_CLAMP; 3882276479Sdim Ops.push_back(TexHandle); 3883276479Sdim Ops.push_back(N->getOperand(2)); 3884276479Sdim Ops.push_back(N->getOperand(3)); 3885276479Sdim Ops.push_back(N->getOperand(4)); 3886276479Sdim Ops.push_back(Chain); 3887276479Sdim break; 3888276479Sdim case NVPTXISD::Suld3DV2I8Clamp: 3889276479Sdim Opc = NVPTX::SULD_3D_V2I8_CLAMP; 3890276479Sdim Ops.push_back(TexHandle); 3891276479Sdim Ops.push_back(N->getOperand(2)); 3892276479Sdim Ops.push_back(N->getOperand(3)); 3893276479Sdim Ops.push_back(N->getOperand(4)); 3894276479Sdim Ops.push_back(Chain); 3895276479Sdim break; 3896276479Sdim case NVPTXISD::Suld3DV2I16Clamp: 3897276479Sdim Opc = NVPTX::SULD_3D_V2I16_CLAMP; 3898276479Sdim Ops.push_back(TexHandle); 3899276479Sdim Ops.push_back(N->getOperand(2)); 3900276479Sdim Ops.push_back(N->getOperand(3)); 3901276479Sdim Ops.push_back(N->getOperand(4)); 3902276479Sdim Ops.push_back(Chain); 3903276479Sdim break; 3904276479Sdim case NVPTXISD::Suld3DV2I32Clamp: 3905276479Sdim Opc = NVPTX::SULD_3D_V2I32_CLAMP; 3906276479Sdim Ops.push_back(TexHandle); 3907276479Sdim Ops.push_back(N->getOperand(2)); 3908276479Sdim Ops.push_back(N->getOperand(3)); 3909276479Sdim Ops.push_back(N->getOperand(4)); 3910276479Sdim Ops.push_back(Chain); 3911276479Sdim break; 3912276479Sdim case NVPTXISD::Suld3DV2I64Clamp: 3913276479Sdim Opc = NVPTX::SULD_3D_V2I64_CLAMP; 3914276479Sdim Ops.push_back(TexHandle); 3915276479Sdim Ops.push_back(N->getOperand(2)); 3916276479Sdim Ops.push_back(N->getOperand(3)); 3917276479Sdim Ops.push_back(N->getOperand(4)); 3918276479Sdim Ops.push_back(Chain); 3919276479Sdim break; 3920276479Sdim case NVPTXISD::Suld3DV4I8Clamp: 3921276479Sdim Opc = NVPTX::SULD_3D_V4I8_CLAMP; 3922276479Sdim Ops.push_back(TexHandle); 3923276479Sdim Ops.push_back(N->getOperand(2)); 3924276479Sdim Ops.push_back(N->getOperand(3)); 3925276479Sdim Ops.push_back(N->getOperand(4)); 3926276479Sdim Ops.push_back(Chain); 3927276479Sdim break; 3928276479Sdim case NVPTXISD::Suld3DV4I16Clamp: 3929276479Sdim Opc = NVPTX::SULD_3D_V4I16_CLAMP; 3930276479Sdim Ops.push_back(TexHandle); 3931276479Sdim Ops.push_back(N->getOperand(2)); 3932276479Sdim Ops.push_back(N->getOperand(3)); 3933276479Sdim Ops.push_back(N->getOperand(4)); 3934276479Sdim Ops.push_back(Chain); 3935276479Sdim break; 3936276479Sdim case NVPTXISD::Suld3DV4I32Clamp: 3937276479Sdim Opc = NVPTX::SULD_3D_V4I32_CLAMP; 3938276479Sdim Ops.push_back(TexHandle); 3939276479Sdim Ops.push_back(N->getOperand(2)); 3940276479Sdim Ops.push_back(N->getOperand(3)); 3941276479Sdim Ops.push_back(N->getOperand(4)); 3942276479Sdim Ops.push_back(Chain); 3943276479Sdim break; 3944276479Sdim case NVPTXISD::Suld1DI8Trap: 3945276479Sdim Opc = NVPTX::SULD_1D_I8_TRAP; 3946276479Sdim Ops.push_back(TexHandle); 3947276479Sdim Ops.push_back(N->getOperand(2)); 3948276479Sdim Ops.push_back(Chain); 3949276479Sdim break; 3950276479Sdim case NVPTXISD::Suld1DI16Trap: 3951276479Sdim Opc = NVPTX::SULD_1D_I16_TRAP; 3952276479Sdim Ops.push_back(TexHandle); 3953276479Sdim Ops.push_back(N->getOperand(2)); 3954276479Sdim Ops.push_back(Chain); 3955276479Sdim break; 3956276479Sdim case NVPTXISD::Suld1DI32Trap: 3957276479Sdim Opc = NVPTX::SULD_1D_I32_TRAP; 3958276479Sdim Ops.push_back(TexHandle); 3959276479Sdim Ops.push_back(N->getOperand(2)); 3960276479Sdim Ops.push_back(Chain); 3961276479Sdim break; 3962276479Sdim case NVPTXISD::Suld1DI64Trap: 3963276479Sdim Opc = NVPTX::SULD_1D_I64_TRAP; 3964276479Sdim Ops.push_back(TexHandle); 3965276479Sdim Ops.push_back(N->getOperand(2)); 3966276479Sdim Ops.push_back(Chain); 3967276479Sdim break; 3968276479Sdim case NVPTXISD::Suld1DV2I8Trap: 3969276479Sdim Opc = NVPTX::SULD_1D_V2I8_TRAP; 3970276479Sdim Ops.push_back(TexHandle); 3971276479Sdim Ops.push_back(N->getOperand(2)); 3972276479Sdim Ops.push_back(Chain); 3973276479Sdim break; 3974276479Sdim case NVPTXISD::Suld1DV2I16Trap: 3975276479Sdim Opc = NVPTX::SULD_1D_V2I16_TRAP; 3976276479Sdim Ops.push_back(TexHandle); 3977276479Sdim Ops.push_back(N->getOperand(2)); 3978276479Sdim Ops.push_back(Chain); 3979276479Sdim break; 3980276479Sdim case NVPTXISD::Suld1DV2I32Trap: 3981276479Sdim Opc = NVPTX::SULD_1D_V2I32_TRAP; 3982276479Sdim Ops.push_back(TexHandle); 3983276479Sdim Ops.push_back(N->getOperand(2)); 3984276479Sdim Ops.push_back(Chain); 3985276479Sdim break; 3986276479Sdim case NVPTXISD::Suld1DV2I64Trap: 3987276479Sdim Opc = NVPTX::SULD_1D_V2I64_TRAP; 3988276479Sdim Ops.push_back(TexHandle); 3989276479Sdim Ops.push_back(N->getOperand(2)); 3990276479Sdim Ops.push_back(Chain); 3991276479Sdim break; 3992276479Sdim case NVPTXISD::Suld1DV4I8Trap: 3993276479Sdim Opc = NVPTX::SULD_1D_V4I8_TRAP; 3994276479Sdim Ops.push_back(TexHandle); 3995276479Sdim Ops.push_back(N->getOperand(2)); 3996276479Sdim Ops.push_back(Chain); 3997276479Sdim break; 3998276479Sdim case NVPTXISD::Suld1DV4I16Trap: 3999276479Sdim Opc = NVPTX::SULD_1D_V4I16_TRAP; 4000276479Sdim Ops.push_back(TexHandle); 4001276479Sdim Ops.push_back(N->getOperand(2)); 4002276479Sdim Ops.push_back(Chain); 4003276479Sdim break; 4004276479Sdim case NVPTXISD::Suld1DV4I32Trap: 4005276479Sdim Opc = NVPTX::SULD_1D_V4I32_TRAP; 4006276479Sdim Ops.push_back(TexHandle); 4007276479Sdim Ops.push_back(N->getOperand(2)); 4008276479Sdim Ops.push_back(Chain); 4009276479Sdim break; 4010276479Sdim case NVPTXISD::Suld1DArrayI8Trap: 4011276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP; 4012276479Sdim Ops.push_back(TexHandle); 4013276479Sdim Ops.push_back(N->getOperand(2)); 4014276479Sdim Ops.push_back(N->getOperand(3)); 4015276479Sdim Ops.push_back(Chain); 4016276479Sdim break; 4017276479Sdim case NVPTXISD::Suld1DArrayI16Trap: 4018276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP; 4019276479Sdim Ops.push_back(TexHandle); 4020276479Sdim Ops.push_back(N->getOperand(2)); 4021276479Sdim Ops.push_back(N->getOperand(3)); 4022276479Sdim Ops.push_back(Chain); 4023276479Sdim break; 4024276479Sdim case NVPTXISD::Suld1DArrayI32Trap: 4025276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP; 4026276479Sdim Ops.push_back(TexHandle); 4027276479Sdim Ops.push_back(N->getOperand(2)); 4028276479Sdim Ops.push_back(N->getOperand(3)); 4029276479Sdim Ops.push_back(Chain); 4030276479Sdim break; 4031276479Sdim case NVPTXISD::Suld1DArrayI64Trap: 4032276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I64_TRAP; 4033276479Sdim Ops.push_back(TexHandle); 4034276479Sdim Ops.push_back(N->getOperand(2)); 4035276479Sdim Ops.push_back(N->getOperand(3)); 4036276479Sdim Ops.push_back(Chain); 4037276479Sdim break; 4038276479Sdim case NVPTXISD::Suld1DArrayV2I8Trap: 4039276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP; 4040276479Sdim Ops.push_back(TexHandle); 4041276479Sdim Ops.push_back(N->getOperand(2)); 4042276479Sdim Ops.push_back(N->getOperand(3)); 4043276479Sdim Ops.push_back(Chain); 4044276479Sdim break; 4045276479Sdim case NVPTXISD::Suld1DArrayV2I16Trap: 4046276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP; 4047276479Sdim Ops.push_back(TexHandle); 4048276479Sdim Ops.push_back(N->getOperand(2)); 4049276479Sdim Ops.push_back(N->getOperand(3)); 4050276479Sdim Ops.push_back(Chain); 4051276479Sdim break; 4052276479Sdim case NVPTXISD::Suld1DArrayV2I32Trap: 4053276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP; 4054276479Sdim Ops.push_back(TexHandle); 4055276479Sdim Ops.push_back(N->getOperand(2)); 4056276479Sdim Ops.push_back(N->getOperand(3)); 4057276479Sdim Ops.push_back(Chain); 4058276479Sdim break; 4059276479Sdim case NVPTXISD::Suld1DArrayV2I64Trap: 4060276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I64_TRAP; 4061276479Sdim Ops.push_back(TexHandle); 4062276479Sdim Ops.push_back(N->getOperand(2)); 4063276479Sdim Ops.push_back(N->getOperand(3)); 4064276479Sdim Ops.push_back(Chain); 4065276479Sdim break; 4066276479Sdim case NVPTXISD::Suld1DArrayV4I8Trap: 4067276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP; 4068276479Sdim Ops.push_back(TexHandle); 4069276479Sdim Ops.push_back(N->getOperand(2)); 4070276479Sdim Ops.push_back(N->getOperand(3)); 4071276479Sdim Ops.push_back(Chain); 4072276479Sdim break; 4073276479Sdim case NVPTXISD::Suld1DArrayV4I16Trap: 4074276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP; 4075276479Sdim Ops.push_back(TexHandle); 4076276479Sdim Ops.push_back(N->getOperand(2)); 4077276479Sdim Ops.push_back(N->getOperand(3)); 4078276479Sdim Ops.push_back(Chain); 4079276479Sdim break; 4080276479Sdim case NVPTXISD::Suld1DArrayV4I32Trap: 4081276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP; 4082276479Sdim Ops.push_back(TexHandle); 4083276479Sdim Ops.push_back(N->getOperand(2)); 4084276479Sdim Ops.push_back(N->getOperand(3)); 4085276479Sdim Ops.push_back(Chain); 4086276479Sdim break; 4087276479Sdim case NVPTXISD::Suld2DI8Trap: 4088276479Sdim Opc = NVPTX::SULD_2D_I8_TRAP; 4089276479Sdim Ops.push_back(TexHandle); 4090276479Sdim Ops.push_back(N->getOperand(2)); 4091276479Sdim Ops.push_back(N->getOperand(3)); 4092276479Sdim Ops.push_back(Chain); 4093276479Sdim break; 4094276479Sdim case NVPTXISD::Suld2DI16Trap: 4095276479Sdim Opc = NVPTX::SULD_2D_I16_TRAP; 4096276479Sdim Ops.push_back(TexHandle); 4097276479Sdim Ops.push_back(N->getOperand(2)); 4098276479Sdim Ops.push_back(N->getOperand(3)); 4099276479Sdim Ops.push_back(Chain); 4100276479Sdim break; 4101276479Sdim case NVPTXISD::Suld2DI32Trap: 4102276479Sdim Opc = NVPTX::SULD_2D_I32_TRAP; 4103276479Sdim Ops.push_back(TexHandle); 4104276479Sdim Ops.push_back(N->getOperand(2)); 4105276479Sdim Ops.push_back(N->getOperand(3)); 4106276479Sdim Ops.push_back(Chain); 4107276479Sdim break; 4108276479Sdim case NVPTXISD::Suld2DI64Trap: 4109276479Sdim Opc = NVPTX::SULD_2D_I64_TRAP; 4110276479Sdim Ops.push_back(TexHandle); 4111276479Sdim Ops.push_back(N->getOperand(2)); 4112276479Sdim Ops.push_back(N->getOperand(3)); 4113276479Sdim Ops.push_back(Chain); 4114276479Sdim break; 4115276479Sdim case NVPTXISD::Suld2DV2I8Trap: 4116276479Sdim Opc = NVPTX::SULD_2D_V2I8_TRAP; 4117276479Sdim Ops.push_back(TexHandle); 4118276479Sdim Ops.push_back(N->getOperand(2)); 4119276479Sdim Ops.push_back(N->getOperand(3)); 4120276479Sdim Ops.push_back(Chain); 4121276479Sdim break; 4122276479Sdim case NVPTXISD::Suld2DV2I16Trap: 4123276479Sdim Opc = NVPTX::SULD_2D_V2I16_TRAP; 4124276479Sdim Ops.push_back(TexHandle); 4125276479Sdim Ops.push_back(N->getOperand(2)); 4126276479Sdim Ops.push_back(N->getOperand(3)); 4127276479Sdim Ops.push_back(Chain); 4128276479Sdim break; 4129276479Sdim case NVPTXISD::Suld2DV2I32Trap: 4130276479Sdim Opc = NVPTX::SULD_2D_V2I32_TRAP; 4131276479Sdim Ops.push_back(TexHandle); 4132276479Sdim Ops.push_back(N->getOperand(2)); 4133276479Sdim Ops.push_back(N->getOperand(3)); 4134276479Sdim Ops.push_back(Chain); 4135276479Sdim break; 4136276479Sdim case NVPTXISD::Suld2DV2I64Trap: 4137276479Sdim Opc = NVPTX::SULD_2D_V2I64_TRAP; 4138276479Sdim Ops.push_back(TexHandle); 4139276479Sdim Ops.push_back(N->getOperand(2)); 4140276479Sdim Ops.push_back(N->getOperand(3)); 4141276479Sdim Ops.push_back(Chain); 4142276479Sdim break; 4143276479Sdim case NVPTXISD::Suld2DV4I8Trap: 4144276479Sdim Opc = NVPTX::SULD_2D_V4I8_TRAP; 4145276479Sdim Ops.push_back(TexHandle); 4146276479Sdim Ops.push_back(N->getOperand(2)); 4147276479Sdim Ops.push_back(N->getOperand(3)); 4148276479Sdim Ops.push_back(Chain); 4149276479Sdim break; 4150276479Sdim case NVPTXISD::Suld2DV4I16Trap: 4151276479Sdim Opc = NVPTX::SULD_2D_V4I16_TRAP; 4152276479Sdim Ops.push_back(TexHandle); 4153276479Sdim Ops.push_back(N->getOperand(2)); 4154276479Sdim Ops.push_back(N->getOperand(3)); 4155276479Sdim Ops.push_back(Chain); 4156276479Sdim break; 4157276479Sdim case NVPTXISD::Suld2DV4I32Trap: 4158276479Sdim Opc = NVPTX::SULD_2D_V4I32_TRAP; 4159276479Sdim Ops.push_back(TexHandle); 4160276479Sdim Ops.push_back(N->getOperand(2)); 4161276479Sdim Ops.push_back(N->getOperand(3)); 4162276479Sdim Ops.push_back(Chain); 4163276479Sdim break; 4164276479Sdim case NVPTXISD::Suld2DArrayI8Trap: 4165276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP; 4166276479Sdim Ops.push_back(TexHandle); 4167276479Sdim Ops.push_back(N->getOperand(2)); 4168276479Sdim Ops.push_back(N->getOperand(3)); 4169276479Sdim Ops.push_back(N->getOperand(4)); 4170276479Sdim Ops.push_back(Chain); 4171276479Sdim break; 4172276479Sdim case NVPTXISD::Suld2DArrayI16Trap: 4173276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP; 4174276479Sdim Ops.push_back(TexHandle); 4175276479Sdim Ops.push_back(N->getOperand(2)); 4176276479Sdim Ops.push_back(N->getOperand(3)); 4177276479Sdim Ops.push_back(N->getOperand(4)); 4178276479Sdim Ops.push_back(Chain); 4179276479Sdim break; 4180276479Sdim case NVPTXISD::Suld2DArrayI32Trap: 4181276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP; 4182276479Sdim Ops.push_back(TexHandle); 4183276479Sdim Ops.push_back(N->getOperand(2)); 4184276479Sdim Ops.push_back(N->getOperand(3)); 4185276479Sdim Ops.push_back(N->getOperand(4)); 4186276479Sdim Ops.push_back(Chain); 4187276479Sdim break; 4188276479Sdim case NVPTXISD::Suld2DArrayI64Trap: 4189276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I64_TRAP; 4190276479Sdim Ops.push_back(TexHandle); 4191276479Sdim Ops.push_back(N->getOperand(2)); 4192276479Sdim Ops.push_back(N->getOperand(3)); 4193276479Sdim Ops.push_back(N->getOperand(4)); 4194276479Sdim Ops.push_back(Chain); 4195276479Sdim break; 4196276479Sdim case NVPTXISD::Suld2DArrayV2I8Trap: 4197276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP; 4198276479Sdim Ops.push_back(TexHandle); 4199276479Sdim Ops.push_back(N->getOperand(2)); 4200276479Sdim Ops.push_back(N->getOperand(3)); 4201276479Sdim Ops.push_back(N->getOperand(4)); 4202276479Sdim Ops.push_back(Chain); 4203276479Sdim break; 4204276479Sdim case NVPTXISD::Suld2DArrayV2I16Trap: 4205276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP; 4206276479Sdim Ops.push_back(TexHandle); 4207276479Sdim Ops.push_back(N->getOperand(2)); 4208276479Sdim Ops.push_back(N->getOperand(3)); 4209276479Sdim Ops.push_back(N->getOperand(4)); 4210276479Sdim Ops.push_back(Chain); 4211276479Sdim break; 4212276479Sdim case NVPTXISD::Suld2DArrayV2I32Trap: 4213276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP; 4214276479Sdim Ops.push_back(TexHandle); 4215276479Sdim Ops.push_back(N->getOperand(2)); 4216276479Sdim Ops.push_back(N->getOperand(3)); 4217276479Sdim Ops.push_back(N->getOperand(4)); 4218276479Sdim Ops.push_back(Chain); 4219276479Sdim break; 4220276479Sdim case NVPTXISD::Suld2DArrayV2I64Trap: 4221276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I64_TRAP; 4222276479Sdim Ops.push_back(TexHandle); 4223276479Sdim Ops.push_back(N->getOperand(2)); 4224276479Sdim Ops.push_back(N->getOperand(3)); 4225276479Sdim Ops.push_back(N->getOperand(4)); 4226276479Sdim Ops.push_back(Chain); 4227276479Sdim break; 4228276479Sdim case NVPTXISD::Suld2DArrayV4I8Trap: 4229276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP; 4230276479Sdim Ops.push_back(TexHandle); 4231276479Sdim Ops.push_back(N->getOperand(2)); 4232276479Sdim Ops.push_back(N->getOperand(3)); 4233276479Sdim Ops.push_back(N->getOperand(4)); 4234276479Sdim Ops.push_back(Chain); 4235276479Sdim break; 4236276479Sdim case NVPTXISD::Suld2DArrayV4I16Trap: 4237276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP; 4238276479Sdim Ops.push_back(TexHandle); 4239276479Sdim Ops.push_back(N->getOperand(2)); 4240276479Sdim Ops.push_back(N->getOperand(3)); 4241276479Sdim Ops.push_back(N->getOperand(4)); 4242276479Sdim Ops.push_back(Chain); 4243276479Sdim break; 4244276479Sdim case NVPTXISD::Suld2DArrayV4I32Trap: 4245276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP; 4246276479Sdim Ops.push_back(TexHandle); 4247276479Sdim Ops.push_back(N->getOperand(2)); 4248276479Sdim Ops.push_back(N->getOperand(3)); 4249276479Sdim Ops.push_back(N->getOperand(4)); 4250276479Sdim Ops.push_back(Chain); 4251276479Sdim break; 4252276479Sdim case NVPTXISD::Suld3DI8Trap: 4253276479Sdim Opc = NVPTX::SULD_3D_I8_TRAP; 4254276479Sdim Ops.push_back(TexHandle); 4255276479Sdim Ops.push_back(N->getOperand(2)); 4256276479Sdim Ops.push_back(N->getOperand(3)); 4257276479Sdim Ops.push_back(N->getOperand(4)); 4258276479Sdim Ops.push_back(Chain); 4259276479Sdim break; 4260276479Sdim case NVPTXISD::Suld3DI16Trap: 4261276479Sdim Opc = NVPTX::SULD_3D_I16_TRAP; 4262276479Sdim Ops.push_back(TexHandle); 4263276479Sdim Ops.push_back(N->getOperand(2)); 4264276479Sdim Ops.push_back(N->getOperand(3)); 4265276479Sdim Ops.push_back(N->getOperand(4)); 4266276479Sdim Ops.push_back(Chain); 4267276479Sdim break; 4268276479Sdim case NVPTXISD::Suld3DI32Trap: 4269276479Sdim Opc = NVPTX::SULD_3D_I32_TRAP; 4270276479Sdim Ops.push_back(TexHandle); 4271276479Sdim Ops.push_back(N->getOperand(2)); 4272276479Sdim Ops.push_back(N->getOperand(3)); 4273276479Sdim Ops.push_back(N->getOperand(4)); 4274276479Sdim Ops.push_back(Chain); 4275276479Sdim break; 4276276479Sdim case NVPTXISD::Suld3DI64Trap: 4277276479Sdim Opc = NVPTX::SULD_3D_I64_TRAP; 4278276479Sdim Ops.push_back(TexHandle); 4279276479Sdim Ops.push_back(N->getOperand(2)); 4280276479Sdim Ops.push_back(N->getOperand(3)); 4281276479Sdim Ops.push_back(N->getOperand(4)); 4282276479Sdim Ops.push_back(Chain); 4283276479Sdim break; 4284276479Sdim case NVPTXISD::Suld3DV2I8Trap: 4285276479Sdim Opc = NVPTX::SULD_3D_V2I8_TRAP; 4286276479Sdim Ops.push_back(TexHandle); 4287276479Sdim Ops.push_back(N->getOperand(2)); 4288276479Sdim Ops.push_back(N->getOperand(3)); 4289276479Sdim Ops.push_back(N->getOperand(4)); 4290276479Sdim Ops.push_back(Chain); 4291276479Sdim break; 4292276479Sdim case NVPTXISD::Suld3DV2I16Trap: 4293276479Sdim Opc = NVPTX::SULD_3D_V2I16_TRAP; 4294276479Sdim Ops.push_back(TexHandle); 4295276479Sdim Ops.push_back(N->getOperand(2)); 4296276479Sdim Ops.push_back(N->getOperand(3)); 4297276479Sdim Ops.push_back(N->getOperand(4)); 4298276479Sdim Ops.push_back(Chain); 4299276479Sdim break; 4300276479Sdim case NVPTXISD::Suld3DV2I32Trap: 4301276479Sdim Opc = NVPTX::SULD_3D_V2I32_TRAP; 4302276479Sdim Ops.push_back(TexHandle); 4303276479Sdim Ops.push_back(N->getOperand(2)); 4304276479Sdim Ops.push_back(N->getOperand(3)); 4305276479Sdim Ops.push_back(N->getOperand(4)); 4306276479Sdim Ops.push_back(Chain); 4307276479Sdim break; 4308276479Sdim case NVPTXISD::Suld3DV2I64Trap: 4309276479Sdim Opc = NVPTX::SULD_3D_V2I64_TRAP; 4310276479Sdim Ops.push_back(TexHandle); 4311276479Sdim Ops.push_back(N->getOperand(2)); 4312276479Sdim Ops.push_back(N->getOperand(3)); 4313276479Sdim Ops.push_back(N->getOperand(4)); 4314276479Sdim Ops.push_back(Chain); 4315276479Sdim break; 4316276479Sdim case NVPTXISD::Suld3DV4I8Trap: 4317276479Sdim Opc = NVPTX::SULD_3D_V4I8_TRAP; 4318276479Sdim Ops.push_back(TexHandle); 4319276479Sdim Ops.push_back(N->getOperand(2)); 4320276479Sdim Ops.push_back(N->getOperand(3)); 4321276479Sdim Ops.push_back(N->getOperand(4)); 4322276479Sdim Ops.push_back(Chain); 4323276479Sdim break; 4324276479Sdim case NVPTXISD::Suld3DV4I16Trap: 4325276479Sdim Opc = NVPTX::SULD_3D_V4I16_TRAP; 4326276479Sdim Ops.push_back(TexHandle); 4327276479Sdim Ops.push_back(N->getOperand(2)); 4328276479Sdim Ops.push_back(N->getOperand(3)); 4329276479Sdim Ops.push_back(N->getOperand(4)); 4330276479Sdim Ops.push_back(Chain); 4331276479Sdim break; 4332276479Sdim case NVPTXISD::Suld3DV4I32Trap: 4333276479Sdim Opc = NVPTX::SULD_3D_V4I32_TRAP; 4334276479Sdim Ops.push_back(TexHandle); 4335276479Sdim Ops.push_back(N->getOperand(2)); 4336276479Sdim Ops.push_back(N->getOperand(3)); 4337276479Sdim Ops.push_back(N->getOperand(4)); 4338276479Sdim Ops.push_back(Chain); 4339276479Sdim break; 4340276479Sdim case NVPTXISD::Suld1DI8Zero: 4341276479Sdim Opc = NVPTX::SULD_1D_I8_ZERO; 4342276479Sdim Ops.push_back(TexHandle); 4343276479Sdim Ops.push_back(N->getOperand(2)); 4344276479Sdim Ops.push_back(Chain); 4345276479Sdim break; 4346276479Sdim case NVPTXISD::Suld1DI16Zero: 4347276479Sdim Opc = NVPTX::SULD_1D_I16_ZERO; 4348276479Sdim Ops.push_back(TexHandle); 4349276479Sdim Ops.push_back(N->getOperand(2)); 4350276479Sdim Ops.push_back(Chain); 4351276479Sdim break; 4352276479Sdim case NVPTXISD::Suld1DI32Zero: 4353276479Sdim Opc = NVPTX::SULD_1D_I32_ZERO; 4354276479Sdim Ops.push_back(TexHandle); 4355276479Sdim Ops.push_back(N->getOperand(2)); 4356276479Sdim Ops.push_back(Chain); 4357276479Sdim break; 4358276479Sdim case NVPTXISD::Suld1DI64Zero: 4359276479Sdim Opc = NVPTX::SULD_1D_I64_ZERO; 4360276479Sdim Ops.push_back(TexHandle); 4361276479Sdim Ops.push_back(N->getOperand(2)); 4362276479Sdim Ops.push_back(Chain); 4363276479Sdim break; 4364276479Sdim case NVPTXISD::Suld1DV2I8Zero: 4365276479Sdim Opc = NVPTX::SULD_1D_V2I8_ZERO; 4366276479Sdim Ops.push_back(TexHandle); 4367276479Sdim Ops.push_back(N->getOperand(2)); 4368276479Sdim Ops.push_back(Chain); 4369276479Sdim break; 4370276479Sdim case NVPTXISD::Suld1DV2I16Zero: 4371276479Sdim Opc = NVPTX::SULD_1D_V2I16_ZERO; 4372276479Sdim Ops.push_back(TexHandle); 4373276479Sdim Ops.push_back(N->getOperand(2)); 4374276479Sdim Ops.push_back(Chain); 4375276479Sdim break; 4376276479Sdim case NVPTXISD::Suld1DV2I32Zero: 4377276479Sdim Opc = NVPTX::SULD_1D_V2I32_ZERO; 4378276479Sdim Ops.push_back(TexHandle); 4379276479Sdim Ops.push_back(N->getOperand(2)); 4380276479Sdim Ops.push_back(Chain); 4381276479Sdim break; 4382276479Sdim case NVPTXISD::Suld1DV2I64Zero: 4383276479Sdim Opc = NVPTX::SULD_1D_V2I64_ZERO; 4384276479Sdim Ops.push_back(TexHandle); 4385276479Sdim Ops.push_back(N->getOperand(2)); 4386276479Sdim Ops.push_back(Chain); 4387276479Sdim break; 4388276479Sdim case NVPTXISD::Suld1DV4I8Zero: 4389276479Sdim Opc = NVPTX::SULD_1D_V4I8_ZERO; 4390276479Sdim Ops.push_back(TexHandle); 4391276479Sdim Ops.push_back(N->getOperand(2)); 4392276479Sdim Ops.push_back(Chain); 4393276479Sdim break; 4394276479Sdim case NVPTXISD::Suld1DV4I16Zero: 4395276479Sdim Opc = NVPTX::SULD_1D_V4I16_ZERO; 4396276479Sdim Ops.push_back(TexHandle); 4397276479Sdim Ops.push_back(N->getOperand(2)); 4398276479Sdim Ops.push_back(Chain); 4399276479Sdim break; 4400276479Sdim case NVPTXISD::Suld1DV4I32Zero: 4401276479Sdim Opc = NVPTX::SULD_1D_V4I32_ZERO; 4402276479Sdim Ops.push_back(TexHandle); 4403276479Sdim Ops.push_back(N->getOperand(2)); 4404276479Sdim Ops.push_back(Chain); 4405276479Sdim break; 4406276479Sdim case NVPTXISD::Suld1DArrayI8Zero: 4407276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I8_ZERO; 4408276479Sdim Ops.push_back(TexHandle); 4409276479Sdim Ops.push_back(N->getOperand(2)); 4410276479Sdim Ops.push_back(N->getOperand(3)); 4411276479Sdim Ops.push_back(Chain); 4412276479Sdim break; 4413276479Sdim case NVPTXISD::Suld1DArrayI16Zero: 4414276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I16_ZERO; 4415276479Sdim Ops.push_back(TexHandle); 4416276479Sdim Ops.push_back(N->getOperand(2)); 4417276479Sdim Ops.push_back(N->getOperand(3)); 4418276479Sdim Ops.push_back(Chain); 4419276479Sdim break; 4420276479Sdim case NVPTXISD::Suld1DArrayI32Zero: 4421276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I32_ZERO; 4422276479Sdim Ops.push_back(TexHandle); 4423276479Sdim Ops.push_back(N->getOperand(2)); 4424276479Sdim Ops.push_back(N->getOperand(3)); 4425276479Sdim Ops.push_back(Chain); 4426276479Sdim break; 4427276479Sdim case NVPTXISD::Suld1DArrayI64Zero: 4428276479Sdim Opc = NVPTX::SULD_1D_ARRAY_I64_ZERO; 4429276479Sdim Ops.push_back(TexHandle); 4430276479Sdim Ops.push_back(N->getOperand(2)); 4431276479Sdim Ops.push_back(N->getOperand(3)); 4432276479Sdim Ops.push_back(Chain); 4433276479Sdim break; 4434276479Sdim case NVPTXISD::Suld1DArrayV2I8Zero: 4435276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I8_ZERO; 4436276479Sdim Ops.push_back(TexHandle); 4437276479Sdim Ops.push_back(N->getOperand(2)); 4438276479Sdim Ops.push_back(N->getOperand(3)); 4439276479Sdim Ops.push_back(Chain); 4440276479Sdim break; 4441276479Sdim case NVPTXISD::Suld1DArrayV2I16Zero: 4442276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I16_ZERO; 4443276479Sdim Ops.push_back(TexHandle); 4444276479Sdim Ops.push_back(N->getOperand(2)); 4445276479Sdim Ops.push_back(N->getOperand(3)); 4446276479Sdim Ops.push_back(Chain); 4447276479Sdim break; 4448276479Sdim case NVPTXISD::Suld1DArrayV2I32Zero: 4449276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I32_ZERO; 4450276479Sdim Ops.push_back(TexHandle); 4451276479Sdim Ops.push_back(N->getOperand(2)); 4452276479Sdim Ops.push_back(N->getOperand(3)); 4453276479Sdim Ops.push_back(Chain); 4454276479Sdim break; 4455276479Sdim case NVPTXISD::Suld1DArrayV2I64Zero: 4456276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V2I64_ZERO; 4457276479Sdim Ops.push_back(TexHandle); 4458276479Sdim Ops.push_back(N->getOperand(2)); 4459276479Sdim Ops.push_back(N->getOperand(3)); 4460276479Sdim Ops.push_back(Chain); 4461276479Sdim break; 4462276479Sdim case NVPTXISD::Suld1DArrayV4I8Zero: 4463276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I8_ZERO; 4464276479Sdim Ops.push_back(TexHandle); 4465276479Sdim Ops.push_back(N->getOperand(2)); 4466276479Sdim Ops.push_back(N->getOperand(3)); 4467276479Sdim Ops.push_back(Chain); 4468276479Sdim break; 4469276479Sdim case NVPTXISD::Suld1DArrayV4I16Zero: 4470276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I16_ZERO; 4471276479Sdim Ops.push_back(TexHandle); 4472276479Sdim Ops.push_back(N->getOperand(2)); 4473276479Sdim Ops.push_back(N->getOperand(3)); 4474276479Sdim Ops.push_back(Chain); 4475276479Sdim break; 4476276479Sdim case NVPTXISD::Suld1DArrayV4I32Zero: 4477276479Sdim Opc = NVPTX::SULD_1D_ARRAY_V4I32_ZERO; 4478276479Sdim Ops.push_back(TexHandle); 4479276479Sdim Ops.push_back(N->getOperand(2)); 4480276479Sdim Ops.push_back(N->getOperand(3)); 4481276479Sdim Ops.push_back(Chain); 4482276479Sdim break; 4483276479Sdim case NVPTXISD::Suld2DI8Zero: 4484276479Sdim Opc = NVPTX::SULD_2D_I8_ZERO; 4485276479Sdim Ops.push_back(TexHandle); 4486276479Sdim Ops.push_back(N->getOperand(2)); 4487276479Sdim Ops.push_back(N->getOperand(3)); 4488276479Sdim Ops.push_back(Chain); 4489276479Sdim break; 4490276479Sdim case NVPTXISD::Suld2DI16Zero: 4491276479Sdim Opc = NVPTX::SULD_2D_I16_ZERO; 4492276479Sdim Ops.push_back(TexHandle); 4493276479Sdim Ops.push_back(N->getOperand(2)); 4494276479Sdim Ops.push_back(N->getOperand(3)); 4495276479Sdim Ops.push_back(Chain); 4496276479Sdim break; 4497276479Sdim case NVPTXISD::Suld2DI32Zero: 4498276479Sdim Opc = NVPTX::SULD_2D_I32_ZERO; 4499276479Sdim Ops.push_back(TexHandle); 4500276479Sdim Ops.push_back(N->getOperand(2)); 4501276479Sdim Ops.push_back(N->getOperand(3)); 4502276479Sdim Ops.push_back(Chain); 4503276479Sdim break; 4504276479Sdim case NVPTXISD::Suld2DI64Zero: 4505276479Sdim Opc = NVPTX::SULD_2D_I64_ZERO; 4506276479Sdim Ops.push_back(TexHandle); 4507276479Sdim Ops.push_back(N->getOperand(2)); 4508276479Sdim Ops.push_back(N->getOperand(3)); 4509276479Sdim Ops.push_back(Chain); 4510276479Sdim break; 4511276479Sdim case NVPTXISD::Suld2DV2I8Zero: 4512276479Sdim Opc = NVPTX::SULD_2D_V2I8_ZERO; 4513276479Sdim Ops.push_back(TexHandle); 4514276479Sdim Ops.push_back(N->getOperand(2)); 4515276479Sdim Ops.push_back(N->getOperand(3)); 4516276479Sdim Ops.push_back(Chain); 4517276479Sdim break; 4518276479Sdim case NVPTXISD::Suld2DV2I16Zero: 4519276479Sdim Opc = NVPTX::SULD_2D_V2I16_ZERO; 4520276479Sdim Ops.push_back(TexHandle); 4521276479Sdim Ops.push_back(N->getOperand(2)); 4522276479Sdim Ops.push_back(N->getOperand(3)); 4523276479Sdim Ops.push_back(Chain); 4524276479Sdim break; 4525276479Sdim case NVPTXISD::Suld2DV2I32Zero: 4526276479Sdim Opc = NVPTX::SULD_2D_V2I32_ZERO; 4527276479Sdim Ops.push_back(TexHandle); 4528276479Sdim Ops.push_back(N->getOperand(2)); 4529276479Sdim Ops.push_back(N->getOperand(3)); 4530276479Sdim Ops.push_back(Chain); 4531276479Sdim break; 4532276479Sdim case NVPTXISD::Suld2DV2I64Zero: 4533276479Sdim Opc = NVPTX::SULD_2D_V2I64_ZERO; 4534276479Sdim Ops.push_back(TexHandle); 4535276479Sdim Ops.push_back(N->getOperand(2)); 4536276479Sdim Ops.push_back(N->getOperand(3)); 4537276479Sdim Ops.push_back(Chain); 4538276479Sdim break; 4539276479Sdim case NVPTXISD::Suld2DV4I8Zero: 4540276479Sdim Opc = NVPTX::SULD_2D_V4I8_ZERO; 4541276479Sdim Ops.push_back(TexHandle); 4542276479Sdim Ops.push_back(N->getOperand(2)); 4543276479Sdim Ops.push_back(N->getOperand(3)); 4544276479Sdim Ops.push_back(Chain); 4545276479Sdim break; 4546276479Sdim case NVPTXISD::Suld2DV4I16Zero: 4547276479Sdim Opc = NVPTX::SULD_2D_V4I16_ZERO; 4548276479Sdim Ops.push_back(TexHandle); 4549276479Sdim Ops.push_back(N->getOperand(2)); 4550276479Sdim Ops.push_back(N->getOperand(3)); 4551276479Sdim Ops.push_back(Chain); 4552276479Sdim break; 4553276479Sdim case NVPTXISD::Suld2DV4I32Zero: 4554276479Sdim Opc = NVPTX::SULD_2D_V4I32_ZERO; 4555276479Sdim Ops.push_back(TexHandle); 4556276479Sdim Ops.push_back(N->getOperand(2)); 4557276479Sdim Ops.push_back(N->getOperand(3)); 4558276479Sdim Ops.push_back(Chain); 4559276479Sdim break; 4560276479Sdim case NVPTXISD::Suld2DArrayI8Zero: 4561276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I8_ZERO; 4562276479Sdim Ops.push_back(TexHandle); 4563276479Sdim Ops.push_back(N->getOperand(2)); 4564276479Sdim Ops.push_back(N->getOperand(3)); 4565276479Sdim Ops.push_back(N->getOperand(4)); 4566276479Sdim Ops.push_back(Chain); 4567276479Sdim break; 4568276479Sdim case NVPTXISD::Suld2DArrayI16Zero: 4569276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I16_ZERO; 4570276479Sdim Ops.push_back(TexHandle); 4571276479Sdim Ops.push_back(N->getOperand(2)); 4572276479Sdim Ops.push_back(N->getOperand(3)); 4573276479Sdim Ops.push_back(N->getOperand(4)); 4574276479Sdim Ops.push_back(Chain); 4575276479Sdim break; 4576276479Sdim case NVPTXISD::Suld2DArrayI32Zero: 4577276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I32_ZERO; 4578276479Sdim Ops.push_back(TexHandle); 4579276479Sdim Ops.push_back(N->getOperand(2)); 4580276479Sdim Ops.push_back(N->getOperand(3)); 4581276479Sdim Ops.push_back(N->getOperand(4)); 4582276479Sdim Ops.push_back(Chain); 4583276479Sdim break; 4584276479Sdim case NVPTXISD::Suld2DArrayI64Zero: 4585276479Sdim Opc = NVPTX::SULD_2D_ARRAY_I64_ZERO; 4586276479Sdim Ops.push_back(TexHandle); 4587276479Sdim Ops.push_back(N->getOperand(2)); 4588276479Sdim Ops.push_back(N->getOperand(3)); 4589276479Sdim Ops.push_back(N->getOperand(4)); 4590276479Sdim Ops.push_back(Chain); 4591276479Sdim break; 4592276479Sdim case NVPTXISD::Suld2DArrayV2I8Zero: 4593276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I8_ZERO; 4594276479Sdim Ops.push_back(TexHandle); 4595276479Sdim Ops.push_back(N->getOperand(2)); 4596276479Sdim Ops.push_back(N->getOperand(3)); 4597276479Sdim Ops.push_back(N->getOperand(4)); 4598276479Sdim Ops.push_back(Chain); 4599276479Sdim break; 4600276479Sdim case NVPTXISD::Suld2DArrayV2I16Zero: 4601276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I16_ZERO; 4602276479Sdim Ops.push_back(TexHandle); 4603276479Sdim Ops.push_back(N->getOperand(2)); 4604276479Sdim Ops.push_back(N->getOperand(3)); 4605276479Sdim Ops.push_back(N->getOperand(4)); 4606276479Sdim Ops.push_back(Chain); 4607276479Sdim break; 4608276479Sdim case NVPTXISD::Suld2DArrayV2I32Zero: 4609276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I32_ZERO; 4610276479Sdim Ops.push_back(TexHandle); 4611276479Sdim Ops.push_back(N->getOperand(2)); 4612276479Sdim Ops.push_back(N->getOperand(3)); 4613276479Sdim Ops.push_back(N->getOperand(4)); 4614276479Sdim Ops.push_back(Chain); 4615276479Sdim break; 4616276479Sdim case NVPTXISD::Suld2DArrayV2I64Zero: 4617276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V2I64_ZERO; 4618276479Sdim Ops.push_back(TexHandle); 4619276479Sdim Ops.push_back(N->getOperand(2)); 4620276479Sdim Ops.push_back(N->getOperand(3)); 4621276479Sdim Ops.push_back(N->getOperand(4)); 4622276479Sdim Ops.push_back(Chain); 4623276479Sdim break; 4624276479Sdim case NVPTXISD::Suld2DArrayV4I8Zero: 4625276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I8_ZERO; 4626276479Sdim Ops.push_back(TexHandle); 4627276479Sdim Ops.push_back(N->getOperand(2)); 4628276479Sdim Ops.push_back(N->getOperand(3)); 4629276479Sdim Ops.push_back(N->getOperand(4)); 4630276479Sdim Ops.push_back(Chain); 4631276479Sdim break; 4632276479Sdim case NVPTXISD::Suld2DArrayV4I16Zero: 4633276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I16_ZERO; 4634276479Sdim Ops.push_back(TexHandle); 4635276479Sdim Ops.push_back(N->getOperand(2)); 4636276479Sdim Ops.push_back(N->getOperand(3)); 4637276479Sdim Ops.push_back(N->getOperand(4)); 4638276479Sdim Ops.push_back(Chain); 4639276479Sdim break; 4640276479Sdim case NVPTXISD::Suld2DArrayV4I32Zero: 4641276479Sdim Opc = NVPTX::SULD_2D_ARRAY_V4I32_ZERO; 4642276479Sdim Ops.push_back(TexHandle); 4643276479Sdim Ops.push_back(N->getOperand(2)); 4644276479Sdim Ops.push_back(N->getOperand(3)); 4645276479Sdim Ops.push_back(N->getOperand(4)); 4646276479Sdim Ops.push_back(Chain); 4647276479Sdim break; 4648276479Sdim case NVPTXISD::Suld3DI8Zero: 4649276479Sdim Opc = NVPTX::SULD_3D_I8_ZERO; 4650276479Sdim Ops.push_back(TexHandle); 4651276479Sdim Ops.push_back(N->getOperand(2)); 4652276479Sdim Ops.push_back(N->getOperand(3)); 4653276479Sdim Ops.push_back(N->getOperand(4)); 4654276479Sdim Ops.push_back(Chain); 4655276479Sdim break; 4656276479Sdim case NVPTXISD::Suld3DI16Zero: 4657276479Sdim Opc = NVPTX::SULD_3D_I16_ZERO; 4658276479Sdim Ops.push_back(TexHandle); 4659276479Sdim Ops.push_back(N->getOperand(2)); 4660276479Sdim Ops.push_back(N->getOperand(3)); 4661276479Sdim Ops.push_back(N->getOperand(4)); 4662276479Sdim Ops.push_back(Chain); 4663276479Sdim break; 4664276479Sdim case NVPTXISD::Suld3DI32Zero: 4665276479Sdim Opc = NVPTX::SULD_3D_I32_ZERO; 4666276479Sdim Ops.push_back(TexHandle); 4667276479Sdim Ops.push_back(N->getOperand(2)); 4668276479Sdim Ops.push_back(N->getOperand(3)); 4669276479Sdim Ops.push_back(N->getOperand(4)); 4670276479Sdim Ops.push_back(Chain); 4671276479Sdim break; 4672276479Sdim case NVPTXISD::Suld3DI64Zero: 4673276479Sdim Opc = NVPTX::SULD_3D_I64_ZERO; 4674276479Sdim Ops.push_back(TexHandle); 4675276479Sdim Ops.push_back(N->getOperand(2)); 4676276479Sdim Ops.push_back(N->getOperand(3)); 4677276479Sdim Ops.push_back(N->getOperand(4)); 4678276479Sdim Ops.push_back(Chain); 4679276479Sdim break; 4680276479Sdim case NVPTXISD::Suld3DV2I8Zero: 4681276479Sdim Opc = NVPTX::SULD_3D_V2I8_ZERO; 4682276479Sdim Ops.push_back(TexHandle); 4683276479Sdim Ops.push_back(N->getOperand(2)); 4684276479Sdim Ops.push_back(N->getOperand(3)); 4685276479Sdim Ops.push_back(N->getOperand(4)); 4686276479Sdim Ops.push_back(Chain); 4687276479Sdim break; 4688276479Sdim case NVPTXISD::Suld3DV2I16Zero: 4689276479Sdim Opc = NVPTX::SULD_3D_V2I16_ZERO; 4690276479Sdim Ops.push_back(TexHandle); 4691276479Sdim Ops.push_back(N->getOperand(2)); 4692276479Sdim Ops.push_back(N->getOperand(3)); 4693276479Sdim Ops.push_back(N->getOperand(4)); 4694276479Sdim Ops.push_back(Chain); 4695276479Sdim break; 4696276479Sdim case NVPTXISD::Suld3DV2I32Zero: 4697276479Sdim Opc = NVPTX::SULD_3D_V2I32_ZERO; 4698276479Sdim Ops.push_back(TexHandle); 4699276479Sdim Ops.push_back(N->getOperand(2)); 4700276479Sdim Ops.push_back(N->getOperand(3)); 4701276479Sdim Ops.push_back(N->getOperand(4)); 4702276479Sdim Ops.push_back(Chain); 4703276479Sdim break; 4704276479Sdim case NVPTXISD::Suld3DV2I64Zero: 4705276479Sdim Opc = NVPTX::SULD_3D_V2I64_ZERO; 4706276479Sdim Ops.push_back(TexHandle); 4707276479Sdim Ops.push_back(N->getOperand(2)); 4708276479Sdim Ops.push_back(N->getOperand(3)); 4709276479Sdim Ops.push_back(N->getOperand(4)); 4710276479Sdim Ops.push_back(Chain); 4711276479Sdim break; 4712276479Sdim case NVPTXISD::Suld3DV4I8Zero: 4713276479Sdim Opc = NVPTX::SULD_3D_V4I8_ZERO; 4714276479Sdim Ops.push_back(TexHandle); 4715276479Sdim Ops.push_back(N->getOperand(2)); 4716276479Sdim Ops.push_back(N->getOperand(3)); 4717276479Sdim Ops.push_back(N->getOperand(4)); 4718276479Sdim Ops.push_back(Chain); 4719276479Sdim break; 4720276479Sdim case NVPTXISD::Suld3DV4I16Zero: 4721276479Sdim Opc = NVPTX::SULD_3D_V4I16_ZERO; 4722276479Sdim Ops.push_back(TexHandle); 4723276479Sdim Ops.push_back(N->getOperand(2)); 4724276479Sdim Ops.push_back(N->getOperand(3)); 4725276479Sdim Ops.push_back(N->getOperand(4)); 4726276479Sdim Ops.push_back(Chain); 4727276479Sdim break; 4728276479Sdim case NVPTXISD::Suld3DV4I32Zero: 4729276479Sdim Opc = NVPTX::SULD_3D_V4I32_ZERO; 4730276479Sdim Ops.push_back(TexHandle); 4731276479Sdim Ops.push_back(N->getOperand(2)); 4732276479Sdim Ops.push_back(N->getOperand(3)); 4733276479Sdim Ops.push_back(N->getOperand(4)); 4734276479Sdim Ops.push_back(Chain); 4735276479Sdim break; 4736276479Sdim } 4737276479Sdim Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops); 4738276479Sdim return Ret; 4739276479Sdim} 4740276479Sdim 4741276479Sdim 4742276479Sdim/// SelectBFE - Look for instruction sequences that can be made more efficient 4743276479Sdim/// by using the 'bfe' (bit-field extract) PTX instruction 4744276479SdimSDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) { 4745276479Sdim SDValue LHS = N->getOperand(0); 4746276479Sdim SDValue RHS = N->getOperand(1); 4747276479Sdim SDValue Len; 4748276479Sdim SDValue Start; 4749276479Sdim SDValue Val; 4750276479Sdim bool IsSigned = false; 4751276479Sdim 4752276479Sdim if (N->getOpcode() == ISD::AND) { 4753276479Sdim // Canonicalize the operands 4754276479Sdim // We want 'and %val, %mask' 4755276479Sdim if (isa<ConstantSDNode>(LHS) && !isa<ConstantSDNode>(RHS)) { 4756276479Sdim std::swap(LHS, RHS); 4757276479Sdim } 4758276479Sdim 4759276479Sdim ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(RHS); 4760276479Sdim if (!Mask) { 4761276479Sdim // We need a constant mask on the RHS of the AND 4762276479Sdim return NULL; 4763276479Sdim } 4764276479Sdim 4765276479Sdim // Extract the mask bits 4766276479Sdim uint64_t MaskVal = Mask->getZExtValue(); 4767276479Sdim if (!isMask_64(MaskVal)) { 4768276479Sdim // We *could* handle shifted masks here, but doing so would require an 4769276479Sdim // 'and' operation to fix up the low-order bits so we would trade 4770276479Sdim // shr+and for bfe+and, which has the same throughput 4771276479Sdim return NULL; 4772276479Sdim } 4773276479Sdim 4774276479Sdim // How many bits are in our mask? 4775276479Sdim uint64_t NumBits = CountTrailingOnes_64(MaskVal); 4776276479Sdim Len = CurDAG->getTargetConstant(NumBits, MVT::i32); 4777276479Sdim 4778276479Sdim if (LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SRA) { 4779276479Sdim // We have a 'srl/and' pair, extract the effective start bit and length 4780276479Sdim Val = LHS.getNode()->getOperand(0); 4781276479Sdim Start = LHS.getNode()->getOperand(1); 4782276479Sdim ConstantSDNode *StartConst = dyn_cast<ConstantSDNode>(Start); 4783276479Sdim if (StartConst) { 4784276479Sdim uint64_t StartVal = StartConst->getZExtValue(); 4785276479Sdim // How many "good" bits do we have left? "good" is defined here as bits 4786276479Sdim // that exist in the original value, not shifted in. 4787276479Sdim uint64_t GoodBits = Start.getValueType().getSizeInBits() - StartVal; 4788276479Sdim if (NumBits > GoodBits) { 4789276479Sdim // Do not handle the case where bits have been shifted in. In theory 4790276479Sdim // we could handle this, but the cost is likely higher than just 4791276479Sdim // emitting the srl/and pair. 4792276479Sdim return NULL; 4793276479Sdim } 4794276479Sdim Start = CurDAG->getTargetConstant(StartVal, MVT::i32); 4795276479Sdim } else { 4796276479Sdim // Do not handle the case where the shift amount (can be zero if no srl 4797276479Sdim // was found) is not constant. We could handle this case, but it would 4798276479Sdim // require run-time logic that would be more expensive than just 4799276479Sdim // emitting the srl/and pair. 4800276479Sdim return NULL; 4801276479Sdim } 4802276479Sdim } else { 4803276479Sdim // Do not handle the case where the LHS of the and is not a shift. While 4804276479Sdim // it would be trivial to handle this case, it would just transform 4805276479Sdim // 'and' -> 'bfe', but 'and' has higher-throughput. 4806276479Sdim return NULL; 4807276479Sdim } 4808276479Sdim } else if (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) { 4809276479Sdim if (LHS->getOpcode() == ISD::AND) { 4810276479Sdim ConstantSDNode *ShiftCnst = dyn_cast<ConstantSDNode>(RHS); 4811276479Sdim if (!ShiftCnst) { 4812276479Sdim // Shift amount must be constant 4813276479Sdim return NULL; 4814276479Sdim } 4815276479Sdim 4816276479Sdim uint64_t ShiftAmt = ShiftCnst->getZExtValue(); 4817276479Sdim 4818276479Sdim SDValue AndLHS = LHS->getOperand(0); 4819276479Sdim SDValue AndRHS = LHS->getOperand(1); 4820276479Sdim 4821276479Sdim // Canonicalize the AND to have the mask on the RHS 4822276479Sdim if (isa<ConstantSDNode>(AndLHS)) { 4823276479Sdim std::swap(AndLHS, AndRHS); 4824276479Sdim } 4825276479Sdim 4826276479Sdim ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(AndRHS); 4827276479Sdim if (!MaskCnst) { 4828276479Sdim // Mask must be constant 4829276479Sdim return NULL; 4830276479Sdim } 4831276479Sdim 4832276479Sdim uint64_t MaskVal = MaskCnst->getZExtValue(); 4833276479Sdim uint64_t NumZeros; 4834276479Sdim uint64_t NumBits; 4835276479Sdim if (isMask_64(MaskVal)) { 4836276479Sdim NumZeros = 0; 4837276479Sdim // The number of bits in the result bitfield will be the number of 4838276479Sdim // trailing ones (the AND) minus the number of bits we shift off 4839276479Sdim NumBits = CountTrailingOnes_64(MaskVal) - ShiftAmt; 4840276479Sdim } else if (isShiftedMask_64(MaskVal)) { 4841276479Sdim NumZeros = countTrailingZeros(MaskVal); 4842276479Sdim unsigned NumOnes = CountTrailingOnes_64(MaskVal >> NumZeros); 4843276479Sdim // The number of bits in the result bitfield will be the number of 4844276479Sdim // trailing zeros plus the number of set bits in the mask minus the 4845276479Sdim // number of bits we shift off 4846276479Sdim NumBits = NumZeros + NumOnes - ShiftAmt; 4847276479Sdim } else { 4848276479Sdim // This is not a mask we can handle 4849276479Sdim return NULL; 4850276479Sdim } 4851276479Sdim 4852276479Sdim if (ShiftAmt < NumZeros) { 4853276479Sdim // Handling this case would require extra logic that would make this 4854276479Sdim // transformation non-profitable 4855276479Sdim return NULL; 4856276479Sdim } 4857276479Sdim 4858276479Sdim Val = AndLHS; 4859276479Sdim Start = CurDAG->getTargetConstant(ShiftAmt, MVT::i32); 4860276479Sdim Len = CurDAG->getTargetConstant(NumBits, MVT::i32); 4861276479Sdim } else if (LHS->getOpcode() == ISD::SHL) { 4862276479Sdim // Here, we have a pattern like: 4863276479Sdim // 4864276479Sdim // (sra (shl val, NN), MM) 4865276479Sdim // or 4866276479Sdim // (srl (shl val, NN), MM) 4867276479Sdim // 4868276479Sdim // If MM >= NN, we can efficiently optimize this with bfe 4869276479Sdim Val = LHS->getOperand(0); 4870276479Sdim 4871276479Sdim SDValue ShlRHS = LHS->getOperand(1); 4872276479Sdim ConstantSDNode *ShlCnst = dyn_cast<ConstantSDNode>(ShlRHS); 4873276479Sdim if (!ShlCnst) { 4874276479Sdim // Shift amount must be constant 4875276479Sdim return NULL; 4876276479Sdim } 4877276479Sdim uint64_t InnerShiftAmt = ShlCnst->getZExtValue(); 4878276479Sdim 4879276479Sdim SDValue ShrRHS = RHS; 4880276479Sdim ConstantSDNode *ShrCnst = dyn_cast<ConstantSDNode>(ShrRHS); 4881276479Sdim if (!ShrCnst) { 4882276479Sdim // Shift amount must be constant 4883276479Sdim return NULL; 4884276479Sdim } 4885276479Sdim uint64_t OuterShiftAmt = ShrCnst->getZExtValue(); 4886276479Sdim 4887276479Sdim // To avoid extra codegen and be profitable, we need Outer >= Inner 4888276479Sdim if (OuterShiftAmt < InnerShiftAmt) { 4889276479Sdim return NULL; 4890276479Sdim } 4891276479Sdim 4892276479Sdim // If the outer shift is more than the type size, we have no bitfield to 4893276479Sdim // extract (since we also check that the inner shift is <= the outer shift 4894276479Sdim // then this also implies that the inner shift is < the type size) 4895276479Sdim if (OuterShiftAmt >= Val.getValueType().getSizeInBits()) { 4896276479Sdim return NULL; 4897276479Sdim } 4898276479Sdim 4899276479Sdim Start = 4900276479Sdim CurDAG->getTargetConstant(OuterShiftAmt - InnerShiftAmt, MVT::i32); 4901276479Sdim Len = 4902276479Sdim CurDAG->getTargetConstant(Val.getValueType().getSizeInBits() - 4903276479Sdim OuterShiftAmt, MVT::i32); 4904276479Sdim 4905276479Sdim if (N->getOpcode() == ISD::SRA) { 4906276479Sdim // If we have a arithmetic right shift, we need to use the signed bfe 4907276479Sdim // variant 4908276479Sdim IsSigned = true; 4909276479Sdim } 4910276479Sdim } else { 4911276479Sdim // No can do... 4912276479Sdim return NULL; 4913276479Sdim } 4914276479Sdim } else { 4915276479Sdim // No can do... 4916276479Sdim return NULL; 4917276479Sdim } 4918276479Sdim 4919276479Sdim 4920276479Sdim unsigned Opc; 4921276479Sdim // For the BFE operations we form here from "and" and "srl", always use the 4922276479Sdim // unsigned variants. 4923276479Sdim if (Val.getValueType() == MVT::i32) { 4924276479Sdim if (IsSigned) { 4925276479Sdim Opc = NVPTX::BFE_S32rii; 4926276479Sdim } else { 4927276479Sdim Opc = NVPTX::BFE_U32rii; 4928276479Sdim } 4929276479Sdim } else if (Val.getValueType() == MVT::i64) { 4930276479Sdim if (IsSigned) { 4931276479Sdim Opc = NVPTX::BFE_S64rii; 4932276479Sdim } else { 4933276479Sdim Opc = NVPTX::BFE_U64rii; 4934276479Sdim } 4935276479Sdim } else { 4936276479Sdim // We cannot handle this type 4937276479Sdim return NULL; 4938276479Sdim } 4939276479Sdim 4940276479Sdim SDValue Ops[] = { 4941276479Sdim Val, Start, Len 4942276479Sdim }; 4943276479Sdim 4944276479Sdim SDNode *Ret = 4945276479Sdim CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops); 4946276479Sdim 4947276479Sdim return Ret; 4948276479Sdim} 4949276479Sdim 4950239310Sdim// SelectDirectAddr - Match a direct address for DAG. 4951239310Sdim// A direct address could be a globaladdress or externalsymbol. 4952239310Sdimbool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) { 4953239310Sdim // Return true if TGA or ES. 4954249423Sdim if (N.getOpcode() == ISD::TargetGlobalAddress || 4955249423Sdim N.getOpcode() == ISD::TargetExternalSymbol) { 4956239310Sdim Address = N; 4957239310Sdim return true; 4958239310Sdim } 4959239310Sdim if (N.getOpcode() == NVPTXISD::Wrapper) { 4960239310Sdim Address = N.getOperand(0); 4961239310Sdim return true; 4962239310Sdim } 4963239310Sdim if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) { 4964239310Sdim unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue(); 4965239310Sdim if (IID == Intrinsic::nvvm_ptr_gen_to_param) 4966239310Sdim if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam) 4967239310Sdim return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address)); 4968239310Sdim } 4969239310Sdim return false; 4970239310Sdim} 4971239310Sdim 4972239310Sdim// symbol+offset 4973249423Sdimbool NVPTXDAGToDAGISel::SelectADDRsi_imp( 4974249423Sdim SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { 4975239310Sdim if (Addr.getOpcode() == ISD::ADD) { 4976239310Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) { 4977249423Sdim SDValue base = Addr.getOperand(0); 4978239310Sdim if (SelectDirectAddr(base, Base)) { 4979239310Sdim Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt); 4980239310Sdim return true; 4981239310Sdim } 4982239310Sdim } 4983239310Sdim } 4984239310Sdim return false; 4985239310Sdim} 4986239310Sdim 4987239310Sdim// symbol+offset 4988239310Sdimbool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr, 4989239310Sdim SDValue &Base, SDValue &Offset) { 4990239310Sdim return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32); 4991239310Sdim} 4992239310Sdim 4993239310Sdim// symbol+offset 4994239310Sdimbool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr, 4995239310Sdim SDValue &Base, SDValue &Offset) { 4996239310Sdim return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64); 4997239310Sdim} 4998239310Sdim 4999239310Sdim// register+offset 5000249423Sdimbool NVPTXDAGToDAGISel::SelectADDRri_imp( 5001249423Sdim SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { 5002239310Sdim if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) { 5003239310Sdim Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt); 5004239310Sdim Offset = CurDAG->getTargetConstant(0, mvt); 5005239310Sdim return true; 5006239310Sdim } 5007239310Sdim if (Addr.getOpcode() == ISD::TargetExternalSymbol || 5008239310Sdim Addr.getOpcode() == ISD::TargetGlobalAddress) 5009249423Sdim return false; // direct calls. 5010239310Sdim 5011239310Sdim if (Addr.getOpcode() == ISD::ADD) { 5012239310Sdim if (SelectDirectAddr(Addr.getOperand(0), Addr)) { 5013239310Sdim return false; 5014239310Sdim } 5015239310Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) { 5016239310Sdim if (FrameIndexSDNode *FIN = 5017249423Sdim dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) 5018239310Sdim // Constant offset from frame ref. 5019239310Sdim Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt); 5020239310Sdim else 5021239310Sdim Base = Addr.getOperand(0); 5022239310Sdim Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt); 5023239310Sdim return true; 5024239310Sdim } 5025239310Sdim } 5026239310Sdim return false; 5027239310Sdim} 5028239310Sdim 5029239310Sdim// register+offset 5030239310Sdimbool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr, 5031239310Sdim SDValue &Base, SDValue &Offset) { 5032239310Sdim return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32); 5033239310Sdim} 5034239310Sdim 5035239310Sdim// register+offset 5036239310Sdimbool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr, 5037239310Sdim SDValue &Base, SDValue &Offset) { 5038239310Sdim return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64); 5039239310Sdim} 5040239310Sdim 5041239310Sdimbool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N, 5042239310Sdim unsigned int spN) const { 5043276479Sdim const Value *Src = nullptr; 5044239310Sdim // Even though MemIntrinsicSDNode is a subclas of MemSDNode, 5045239310Sdim // the classof() for MemSDNode does not include MemIntrinsicSDNode 5046239310Sdim // (See SelectionDAGNodes.h). So we need to check for both. 5047239310Sdim if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) { 5048276479Sdim if (spN == 0 && mN->getMemOperand()->getPseudoValue()) 5049276479Sdim return true; 5050276479Sdim Src = mN->getMemOperand()->getValue(); 5051249423Sdim } else if (MemSDNode *mN = dyn_cast<MemIntrinsicSDNode>(N)) { 5052276479Sdim if (spN == 0 && mN->getMemOperand()->getPseudoValue()) 5053276479Sdim return true; 5054276479Sdim Src = mN->getMemOperand()->getValue(); 5055239310Sdim } 5056239310Sdim if (!Src) 5057239310Sdim return false; 5058239310Sdim if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) 5059239310Sdim return (PT->getAddressSpace() == spN); 5060239310Sdim return false; 5061239310Sdim} 5062239310Sdim 5063239310Sdim/// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 5064239310Sdim/// inline asm expressions. 5065249423Sdimbool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand( 5066249423Sdim const SDValue &Op, char ConstraintCode, std::vector<SDValue> &OutOps) { 5067239310Sdim SDValue Op0, Op1; 5068239310Sdim switch (ConstraintCode) { 5069249423Sdim default: 5070249423Sdim return true; 5071249423Sdim case 'm': // memory 5072239310Sdim if (SelectDirectAddr(Op, Op0)) { 5073239310Sdim OutOps.push_back(Op0); 5074239310Sdim OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32)); 5075239310Sdim return false; 5076239310Sdim } 5077239310Sdim if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) { 5078239310Sdim OutOps.push_back(Op0); 5079239310Sdim OutOps.push_back(Op1); 5080239310Sdim return false; 5081239310Sdim } 5082239310Sdim break; 5083239310Sdim } 5084239310Sdim return true; 5085239310Sdim} 5086