AMDGPUISelLowering.cpp revision 284677
1//===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10/// \file 11/// \brief This is the parent TargetLowering class for hardware code gen 12/// targets. 13// 14//===----------------------------------------------------------------------===// 15 16#include "AMDGPUISelLowering.h" 17#include "AMDGPU.h" 18#include "AMDGPUFrameLowering.h" 19#include "AMDGPUIntrinsicInfo.h" 20#include "AMDGPURegisterInfo.h" 21#include "AMDGPUSubtarget.h" 22#include "R600MachineFunctionInfo.h" 23#include "SIMachineFunctionInfo.h" 24#include "llvm/CodeGen/CallingConvLower.h" 25#include "llvm/CodeGen/MachineFunction.h" 26#include "llvm/CodeGen/MachineRegisterInfo.h" 27#include "llvm/CodeGen/SelectionDAG.h" 28#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 29#include "llvm/IR/DataLayout.h" 30#include "llvm/IR/DiagnosticInfo.h" 31#include "llvm/IR/DiagnosticPrinter.h" 32 33using namespace llvm; 34 35namespace { 36 37/// Diagnostic information for unimplemented or unsupported feature reporting. 38class DiagnosticInfoUnsupported : public DiagnosticInfo { 39private: 40 const Twine &Description; 41 const Function &Fn; 42 43 static int KindID; 44 45 static int getKindID() { 46 if (KindID == 0) 47 KindID = llvm::getNextAvailablePluginDiagnosticKind(); 48 return KindID; 49 } 50 51public: 52 DiagnosticInfoUnsupported(const Function &Fn, const Twine &Desc, 53 DiagnosticSeverity Severity = DS_Error) 54 : DiagnosticInfo(getKindID(), Severity), 55 Description(Desc), 56 Fn(Fn) { } 57 58 const Function &getFunction() const { return Fn; } 59 const Twine &getDescription() const { return Description; } 60 61 void print(DiagnosticPrinter &DP) const override { 62 DP << "unsupported " << getDescription() << " in " << Fn.getName(); 63 } 64 65 static bool classof(const DiagnosticInfo *DI) { 66 return DI->getKind() == getKindID(); 67 } 68}; 69 70int DiagnosticInfoUnsupported::KindID = 0; 71} // namespace 72 73 74static bool allocateStack(unsigned ValNo, MVT ValVT, MVT LocVT, 75 CCValAssign::LocInfo LocInfo, 76 ISD::ArgFlagsTy ArgFlags, CCState &State) { 77 unsigned Offset = State.AllocateStack(ValVT.getStoreSize(), 78 ArgFlags.getOrigAlign()); 79 State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo)); 80 81 return true; 82} 83 84#include "AMDGPUGenCallingConv.inc" 85 86// Find a larger type to do a load / store of a vector with. 87EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) { 88 unsigned StoreSize = VT.getStoreSizeInBits(); 89 if (StoreSize <= 32) 90 return EVT::getIntegerVT(Ctx, StoreSize); 91 92 assert(StoreSize % 32 == 0 && "Store size not a multiple of 32"); 93 return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32); 94} 95 96// Type for a vector that will be loaded to. 97EVT AMDGPUTargetLowering::getEquivalentLoadRegType(LLVMContext &Ctx, EVT VT) { 98 unsigned StoreSize = VT.getStoreSizeInBits(); 99 if (StoreSize <= 32) 100 return EVT::getIntegerVT(Ctx, 32); 101 102 return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32); 103} 104 105AMDGPUTargetLowering::AMDGPUTargetLowering(TargetMachine &TM, 106 const AMDGPUSubtarget &STI) 107 : TargetLowering(TM), Subtarget(&STI) { 108 setOperationAction(ISD::Constant, MVT::i32, Legal); 109 setOperationAction(ISD::Constant, MVT::i64, Legal); 110 setOperationAction(ISD::ConstantFP, MVT::f32, Legal); 111 setOperationAction(ISD::ConstantFP, MVT::f64, Legal); 112 113 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 114 setOperationAction(ISD::BRIND, MVT::Other, Expand); 115 116 // We need to custom lower some of the intrinsics 117 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 118 119 // Library functions. These default to Expand, but we have instructions 120 // for them. 121 setOperationAction(ISD::FCEIL, MVT::f32, Legal); 122 setOperationAction(ISD::FEXP2, MVT::f32, Legal); 123 setOperationAction(ISD::FPOW, MVT::f32, Legal); 124 setOperationAction(ISD::FLOG2, MVT::f32, Legal); 125 setOperationAction(ISD::FABS, MVT::f32, Legal); 126 setOperationAction(ISD::FFLOOR, MVT::f32, Legal); 127 setOperationAction(ISD::FRINT, MVT::f32, Legal); 128 setOperationAction(ISD::FTRUNC, MVT::f32, Legal); 129 setOperationAction(ISD::FMINNUM, MVT::f32, Legal); 130 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal); 131 132 setOperationAction(ISD::FROUND, MVT::f32, Custom); 133 setOperationAction(ISD::FROUND, MVT::f64, Custom); 134 135 setOperationAction(ISD::FREM, MVT::f32, Custom); 136 setOperationAction(ISD::FREM, MVT::f64, Custom); 137 138 // v_mad_f32 does not support denormals according to some sources. 139 if (!Subtarget->hasFP32Denormals()) 140 setOperationAction(ISD::FMAD, MVT::f32, Legal); 141 142 // Expand to fneg + fadd. 143 setOperationAction(ISD::FSUB, MVT::f64, Expand); 144 145 // Lower floating point store/load to integer store/load to reduce the number 146 // of patterns in tablegen. 147 setOperationAction(ISD::STORE, MVT::f32, Promote); 148 AddPromotedToType(ISD::STORE, MVT::f32, MVT::i32); 149 150 setOperationAction(ISD::STORE, MVT::v2f32, Promote); 151 AddPromotedToType(ISD::STORE, MVT::v2f32, MVT::v2i32); 152 153 setOperationAction(ISD::STORE, MVT::v4f32, Promote); 154 AddPromotedToType(ISD::STORE, MVT::v4f32, MVT::v4i32); 155 156 setOperationAction(ISD::STORE, MVT::v8f32, Promote); 157 AddPromotedToType(ISD::STORE, MVT::v8f32, MVT::v8i32); 158 159 setOperationAction(ISD::STORE, MVT::v16f32, Promote); 160 AddPromotedToType(ISD::STORE, MVT::v16f32, MVT::v16i32); 161 162 setOperationAction(ISD::STORE, MVT::f64, Promote); 163 AddPromotedToType(ISD::STORE, MVT::f64, MVT::i64); 164 165 setOperationAction(ISD::STORE, MVT::v2f64, Promote); 166 AddPromotedToType(ISD::STORE, MVT::v2f64, MVT::v2i64); 167 168 // Custom lowering of vector stores is required for local address space 169 // stores. 170 setOperationAction(ISD::STORE, MVT::v4i32, Custom); 171 172 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Custom); 173 setTruncStoreAction(MVT::v2i32, MVT::v2i8, Custom); 174 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Custom); 175 176 // XXX: This can be change to Custom, once ExpandVectorStores can 177 // handle 64-bit stores. 178 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand); 179 180 setTruncStoreAction(MVT::i64, MVT::i16, Expand); 181 setTruncStoreAction(MVT::i64, MVT::i8, Expand); 182 setTruncStoreAction(MVT::i64, MVT::i1, Expand); 183 setTruncStoreAction(MVT::v2i64, MVT::v2i1, Expand); 184 setTruncStoreAction(MVT::v4i64, MVT::v4i1, Expand); 185 186 187 setOperationAction(ISD::LOAD, MVT::f32, Promote); 188 AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32); 189 190 setOperationAction(ISD::LOAD, MVT::v2f32, Promote); 191 AddPromotedToType(ISD::LOAD, MVT::v2f32, MVT::v2i32); 192 193 setOperationAction(ISD::LOAD, MVT::v4f32, Promote); 194 AddPromotedToType(ISD::LOAD, MVT::v4f32, MVT::v4i32); 195 196 setOperationAction(ISD::LOAD, MVT::v8f32, Promote); 197 AddPromotedToType(ISD::LOAD, MVT::v8f32, MVT::v8i32); 198 199 setOperationAction(ISD::LOAD, MVT::v16f32, Promote); 200 AddPromotedToType(ISD::LOAD, MVT::v16f32, MVT::v16i32); 201 202 setOperationAction(ISD::LOAD, MVT::f64, Promote); 203 AddPromotedToType(ISD::LOAD, MVT::f64, MVT::i64); 204 205 setOperationAction(ISD::LOAD, MVT::v2f64, Promote); 206 AddPromotedToType(ISD::LOAD, MVT::v2f64, MVT::v2i64); 207 208 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i32, Custom); 209 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4f32, Custom); 210 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i32, Custom); 211 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f32, Custom); 212 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2f32, Custom); 213 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2i32, Custom); 214 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4f32, Custom); 215 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4i32, Custom); 216 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8f32, Custom); 217 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8i32, Custom); 218 219 // There are no 64-bit extloads. These should be done as a 32-bit extload and 220 // an extension to 64-bit. 221 for (MVT VT : MVT::integer_valuetypes()) { 222 setLoadExtAction(ISD::EXTLOAD, MVT::i64, VT, Expand); 223 setLoadExtAction(ISD::SEXTLOAD, MVT::i64, VT, Expand); 224 setLoadExtAction(ISD::ZEXTLOAD, MVT::i64, VT, Expand); 225 } 226 227 for (MVT VT : MVT::integer_vector_valuetypes()) { 228 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Expand); 229 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Expand); 230 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i8, Expand); 231 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Expand); 232 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i8, Expand); 233 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v4i8, Expand); 234 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Expand); 235 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Expand); 236 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i16, Expand); 237 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Expand); 238 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i16, Expand); 239 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v4i16, Expand); 240 } 241 242 setOperationAction(ISD::BR_CC, MVT::i1, Expand); 243 244 if (Subtarget->getGeneration() < AMDGPUSubtarget::SEA_ISLANDS) { 245 setOperationAction(ISD::FCEIL, MVT::f64, Custom); 246 setOperationAction(ISD::FTRUNC, MVT::f64, Custom); 247 setOperationAction(ISD::FRINT, MVT::f64, Custom); 248 setOperationAction(ISD::FFLOOR, MVT::f64, Custom); 249 } 250 251 if (!Subtarget->hasBFI()) { 252 // fcopysign can be done in a single instruction with BFI. 253 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 254 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 255 } 256 257 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); 258 259 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); 260 setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand); 261 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand); 262 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Expand); 263 264 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); 265 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand); 266 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand); 267 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Expand); 268 269 setTruncStoreAction(MVT::f32, MVT::f16, Expand); 270 setTruncStoreAction(MVT::v2f32, MVT::v2f16, Expand); 271 setTruncStoreAction(MVT::v4f32, MVT::v4f16, Expand); 272 setTruncStoreAction(MVT::v8f32, MVT::v8f16, Expand); 273 274 setTruncStoreAction(MVT::f64, MVT::f16, Expand); 275 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 276 277 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; 278 for (MVT VT : ScalarIntVTs) { 279 setOperationAction(ISD::SREM, VT, Expand); 280 setOperationAction(ISD::SDIV, VT, Expand); 281 282 // GPU does not have divrem function for signed or unsigned. 283 setOperationAction(ISD::SDIVREM, VT, Custom); 284 setOperationAction(ISD::UDIVREM, VT, Custom); 285 286 // GPU does not have [S|U]MUL_LOHI functions as a single instruction. 287 setOperationAction(ISD::SMUL_LOHI, VT, Expand); 288 setOperationAction(ISD::UMUL_LOHI, VT, Expand); 289 290 setOperationAction(ISD::BSWAP, VT, Expand); 291 setOperationAction(ISD::CTTZ, VT, Expand); 292 setOperationAction(ISD::CTLZ, VT, Expand); 293 } 294 295 if (!Subtarget->hasBCNT(32)) 296 setOperationAction(ISD::CTPOP, MVT::i32, Expand); 297 298 if (!Subtarget->hasBCNT(64)) 299 setOperationAction(ISD::CTPOP, MVT::i64, Expand); 300 301 // The hardware supports 32-bit ROTR, but not ROTL. 302 setOperationAction(ISD::ROTL, MVT::i32, Expand); 303 setOperationAction(ISD::ROTL, MVT::i64, Expand); 304 setOperationAction(ISD::ROTR, MVT::i64, Expand); 305 306 setOperationAction(ISD::MUL, MVT::i64, Expand); 307 setOperationAction(ISD::MULHU, MVT::i64, Expand); 308 setOperationAction(ISD::MULHS, MVT::i64, Expand); 309 setOperationAction(ISD::UDIV, MVT::i32, Expand); 310 setOperationAction(ISD::UREM, MVT::i32, Expand); 311 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); 312 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); 313 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); 314 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); 315 setOperationAction(ISD::SELECT_CC, MVT::i64, Expand); 316 317 setOperationAction(ISD::SMIN, MVT::i32, Legal); 318 setOperationAction(ISD::UMIN, MVT::i32, Legal); 319 setOperationAction(ISD::SMAX, MVT::i32, Legal); 320 setOperationAction(ISD::UMAX, MVT::i32, Legal); 321 322 if (!Subtarget->hasFFBH()) 323 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Expand); 324 325 if (!Subtarget->hasFFBL()) 326 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Expand); 327 328 static const MVT::SimpleValueType VectorIntTypes[] = { 329 MVT::v2i32, MVT::v4i32 330 }; 331 332 for (MVT VT : VectorIntTypes) { 333 // Expand the following operations for the current type by default. 334 setOperationAction(ISD::ADD, VT, Expand); 335 setOperationAction(ISD::AND, VT, Expand); 336 setOperationAction(ISD::FP_TO_SINT, VT, Expand); 337 setOperationAction(ISD::FP_TO_UINT, VT, Expand); 338 setOperationAction(ISD::MUL, VT, Expand); 339 setOperationAction(ISD::OR, VT, Expand); 340 setOperationAction(ISD::SHL, VT, Expand); 341 setOperationAction(ISD::SRA, VT, Expand); 342 setOperationAction(ISD::SRL, VT, Expand); 343 setOperationAction(ISD::ROTL, VT, Expand); 344 setOperationAction(ISD::ROTR, VT, Expand); 345 setOperationAction(ISD::SUB, VT, Expand); 346 setOperationAction(ISD::SINT_TO_FP, VT, Expand); 347 setOperationAction(ISD::UINT_TO_FP, VT, Expand); 348 setOperationAction(ISD::SDIV, VT, Expand); 349 setOperationAction(ISD::UDIV, VT, Expand); 350 setOperationAction(ISD::SREM, VT, Expand); 351 setOperationAction(ISD::UREM, VT, Expand); 352 setOperationAction(ISD::SMUL_LOHI, VT, Expand); 353 setOperationAction(ISD::UMUL_LOHI, VT, Expand); 354 setOperationAction(ISD::SDIVREM, VT, Custom); 355 setOperationAction(ISD::UDIVREM, VT, Custom); 356 setOperationAction(ISD::ADDC, VT, Expand); 357 setOperationAction(ISD::SUBC, VT, Expand); 358 setOperationAction(ISD::ADDE, VT, Expand); 359 setOperationAction(ISD::SUBE, VT, Expand); 360 setOperationAction(ISD::SELECT, VT, Expand); 361 setOperationAction(ISD::VSELECT, VT, Expand); 362 setOperationAction(ISD::SELECT_CC, VT, Expand); 363 setOperationAction(ISD::XOR, VT, Expand); 364 setOperationAction(ISD::BSWAP, VT, Expand); 365 setOperationAction(ISD::CTPOP, VT, Expand); 366 setOperationAction(ISD::CTTZ, VT, Expand); 367 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); 368 setOperationAction(ISD::CTLZ, VT, Expand); 369 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); 370 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand); 371 } 372 373 static const MVT::SimpleValueType FloatVectorTypes[] = { 374 MVT::v2f32, MVT::v4f32 375 }; 376 377 for (MVT VT : FloatVectorTypes) { 378 setOperationAction(ISD::FABS, VT, Expand); 379 setOperationAction(ISD::FMINNUM, VT, Expand); 380 setOperationAction(ISD::FMAXNUM, VT, Expand); 381 setOperationAction(ISD::FADD, VT, Expand); 382 setOperationAction(ISD::FCEIL, VT, Expand); 383 setOperationAction(ISD::FCOS, VT, Expand); 384 setOperationAction(ISD::FDIV, VT, Expand); 385 setOperationAction(ISD::FEXP2, VT, Expand); 386 setOperationAction(ISD::FLOG2, VT, Expand); 387 setOperationAction(ISD::FREM, VT, Expand); 388 setOperationAction(ISD::FPOW, VT, Expand); 389 setOperationAction(ISD::FFLOOR, VT, Expand); 390 setOperationAction(ISD::FTRUNC, VT, Expand); 391 setOperationAction(ISD::FMUL, VT, Expand); 392 setOperationAction(ISD::FMA, VT, Expand); 393 setOperationAction(ISD::FRINT, VT, Expand); 394 setOperationAction(ISD::FNEARBYINT, VT, Expand); 395 setOperationAction(ISD::FSQRT, VT, Expand); 396 setOperationAction(ISD::FSIN, VT, Expand); 397 setOperationAction(ISD::FSUB, VT, Expand); 398 setOperationAction(ISD::FNEG, VT, Expand); 399 setOperationAction(ISD::SELECT, VT, Expand); 400 setOperationAction(ISD::VSELECT, VT, Expand); 401 setOperationAction(ISD::SELECT_CC, VT, Expand); 402 setOperationAction(ISD::FCOPYSIGN, VT, Expand); 403 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand); 404 } 405 406 setOperationAction(ISD::FNEARBYINT, MVT::f32, Custom); 407 setOperationAction(ISD::FNEARBYINT, MVT::f64, Custom); 408 409 setTargetDAGCombine(ISD::MUL); 410 setTargetDAGCombine(ISD::SELECT); 411 setTargetDAGCombine(ISD::SELECT_CC); 412 setTargetDAGCombine(ISD::STORE); 413 414 setTargetDAGCombine(ISD::FADD); 415 setTargetDAGCombine(ISD::FSUB); 416 417 setBooleanContents(ZeroOrNegativeOneBooleanContent); 418 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 419 420 setSchedulingPreference(Sched::RegPressure); 421 setJumpIsExpensive(true); 422 423 // SI at least has hardware support for floating point exceptions, but no way 424 // of using or handling them is implemented. They are also optional in OpenCL 425 // (Section 7.3) 426 setHasFloatingPointExceptions(false); 427 428 setSelectIsExpensive(false); 429 PredictableSelectIsExpensive = false; 430 431 // There are no integer divide instructions, and these expand to a pretty 432 // large sequence of instructions. 433 setIntDivIsCheap(false); 434 setPow2SDivIsCheap(false); 435 setFsqrtIsCheap(true); 436 437 // FIXME: Need to really handle these. 438 MaxStoresPerMemcpy = 4096; 439 MaxStoresPerMemmove = 4096; 440 MaxStoresPerMemset = 4096; 441} 442 443//===----------------------------------------------------------------------===// 444// Target Information 445//===----------------------------------------------------------------------===// 446 447MVT AMDGPUTargetLowering::getVectorIdxTy() const { 448 return MVT::i32; 449} 450 451bool AMDGPUTargetLowering::isSelectSupported(SelectSupportKind SelType) const { 452 return true; 453} 454 455// The backend supports 32 and 64 bit floating point immediates. 456// FIXME: Why are we reporting vectors of FP immediates as legal? 457bool AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { 458 EVT ScalarVT = VT.getScalarType(); 459 return (ScalarVT == MVT::f32 || ScalarVT == MVT::f64); 460} 461 462// We don't want to shrink f64 / f32 constants. 463bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const { 464 EVT ScalarVT = VT.getScalarType(); 465 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64); 466} 467 468bool AMDGPUTargetLowering::shouldReduceLoadWidth(SDNode *N, 469 ISD::LoadExtType, 470 EVT NewVT) const { 471 472 unsigned NewSize = NewVT.getStoreSizeInBits(); 473 474 // If we are reducing to a 32-bit load, this is always better. 475 if (NewSize == 32) 476 return true; 477 478 EVT OldVT = N->getValueType(0); 479 unsigned OldSize = OldVT.getStoreSizeInBits(); 480 481 // Don't produce extloads from sub 32-bit types. SI doesn't have scalar 482 // extloads, so doing one requires using a buffer_load. In cases where we 483 // still couldn't use a scalar load, using the wider load shouldn't really 484 // hurt anything. 485 486 // If the old size already had to be an extload, there's no harm in continuing 487 // to reduce the width. 488 return (OldSize < 32); 489} 490 491bool AMDGPUTargetLowering::isLoadBitCastBeneficial(EVT LoadTy, 492 EVT CastTy) const { 493 if (LoadTy.getSizeInBits() != CastTy.getSizeInBits()) 494 return true; 495 496 unsigned LScalarSize = LoadTy.getScalarType().getSizeInBits(); 497 unsigned CastScalarSize = CastTy.getScalarType().getSizeInBits(); 498 499 return ((LScalarSize <= CastScalarSize) || 500 (CastScalarSize >= 32) || 501 (LScalarSize < 32)); 502} 503 504// SI+ has instructions for cttz / ctlz for 32-bit values. This is probably also 505// profitable with the expansion for 64-bit since it's generally good to 506// speculate things. 507// FIXME: These should really have the size as a parameter. 508bool AMDGPUTargetLowering::isCheapToSpeculateCttz() const { 509 return true; 510} 511 512bool AMDGPUTargetLowering::isCheapToSpeculateCtlz() const { 513 return true; 514} 515 516//===---------------------------------------------------------------------===// 517// Target Properties 518//===---------------------------------------------------------------------===// 519 520bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const { 521 assert(VT.isFloatingPoint()); 522 return VT == MVT::f32 || VT == MVT::f64; 523} 524 525bool AMDGPUTargetLowering::isFNegFree(EVT VT) const { 526 assert(VT.isFloatingPoint()); 527 return VT == MVT::f32 || VT == MVT::f64; 528} 529 530bool AMDGPUTargetLowering:: storeOfVectorConstantIsCheap(EVT MemVT, 531 unsigned NumElem, 532 unsigned AS) const { 533 return true; 534} 535 536bool AMDGPUTargetLowering::isTruncateFree(EVT Source, EVT Dest) const { 537 // Truncate is just accessing a subregister. 538 return Dest.bitsLT(Source) && (Dest.getSizeInBits() % 32 == 0); 539} 540 541bool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const { 542 // Truncate is just accessing a subregister. 543 return Dest->getPrimitiveSizeInBits() < Source->getPrimitiveSizeInBits() && 544 (Dest->getPrimitiveSizeInBits() % 32 == 0); 545} 546 547bool AMDGPUTargetLowering::isZExtFree(Type *Src, Type *Dest) const { 548 const DataLayout *DL = getDataLayout(); 549 unsigned SrcSize = DL->getTypeSizeInBits(Src->getScalarType()); 550 unsigned DestSize = DL->getTypeSizeInBits(Dest->getScalarType()); 551 552 return SrcSize == 32 && DestSize == 64; 553} 554 555bool AMDGPUTargetLowering::isZExtFree(EVT Src, EVT Dest) const { 556 // Any register load of a 64-bit value really requires 2 32-bit moves. For all 557 // practical purposes, the extra mov 0 to load a 64-bit is free. As used, 558 // this will enable reducing 64-bit operations the 32-bit, which is always 559 // good. 560 return Src == MVT::i32 && Dest == MVT::i64; 561} 562 563bool AMDGPUTargetLowering::isZExtFree(SDValue Val, EVT VT2) const { 564 return isZExtFree(Val.getValueType(), VT2); 565} 566 567bool AMDGPUTargetLowering::isNarrowingProfitable(EVT SrcVT, EVT DestVT) const { 568 // There aren't really 64-bit registers, but pairs of 32-bit ones and only a 569 // limited number of native 64-bit operations. Shrinking an operation to fit 570 // in a single 32-bit register should always be helpful. As currently used, 571 // this is much less general than the name suggests, and is only used in 572 // places trying to reduce the sizes of loads. Shrinking loads to < 32-bits is 573 // not profitable, and may actually be harmful. 574 return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32; 575} 576 577//===---------------------------------------------------------------------===// 578// TargetLowering Callbacks 579//===---------------------------------------------------------------------===// 580 581void AMDGPUTargetLowering::AnalyzeFormalArguments(CCState &State, 582 const SmallVectorImpl<ISD::InputArg> &Ins) const { 583 584 State.AnalyzeFormalArguments(Ins, CC_AMDGPU); 585} 586 587SDValue AMDGPUTargetLowering::LowerReturn( 588 SDValue Chain, 589 CallingConv::ID CallConv, 590 bool isVarArg, 591 const SmallVectorImpl<ISD::OutputArg> &Outs, 592 const SmallVectorImpl<SDValue> &OutVals, 593 SDLoc DL, SelectionDAG &DAG) const { 594 return DAG.getNode(AMDGPUISD::RET_FLAG, DL, MVT::Other, Chain); 595} 596 597//===---------------------------------------------------------------------===// 598// Target specific lowering 599//===---------------------------------------------------------------------===// 600 601SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI, 602 SmallVectorImpl<SDValue> &InVals) const { 603 SDValue Callee = CLI.Callee; 604 SelectionDAG &DAG = CLI.DAG; 605 606 const Function &Fn = *DAG.getMachineFunction().getFunction(); 607 608 StringRef FuncName("<unknown>"); 609 610 if (const ExternalSymbolSDNode *G = dyn_cast<ExternalSymbolSDNode>(Callee)) 611 FuncName = G->getSymbol(); 612 else if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 613 FuncName = G->getGlobal()->getName(); 614 615 DiagnosticInfoUnsupported NoCalls(Fn, "call to function " + FuncName); 616 DAG.getContext()->diagnose(NoCalls); 617 return SDValue(); 618} 619 620SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op, 621 SelectionDAG &DAG) const { 622 switch (Op.getOpcode()) { 623 default: 624 Op.getNode()->dump(); 625 llvm_unreachable("Custom lowering code for this" 626 "instruction is not implemented yet!"); 627 break; 628 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG); 629 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); 630 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG); 631 case ISD::FrameIndex: return LowerFrameIndex(Op, DAG); 632 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 633 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG); 634 case ISD::SDIVREM: return LowerSDIVREM(Op, DAG); 635 case ISD::FREM: return LowerFREM(Op, DAG); 636 case ISD::FCEIL: return LowerFCEIL(Op, DAG); 637 case ISD::FTRUNC: return LowerFTRUNC(Op, DAG); 638 case ISD::FRINT: return LowerFRINT(Op, DAG); 639 case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG); 640 case ISD::FROUND: return LowerFROUND(Op, DAG); 641 case ISD::FFLOOR: return LowerFFLOOR(Op, DAG); 642 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); 643 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG); 644 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG); 645 case ISD::FP_TO_UINT: return LowerFP_TO_UINT(Op, DAG); 646 } 647 return Op; 648} 649 650void AMDGPUTargetLowering::ReplaceNodeResults(SDNode *N, 651 SmallVectorImpl<SDValue> &Results, 652 SelectionDAG &DAG) const { 653 switch (N->getOpcode()) { 654 case ISD::SIGN_EXTEND_INREG: 655 // Different parts of legalization seem to interpret which type of 656 // sign_extend_inreg is the one to check for custom lowering. The extended 657 // from type is what really matters, but some places check for custom 658 // lowering of the result type. This results in trying to use 659 // ReplaceNodeResults to sext_in_reg to an illegal type, so we'll just do 660 // nothing here and let the illegal result integer be handled normally. 661 return; 662 case ISD::LOAD: { 663 SDNode *Node = LowerLOAD(SDValue(N, 0), DAG).getNode(); 664 if (!Node) 665 return; 666 667 Results.push_back(SDValue(Node, 0)); 668 Results.push_back(SDValue(Node, 1)); 669 // XXX: LLVM seems not to replace Chain Value inside CustomWidenLowerNode 670 // function 671 DAG.ReplaceAllUsesOfValueWith(SDValue(N,1), SDValue(Node, 1)); 672 return; 673 } 674 case ISD::STORE: { 675 SDValue Lowered = LowerSTORE(SDValue(N, 0), DAG); 676 if (Lowered.getNode()) 677 Results.push_back(Lowered); 678 return; 679 } 680 default: 681 return; 682 } 683} 684 685// FIXME: This implements accesses to initialized globals in the constant 686// address space by copying them to private and accessing that. It does not 687// properly handle illegal types or vectors. The private vector loads are not 688// scalarized, and the illegal scalars hit an assertion. This technique will not 689// work well with large initializers, and this should eventually be 690// removed. Initialized globals should be placed into a data section that the 691// runtime will load into a buffer before the kernel is executed. Uses of the 692// global need to be replaced with a pointer loaded from an implicit kernel 693// argument into this buffer holding the copy of the data, which will remove the 694// need for any of this. 695SDValue AMDGPUTargetLowering::LowerConstantInitializer(const Constant* Init, 696 const GlobalValue *GV, 697 const SDValue &InitPtr, 698 SDValue Chain, 699 SelectionDAG &DAG) const { 700 const DataLayout *TD = getDataLayout(); 701 SDLoc DL(InitPtr); 702 Type *InitTy = Init->getType(); 703 704 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Init)) { 705 EVT VT = EVT::getEVT(InitTy); 706 PointerType *PtrTy = PointerType::get(InitTy, AMDGPUAS::PRIVATE_ADDRESS); 707 return DAG.getStore(Chain, DL, DAG.getConstant(*CI, DL, VT), InitPtr, 708 MachinePointerInfo(UndefValue::get(PtrTy)), false, false, 709 TD->getPrefTypeAlignment(InitTy)); 710 } 711 712 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Init)) { 713 EVT VT = EVT::getEVT(CFP->getType()); 714 PointerType *PtrTy = PointerType::get(CFP->getType(), 0); 715 return DAG.getStore(Chain, DL, DAG.getConstantFP(*CFP, DL, VT), InitPtr, 716 MachinePointerInfo(UndefValue::get(PtrTy)), false, false, 717 TD->getPrefTypeAlignment(CFP->getType())); 718 } 719 720 if (StructType *ST = dyn_cast<StructType>(InitTy)) { 721 const StructLayout *SL = TD->getStructLayout(ST); 722 723 EVT PtrVT = InitPtr.getValueType(); 724 SmallVector<SDValue, 8> Chains; 725 726 for (unsigned I = 0, N = ST->getNumElements(); I != N; ++I) { 727 SDValue Offset = DAG.getConstant(SL->getElementOffset(I), DL, PtrVT); 728 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, InitPtr, Offset); 729 730 Constant *Elt = Init->getAggregateElement(I); 731 Chains.push_back(LowerConstantInitializer(Elt, GV, Ptr, Chain, DAG)); 732 } 733 734 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); 735 } 736 737 if (SequentialType *SeqTy = dyn_cast<SequentialType>(InitTy)) { 738 EVT PtrVT = InitPtr.getValueType(); 739 740 unsigned NumElements; 741 if (ArrayType *AT = dyn_cast<ArrayType>(SeqTy)) 742 NumElements = AT->getNumElements(); 743 else if (VectorType *VT = dyn_cast<VectorType>(SeqTy)) 744 NumElements = VT->getNumElements(); 745 else 746 llvm_unreachable("Unexpected type"); 747 748 unsigned EltSize = TD->getTypeAllocSize(SeqTy->getElementType()); 749 SmallVector<SDValue, 8> Chains; 750 for (unsigned i = 0; i < NumElements; ++i) { 751 SDValue Offset = DAG.getConstant(i * EltSize, DL, PtrVT); 752 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, InitPtr, Offset); 753 754 Constant *Elt = Init->getAggregateElement(i); 755 Chains.push_back(LowerConstantInitializer(Elt, GV, Ptr, Chain, DAG)); 756 } 757 758 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); 759 } 760 761 if (isa<UndefValue>(Init)) { 762 EVT VT = EVT::getEVT(InitTy); 763 PointerType *PtrTy = PointerType::get(InitTy, AMDGPUAS::PRIVATE_ADDRESS); 764 return DAG.getStore(Chain, DL, DAG.getUNDEF(VT), InitPtr, 765 MachinePointerInfo(UndefValue::get(PtrTy)), false, false, 766 TD->getPrefTypeAlignment(InitTy)); 767 } 768 769 Init->dump(); 770 llvm_unreachable("Unhandled constant initializer"); 771} 772 773static bool hasDefinedInitializer(const GlobalValue *GV) { 774 const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV); 775 if (!GVar || !GVar->hasInitializer()) 776 return false; 777 778 if (isa<UndefValue>(GVar->getInitializer())) 779 return false; 780 781 return true; 782} 783 784SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI, 785 SDValue Op, 786 SelectionDAG &DAG) const { 787 788 const DataLayout *TD = getDataLayout(); 789 GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Op); 790 const GlobalValue *GV = G->getGlobal(); 791 792 switch (G->getAddressSpace()) { 793 case AMDGPUAS::LOCAL_ADDRESS: { 794 // XXX: What does the value of G->getOffset() mean? 795 assert(G->getOffset() == 0 && 796 "Do not know what to do with an non-zero offset"); 797 798 // TODO: We could emit code to handle the initialization somewhere. 799 if (hasDefinedInitializer(GV)) 800 break; 801 802 unsigned Offset; 803 if (MFI->LocalMemoryObjects.count(GV) == 0) { 804 uint64_t Size = TD->getTypeAllocSize(GV->getType()->getElementType()); 805 Offset = MFI->LDSSize; 806 MFI->LocalMemoryObjects[GV] = Offset; 807 // XXX: Account for alignment? 808 MFI->LDSSize += Size; 809 } else { 810 Offset = MFI->LocalMemoryObjects[GV]; 811 } 812 813 return DAG.getConstant(Offset, SDLoc(Op), 814 getPointerTy(AMDGPUAS::LOCAL_ADDRESS)); 815 } 816 case AMDGPUAS::CONSTANT_ADDRESS: { 817 MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo(); 818 Type *EltType = GV->getType()->getElementType(); 819 unsigned Size = TD->getTypeAllocSize(EltType); 820 unsigned Alignment = TD->getPrefTypeAlignment(EltType); 821 822 MVT PrivPtrVT = getPointerTy(AMDGPUAS::PRIVATE_ADDRESS); 823 MVT ConstPtrVT = getPointerTy(AMDGPUAS::CONSTANT_ADDRESS); 824 825 int FI = FrameInfo->CreateStackObject(Size, Alignment, false); 826 SDValue InitPtr = DAG.getFrameIndex(FI, PrivPtrVT); 827 828 const GlobalVariable *Var = cast<GlobalVariable>(GV); 829 if (!Var->hasInitializer()) { 830 // This has no use, but bugpoint will hit it. 831 return DAG.getZExtOrTrunc(InitPtr, SDLoc(Op), ConstPtrVT); 832 } 833 834 const Constant *Init = Var->getInitializer(); 835 SmallVector<SDNode*, 8> WorkList; 836 837 for (SDNode::use_iterator I = DAG.getEntryNode()->use_begin(), 838 E = DAG.getEntryNode()->use_end(); I != E; ++I) { 839 if (I->getOpcode() != AMDGPUISD::REGISTER_LOAD && I->getOpcode() != ISD::LOAD) 840 continue; 841 WorkList.push_back(*I); 842 } 843 SDValue Chain = LowerConstantInitializer(Init, GV, InitPtr, DAG.getEntryNode(), DAG); 844 for (SmallVector<SDNode*, 8>::iterator I = WorkList.begin(), 845 E = WorkList.end(); I != E; ++I) { 846 SmallVector<SDValue, 8> Ops; 847 Ops.push_back(Chain); 848 for (unsigned i = 1; i < (*I)->getNumOperands(); ++i) { 849 Ops.push_back((*I)->getOperand(i)); 850 } 851 DAG.UpdateNodeOperands(*I, Ops); 852 } 853 return DAG.getZExtOrTrunc(InitPtr, SDLoc(Op), ConstPtrVT); 854 } 855 } 856 857 const Function &Fn = *DAG.getMachineFunction().getFunction(); 858 DiagnosticInfoUnsupported BadInit(Fn, 859 "initializer for address space"); 860 DAG.getContext()->diagnose(BadInit); 861 return SDValue(); 862} 863 864SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op, 865 SelectionDAG &DAG) const { 866 SmallVector<SDValue, 8> Args; 867 868 for (const SDUse &U : Op->ops()) 869 DAG.ExtractVectorElements(U.get(), Args); 870 871 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), Op.getValueType(), Args); 872} 873 874SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op, 875 SelectionDAG &DAG) const { 876 877 SmallVector<SDValue, 8> Args; 878 unsigned Start = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 879 EVT VT = Op.getValueType(); 880 DAG.ExtractVectorElements(Op.getOperand(0), Args, Start, 881 VT.getVectorNumElements()); 882 883 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), Op.getValueType(), Args); 884} 885 886SDValue AMDGPUTargetLowering::LowerFrameIndex(SDValue Op, 887 SelectionDAG &DAG) const { 888 889 MachineFunction &MF = DAG.getMachineFunction(); 890 const AMDGPUFrameLowering *TFL = Subtarget->getFrameLowering(); 891 892 FrameIndexSDNode *FIN = cast<FrameIndexSDNode>(Op); 893 894 unsigned FrameIndex = FIN->getIndex(); 895 unsigned Offset = TFL->getFrameIndexOffset(MF, FrameIndex); 896 return DAG.getConstant(Offset * 4 * TFL->getStackWidth(MF), SDLoc(Op), 897 Op.getValueType()); 898} 899 900SDValue AMDGPUTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, 901 SelectionDAG &DAG) const { 902 unsigned IntrinsicID = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 903 SDLoc DL(Op); 904 EVT VT = Op.getValueType(); 905 906 switch (IntrinsicID) { 907 default: return Op; 908 case AMDGPUIntrinsic::AMDGPU_abs: 909 case AMDGPUIntrinsic::AMDIL_abs: // Legacy name. 910 return LowerIntrinsicIABS(Op, DAG); 911 case AMDGPUIntrinsic::AMDGPU_lrp: 912 return LowerIntrinsicLRP(Op, DAG); 913 914 case AMDGPUIntrinsic::AMDGPU_clamp: 915 case AMDGPUIntrinsic::AMDIL_clamp: // Legacy name. 916 return DAG.getNode(AMDGPUISD::CLAMP, DL, VT, 917 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 918 919 case Intrinsic::AMDGPU_div_scale: { 920 // 3rd parameter required to be a constant. 921 const ConstantSDNode *Param = dyn_cast<ConstantSDNode>(Op.getOperand(3)); 922 if (!Param) 923 return DAG.getUNDEF(VT); 924 925 // Translate to the operands expected by the machine instruction. The 926 // first parameter must be the same as the first instruction. 927 SDValue Numerator = Op.getOperand(1); 928 SDValue Denominator = Op.getOperand(2); 929 930 // Note this order is opposite of the machine instruction's operations, 931 // which is s0.f = Quotient, s1.f = Denominator, s2.f = Numerator. The 932 // intrinsic has the numerator as the first operand to match a normal 933 // division operation. 934 935 SDValue Src0 = Param->isAllOnesValue() ? Numerator : Denominator; 936 937 return DAG.getNode(AMDGPUISD::DIV_SCALE, DL, Op->getVTList(), Src0, 938 Denominator, Numerator); 939 } 940 941 case Intrinsic::AMDGPU_div_fmas: 942 return DAG.getNode(AMDGPUISD::DIV_FMAS, DL, VT, 943 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3), 944 Op.getOperand(4)); 945 946 case Intrinsic::AMDGPU_div_fixup: 947 return DAG.getNode(AMDGPUISD::DIV_FIXUP, DL, VT, 948 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 949 950 case Intrinsic::AMDGPU_trig_preop: 951 return DAG.getNode(AMDGPUISD::TRIG_PREOP, DL, VT, 952 Op.getOperand(1), Op.getOperand(2)); 953 954 case Intrinsic::AMDGPU_rcp: 955 return DAG.getNode(AMDGPUISD::RCP, DL, VT, Op.getOperand(1)); 956 957 case Intrinsic::AMDGPU_rsq: 958 return DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1)); 959 960 case AMDGPUIntrinsic::AMDGPU_legacy_rsq: 961 return DAG.getNode(AMDGPUISD::RSQ_LEGACY, DL, VT, Op.getOperand(1)); 962 963 case Intrinsic::AMDGPU_rsq_clamped: 964 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) { 965 Type *Type = VT.getTypeForEVT(*DAG.getContext()); 966 APFloat Max = APFloat::getLargest(Type->getFltSemantics()); 967 APFloat Min = APFloat::getLargest(Type->getFltSemantics(), true); 968 969 SDValue Rsq = DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1)); 970 SDValue Tmp = DAG.getNode(ISD::FMINNUM, DL, VT, Rsq, 971 DAG.getConstantFP(Max, DL, VT)); 972 return DAG.getNode(ISD::FMAXNUM, DL, VT, Tmp, 973 DAG.getConstantFP(Min, DL, VT)); 974 } else { 975 return DAG.getNode(AMDGPUISD::RSQ_CLAMPED, DL, VT, Op.getOperand(1)); 976 } 977 978 case Intrinsic::AMDGPU_ldexp: 979 return DAG.getNode(AMDGPUISD::LDEXP, DL, VT, Op.getOperand(1), 980 Op.getOperand(2)); 981 982 case AMDGPUIntrinsic::AMDGPU_imax: 983 return DAG.getNode(ISD::SMAX, DL, VT, Op.getOperand(1), 984 Op.getOperand(2)); 985 case AMDGPUIntrinsic::AMDGPU_umax: 986 return DAG.getNode(ISD::UMAX, DL, VT, Op.getOperand(1), 987 Op.getOperand(2)); 988 case AMDGPUIntrinsic::AMDGPU_imin: 989 return DAG.getNode(ISD::SMIN, DL, VT, Op.getOperand(1), 990 Op.getOperand(2)); 991 case AMDGPUIntrinsic::AMDGPU_umin: 992 return DAG.getNode(ISD::UMIN, DL, VT, Op.getOperand(1), 993 Op.getOperand(2)); 994 995 case AMDGPUIntrinsic::AMDGPU_umul24: 996 return DAG.getNode(AMDGPUISD::MUL_U24, DL, VT, 997 Op.getOperand(1), Op.getOperand(2)); 998 999 case AMDGPUIntrinsic::AMDGPU_imul24: 1000 return DAG.getNode(AMDGPUISD::MUL_I24, DL, VT, 1001 Op.getOperand(1), Op.getOperand(2)); 1002 1003 case AMDGPUIntrinsic::AMDGPU_umad24: 1004 return DAG.getNode(AMDGPUISD::MAD_U24, DL, VT, 1005 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 1006 1007 case AMDGPUIntrinsic::AMDGPU_imad24: 1008 return DAG.getNode(AMDGPUISD::MAD_I24, DL, VT, 1009 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); 1010 1011 case AMDGPUIntrinsic::AMDGPU_cvt_f32_ubyte0: 1012 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0, DL, VT, Op.getOperand(1)); 1013 1014 case AMDGPUIntrinsic::AMDGPU_cvt_f32_ubyte1: 1015 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE1, DL, VT, Op.getOperand(1)); 1016 1017 case AMDGPUIntrinsic::AMDGPU_cvt_f32_ubyte2: 1018 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE2, DL, VT, Op.getOperand(1)); 1019 1020 case AMDGPUIntrinsic::AMDGPU_cvt_f32_ubyte3: 1021 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE3, DL, VT, Op.getOperand(1)); 1022 1023 case AMDGPUIntrinsic::AMDGPU_bfe_i32: 1024 return DAG.getNode(AMDGPUISD::BFE_I32, DL, VT, 1025 Op.getOperand(1), 1026 Op.getOperand(2), 1027 Op.getOperand(3)); 1028 1029 case AMDGPUIntrinsic::AMDGPU_bfe_u32: 1030 return DAG.getNode(AMDGPUISD::BFE_U32, DL, VT, 1031 Op.getOperand(1), 1032 Op.getOperand(2), 1033 Op.getOperand(3)); 1034 1035 case AMDGPUIntrinsic::AMDGPU_bfi: 1036 return DAG.getNode(AMDGPUISD::BFI, DL, VT, 1037 Op.getOperand(1), 1038 Op.getOperand(2), 1039 Op.getOperand(3)); 1040 1041 case AMDGPUIntrinsic::AMDGPU_bfm: 1042 return DAG.getNode(AMDGPUISD::BFM, DL, VT, 1043 Op.getOperand(1), 1044 Op.getOperand(2)); 1045 1046 case AMDGPUIntrinsic::AMDGPU_brev: 1047 return DAG.getNode(AMDGPUISD::BREV, DL, VT, Op.getOperand(1)); 1048 1049 case Intrinsic::AMDGPU_class: 1050 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, VT, 1051 Op.getOperand(1), Op.getOperand(2)); 1052 1053 case AMDGPUIntrinsic::AMDIL_exp: // Legacy name. 1054 return DAG.getNode(ISD::FEXP2, DL, VT, Op.getOperand(1)); 1055 1056 case AMDGPUIntrinsic::AMDIL_round_nearest: // Legacy name. 1057 return DAG.getNode(ISD::FRINT, DL, VT, Op.getOperand(1)); 1058 case AMDGPUIntrinsic::AMDGPU_trunc: // Legacy name. 1059 return DAG.getNode(ISD::FTRUNC, DL, VT, Op.getOperand(1)); 1060 } 1061} 1062 1063///IABS(a) = SMAX(sub(0, a), a) 1064SDValue AMDGPUTargetLowering::LowerIntrinsicIABS(SDValue Op, 1065 SelectionDAG &DAG) const { 1066 SDLoc DL(Op); 1067 EVT VT = Op.getValueType(); 1068 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), 1069 Op.getOperand(1)); 1070 1071 return DAG.getNode(ISD::SMAX, DL, VT, Neg, Op.getOperand(1)); 1072} 1073 1074/// Linear Interpolation 1075/// LRP(a, b, c) = muladd(a, b, (1 - a) * c) 1076SDValue AMDGPUTargetLowering::LowerIntrinsicLRP(SDValue Op, 1077 SelectionDAG &DAG) const { 1078 SDLoc DL(Op); 1079 EVT VT = Op.getValueType(); 1080 SDValue OneSubA = DAG.getNode(ISD::FSUB, DL, VT, 1081 DAG.getConstantFP(1.0f, DL, MVT::f32), 1082 Op.getOperand(1)); 1083 SDValue OneSubAC = DAG.getNode(ISD::FMUL, DL, VT, OneSubA, 1084 Op.getOperand(3)); 1085 return DAG.getNode(ISD::FADD, DL, VT, 1086 DAG.getNode(ISD::FMUL, DL, VT, Op.getOperand(1), Op.getOperand(2)), 1087 OneSubAC); 1088} 1089 1090/// \brief Generate Min/Max node 1091SDValue AMDGPUTargetLowering::CombineFMinMaxLegacy(SDLoc DL, 1092 EVT VT, 1093 SDValue LHS, 1094 SDValue RHS, 1095 SDValue True, 1096 SDValue False, 1097 SDValue CC, 1098 DAGCombinerInfo &DCI) const { 1099 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) 1100 return SDValue(); 1101 1102 if (!(LHS == True && RHS == False) && !(LHS == False && RHS == True)) 1103 return SDValue(); 1104 1105 SelectionDAG &DAG = DCI.DAG; 1106 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 1107 switch (CCOpcode) { 1108 case ISD::SETOEQ: 1109 case ISD::SETONE: 1110 case ISD::SETUNE: 1111 case ISD::SETNE: 1112 case ISD::SETUEQ: 1113 case ISD::SETEQ: 1114 case ISD::SETFALSE: 1115 case ISD::SETFALSE2: 1116 case ISD::SETTRUE: 1117 case ISD::SETTRUE2: 1118 case ISD::SETUO: 1119 case ISD::SETO: 1120 break; 1121 case ISD::SETULE: 1122 case ISD::SETULT: { 1123 if (LHS == True) 1124 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS); 1125 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS); 1126 } 1127 case ISD::SETOLE: 1128 case ISD::SETOLT: 1129 case ISD::SETLE: 1130 case ISD::SETLT: { 1131 // Ordered. Assume ordered for undefined. 1132 1133 // Only do this after legalization to avoid interfering with other combines 1134 // which might occur. 1135 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG && 1136 !DCI.isCalledByLegalizer()) 1137 return SDValue(); 1138 1139 // We need to permute the operands to get the correct NaN behavior. The 1140 // selected operand is the second one based on the failing compare with NaN, 1141 // so permute it based on the compare type the hardware uses. 1142 if (LHS == True) 1143 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS); 1144 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS); 1145 } 1146 case ISD::SETUGE: 1147 case ISD::SETUGT: { 1148 if (LHS == True) 1149 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS); 1150 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS); 1151 } 1152 case ISD::SETGT: 1153 case ISD::SETGE: 1154 case ISD::SETOGE: 1155 case ISD::SETOGT: { 1156 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG && 1157 !DCI.isCalledByLegalizer()) 1158 return SDValue(); 1159 1160 if (LHS == True) 1161 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS); 1162 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS); 1163 } 1164 case ISD::SETCC_INVALID: 1165 llvm_unreachable("Invalid setcc condcode!"); 1166 } 1167 return SDValue(); 1168} 1169 1170// FIXME: Remove this when combines added to DAGCombiner. 1171SDValue AMDGPUTargetLowering::CombineIMinMax(SDLoc DL, 1172 EVT VT, 1173 SDValue LHS, 1174 SDValue RHS, 1175 SDValue True, 1176 SDValue False, 1177 SDValue CC, 1178 SelectionDAG &DAG) const { 1179 if (!(LHS == True && RHS == False) && !(LHS == False && RHS == True)) 1180 return SDValue(); 1181 1182 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 1183 switch (CCOpcode) { 1184 case ISD::SETULE: 1185 case ISD::SETULT: { 1186 unsigned Opc = (LHS == True) ? ISD::UMIN : ISD::UMAX; 1187 return DAG.getNode(Opc, DL, VT, LHS, RHS); 1188 } 1189 case ISD::SETLE: 1190 case ISD::SETLT: { 1191 unsigned Opc = (LHS == True) ? ISD::SMIN : ISD::SMAX; 1192 return DAG.getNode(Opc, DL, VT, LHS, RHS); 1193 } 1194 case ISD::SETGT: 1195 case ISD::SETGE: { 1196 unsigned Opc = (LHS == True) ? ISD::SMAX : ISD::SMIN; 1197 return DAG.getNode(Opc, DL, VT, LHS, RHS); 1198 } 1199 case ISD::SETUGE: 1200 case ISD::SETUGT: { 1201 unsigned Opc = (LHS == True) ? ISD::UMAX : ISD::UMIN; 1202 return DAG.getNode(Opc, DL, VT, LHS, RHS); 1203 } 1204 default: 1205 return SDValue(); 1206 } 1207} 1208 1209SDValue AMDGPUTargetLowering::ScalarizeVectorLoad(const SDValue Op, 1210 SelectionDAG &DAG) const { 1211 LoadSDNode *Load = cast<LoadSDNode>(Op); 1212 EVT MemVT = Load->getMemoryVT(); 1213 EVT MemEltVT = MemVT.getVectorElementType(); 1214 1215 EVT LoadVT = Op.getValueType(); 1216 EVT EltVT = LoadVT.getVectorElementType(); 1217 EVT PtrVT = Load->getBasePtr().getValueType(); 1218 1219 unsigned NumElts = Load->getMemoryVT().getVectorNumElements(); 1220 SmallVector<SDValue, 8> Loads; 1221 SmallVector<SDValue, 8> Chains; 1222 1223 SDLoc SL(Op); 1224 unsigned MemEltSize = MemEltVT.getStoreSize(); 1225 MachinePointerInfo SrcValue(Load->getMemOperand()->getValue()); 1226 1227 for (unsigned i = 0; i < NumElts; ++i) { 1228 SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT, Load->getBasePtr(), 1229 DAG.getConstant(i * MemEltSize, SL, PtrVT)); 1230 1231 SDValue NewLoad 1232 = DAG.getExtLoad(Load->getExtensionType(), SL, EltVT, 1233 Load->getChain(), Ptr, 1234 SrcValue.getWithOffset(i * MemEltSize), 1235 MemEltVT, Load->isVolatile(), Load->isNonTemporal(), 1236 Load->isInvariant(), Load->getAlignment()); 1237 Loads.push_back(NewLoad.getValue(0)); 1238 Chains.push_back(NewLoad.getValue(1)); 1239 } 1240 1241 SDValue Ops[] = { 1242 DAG.getNode(ISD::BUILD_VECTOR, SL, LoadVT, Loads), 1243 DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Chains) 1244 }; 1245 1246 return DAG.getMergeValues(Ops, SL); 1247} 1248 1249SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op, 1250 SelectionDAG &DAG) const { 1251 EVT VT = Op.getValueType(); 1252 1253 // If this is a 2 element vector, we really want to scalarize and not create 1254 // weird 1 element vectors. 1255 if (VT.getVectorNumElements() == 2) 1256 return ScalarizeVectorLoad(Op, DAG); 1257 1258 LoadSDNode *Load = cast<LoadSDNode>(Op); 1259 SDValue BasePtr = Load->getBasePtr(); 1260 EVT PtrVT = BasePtr.getValueType(); 1261 EVT MemVT = Load->getMemoryVT(); 1262 SDLoc SL(Op); 1263 MachinePointerInfo SrcValue(Load->getMemOperand()->getValue()); 1264 1265 EVT LoVT, HiVT; 1266 EVT LoMemVT, HiMemVT; 1267 SDValue Lo, Hi; 1268 1269 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); 1270 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemVT); 1271 std::tie(Lo, Hi) = DAG.SplitVector(Op, SL, LoVT, HiVT); 1272 SDValue LoLoad 1273 = DAG.getExtLoad(Load->getExtensionType(), SL, LoVT, 1274 Load->getChain(), BasePtr, 1275 SrcValue, 1276 LoMemVT, Load->isVolatile(), Load->isNonTemporal(), 1277 Load->isInvariant(), Load->getAlignment()); 1278 1279 SDValue HiPtr = DAG.getNode(ISD::ADD, SL, PtrVT, BasePtr, 1280 DAG.getConstant(LoMemVT.getStoreSize(), SL, 1281 PtrVT)); 1282 1283 SDValue HiLoad 1284 = DAG.getExtLoad(Load->getExtensionType(), SL, HiVT, 1285 Load->getChain(), HiPtr, 1286 SrcValue.getWithOffset(LoMemVT.getStoreSize()), 1287 HiMemVT, Load->isVolatile(), Load->isNonTemporal(), 1288 Load->isInvariant(), Load->getAlignment()); 1289 1290 SDValue Ops[] = { 1291 DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad), 1292 DAG.getNode(ISD::TokenFactor, SL, MVT::Other, 1293 LoLoad.getValue(1), HiLoad.getValue(1)) 1294 }; 1295 1296 return DAG.getMergeValues(Ops, SL); 1297} 1298 1299SDValue AMDGPUTargetLowering::MergeVectorStore(const SDValue &Op, 1300 SelectionDAG &DAG) const { 1301 StoreSDNode *Store = cast<StoreSDNode>(Op); 1302 EVT MemVT = Store->getMemoryVT(); 1303 unsigned MemBits = MemVT.getSizeInBits(); 1304 1305 // Byte stores are really expensive, so if possible, try to pack 32-bit vector 1306 // truncating store into an i32 store. 1307 // XXX: We could also handle optimize other vector bitwidths. 1308 if (!MemVT.isVector() || MemBits > 32) { 1309 return SDValue(); 1310 } 1311 1312 SDLoc DL(Op); 1313 SDValue Value = Store->getValue(); 1314 EVT VT = Value.getValueType(); 1315 EVT ElemVT = VT.getVectorElementType(); 1316 SDValue Ptr = Store->getBasePtr(); 1317 EVT MemEltVT = MemVT.getVectorElementType(); 1318 unsigned MemEltBits = MemEltVT.getSizeInBits(); 1319 unsigned MemNumElements = MemVT.getVectorNumElements(); 1320 unsigned PackedSize = MemVT.getStoreSizeInBits(); 1321 SDValue Mask = DAG.getConstant((1 << MemEltBits) - 1, DL, MVT::i32); 1322 1323 assert(Value.getValueType().getScalarSizeInBits() >= 32); 1324 1325 SDValue PackedValue; 1326 for (unsigned i = 0; i < MemNumElements; ++i) { 1327 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElemVT, Value, 1328 DAG.getConstant(i, DL, MVT::i32)); 1329 Elt = DAG.getZExtOrTrunc(Elt, DL, MVT::i32); 1330 Elt = DAG.getNode(ISD::AND, DL, MVT::i32, Elt, Mask); // getZeroExtendInReg 1331 1332 SDValue Shift = DAG.getConstant(MemEltBits * i, DL, MVT::i32); 1333 Elt = DAG.getNode(ISD::SHL, DL, MVT::i32, Elt, Shift); 1334 1335 if (i == 0) { 1336 PackedValue = Elt; 1337 } else { 1338 PackedValue = DAG.getNode(ISD::OR, DL, MVT::i32, PackedValue, Elt); 1339 } 1340 } 1341 1342 if (PackedSize < 32) { 1343 EVT PackedVT = EVT::getIntegerVT(*DAG.getContext(), PackedSize); 1344 return DAG.getTruncStore(Store->getChain(), DL, PackedValue, Ptr, 1345 Store->getMemOperand()->getPointerInfo(), 1346 PackedVT, 1347 Store->isNonTemporal(), Store->isVolatile(), 1348 Store->getAlignment()); 1349 } 1350 1351 return DAG.getStore(Store->getChain(), DL, PackedValue, Ptr, 1352 Store->getMemOperand()->getPointerInfo(), 1353 Store->isVolatile(), Store->isNonTemporal(), 1354 Store->getAlignment()); 1355} 1356 1357SDValue AMDGPUTargetLowering::ScalarizeVectorStore(SDValue Op, 1358 SelectionDAG &DAG) const { 1359 StoreSDNode *Store = cast<StoreSDNode>(Op); 1360 EVT MemEltVT = Store->getMemoryVT().getVectorElementType(); 1361 EVT EltVT = Store->getValue().getValueType().getVectorElementType(); 1362 EVT PtrVT = Store->getBasePtr().getValueType(); 1363 unsigned NumElts = Store->getMemoryVT().getVectorNumElements(); 1364 SDLoc SL(Op); 1365 1366 SmallVector<SDValue, 8> Chains; 1367 1368 unsigned EltSize = MemEltVT.getStoreSize(); 1369 MachinePointerInfo SrcValue(Store->getMemOperand()->getValue()); 1370 1371 for (unsigned i = 0, e = NumElts; i != e; ++i) { 1372 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, 1373 Store->getValue(), 1374 DAG.getConstant(i, SL, MVT::i32)); 1375 1376 SDValue Offset = DAG.getConstant(i * MemEltVT.getStoreSize(), SL, PtrVT); 1377 SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT, Store->getBasePtr(), Offset); 1378 SDValue NewStore = 1379 DAG.getTruncStore(Store->getChain(), SL, Val, Ptr, 1380 SrcValue.getWithOffset(i * EltSize), 1381 MemEltVT, Store->isNonTemporal(), Store->isVolatile(), 1382 Store->getAlignment()); 1383 Chains.push_back(NewStore); 1384 } 1385 1386 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Chains); 1387} 1388 1389SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op, 1390 SelectionDAG &DAG) const { 1391 StoreSDNode *Store = cast<StoreSDNode>(Op); 1392 SDValue Val = Store->getValue(); 1393 EVT VT = Val.getValueType(); 1394 1395 // If this is a 2 element vector, we really want to scalarize and not create 1396 // weird 1 element vectors. 1397 if (VT.getVectorNumElements() == 2) 1398 return ScalarizeVectorStore(Op, DAG); 1399 1400 EVT MemVT = Store->getMemoryVT(); 1401 SDValue Chain = Store->getChain(); 1402 SDValue BasePtr = Store->getBasePtr(); 1403 SDLoc SL(Op); 1404 1405 EVT LoVT, HiVT; 1406 EVT LoMemVT, HiMemVT; 1407 SDValue Lo, Hi; 1408 1409 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); 1410 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemVT); 1411 std::tie(Lo, Hi) = DAG.SplitVector(Val, SL, LoVT, HiVT); 1412 1413 EVT PtrVT = BasePtr.getValueType(); 1414 SDValue HiPtr = DAG.getNode(ISD::ADD, SL, PtrVT, BasePtr, 1415 DAG.getConstant(LoMemVT.getStoreSize(), SL, 1416 PtrVT)); 1417 1418 MachinePointerInfo SrcValue(Store->getMemOperand()->getValue()); 1419 SDValue LoStore 1420 = DAG.getTruncStore(Chain, SL, Lo, 1421 BasePtr, 1422 SrcValue, 1423 LoMemVT, 1424 Store->isNonTemporal(), 1425 Store->isVolatile(), 1426 Store->getAlignment()); 1427 SDValue HiStore 1428 = DAG.getTruncStore(Chain, SL, Hi, 1429 HiPtr, 1430 SrcValue.getWithOffset(LoMemVT.getStoreSize()), 1431 HiMemVT, 1432 Store->isNonTemporal(), 1433 Store->isVolatile(), 1434 Store->getAlignment()); 1435 1436 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore); 1437} 1438 1439 1440SDValue AMDGPUTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { 1441 SDLoc DL(Op); 1442 LoadSDNode *Load = cast<LoadSDNode>(Op); 1443 ISD::LoadExtType ExtType = Load->getExtensionType(); 1444 EVT VT = Op.getValueType(); 1445 EVT MemVT = Load->getMemoryVT(); 1446 1447 if (ExtType == ISD::NON_EXTLOAD && VT.getSizeInBits() < 32) { 1448 assert(VT == MVT::i1 && "Only i1 non-extloads expected"); 1449 // FIXME: Copied from PPC 1450 // First, load into 32 bits, then truncate to 1 bit. 1451 1452 SDValue Chain = Load->getChain(); 1453 SDValue BasePtr = Load->getBasePtr(); 1454 MachineMemOperand *MMO = Load->getMemOperand(); 1455 1456 SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, DL, MVT::i32, Chain, 1457 BasePtr, MVT::i8, MMO); 1458 1459 SDValue Ops[] = { 1460 DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD), 1461 NewLD.getValue(1) 1462 }; 1463 1464 return DAG.getMergeValues(Ops, DL); 1465 } 1466 1467 if (Subtarget->getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS || 1468 Load->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS || 1469 ExtType == ISD::NON_EXTLOAD || Load->getMemoryVT().bitsGE(MVT::i32)) 1470 return SDValue(); 1471 1472 // <SI && AS=PRIVATE && EXTLOAD && size < 32bit, 1473 // register (2-)byte extract. 1474 1475 // Get Register holding the target. 1476 SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Load->getBasePtr(), 1477 DAG.getConstant(2, DL, MVT::i32)); 1478 // Load the Register. 1479 SDValue Ret = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, Op.getValueType(), 1480 Load->getChain(), Ptr, 1481 DAG.getTargetConstant(0, DL, MVT::i32), 1482 Op.getOperand(2)); 1483 1484 // Get offset within the register. 1485 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, 1486 Load->getBasePtr(), 1487 DAG.getConstant(0x3, DL, MVT::i32)); 1488 1489 // Bit offset of target byte (byteIdx * 8). 1490 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx, 1491 DAG.getConstant(3, DL, MVT::i32)); 1492 1493 // Shift to the right. 1494 Ret = DAG.getNode(ISD::SRL, DL, MVT::i32, Ret, ShiftAmt); 1495 1496 // Eliminate the upper bits by setting them to ... 1497 EVT MemEltVT = MemVT.getScalarType(); 1498 1499 // ... ones. 1500 if (ExtType == ISD::SEXTLOAD) { 1501 SDValue MemEltVTNode = DAG.getValueType(MemEltVT); 1502 1503 SDValue Ops[] = { 1504 DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, Ret, MemEltVTNode), 1505 Load->getChain() 1506 }; 1507 1508 return DAG.getMergeValues(Ops, DL); 1509 } 1510 1511 // ... or zeros. 1512 SDValue Ops[] = { 1513 DAG.getZeroExtendInReg(Ret, DL, MemEltVT), 1514 Load->getChain() 1515 }; 1516 1517 return DAG.getMergeValues(Ops, DL); 1518} 1519 1520SDValue AMDGPUTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { 1521 SDLoc DL(Op); 1522 SDValue Result = AMDGPUTargetLowering::MergeVectorStore(Op, DAG); 1523 if (Result.getNode()) { 1524 return Result; 1525 } 1526 1527 StoreSDNode *Store = cast<StoreSDNode>(Op); 1528 SDValue Chain = Store->getChain(); 1529 if ((Store->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS || 1530 Store->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) && 1531 Store->getValue().getValueType().isVector()) { 1532 return ScalarizeVectorStore(Op, DAG); 1533 } 1534 1535 EVT MemVT = Store->getMemoryVT(); 1536 if (Store->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS && 1537 MemVT.bitsLT(MVT::i32)) { 1538 unsigned Mask = 0; 1539 if (Store->getMemoryVT() == MVT::i8) { 1540 Mask = 0xff; 1541 } else if (Store->getMemoryVT() == MVT::i16) { 1542 Mask = 0xffff; 1543 } 1544 SDValue BasePtr = Store->getBasePtr(); 1545 SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, BasePtr, 1546 DAG.getConstant(2, DL, MVT::i32)); 1547 SDValue Dst = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, MVT::i32, 1548 Chain, Ptr, 1549 DAG.getTargetConstant(0, DL, MVT::i32)); 1550 1551 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, BasePtr, 1552 DAG.getConstant(0x3, DL, MVT::i32)); 1553 1554 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx, 1555 DAG.getConstant(3, DL, MVT::i32)); 1556 1557 SDValue SExtValue = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, 1558 Store->getValue()); 1559 1560 SDValue MaskedValue = DAG.getZeroExtendInReg(SExtValue, DL, MemVT); 1561 1562 SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, MVT::i32, 1563 MaskedValue, ShiftAmt); 1564 1565 SDValue DstMask = DAG.getNode(ISD::SHL, DL, MVT::i32, 1566 DAG.getConstant(Mask, DL, MVT::i32), 1567 ShiftAmt); 1568 DstMask = DAG.getNode(ISD::XOR, DL, MVT::i32, DstMask, 1569 DAG.getConstant(0xffffffff, DL, MVT::i32)); 1570 Dst = DAG.getNode(ISD::AND, DL, MVT::i32, Dst, DstMask); 1571 1572 SDValue Value = DAG.getNode(ISD::OR, DL, MVT::i32, Dst, ShiftedValue); 1573 return DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other, 1574 Chain, Value, Ptr, 1575 DAG.getTargetConstant(0, DL, MVT::i32)); 1576 } 1577 return SDValue(); 1578} 1579 1580// This is a shortcut for integer division because we have fast i32<->f32 1581// conversions, and fast f32 reciprocal instructions. The fractional part of a 1582// float is enough to accurately represent up to a 24-bit integer. 1583SDValue AMDGPUTargetLowering::LowerDIVREM24(SDValue Op, SelectionDAG &DAG, bool sign) const { 1584 SDLoc DL(Op); 1585 EVT VT = Op.getValueType(); 1586 SDValue LHS = Op.getOperand(0); 1587 SDValue RHS = Op.getOperand(1); 1588 MVT IntVT = MVT::i32; 1589 MVT FltVT = MVT::f32; 1590 1591 ISD::NodeType ToFp = sign ? ISD::SINT_TO_FP : ISD::UINT_TO_FP; 1592 ISD::NodeType ToInt = sign ? ISD::FP_TO_SINT : ISD::FP_TO_UINT; 1593 1594 if (VT.isVector()) { 1595 unsigned NElts = VT.getVectorNumElements(); 1596 IntVT = MVT::getVectorVT(MVT::i32, NElts); 1597 FltVT = MVT::getVectorVT(MVT::f32, NElts); 1598 } 1599 1600 unsigned BitSize = VT.getScalarType().getSizeInBits(); 1601 1602 SDValue jq = DAG.getConstant(1, DL, IntVT); 1603 1604 if (sign) { 1605 // char|short jq = ia ^ ib; 1606 jq = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS); 1607 1608 // jq = jq >> (bitsize - 2) 1609 jq = DAG.getNode(ISD::SRA, DL, VT, jq, 1610 DAG.getConstant(BitSize - 2, DL, VT)); 1611 1612 // jq = jq | 0x1 1613 jq = DAG.getNode(ISD::OR, DL, VT, jq, DAG.getConstant(1, DL, VT)); 1614 1615 // jq = (int)jq 1616 jq = DAG.getSExtOrTrunc(jq, DL, IntVT); 1617 } 1618 1619 // int ia = (int)LHS; 1620 SDValue ia = sign ? 1621 DAG.getSExtOrTrunc(LHS, DL, IntVT) : DAG.getZExtOrTrunc(LHS, DL, IntVT); 1622 1623 // int ib, (int)RHS; 1624 SDValue ib = sign ? 1625 DAG.getSExtOrTrunc(RHS, DL, IntVT) : DAG.getZExtOrTrunc(RHS, DL, IntVT); 1626 1627 // float fa = (float)ia; 1628 SDValue fa = DAG.getNode(ToFp, DL, FltVT, ia); 1629 1630 // float fb = (float)ib; 1631 SDValue fb = DAG.getNode(ToFp, DL, FltVT, ib); 1632 1633 // float fq = native_divide(fa, fb); 1634 SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT, 1635 fa, DAG.getNode(AMDGPUISD::RCP, DL, FltVT, fb)); 1636 1637 // fq = trunc(fq); 1638 fq = DAG.getNode(ISD::FTRUNC, DL, FltVT, fq); 1639 1640 // float fqneg = -fq; 1641 SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FltVT, fq); 1642 1643 // float fr = mad(fqneg, fb, fa); 1644 SDValue fr = DAG.getNode(ISD::FADD, DL, FltVT, 1645 DAG.getNode(ISD::FMUL, DL, FltVT, fqneg, fb), fa); 1646 1647 // int iq = (int)fq; 1648 SDValue iq = DAG.getNode(ToInt, DL, IntVT, fq); 1649 1650 // fr = fabs(fr); 1651 fr = DAG.getNode(ISD::FABS, DL, FltVT, fr); 1652 1653 // fb = fabs(fb); 1654 fb = DAG.getNode(ISD::FABS, DL, FltVT, fb); 1655 1656 EVT SetCCVT = getSetCCResultType(*DAG.getContext(), VT); 1657 1658 // int cv = fr >= fb; 1659 SDValue cv = DAG.getSetCC(DL, SetCCVT, fr, fb, ISD::SETOGE); 1660 1661 // jq = (cv ? jq : 0); 1662 jq = DAG.getNode(ISD::SELECT, DL, VT, cv, jq, DAG.getConstant(0, DL, VT)); 1663 1664 // dst = trunc/extend to legal type 1665 iq = sign ? DAG.getSExtOrTrunc(iq, DL, VT) : DAG.getZExtOrTrunc(iq, DL, VT); 1666 1667 // dst = iq + jq; 1668 SDValue Div = DAG.getNode(ISD::ADD, DL, VT, iq, jq); 1669 1670 // Rem needs compensation, it's easier to recompute it 1671 SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS); 1672 Rem = DAG.getNode(ISD::SUB, DL, VT, LHS, Rem); 1673 1674 SDValue Res[2] = { 1675 Div, 1676 Rem 1677 }; 1678 return DAG.getMergeValues(Res, DL); 1679} 1680 1681void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op, 1682 SelectionDAG &DAG, 1683 SmallVectorImpl<SDValue> &Results) const { 1684 assert(Op.getValueType() == MVT::i64); 1685 1686 SDLoc DL(Op); 1687 EVT VT = Op.getValueType(); 1688 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext()); 1689 1690 SDValue one = DAG.getConstant(1, DL, HalfVT); 1691 SDValue zero = DAG.getConstant(0, DL, HalfVT); 1692 1693 //HiLo split 1694 SDValue LHS = Op.getOperand(0); 1695 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, zero); 1696 SDValue LHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, one); 1697 1698 SDValue RHS = Op.getOperand(1); 1699 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, zero); 1700 SDValue RHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, one); 1701 1702 if (VT == MVT::i64 && 1703 DAG.MaskedValueIsZero(RHS, APInt::getHighBitsSet(64, 32)) && 1704 DAG.MaskedValueIsZero(LHS, APInt::getHighBitsSet(64, 32))) { 1705 1706 SDValue Res = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(HalfVT, HalfVT), 1707 LHS_Lo, RHS_Lo); 1708 1709 SDValue DIV = DAG.getNode(ISD::BUILD_PAIR, DL, VT, Res.getValue(0), zero); 1710 SDValue REM = DAG.getNode(ISD::BUILD_PAIR, DL, VT, Res.getValue(1), zero); 1711 Results.push_back(DIV); 1712 Results.push_back(REM); 1713 return; 1714 } 1715 1716 // Get Speculative values 1717 SDValue DIV_Part = DAG.getNode(ISD::UDIV, DL, HalfVT, LHS_Hi, RHS_Lo); 1718 SDValue REM_Part = DAG.getNode(ISD::UREM, DL, HalfVT, LHS_Hi, RHS_Lo); 1719 1720 SDValue REM_Lo = DAG.getSelectCC(DL, RHS_Hi, zero, REM_Part, LHS_Hi, ISD::SETEQ); 1721 SDValue REM = DAG.getNode(ISD::BUILD_PAIR, DL, VT, REM_Lo, zero); 1722 1723 SDValue DIV_Hi = DAG.getSelectCC(DL, RHS_Hi, zero, DIV_Part, zero, ISD::SETEQ); 1724 SDValue DIV_Lo = zero; 1725 1726 const unsigned halfBitWidth = HalfVT.getSizeInBits(); 1727 1728 for (unsigned i = 0; i < halfBitWidth; ++i) { 1729 const unsigned bitPos = halfBitWidth - i - 1; 1730 SDValue POS = DAG.getConstant(bitPos, DL, HalfVT); 1731 // Get value of high bit 1732 SDValue HBit = DAG.getNode(ISD::SRL, DL, HalfVT, LHS_Lo, POS); 1733 HBit = DAG.getNode(ISD::AND, DL, HalfVT, HBit, one); 1734 HBit = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, HBit); 1735 1736 // Shift 1737 REM = DAG.getNode(ISD::SHL, DL, VT, REM, DAG.getConstant(1, DL, VT)); 1738 // Add LHS high bit 1739 REM = DAG.getNode(ISD::OR, DL, VT, REM, HBit); 1740 1741 SDValue BIT = DAG.getConstant(1 << bitPos, DL, HalfVT); 1742 SDValue realBIT = DAG.getSelectCC(DL, REM, RHS, BIT, zero, ISD::SETUGE); 1743 1744 DIV_Lo = DAG.getNode(ISD::OR, DL, HalfVT, DIV_Lo, realBIT); 1745 1746 // Update REM 1747 SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS); 1748 REM = DAG.getSelectCC(DL, REM, RHS, REM_sub, REM, ISD::SETUGE); 1749 } 1750 1751 SDValue DIV = DAG.getNode(ISD::BUILD_PAIR, DL, VT, DIV_Lo, DIV_Hi); 1752 Results.push_back(DIV); 1753 Results.push_back(REM); 1754} 1755 1756SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op, 1757 SelectionDAG &DAG) const { 1758 SDLoc DL(Op); 1759 EVT VT = Op.getValueType(); 1760 1761 if (VT == MVT::i64) { 1762 SmallVector<SDValue, 2> Results; 1763 LowerUDIVREM64(Op, DAG, Results); 1764 return DAG.getMergeValues(Results, DL); 1765 } 1766 1767 SDValue Num = Op.getOperand(0); 1768 SDValue Den = Op.getOperand(1); 1769 1770 if (VT == MVT::i32) { 1771 if (DAG.MaskedValueIsZero(Num, APInt::getHighBitsSet(32, 8)) && 1772 DAG.MaskedValueIsZero(Den, APInt::getHighBitsSet(32, 8))) { 1773 // TODO: We technically could do this for i64, but shouldn't that just be 1774 // handled by something generally reducing 64-bit division on 32-bit 1775 // values to 32-bit? 1776 return LowerDIVREM24(Op, DAG, false); 1777 } 1778 } 1779 1780 // RCP = URECIP(Den) = 2^32 / Den + e 1781 // e is rounding error. 1782 SDValue RCP = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Den); 1783 1784 // RCP_LO = mul(RCP, Den) */ 1785 SDValue RCP_LO = DAG.getNode(ISD::MUL, DL, VT, RCP, Den); 1786 1787 // RCP_HI = mulhu (RCP, Den) */ 1788 SDValue RCP_HI = DAG.getNode(ISD::MULHU, DL, VT, RCP, Den); 1789 1790 // NEG_RCP_LO = -RCP_LO 1791 SDValue NEG_RCP_LO = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), 1792 RCP_LO); 1793 1794 // ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO) 1795 SDValue ABS_RCP_LO = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, DL, VT), 1796 NEG_RCP_LO, RCP_LO, 1797 ISD::SETEQ); 1798 // Calculate the rounding error from the URECIP instruction 1799 // E = mulhu(ABS_RCP_LO, RCP) 1800 SDValue E = DAG.getNode(ISD::MULHU, DL, VT, ABS_RCP_LO, RCP); 1801 1802 // RCP_A_E = RCP + E 1803 SDValue RCP_A_E = DAG.getNode(ISD::ADD, DL, VT, RCP, E); 1804 1805 // RCP_S_E = RCP - E 1806 SDValue RCP_S_E = DAG.getNode(ISD::SUB, DL, VT, RCP, E); 1807 1808 // Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E) 1809 SDValue Tmp0 = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, DL, VT), 1810 RCP_A_E, RCP_S_E, 1811 ISD::SETEQ); 1812 // Quotient = mulhu(Tmp0, Num) 1813 SDValue Quotient = DAG.getNode(ISD::MULHU, DL, VT, Tmp0, Num); 1814 1815 // Num_S_Remainder = Quotient * Den 1816 SDValue Num_S_Remainder = DAG.getNode(ISD::MUL, DL, VT, Quotient, Den); 1817 1818 // Remainder = Num - Num_S_Remainder 1819 SDValue Remainder = DAG.getNode(ISD::SUB, DL, VT, Num, Num_S_Remainder); 1820 1821 // Remainder_GE_Den = (Remainder >= Den ? -1 : 0) 1822 SDValue Remainder_GE_Den = DAG.getSelectCC(DL, Remainder, Den, 1823 DAG.getConstant(-1, DL, VT), 1824 DAG.getConstant(0, DL, VT), 1825 ISD::SETUGE); 1826 // Remainder_GE_Zero = (Num >= Num_S_Remainder ? -1 : 0) 1827 SDValue Remainder_GE_Zero = DAG.getSelectCC(DL, Num, 1828 Num_S_Remainder, 1829 DAG.getConstant(-1, DL, VT), 1830 DAG.getConstant(0, DL, VT), 1831 ISD::SETUGE); 1832 // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero 1833 SDValue Tmp1 = DAG.getNode(ISD::AND, DL, VT, Remainder_GE_Den, 1834 Remainder_GE_Zero); 1835 1836 // Calculate Division result: 1837 1838 // Quotient_A_One = Quotient + 1 1839 SDValue Quotient_A_One = DAG.getNode(ISD::ADD, DL, VT, Quotient, 1840 DAG.getConstant(1, DL, VT)); 1841 1842 // Quotient_S_One = Quotient - 1 1843 SDValue Quotient_S_One = DAG.getNode(ISD::SUB, DL, VT, Quotient, 1844 DAG.getConstant(1, DL, VT)); 1845 1846 // Div = (Tmp1 == 0 ? Quotient : Quotient_A_One) 1847 SDValue Div = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, DL, VT), 1848 Quotient, Quotient_A_One, ISD::SETEQ); 1849 1850 // Div = (Remainder_GE_Zero == 0 ? Quotient_S_One : Div) 1851 Div = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, DL, VT), 1852 Quotient_S_One, Div, ISD::SETEQ); 1853 1854 // Calculate Rem result: 1855 1856 // Remainder_S_Den = Remainder - Den 1857 SDValue Remainder_S_Den = DAG.getNode(ISD::SUB, DL, VT, Remainder, Den); 1858 1859 // Remainder_A_Den = Remainder + Den 1860 SDValue Remainder_A_Den = DAG.getNode(ISD::ADD, DL, VT, Remainder, Den); 1861 1862 // Rem = (Tmp1 == 0 ? Remainder : Remainder_S_Den) 1863 SDValue Rem = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, DL, VT), 1864 Remainder, Remainder_S_Den, ISD::SETEQ); 1865 1866 // Rem = (Remainder_GE_Zero == 0 ? Remainder_A_Den : Rem) 1867 Rem = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, DL, VT), 1868 Remainder_A_Den, Rem, ISD::SETEQ); 1869 SDValue Ops[2] = { 1870 Div, 1871 Rem 1872 }; 1873 return DAG.getMergeValues(Ops, DL); 1874} 1875 1876SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op, 1877 SelectionDAG &DAG) const { 1878 SDLoc DL(Op); 1879 EVT VT = Op.getValueType(); 1880 1881 SDValue LHS = Op.getOperand(0); 1882 SDValue RHS = Op.getOperand(1); 1883 1884 SDValue Zero = DAG.getConstant(0, DL, VT); 1885 SDValue NegOne = DAG.getConstant(-1, DL, VT); 1886 1887 if (VT == MVT::i32 && 1888 DAG.ComputeNumSignBits(LHS) > 8 && 1889 DAG.ComputeNumSignBits(RHS) > 8) { 1890 return LowerDIVREM24(Op, DAG, true); 1891 } 1892 if (VT == MVT::i64 && 1893 DAG.ComputeNumSignBits(LHS) > 32 && 1894 DAG.ComputeNumSignBits(RHS) > 32) { 1895 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext()); 1896 1897 //HiLo split 1898 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero); 1899 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero); 1900 SDValue DIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(HalfVT, HalfVT), 1901 LHS_Lo, RHS_Lo); 1902 SDValue Res[2] = { 1903 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(0)), 1904 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(1)) 1905 }; 1906 return DAG.getMergeValues(Res, DL); 1907 } 1908 1909 SDValue LHSign = DAG.getSelectCC(DL, LHS, Zero, NegOne, Zero, ISD::SETLT); 1910 SDValue RHSign = DAG.getSelectCC(DL, RHS, Zero, NegOne, Zero, ISD::SETLT); 1911 SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign); 1912 SDValue RSign = LHSign; // Remainder sign is the same as LHS 1913 1914 LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign); 1915 RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign); 1916 1917 LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign); 1918 RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign); 1919 1920 SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS); 1921 SDValue Rem = Div.getValue(1); 1922 1923 Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign); 1924 Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign); 1925 1926 Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign); 1927 Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign); 1928 1929 SDValue Res[2] = { 1930 Div, 1931 Rem 1932 }; 1933 return DAG.getMergeValues(Res, DL); 1934} 1935 1936// (frem x, y) -> (fsub x, (fmul (ftrunc (fdiv x, y)), y)) 1937SDValue AMDGPUTargetLowering::LowerFREM(SDValue Op, SelectionDAG &DAG) const { 1938 SDLoc SL(Op); 1939 EVT VT = Op.getValueType(); 1940 SDValue X = Op.getOperand(0); 1941 SDValue Y = Op.getOperand(1); 1942 1943 SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y); 1944 SDValue Floor = DAG.getNode(ISD::FTRUNC, SL, VT, Div); 1945 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Floor, Y); 1946 1947 return DAG.getNode(ISD::FSUB, SL, VT, X, Mul); 1948} 1949 1950SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const { 1951 SDLoc SL(Op); 1952 SDValue Src = Op.getOperand(0); 1953 1954 // result = trunc(src) 1955 // if (src > 0.0 && src != result) 1956 // result += 1.0 1957 1958 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); 1959 1960 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64); 1961 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64); 1962 1963 EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f64); 1964 1965 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT); 1966 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE); 1967 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc); 1968 1969 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero); 1970 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); 1971} 1972 1973static SDValue extractF64Exponent(SDValue Hi, SDLoc SL, SelectionDAG &DAG) { 1974 const unsigned FractBits = 52; 1975 const unsigned ExpBits = 11; 1976 1977 SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, 1978 Hi, 1979 DAG.getConstant(FractBits - 32, SL, MVT::i32), 1980 DAG.getConstant(ExpBits, SL, MVT::i32)); 1981 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart, 1982 DAG.getConstant(1023, SL, MVT::i32)); 1983 1984 return Exp; 1985} 1986 1987SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const { 1988 SDLoc SL(Op); 1989 SDValue Src = Op.getOperand(0); 1990 1991 assert(Op.getValueType() == MVT::f64); 1992 1993 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); 1994 const SDValue One = DAG.getConstant(1, SL, MVT::i32); 1995 1996 SDValue VecSrc = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src); 1997 1998 // Extract the upper half, since this is where we will find the sign and 1999 // exponent. 2000 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, VecSrc, One); 2001 2002 SDValue Exp = extractF64Exponent(Hi, SL, DAG); 2003 2004 const unsigned FractBits = 52; 2005 2006 // Extract the sign bit. 2007 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, SL, MVT::i32); 2008 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask); 2009 2010 // Extend back to to 64-bits. 2011 SDValue SignBit64 = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, 2012 Zero, SignBit); 2013 SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64); 2014 2015 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src); 2016 const SDValue FractMask 2017 = DAG.getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64); 2018 2019 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp); 2020 SDValue Not = DAG.getNOT(SL, Shr, MVT::i64); 2021 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not); 2022 2023 EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::i32); 2024 2025 const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32); 2026 2027 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT); 2028 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT); 2029 2030 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0); 2031 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1); 2032 2033 return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2); 2034} 2035 2036SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const { 2037 SDLoc SL(Op); 2038 SDValue Src = Op.getOperand(0); 2039 2040 assert(Op.getValueType() == MVT::f64); 2041 2042 APFloat C1Val(APFloat::IEEEdouble, "0x1.0p+52"); 2043 SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64); 2044 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src); 2045 2046 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign); 2047 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign); 2048 2049 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src); 2050 2051 APFloat C2Val(APFloat::IEEEdouble, "0x1.fffffffffffffp+51"); 2052 SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64); 2053 2054 EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f64); 2055 SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT); 2056 2057 return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2); 2058} 2059 2060SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const { 2061 // FNEARBYINT and FRINT are the same, except in their handling of FP 2062 // exceptions. Those aren't really meaningful for us, and OpenCL only has 2063 // rint, so just treat them as equivalent. 2064 return DAG.getNode(ISD::FRINT, SDLoc(Op), Op.getValueType(), Op.getOperand(0)); 2065} 2066 2067// XXX - May require not supporting f32 denormals? 2068SDValue AMDGPUTargetLowering::LowerFROUND32(SDValue Op, SelectionDAG &DAG) const { 2069 SDLoc SL(Op); 2070 SDValue X = Op.getOperand(0); 2071 2072 SDValue T = DAG.getNode(ISD::FTRUNC, SL, MVT::f32, X); 2073 2074 SDValue Diff = DAG.getNode(ISD::FSUB, SL, MVT::f32, X, T); 2075 2076 SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, MVT::f32, Diff); 2077 2078 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f32); 2079 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); 2080 const SDValue Half = DAG.getConstantFP(0.5, SL, MVT::f32); 2081 2082 SDValue SignOne = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f32, One, X); 2083 2084 EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f32); 2085 2086 SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE); 2087 2088 SDValue Sel = DAG.getNode(ISD::SELECT, SL, MVT::f32, Cmp, SignOne, Zero); 2089 2090 return DAG.getNode(ISD::FADD, SL, MVT::f32, T, Sel); 2091} 2092 2093SDValue AMDGPUTargetLowering::LowerFROUND64(SDValue Op, SelectionDAG &DAG) const { 2094 SDLoc SL(Op); 2095 SDValue X = Op.getOperand(0); 2096 2097 SDValue L = DAG.getNode(ISD::BITCAST, SL, MVT::i64, X); 2098 2099 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); 2100 const SDValue One = DAG.getConstant(1, SL, MVT::i32); 2101 const SDValue NegOne = DAG.getConstant(-1, SL, MVT::i32); 2102 const SDValue FiftyOne = DAG.getConstant(51, SL, MVT::i32); 2103 EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::i32); 2104 2105 2106 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X); 2107 2108 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC, One); 2109 2110 SDValue Exp = extractF64Exponent(Hi, SL, DAG); 2111 2112 const SDValue Mask = DAG.getConstant(INT64_C(0x000fffffffffffff), SL, 2113 MVT::i64); 2114 2115 SDValue M = DAG.getNode(ISD::SRA, SL, MVT::i64, Mask, Exp); 2116 SDValue D = DAG.getNode(ISD::SRA, SL, MVT::i64, 2117 DAG.getConstant(INT64_C(0x0008000000000000), SL, 2118 MVT::i64), 2119 Exp); 2120 2121 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, L, M); 2122 SDValue Tmp1 = DAG.getSetCC(SL, SetCCVT, 2123 DAG.getConstant(0, SL, MVT::i64), Tmp0, 2124 ISD::SETNE); 2125 2126 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, Tmp1, 2127 D, DAG.getConstant(0, SL, MVT::i64)); 2128 SDValue K = DAG.getNode(ISD::ADD, SL, MVT::i64, L, Tmp2); 2129 2130 K = DAG.getNode(ISD::AND, SL, MVT::i64, K, DAG.getNOT(SL, M, MVT::i64)); 2131 K = DAG.getNode(ISD::BITCAST, SL, MVT::f64, K); 2132 2133 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT); 2134 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT); 2135 SDValue ExpEqNegOne = DAG.getSetCC(SL, SetCCVT, NegOne, Exp, ISD::SETEQ); 2136 2137 SDValue Mag = DAG.getNode(ISD::SELECT, SL, MVT::f64, 2138 ExpEqNegOne, 2139 DAG.getConstantFP(1.0, SL, MVT::f64), 2140 DAG.getConstantFP(0.0, SL, MVT::f64)); 2141 2142 SDValue S = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, Mag, X); 2143 2144 K = DAG.getNode(ISD::SELECT, SL, MVT::f64, ExpLt0, S, K); 2145 K = DAG.getNode(ISD::SELECT, SL, MVT::f64, ExpGt51, X, K); 2146 2147 return K; 2148} 2149 2150SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const { 2151 EVT VT = Op.getValueType(); 2152 2153 if (VT == MVT::f32) 2154 return LowerFROUND32(Op, DAG); 2155 2156 if (VT == MVT::f64) 2157 return LowerFROUND64(Op, DAG); 2158 2159 llvm_unreachable("unhandled type"); 2160} 2161 2162SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const { 2163 SDLoc SL(Op); 2164 SDValue Src = Op.getOperand(0); 2165 2166 // result = trunc(src); 2167 // if (src < 0.0 && src != result) 2168 // result += -1.0. 2169 2170 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); 2171 2172 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64); 2173 const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64); 2174 2175 EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f64); 2176 2177 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT); 2178 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE); 2179 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc); 2180 2181 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero); 2182 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); 2183} 2184 2185SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG, 2186 bool Signed) const { 2187 SDLoc SL(Op); 2188 SDValue Src = Op.getOperand(0); 2189 2190 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src); 2191 2192 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC, 2193 DAG.getConstant(0, SL, MVT::i32)); 2194 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC, 2195 DAG.getConstant(1, SL, MVT::i32)); 2196 2197 SDValue CvtHi = DAG.getNode(Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, 2198 SL, MVT::f64, Hi); 2199 2200 SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo); 2201 2202 SDValue LdExp = DAG.getNode(AMDGPUISD::LDEXP, SL, MVT::f64, CvtHi, 2203 DAG.getConstant(32, SL, MVT::i32)); 2204 2205 return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo); 2206} 2207 2208SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op, 2209 SelectionDAG &DAG) const { 2210 SDValue S0 = Op.getOperand(0); 2211 if (S0.getValueType() != MVT::i64) 2212 return SDValue(); 2213 2214 EVT DestVT = Op.getValueType(); 2215 if (DestVT == MVT::f64) 2216 return LowerINT_TO_FP64(Op, DAG, false); 2217 2218 assert(DestVT == MVT::f32); 2219 2220 SDLoc DL(Op); 2221 2222 // f32 uint_to_fp i64 2223 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, S0, 2224 DAG.getConstant(0, DL, MVT::i32)); 2225 SDValue FloatLo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, Lo); 2226 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, S0, 2227 DAG.getConstant(1, DL, MVT::i32)); 2228 SDValue FloatHi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, Hi); 2229 FloatHi = DAG.getNode(ISD::FMUL, DL, MVT::f32, FloatHi, 2230 DAG.getConstantFP(4294967296.0f, DL, MVT::f32)); // 2^32 2231 return DAG.getNode(ISD::FADD, DL, MVT::f32, FloatLo, FloatHi); 2232} 2233 2234SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op, 2235 SelectionDAG &DAG) const { 2236 SDValue Src = Op.getOperand(0); 2237 if (Src.getValueType() == MVT::i64 && Op.getValueType() == MVT::f64) 2238 return LowerINT_TO_FP64(Op, DAG, true); 2239 2240 return SDValue(); 2241} 2242 2243SDValue AMDGPUTargetLowering::LowerFP64_TO_INT(SDValue Op, SelectionDAG &DAG, 2244 bool Signed) const { 2245 SDLoc SL(Op); 2246 2247 SDValue Src = Op.getOperand(0); 2248 2249 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); 2250 2251 SDValue K0 = DAG.getConstantFP(BitsToDouble(UINT64_C(0x3df0000000000000)), SL, 2252 MVT::f64); 2253 SDValue K1 = DAG.getConstantFP(BitsToDouble(UINT64_C(0xc1f0000000000000)), SL, 2254 MVT::f64); 2255 2256 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, Trunc, K0); 2257 2258 SDValue FloorMul = DAG.getNode(ISD::FFLOOR, SL, MVT::f64, Mul); 2259 2260 2261 SDValue Fma = DAG.getNode(ISD::FMA, SL, MVT::f64, FloorMul, K1, Trunc); 2262 2263 SDValue Hi = DAG.getNode(Signed ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, SL, 2264 MVT::i32, FloorMul); 2265 SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma); 2266 2267 SDValue Result = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Lo, Hi); 2268 2269 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Result); 2270} 2271 2272SDValue AMDGPUTargetLowering::LowerFP_TO_SINT(SDValue Op, 2273 SelectionDAG &DAG) const { 2274 SDValue Src = Op.getOperand(0); 2275 2276 if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64) 2277 return LowerFP64_TO_INT(Op, DAG, true); 2278 2279 return SDValue(); 2280} 2281 2282SDValue AMDGPUTargetLowering::LowerFP_TO_UINT(SDValue Op, 2283 SelectionDAG &DAG) const { 2284 SDValue Src = Op.getOperand(0); 2285 2286 if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64) 2287 return LowerFP64_TO_INT(Op, DAG, false); 2288 2289 return SDValue(); 2290} 2291 2292SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, 2293 SelectionDAG &DAG) const { 2294 EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 2295 MVT VT = Op.getSimpleValueType(); 2296 MVT ScalarVT = VT.getScalarType(); 2297 2298 if (!VT.isVector()) 2299 return SDValue(); 2300 2301 SDValue Src = Op.getOperand(0); 2302 SDLoc DL(Op); 2303 2304 // TODO: Don't scalarize on Evergreen? 2305 unsigned NElts = VT.getVectorNumElements(); 2306 SmallVector<SDValue, 8> Args; 2307 DAG.ExtractVectorElements(Src, Args, 0, NElts); 2308 2309 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType()); 2310 for (unsigned I = 0; I < NElts; ++I) 2311 Args[I] = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ScalarVT, Args[I], VTOp); 2312 2313 return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Args); 2314} 2315 2316//===----------------------------------------------------------------------===// 2317// Custom DAG optimizations 2318//===----------------------------------------------------------------------===// 2319 2320static bool isU24(SDValue Op, SelectionDAG &DAG) { 2321 APInt KnownZero, KnownOne; 2322 EVT VT = Op.getValueType(); 2323 DAG.computeKnownBits(Op, KnownZero, KnownOne); 2324 2325 return (VT.getSizeInBits() - KnownZero.countLeadingOnes()) <= 24; 2326} 2327 2328static bool isI24(SDValue Op, SelectionDAG &DAG) { 2329 EVT VT = Op.getValueType(); 2330 2331 // In order for this to be a signed 24-bit value, bit 23, must 2332 // be a sign bit. 2333 return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated 2334 // as unsigned 24-bit values. 2335 (VT.getSizeInBits() - DAG.ComputeNumSignBits(Op)) < 24; 2336} 2337 2338static void simplifyI24(SDValue Op, TargetLowering::DAGCombinerInfo &DCI) { 2339 2340 SelectionDAG &DAG = DCI.DAG; 2341 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2342 EVT VT = Op.getValueType(); 2343 2344 APInt Demanded = APInt::getLowBitsSet(VT.getSizeInBits(), 24); 2345 APInt KnownZero, KnownOne; 2346 TargetLowering::TargetLoweringOpt TLO(DAG, true, true); 2347 if (TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO)) 2348 DCI.CommitTargetLoweringOpt(TLO); 2349} 2350 2351template <typename IntTy> 2352static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, 2353 uint32_t Offset, uint32_t Width, SDLoc DL) { 2354 if (Width + Offset < 32) { 2355 uint32_t Shl = static_cast<uint32_t>(Src0) << (32 - Offset - Width); 2356 IntTy Result = static_cast<IntTy>(Shl) >> (32 - Width); 2357 return DAG.getConstant(Result, DL, MVT::i32); 2358 } 2359 2360 return DAG.getConstant(Src0 >> Offset, DL, MVT::i32); 2361} 2362 2363static bool usesAllNormalStores(SDNode *LoadVal) { 2364 for (SDNode::use_iterator I = LoadVal->use_begin(); !I.atEnd(); ++I) { 2365 if (!ISD::isNormalStore(*I)) 2366 return false; 2367 } 2368 2369 return true; 2370} 2371 2372// If we have a copy of an illegal type, replace it with a load / store of an 2373// equivalently sized legal type. This avoids intermediate bit pack / unpack 2374// instructions emitted when handling extloads and truncstores. Ideally we could 2375// recognize the pack / unpack pattern to eliminate it. 2376SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N, 2377 DAGCombinerInfo &DCI) const { 2378 if (!DCI.isBeforeLegalize()) 2379 return SDValue(); 2380 2381 StoreSDNode *SN = cast<StoreSDNode>(N); 2382 SDValue Value = SN->getValue(); 2383 EVT VT = Value.getValueType(); 2384 2385 if (isTypeLegal(VT) || SN->isVolatile() || 2386 !ISD::isNormalLoad(Value.getNode()) || VT.getSizeInBits() < 8) 2387 return SDValue(); 2388 2389 LoadSDNode *LoadVal = cast<LoadSDNode>(Value); 2390 if (LoadVal->isVolatile() || !usesAllNormalStores(LoadVal)) 2391 return SDValue(); 2392 2393 EVT MemVT = LoadVal->getMemoryVT(); 2394 2395 SDLoc SL(N); 2396 SelectionDAG &DAG = DCI.DAG; 2397 EVT LoadVT = getEquivalentMemType(*DAG.getContext(), MemVT); 2398 2399 SDValue NewLoad = DAG.getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, 2400 LoadVT, SL, 2401 LoadVal->getChain(), 2402 LoadVal->getBasePtr(), 2403 LoadVal->getOffset(), 2404 LoadVT, 2405 LoadVal->getMemOperand()); 2406 2407 SDValue CastLoad = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad.getValue(0)); 2408 DCI.CombineTo(LoadVal, CastLoad, NewLoad.getValue(1), false); 2409 2410 return DAG.getStore(SN->getChain(), SL, NewLoad, 2411 SN->getBasePtr(), SN->getMemOperand()); 2412} 2413 2414SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N, 2415 DAGCombinerInfo &DCI) const { 2416 EVT VT = N->getValueType(0); 2417 2418 if (VT.isVector() || VT.getSizeInBits() > 32) 2419 return SDValue(); 2420 2421 SelectionDAG &DAG = DCI.DAG; 2422 SDLoc DL(N); 2423 2424 SDValue N0 = N->getOperand(0); 2425 SDValue N1 = N->getOperand(1); 2426 SDValue Mul; 2427 2428 if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) { 2429 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); 2430 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); 2431 Mul = DAG.getNode(AMDGPUISD::MUL_U24, DL, MVT::i32, N0, N1); 2432 } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) { 2433 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); 2434 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); 2435 Mul = DAG.getNode(AMDGPUISD::MUL_I24, DL, MVT::i32, N0, N1); 2436 } else { 2437 return SDValue(); 2438 } 2439 2440 // We need to use sext even for MUL_U24, because MUL_U24 is used 2441 // for signed multiply of 8 and 16-bit types. 2442 return DAG.getSExtOrTrunc(Mul, DL, VT); 2443} 2444 2445SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N, 2446 DAGCombinerInfo &DCI) const { 2447 SelectionDAG &DAG = DCI.DAG; 2448 SDLoc DL(N); 2449 2450 switch(N->getOpcode()) { 2451 default: break; 2452 case ISD::MUL: 2453 return performMulCombine(N, DCI); 2454 case AMDGPUISD::MUL_I24: 2455 case AMDGPUISD::MUL_U24: { 2456 SDValue N0 = N->getOperand(0); 2457 SDValue N1 = N->getOperand(1); 2458 simplifyI24(N0, DCI); 2459 simplifyI24(N1, DCI); 2460 return SDValue(); 2461 } 2462 case ISD::SELECT: { 2463 SDValue Cond = N->getOperand(0); 2464 if (Cond.getOpcode() == ISD::SETCC && Cond.hasOneUse()) { 2465 EVT VT = N->getValueType(0); 2466 SDValue LHS = Cond.getOperand(0); 2467 SDValue RHS = Cond.getOperand(1); 2468 SDValue CC = Cond.getOperand(2); 2469 2470 SDValue True = N->getOperand(1); 2471 SDValue False = N->getOperand(2); 2472 2473 if (VT == MVT::f32) 2474 return CombineFMinMaxLegacy(DL, VT, LHS, RHS, True, False, CC, DCI); 2475 2476 // TODO: Implement min / max Evergreen instructions. 2477 if (VT == MVT::i32 && 2478 Subtarget->getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS) { 2479 return CombineIMinMax(DL, VT, LHS, RHS, True, False, CC, DAG); 2480 } 2481 } 2482 2483 break; 2484 } 2485 case AMDGPUISD::BFE_I32: 2486 case AMDGPUISD::BFE_U32: { 2487 assert(!N->getValueType(0).isVector() && 2488 "Vector handling of BFE not implemented"); 2489 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2)); 2490 if (!Width) 2491 break; 2492 2493 uint32_t WidthVal = Width->getZExtValue() & 0x1f; 2494 if (WidthVal == 0) 2495 return DAG.getConstant(0, DL, MVT::i32); 2496 2497 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1)); 2498 if (!Offset) 2499 break; 2500 2501 SDValue BitsFrom = N->getOperand(0); 2502 uint32_t OffsetVal = Offset->getZExtValue() & 0x1f; 2503 2504 bool Signed = N->getOpcode() == AMDGPUISD::BFE_I32; 2505 2506 if (OffsetVal == 0) { 2507 // This is already sign / zero extended, so try to fold away extra BFEs. 2508 unsigned SignBits = Signed ? (32 - WidthVal + 1) : (32 - WidthVal); 2509 2510 unsigned OpSignBits = DAG.ComputeNumSignBits(BitsFrom); 2511 if (OpSignBits >= SignBits) 2512 return BitsFrom; 2513 2514 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), WidthVal); 2515 if (Signed) { 2516 // This is a sign_extend_inreg. Replace it to take advantage of existing 2517 // DAG Combines. If not eliminated, we will match back to BFE during 2518 // selection. 2519 2520 // TODO: The sext_inreg of extended types ends, although we can could 2521 // handle them in a single BFE. 2522 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom, 2523 DAG.getValueType(SmallVT)); 2524 } 2525 2526 return DAG.getZeroExtendInReg(BitsFrom, DL, SmallVT); 2527 } 2528 2529 if (ConstantSDNode *CVal = dyn_cast<ConstantSDNode>(BitsFrom)) { 2530 if (Signed) { 2531 return constantFoldBFE<int32_t>(DAG, 2532 CVal->getSExtValue(), 2533 OffsetVal, 2534 WidthVal, 2535 DL); 2536 } 2537 2538 return constantFoldBFE<uint32_t>(DAG, 2539 CVal->getZExtValue(), 2540 OffsetVal, 2541 WidthVal, 2542 DL); 2543 } 2544 2545 if ((OffsetVal + WidthVal) >= 32) { 2546 SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32); 2547 return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32, 2548 BitsFrom, ShiftVal); 2549 } 2550 2551 if (BitsFrom.hasOneUse()) { 2552 APInt Demanded = APInt::getBitsSet(32, 2553 OffsetVal, 2554 OffsetVal + WidthVal); 2555 2556 APInt KnownZero, KnownOne; 2557 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(), 2558 !DCI.isBeforeLegalizeOps()); 2559 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2560 if (TLO.ShrinkDemandedConstant(BitsFrom, Demanded) || 2561 TLI.SimplifyDemandedBits(BitsFrom, Demanded, 2562 KnownZero, KnownOne, TLO)) { 2563 DCI.CommitTargetLoweringOpt(TLO); 2564 } 2565 } 2566 2567 break; 2568 } 2569 2570 case ISD::STORE: 2571 return performStoreCombine(N, DCI); 2572 } 2573 return SDValue(); 2574} 2575 2576//===----------------------------------------------------------------------===// 2577// Helper functions 2578//===----------------------------------------------------------------------===// 2579 2580void AMDGPUTargetLowering::getOriginalFunctionArgs( 2581 SelectionDAG &DAG, 2582 const Function *F, 2583 const SmallVectorImpl<ISD::InputArg> &Ins, 2584 SmallVectorImpl<ISD::InputArg> &OrigIns) const { 2585 2586 for (unsigned i = 0, e = Ins.size(); i < e; ++i) { 2587 if (Ins[i].ArgVT == Ins[i].VT) { 2588 OrigIns.push_back(Ins[i]); 2589 continue; 2590 } 2591 2592 EVT VT; 2593 if (Ins[i].ArgVT.isVector() && !Ins[i].VT.isVector()) { 2594 // Vector has been split into scalars. 2595 VT = Ins[i].ArgVT.getVectorElementType(); 2596 } else if (Ins[i].VT.isVector() && Ins[i].ArgVT.isVector() && 2597 Ins[i].ArgVT.getVectorElementType() != 2598 Ins[i].VT.getVectorElementType()) { 2599 // Vector elements have been promoted 2600 VT = Ins[i].ArgVT; 2601 } else { 2602 // Vector has been spilt into smaller vectors. 2603 VT = Ins[i].VT; 2604 } 2605 2606 ISD::InputArg Arg(Ins[i].Flags, VT, VT, Ins[i].Used, 2607 Ins[i].OrigArgIndex, Ins[i].PartOffset); 2608 OrigIns.push_back(Arg); 2609 } 2610} 2611 2612bool AMDGPUTargetLowering::isHWTrueValue(SDValue Op) const { 2613 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) { 2614 return CFP->isExactlyValue(1.0); 2615 } 2616 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 2617 return C->isAllOnesValue(); 2618 } 2619 return false; 2620} 2621 2622bool AMDGPUTargetLowering::isHWFalseValue(SDValue Op) const { 2623 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) { 2624 return CFP->getValueAPF().isZero(); 2625 } 2626 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 2627 return C->isNullValue(); 2628 } 2629 return false; 2630} 2631 2632SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG, 2633 const TargetRegisterClass *RC, 2634 unsigned Reg, EVT VT) const { 2635 MachineFunction &MF = DAG.getMachineFunction(); 2636 MachineRegisterInfo &MRI = MF.getRegInfo(); 2637 unsigned VirtualRegister; 2638 if (!MRI.isLiveIn(Reg)) { 2639 VirtualRegister = MRI.createVirtualRegister(RC); 2640 MRI.addLiveIn(Reg, VirtualRegister); 2641 } else { 2642 VirtualRegister = MRI.getLiveInVirtReg(Reg); 2643 } 2644 return DAG.getRegister(VirtualRegister, VT); 2645} 2646 2647#define NODE_NAME_CASE(node) case AMDGPUISD::node: return #node; 2648 2649const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const { 2650 switch ((AMDGPUISD::NodeType)Opcode) { 2651 case AMDGPUISD::FIRST_NUMBER: break; 2652 // AMDIL DAG nodes 2653 NODE_NAME_CASE(CALL); 2654 NODE_NAME_CASE(UMUL); 2655 NODE_NAME_CASE(RET_FLAG); 2656 NODE_NAME_CASE(BRANCH_COND); 2657 2658 // AMDGPU DAG nodes 2659 NODE_NAME_CASE(DWORDADDR) 2660 NODE_NAME_CASE(FRACT) 2661 NODE_NAME_CASE(CLAMP) 2662 NODE_NAME_CASE(COS_HW) 2663 NODE_NAME_CASE(SIN_HW) 2664 NODE_NAME_CASE(FMAX_LEGACY) 2665 NODE_NAME_CASE(FMIN_LEGACY) 2666 NODE_NAME_CASE(FMAX3) 2667 NODE_NAME_CASE(SMAX3) 2668 NODE_NAME_CASE(UMAX3) 2669 NODE_NAME_CASE(FMIN3) 2670 NODE_NAME_CASE(SMIN3) 2671 NODE_NAME_CASE(UMIN3) 2672 NODE_NAME_CASE(URECIP) 2673 NODE_NAME_CASE(DIV_SCALE) 2674 NODE_NAME_CASE(DIV_FMAS) 2675 NODE_NAME_CASE(DIV_FIXUP) 2676 NODE_NAME_CASE(TRIG_PREOP) 2677 NODE_NAME_CASE(RCP) 2678 NODE_NAME_CASE(RSQ) 2679 NODE_NAME_CASE(RSQ_LEGACY) 2680 NODE_NAME_CASE(RSQ_CLAMPED) 2681 NODE_NAME_CASE(LDEXP) 2682 NODE_NAME_CASE(FP_CLASS) 2683 NODE_NAME_CASE(DOT4) 2684 NODE_NAME_CASE(CARRY) 2685 NODE_NAME_CASE(BORROW) 2686 NODE_NAME_CASE(BFE_U32) 2687 NODE_NAME_CASE(BFE_I32) 2688 NODE_NAME_CASE(BFI) 2689 NODE_NAME_CASE(BFM) 2690 NODE_NAME_CASE(BREV) 2691 NODE_NAME_CASE(MUL_U24) 2692 NODE_NAME_CASE(MUL_I24) 2693 NODE_NAME_CASE(MAD_U24) 2694 NODE_NAME_CASE(MAD_I24) 2695 NODE_NAME_CASE(TEXTURE_FETCH) 2696 NODE_NAME_CASE(EXPORT) 2697 NODE_NAME_CASE(CONST_ADDRESS) 2698 NODE_NAME_CASE(REGISTER_LOAD) 2699 NODE_NAME_CASE(REGISTER_STORE) 2700 NODE_NAME_CASE(LOAD_CONSTANT) 2701 NODE_NAME_CASE(LOAD_INPUT) 2702 NODE_NAME_CASE(SAMPLE) 2703 NODE_NAME_CASE(SAMPLEB) 2704 NODE_NAME_CASE(SAMPLED) 2705 NODE_NAME_CASE(SAMPLEL) 2706 NODE_NAME_CASE(CVT_F32_UBYTE0) 2707 NODE_NAME_CASE(CVT_F32_UBYTE1) 2708 NODE_NAME_CASE(CVT_F32_UBYTE2) 2709 NODE_NAME_CASE(CVT_F32_UBYTE3) 2710 NODE_NAME_CASE(BUILD_VERTICAL_VECTOR) 2711 NODE_NAME_CASE(CONST_DATA_PTR) 2712 case AMDGPUISD::FIRST_MEM_OPCODE_NUMBER: break; 2713 NODE_NAME_CASE(SENDMSG) 2714 NODE_NAME_CASE(INTERP_MOV) 2715 NODE_NAME_CASE(INTERP_P1) 2716 NODE_NAME_CASE(INTERP_P2) 2717 NODE_NAME_CASE(STORE_MSKOR) 2718 NODE_NAME_CASE(TBUFFER_STORE_FORMAT) 2719 case AMDGPUISD::LAST_AMDGPU_ISD_NUMBER: break; 2720 } 2721 return nullptr; 2722} 2723 2724SDValue AMDGPUTargetLowering::getRsqrtEstimate(SDValue Operand, 2725 DAGCombinerInfo &DCI, 2726 unsigned &RefinementSteps, 2727 bool &UseOneConstNR) const { 2728 SelectionDAG &DAG = DCI.DAG; 2729 EVT VT = Operand.getValueType(); 2730 2731 if (VT == MVT::f32) { 2732 RefinementSteps = 0; 2733 return DAG.getNode(AMDGPUISD::RSQ, SDLoc(Operand), VT, Operand); 2734 } 2735 2736 // TODO: There is also f64 rsq instruction, but the documentation is less 2737 // clear on its precision. 2738 2739 return SDValue(); 2740} 2741 2742SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand, 2743 DAGCombinerInfo &DCI, 2744 unsigned &RefinementSteps) const { 2745 SelectionDAG &DAG = DCI.DAG; 2746 EVT VT = Operand.getValueType(); 2747 2748 if (VT == MVT::f32) { 2749 // Reciprocal, < 1 ulp error. 2750 // 2751 // This reciprocal approximation converges to < 0.5 ulp error with one 2752 // newton rhapson performed with two fused multiple adds (FMAs). 2753 2754 RefinementSteps = 0; 2755 return DAG.getNode(AMDGPUISD::RCP, SDLoc(Operand), VT, Operand); 2756 } 2757 2758 // TODO: There is also f64 rcp instruction, but the documentation is less 2759 // clear on its precision. 2760 2761 return SDValue(); 2762} 2763 2764static void computeKnownBitsForMinMax(const SDValue Op0, 2765 const SDValue Op1, 2766 APInt &KnownZero, 2767 APInt &KnownOne, 2768 const SelectionDAG &DAG, 2769 unsigned Depth) { 2770 APInt Op0Zero, Op0One; 2771 APInt Op1Zero, Op1One; 2772 DAG.computeKnownBits(Op0, Op0Zero, Op0One, Depth); 2773 DAG.computeKnownBits(Op1, Op1Zero, Op1One, Depth); 2774 2775 KnownZero = Op0Zero & Op1Zero; 2776 KnownOne = Op0One & Op1One; 2777} 2778 2779void AMDGPUTargetLowering::computeKnownBitsForTargetNode( 2780 const SDValue Op, 2781 APInt &KnownZero, 2782 APInt &KnownOne, 2783 const SelectionDAG &DAG, 2784 unsigned Depth) const { 2785 2786 KnownZero = KnownOne = APInt(KnownOne.getBitWidth(), 0); // Don't know anything. 2787 2788 APInt KnownZero2; 2789 APInt KnownOne2; 2790 unsigned Opc = Op.getOpcode(); 2791 2792 switch (Opc) { 2793 default: 2794 break; 2795 case ISD::INTRINSIC_WO_CHAIN: { 2796 // FIXME: The intrinsic should just use the node. 2797 switch (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue()) { 2798 case AMDGPUIntrinsic::AMDGPU_imax: 2799 case AMDGPUIntrinsic::AMDGPU_umax: 2800 case AMDGPUIntrinsic::AMDGPU_imin: 2801 case AMDGPUIntrinsic::AMDGPU_umin: 2802 computeKnownBitsForMinMax(Op.getOperand(1), Op.getOperand(2), 2803 KnownZero, KnownOne, DAG, Depth); 2804 break; 2805 default: 2806 break; 2807 } 2808 2809 break; 2810 } 2811 case AMDGPUISD::CARRY: 2812 case AMDGPUISD::BORROW: { 2813 KnownZero = APInt::getHighBitsSet(32, 31); 2814 break; 2815 } 2816 2817 case AMDGPUISD::BFE_I32: 2818 case AMDGPUISD::BFE_U32: { 2819 ConstantSDNode *CWidth = dyn_cast<ConstantSDNode>(Op.getOperand(2)); 2820 if (!CWidth) 2821 return; 2822 2823 unsigned BitWidth = 32; 2824 uint32_t Width = CWidth->getZExtValue() & 0x1f; 2825 2826 if (Opc == AMDGPUISD::BFE_U32) 2827 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - Width); 2828 2829 break; 2830 } 2831 } 2832} 2833 2834unsigned AMDGPUTargetLowering::ComputeNumSignBitsForTargetNode( 2835 SDValue Op, 2836 const SelectionDAG &DAG, 2837 unsigned Depth) const { 2838 switch (Op.getOpcode()) { 2839 case AMDGPUISD::BFE_I32: { 2840 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Op.getOperand(2)); 2841 if (!Width) 2842 return 1; 2843 2844 unsigned SignBits = 32 - Width->getZExtValue() + 1; 2845 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 2846 if (!Offset || !Offset->isNullValue()) 2847 return SignBits; 2848 2849 // TODO: Could probably figure something out with non-0 offsets. 2850 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1); 2851 return std::max(SignBits, Op0SignBits); 2852 } 2853 2854 case AMDGPUISD::BFE_U32: { 2855 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Op.getOperand(2)); 2856 return Width ? 32 - (Width->getZExtValue() & 0x1f) : 1; 2857 } 2858 2859 case AMDGPUISD::CARRY: 2860 case AMDGPUISD::BORROW: 2861 return 31; 2862 2863 default: 2864 return 1; 2865 } 2866} 2867