1//===- llvm/Analysis/TargetTransformInfo.cpp ------------------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9#include "llvm/Analysis/TargetTransformInfo.h" 10#include "llvm/Analysis/CFG.h" 11#include "llvm/Analysis/LoopIterator.h" 12#include "llvm/Analysis/TargetTransformInfoImpl.h" 13#include "llvm/IR/CFG.h" 14#include "llvm/IR/CallSite.h" 15#include "llvm/IR/DataLayout.h" 16#include "llvm/IR/Instruction.h" 17#include "llvm/IR/Instructions.h" 18#include "llvm/IR/IntrinsicInst.h" 19#include "llvm/IR/Module.h" 20#include "llvm/IR/Operator.h" 21#include "llvm/IR/PatternMatch.h" 22#include "llvm/InitializePasses.h" 23#include "llvm/Support/CommandLine.h" 24#include "llvm/Support/ErrorHandling.h" 25#include <utility> 26 27using namespace llvm; 28using namespace PatternMatch; 29 30#define DEBUG_TYPE "tti" 31 32static cl::opt<bool> EnableReduxCost("costmodel-reduxcost", cl::init(false), 33 cl::Hidden, 34 cl::desc("Recognize reduction patterns.")); 35 36namespace { 37/// No-op implementation of the TTI interface using the utility base 38/// classes. 39/// 40/// This is used when no target specific information is available. 41struct NoTTIImpl : TargetTransformInfoImplCRTPBase<NoTTIImpl> { 42 explicit NoTTIImpl(const DataLayout &DL) 43 : TargetTransformInfoImplCRTPBase<NoTTIImpl>(DL) {} 44}; 45} 46 47bool HardwareLoopInfo::canAnalyze(LoopInfo &LI) { 48 // If the loop has irreducible control flow, it can not be converted to 49 // Hardware loop. 50 LoopBlocksRPO RPOT(L); 51 RPOT.perform(&LI); 52 if (containsIrreducibleCFG<const BasicBlock *>(RPOT, LI)) 53 return false; 54 return true; 55} 56 57bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE, 58 LoopInfo &LI, DominatorTree &DT, 59 bool ForceNestedLoop, 60 bool ForceHardwareLoopPHI) { 61 SmallVector<BasicBlock *, 4> ExitingBlocks; 62 L->getExitingBlocks(ExitingBlocks); 63 64 for (BasicBlock *BB : ExitingBlocks) { 65 // If we pass the updated counter back through a phi, we need to know 66 // which latch the updated value will be coming from. 67 if (!L->isLoopLatch(BB)) { 68 if (ForceHardwareLoopPHI || CounterInReg) 69 continue; 70 } 71 72 const SCEV *EC = SE.getExitCount(L, BB); 73 if (isa<SCEVCouldNotCompute>(EC)) 74 continue; 75 if (const SCEVConstant *ConstEC = dyn_cast<SCEVConstant>(EC)) { 76 if (ConstEC->getValue()->isZero()) 77 continue; 78 } else if (!SE.isLoopInvariant(EC, L)) 79 continue; 80 81 if (SE.getTypeSizeInBits(EC->getType()) > CountType->getBitWidth()) 82 continue; 83 84 // If this exiting block is contained in a nested loop, it is not eligible 85 // for insertion of the branch-and-decrement since the inner loop would 86 // end up messing up the value in the CTR. 87 if (!IsNestingLegal && LI.getLoopFor(BB) != L && !ForceNestedLoop) 88 continue; 89 90 // We now have a loop-invariant count of loop iterations (which is not the 91 // constant zero) for which we know that this loop will not exit via this 92 // existing block. 93 94 // We need to make sure that this block will run on every loop iteration. 95 // For this to be true, we must dominate all blocks with backedges. Such 96 // blocks are in-loop predecessors to the header block. 97 bool NotAlways = false; 98 for (BasicBlock *Pred : predecessors(L->getHeader())) { 99 if (!L->contains(Pred)) 100 continue; 101 102 if (!DT.dominates(BB, Pred)) { 103 NotAlways = true; 104 break; 105 } 106 } 107 108 if (NotAlways) 109 continue; 110 111 // Make sure this blocks ends with a conditional branch. 112 Instruction *TI = BB->getTerminator(); 113 if (!TI) 114 continue; 115 116 if (BranchInst *BI = dyn_cast<BranchInst>(TI)) { 117 if (!BI->isConditional()) 118 continue; 119 120 ExitBranch = BI; 121 } else 122 continue; 123 124 // Note that this block may not be the loop latch block, even if the loop 125 // has a latch block. 126 ExitBlock = BB; 127 ExitCount = EC; 128 break; 129 } 130 131 if (!ExitBlock) 132 return false; 133 return true; 134} 135 136TargetTransformInfo::TargetTransformInfo(const DataLayout &DL) 137 : TTIImpl(new Model<NoTTIImpl>(NoTTIImpl(DL))) {} 138 139TargetTransformInfo::~TargetTransformInfo() {} 140 141TargetTransformInfo::TargetTransformInfo(TargetTransformInfo &&Arg) 142 : TTIImpl(std::move(Arg.TTIImpl)) {} 143 144TargetTransformInfo &TargetTransformInfo::operator=(TargetTransformInfo &&RHS) { 145 TTIImpl = std::move(RHS.TTIImpl); 146 return *this; 147} 148 149int TargetTransformInfo::getOperationCost(unsigned Opcode, Type *Ty, 150 Type *OpTy) const { 151 int Cost = TTIImpl->getOperationCost(Opcode, Ty, OpTy); 152 assert(Cost >= 0 && "TTI should not produce negative costs!"); 153 return Cost; 154} 155 156int TargetTransformInfo::getCallCost(FunctionType *FTy, int NumArgs, 157 const User *U) const { 158 int Cost = TTIImpl->getCallCost(FTy, NumArgs, U); 159 assert(Cost >= 0 && "TTI should not produce negative costs!"); 160 return Cost; 161} 162 163int TargetTransformInfo::getCallCost(const Function *F, 164 ArrayRef<const Value *> Arguments, 165 const User *U) const { 166 int Cost = TTIImpl->getCallCost(F, Arguments, U); 167 assert(Cost >= 0 && "TTI should not produce negative costs!"); 168 return Cost; 169} 170 171unsigned TargetTransformInfo::getInliningThresholdMultiplier() const { 172 return TTIImpl->getInliningThresholdMultiplier(); 173} 174 175int TargetTransformInfo::getInlinerVectorBonusPercent() const { 176 return TTIImpl->getInlinerVectorBonusPercent(); 177} 178 179int TargetTransformInfo::getGEPCost(Type *PointeeType, const Value *Ptr, 180 ArrayRef<const Value *> Operands) const { 181 return TTIImpl->getGEPCost(PointeeType, Ptr, Operands); 182} 183 184int TargetTransformInfo::getExtCost(const Instruction *I, 185 const Value *Src) const { 186 return TTIImpl->getExtCost(I, Src); 187} 188 189int TargetTransformInfo::getIntrinsicCost( 190 Intrinsic::ID IID, Type *RetTy, ArrayRef<const Value *> Arguments, 191 const User *U) const { 192 int Cost = TTIImpl->getIntrinsicCost(IID, RetTy, Arguments, U); 193 assert(Cost >= 0 && "TTI should not produce negative costs!"); 194 return Cost; 195} 196 197unsigned 198TargetTransformInfo::getEstimatedNumberOfCaseClusters( 199 const SwitchInst &SI, unsigned &JTSize, ProfileSummaryInfo *PSI, 200 BlockFrequencyInfo *BFI) const { 201 return TTIImpl->getEstimatedNumberOfCaseClusters(SI, JTSize, PSI, BFI); 202} 203 204int TargetTransformInfo::getUserCost(const User *U, 205 ArrayRef<const Value *> Operands) const { 206 int Cost = TTIImpl->getUserCost(U, Operands); 207 assert(Cost >= 0 && "TTI should not produce negative costs!"); 208 return Cost; 209} 210 211bool TargetTransformInfo::hasBranchDivergence() const { 212 return TTIImpl->hasBranchDivergence(); 213} 214 215bool TargetTransformInfo::isSourceOfDivergence(const Value *V) const { 216 return TTIImpl->isSourceOfDivergence(V); 217} 218 219bool llvm::TargetTransformInfo::isAlwaysUniform(const Value *V) const { 220 return TTIImpl->isAlwaysUniform(V); 221} 222 223unsigned TargetTransformInfo::getFlatAddressSpace() const { 224 return TTIImpl->getFlatAddressSpace(); 225} 226 227bool TargetTransformInfo::collectFlatAddressOperands( 228 SmallVectorImpl<int> &OpIndexes, Intrinsic::ID IID) const { 229 return TTIImpl->collectFlatAddressOperands(OpIndexes, IID); 230} 231 232bool TargetTransformInfo::rewriteIntrinsicWithAddressSpace( 233 IntrinsicInst *II, Value *OldV, Value *NewV) const { 234 return TTIImpl->rewriteIntrinsicWithAddressSpace(II, OldV, NewV); 235} 236 237bool TargetTransformInfo::isLoweredToCall(const Function *F) const { 238 return TTIImpl->isLoweredToCall(F); 239} 240 241bool TargetTransformInfo::isHardwareLoopProfitable( 242 Loop *L, ScalarEvolution &SE, AssumptionCache &AC, 243 TargetLibraryInfo *LibInfo, HardwareLoopInfo &HWLoopInfo) const { 244 return TTIImpl->isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo); 245} 246 247bool TargetTransformInfo::preferPredicateOverEpilogue(Loop *L, LoopInfo *LI, 248 ScalarEvolution &SE, AssumptionCache &AC, TargetLibraryInfo *TLI, 249 DominatorTree *DT, const LoopAccessInfo *LAI) const { 250 return TTIImpl->preferPredicateOverEpilogue(L, LI, SE, AC, TLI, DT, LAI); 251} 252 253void TargetTransformInfo::getUnrollingPreferences( 254 Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP) const { 255 return TTIImpl->getUnrollingPreferences(L, SE, UP); 256} 257 258bool TargetTransformInfo::isLegalAddImmediate(int64_t Imm) const { 259 return TTIImpl->isLegalAddImmediate(Imm); 260} 261 262bool TargetTransformInfo::isLegalICmpImmediate(int64_t Imm) const { 263 return TTIImpl->isLegalICmpImmediate(Imm); 264} 265 266bool TargetTransformInfo::isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, 267 int64_t BaseOffset, 268 bool HasBaseReg, 269 int64_t Scale, 270 unsigned AddrSpace, 271 Instruction *I) const { 272 return TTIImpl->isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg, 273 Scale, AddrSpace, I); 274} 275 276bool TargetTransformInfo::isLSRCostLess(LSRCost &C1, LSRCost &C2) const { 277 return TTIImpl->isLSRCostLess(C1, C2); 278} 279 280bool TargetTransformInfo::canMacroFuseCmp() const { 281 return TTIImpl->canMacroFuseCmp(); 282} 283 284bool TargetTransformInfo::canSaveCmp(Loop *L, BranchInst **BI, 285 ScalarEvolution *SE, LoopInfo *LI, 286 DominatorTree *DT, AssumptionCache *AC, 287 TargetLibraryInfo *LibInfo) const { 288 return TTIImpl->canSaveCmp(L, BI, SE, LI, DT, AC, LibInfo); 289} 290 291bool TargetTransformInfo::shouldFavorPostInc() const { 292 return TTIImpl->shouldFavorPostInc(); 293} 294 295bool TargetTransformInfo::shouldFavorBackedgeIndex(const Loop *L) const { 296 return TTIImpl->shouldFavorBackedgeIndex(L); 297} 298 299bool TargetTransformInfo::isLegalMaskedStore(Type *DataType, 300 MaybeAlign Alignment) const { 301 return TTIImpl->isLegalMaskedStore(DataType, Alignment); 302} 303 304bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType, 305 MaybeAlign Alignment) const { 306 return TTIImpl->isLegalMaskedLoad(DataType, Alignment); 307} 308 309bool TargetTransformInfo::isLegalNTStore(Type *DataType, 310 Align Alignment) const { 311 return TTIImpl->isLegalNTStore(DataType, Alignment); 312} 313 314bool TargetTransformInfo::isLegalNTLoad(Type *DataType, Align Alignment) const { 315 return TTIImpl->isLegalNTLoad(DataType, Alignment); 316} 317 318bool TargetTransformInfo::isLegalMaskedGather(Type *DataType, 319 MaybeAlign Alignment) const { 320 return TTIImpl->isLegalMaskedGather(DataType, Alignment); 321} 322 323bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType, 324 MaybeAlign Alignment) const { 325 return TTIImpl->isLegalMaskedScatter(DataType, Alignment); 326} 327 328bool TargetTransformInfo::isLegalMaskedCompressStore(Type *DataType) const { 329 return TTIImpl->isLegalMaskedCompressStore(DataType); 330} 331 332bool TargetTransformInfo::isLegalMaskedExpandLoad(Type *DataType) const { 333 return TTIImpl->isLegalMaskedExpandLoad(DataType); 334} 335 336bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const { 337 return TTIImpl->hasDivRemOp(DataType, IsSigned); 338} 339 340bool TargetTransformInfo::hasVolatileVariant(Instruction *I, 341 unsigned AddrSpace) const { 342 return TTIImpl->hasVolatileVariant(I, AddrSpace); 343} 344 345bool TargetTransformInfo::prefersVectorizedAddressing() const { 346 return TTIImpl->prefersVectorizedAddressing(); 347} 348 349int TargetTransformInfo::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, 350 int64_t BaseOffset, 351 bool HasBaseReg, 352 int64_t Scale, 353 unsigned AddrSpace) const { 354 int Cost = TTIImpl->getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg, 355 Scale, AddrSpace); 356 assert(Cost >= 0 && "TTI should not produce negative costs!"); 357 return Cost; 358} 359 360bool TargetTransformInfo::LSRWithInstrQueries() const { 361 return TTIImpl->LSRWithInstrQueries(); 362} 363 364bool TargetTransformInfo::isTruncateFree(Type *Ty1, Type *Ty2) const { 365 return TTIImpl->isTruncateFree(Ty1, Ty2); 366} 367 368bool TargetTransformInfo::isProfitableToHoist(Instruction *I) const { 369 return TTIImpl->isProfitableToHoist(I); 370} 371 372bool TargetTransformInfo::useAA() const { return TTIImpl->useAA(); } 373 374bool TargetTransformInfo::isTypeLegal(Type *Ty) const { 375 return TTIImpl->isTypeLegal(Ty); 376} 377 378bool TargetTransformInfo::shouldBuildLookupTables() const { 379 return TTIImpl->shouldBuildLookupTables(); 380} 381bool TargetTransformInfo::shouldBuildLookupTablesForConstant(Constant *C) const { 382 return TTIImpl->shouldBuildLookupTablesForConstant(C); 383} 384 385bool TargetTransformInfo::useColdCCForColdCall(Function &F) const { 386 return TTIImpl->useColdCCForColdCall(F); 387} 388 389unsigned TargetTransformInfo:: 390getScalarizationOverhead(Type *Ty, bool Insert, bool Extract) const { 391 return TTIImpl->getScalarizationOverhead(Ty, Insert, Extract); 392} 393 394unsigned TargetTransformInfo:: 395getOperandsScalarizationOverhead(ArrayRef<const Value *> Args, 396 unsigned VF) const { 397 return TTIImpl->getOperandsScalarizationOverhead(Args, VF); 398} 399 400bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const { 401 return TTIImpl->supportsEfficientVectorElementLoadStore(); 402} 403 404bool TargetTransformInfo::enableAggressiveInterleaving(bool LoopHasReductions) const { 405 return TTIImpl->enableAggressiveInterleaving(LoopHasReductions); 406} 407 408TargetTransformInfo::MemCmpExpansionOptions 409TargetTransformInfo::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const { 410 return TTIImpl->enableMemCmpExpansion(OptSize, IsZeroCmp); 411} 412 413bool TargetTransformInfo::enableInterleavedAccessVectorization() const { 414 return TTIImpl->enableInterleavedAccessVectorization(); 415} 416 417bool TargetTransformInfo::enableMaskedInterleavedAccessVectorization() const { 418 return TTIImpl->enableMaskedInterleavedAccessVectorization(); 419} 420 421bool TargetTransformInfo::isFPVectorizationPotentiallyUnsafe() const { 422 return TTIImpl->isFPVectorizationPotentiallyUnsafe(); 423} 424 425bool TargetTransformInfo::allowsMisalignedMemoryAccesses(LLVMContext &Context, 426 unsigned BitWidth, 427 unsigned AddressSpace, 428 unsigned Alignment, 429 bool *Fast) const { 430 return TTIImpl->allowsMisalignedMemoryAccesses(Context, BitWidth, AddressSpace, 431 Alignment, Fast); 432} 433 434TargetTransformInfo::PopcntSupportKind 435TargetTransformInfo::getPopcntSupport(unsigned IntTyWidthInBit) const { 436 return TTIImpl->getPopcntSupport(IntTyWidthInBit); 437} 438 439bool TargetTransformInfo::haveFastSqrt(Type *Ty) const { 440 return TTIImpl->haveFastSqrt(Ty); 441} 442 443bool TargetTransformInfo::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const { 444 return TTIImpl->isFCmpOrdCheaperThanFCmpZero(Ty); 445} 446 447int TargetTransformInfo::getFPOpCost(Type *Ty) const { 448 int Cost = TTIImpl->getFPOpCost(Ty); 449 assert(Cost >= 0 && "TTI should not produce negative costs!"); 450 return Cost; 451} 452 453int TargetTransformInfo::getIntImmCodeSizeCost(unsigned Opcode, unsigned Idx, 454 const APInt &Imm, 455 Type *Ty) const { 456 int Cost = TTIImpl->getIntImmCodeSizeCost(Opcode, Idx, Imm, Ty); 457 assert(Cost >= 0 && "TTI should not produce negative costs!"); 458 return Cost; 459} 460 461int TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty) const { 462 int Cost = TTIImpl->getIntImmCost(Imm, Ty); 463 assert(Cost >= 0 && "TTI should not produce negative costs!"); 464 return Cost; 465} 466 467int TargetTransformInfo::getIntImmCostInst(unsigned Opcode, unsigned Idx, 468 const APInt &Imm, Type *Ty) const { 469 int Cost = TTIImpl->getIntImmCostInst(Opcode, Idx, Imm, Ty); 470 assert(Cost >= 0 && "TTI should not produce negative costs!"); 471 return Cost; 472} 473 474int TargetTransformInfo::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, 475 const APInt &Imm, Type *Ty) const { 476 int Cost = TTIImpl->getIntImmCostIntrin(IID, Idx, Imm, Ty); 477 assert(Cost >= 0 && "TTI should not produce negative costs!"); 478 return Cost; 479} 480 481unsigned TargetTransformInfo::getNumberOfRegisters(unsigned ClassID) const { 482 return TTIImpl->getNumberOfRegisters(ClassID); 483} 484 485unsigned TargetTransformInfo::getRegisterClassForType(bool Vector, Type *Ty) const { 486 return TTIImpl->getRegisterClassForType(Vector, Ty); 487} 488 489const char* TargetTransformInfo::getRegisterClassName(unsigned ClassID) const { 490 return TTIImpl->getRegisterClassName(ClassID); 491} 492 493unsigned TargetTransformInfo::getRegisterBitWidth(bool Vector) const { 494 return TTIImpl->getRegisterBitWidth(Vector); 495} 496 497unsigned TargetTransformInfo::getMinVectorRegisterBitWidth() const { 498 return TTIImpl->getMinVectorRegisterBitWidth(); 499} 500 501bool TargetTransformInfo::shouldMaximizeVectorBandwidth(bool OptSize) const { 502 return TTIImpl->shouldMaximizeVectorBandwidth(OptSize); 503} 504 505unsigned TargetTransformInfo::getMinimumVF(unsigned ElemWidth) const { 506 return TTIImpl->getMinimumVF(ElemWidth); 507} 508 509bool TargetTransformInfo::shouldConsiderAddressTypePromotion( 510 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const { 511 return TTIImpl->shouldConsiderAddressTypePromotion( 512 I, AllowPromotionWithoutCommonHeader); 513} 514 515unsigned TargetTransformInfo::getCacheLineSize() const { 516 return TTIImpl->getCacheLineSize(); 517} 518 519llvm::Optional<unsigned> TargetTransformInfo::getCacheSize(CacheLevel Level) 520 const { 521 return TTIImpl->getCacheSize(Level); 522} 523 524llvm::Optional<unsigned> TargetTransformInfo::getCacheAssociativity( 525 CacheLevel Level) const { 526 return TTIImpl->getCacheAssociativity(Level); 527} 528 529unsigned TargetTransformInfo::getPrefetchDistance() const { 530 return TTIImpl->getPrefetchDistance(); 531} 532 533unsigned TargetTransformInfo::getMinPrefetchStride() const { 534 return TTIImpl->getMinPrefetchStride(); 535} 536 537unsigned TargetTransformInfo::getMaxPrefetchIterationsAhead() const { 538 return TTIImpl->getMaxPrefetchIterationsAhead(); 539} 540 541unsigned TargetTransformInfo::getMaxInterleaveFactor(unsigned VF) const { 542 return TTIImpl->getMaxInterleaveFactor(VF); 543} 544 545TargetTransformInfo::OperandValueKind 546TargetTransformInfo::getOperandInfo(Value *V, OperandValueProperties &OpProps) { 547 OperandValueKind OpInfo = OK_AnyValue; 548 OpProps = OP_None; 549 550 if (auto *CI = dyn_cast<ConstantInt>(V)) { 551 if (CI->getValue().isPowerOf2()) 552 OpProps = OP_PowerOf2; 553 return OK_UniformConstantValue; 554 } 555 556 // A broadcast shuffle creates a uniform value. 557 // TODO: Add support for non-zero index broadcasts. 558 // TODO: Add support for different source vector width. 559 if (auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(V)) 560 if (ShuffleInst->isZeroEltSplat()) 561 OpInfo = OK_UniformValue; 562 563 const Value *Splat = getSplatValue(V); 564 565 // Check for a splat of a constant or for a non uniform vector of constants 566 // and check if the constant(s) are all powers of two. 567 if (isa<ConstantVector>(V) || isa<ConstantDataVector>(V)) { 568 OpInfo = OK_NonUniformConstantValue; 569 if (Splat) { 570 OpInfo = OK_UniformConstantValue; 571 if (auto *CI = dyn_cast<ConstantInt>(Splat)) 572 if (CI->getValue().isPowerOf2()) 573 OpProps = OP_PowerOf2; 574 } else if (auto *CDS = dyn_cast<ConstantDataSequential>(V)) { 575 OpProps = OP_PowerOf2; 576 for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) { 577 if (auto *CI = dyn_cast<ConstantInt>(CDS->getElementAsConstant(I))) 578 if (CI->getValue().isPowerOf2()) 579 continue; 580 OpProps = OP_None; 581 break; 582 } 583 } 584 } 585 586 // Check for a splat of a uniform value. This is not loop aware, so return 587 // true only for the obviously uniform cases (argument, globalvalue) 588 if (Splat && (isa<Argument>(Splat) || isa<GlobalValue>(Splat))) 589 OpInfo = OK_UniformValue; 590 591 return OpInfo; 592} 593 594int TargetTransformInfo::getArithmeticInstrCost( 595 unsigned Opcode, Type *Ty, OperandValueKind Opd1Info, 596 OperandValueKind Opd2Info, OperandValueProperties Opd1PropInfo, 597 OperandValueProperties Opd2PropInfo, ArrayRef<const Value *> Args, 598 const Instruction *CxtI) const { 599 int Cost = TTIImpl->getArithmeticInstrCost( 600 Opcode, Ty, Opd1Info, Opd2Info, Opd1PropInfo, Opd2PropInfo, Args, CxtI); 601 assert(Cost >= 0 && "TTI should not produce negative costs!"); 602 return Cost; 603} 604 605int TargetTransformInfo::getShuffleCost(ShuffleKind Kind, Type *Ty, int Index, 606 Type *SubTp) const { 607 int Cost = TTIImpl->getShuffleCost(Kind, Ty, Index, SubTp); 608 assert(Cost >= 0 && "TTI should not produce negative costs!"); 609 return Cost; 610} 611 612int TargetTransformInfo::getCastInstrCost(unsigned Opcode, Type *Dst, 613 Type *Src, const Instruction *I) const { 614 assert ((I == nullptr || I->getOpcode() == Opcode) && 615 "Opcode should reflect passed instruction."); 616 int Cost = TTIImpl->getCastInstrCost(Opcode, Dst, Src, I); 617 assert(Cost >= 0 && "TTI should not produce negative costs!"); 618 return Cost; 619} 620 621int TargetTransformInfo::getExtractWithExtendCost(unsigned Opcode, Type *Dst, 622 VectorType *VecTy, 623 unsigned Index) const { 624 int Cost = TTIImpl->getExtractWithExtendCost(Opcode, Dst, VecTy, Index); 625 assert(Cost >= 0 && "TTI should not produce negative costs!"); 626 return Cost; 627} 628 629int TargetTransformInfo::getCFInstrCost(unsigned Opcode) const { 630 int Cost = TTIImpl->getCFInstrCost(Opcode); 631 assert(Cost >= 0 && "TTI should not produce negative costs!"); 632 return Cost; 633} 634 635int TargetTransformInfo::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, 636 Type *CondTy, const Instruction *I) const { 637 assert ((I == nullptr || I->getOpcode() == Opcode) && 638 "Opcode should reflect passed instruction."); 639 int Cost = TTIImpl->getCmpSelInstrCost(Opcode, ValTy, CondTy, I); 640 assert(Cost >= 0 && "TTI should not produce negative costs!"); 641 return Cost; 642} 643 644int TargetTransformInfo::getVectorInstrCost(unsigned Opcode, Type *Val, 645 unsigned Index) const { 646 int Cost = TTIImpl->getVectorInstrCost(Opcode, Val, Index); 647 assert(Cost >= 0 && "TTI should not produce negative costs!"); 648 return Cost; 649} 650 651int TargetTransformInfo::getMemoryOpCost(unsigned Opcode, Type *Src, 652 MaybeAlign Alignment, 653 unsigned AddressSpace, 654 const Instruction *I) const { 655 assert ((I == nullptr || I->getOpcode() == Opcode) && 656 "Opcode should reflect passed instruction."); 657 int Cost = TTIImpl->getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, I); 658 assert(Cost >= 0 && "TTI should not produce negative costs!"); 659 return Cost; 660} 661 662int TargetTransformInfo::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, 663 unsigned Alignment, 664 unsigned AddressSpace) const { 665 int Cost = 666 TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace); 667 assert(Cost >= 0 && "TTI should not produce negative costs!"); 668 return Cost; 669} 670 671int TargetTransformInfo::getGatherScatterOpCost(unsigned Opcode, Type *DataTy, 672 Value *Ptr, bool VariableMask, 673 unsigned Alignment) const { 674 int Cost = TTIImpl->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask, 675 Alignment); 676 assert(Cost >= 0 && "TTI should not produce negative costs!"); 677 return Cost; 678} 679 680int TargetTransformInfo::getInterleavedMemoryOpCost( 681 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices, 682 unsigned Alignment, unsigned AddressSpace, bool UseMaskForCond, 683 bool UseMaskForGaps) const { 684 int Cost = TTIImpl->getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices, 685 Alignment, AddressSpace, 686 UseMaskForCond, 687 UseMaskForGaps); 688 assert(Cost >= 0 && "TTI should not produce negative costs!"); 689 return Cost; 690} 691 692int TargetTransformInfo::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, 693 ArrayRef<Type *> Tys, FastMathFlags FMF, 694 unsigned ScalarizationCostPassed) const { 695 int Cost = TTIImpl->getIntrinsicInstrCost(ID, RetTy, Tys, FMF, 696 ScalarizationCostPassed); 697 assert(Cost >= 0 && "TTI should not produce negative costs!"); 698 return Cost; 699} 700 701int TargetTransformInfo::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, 702 ArrayRef<Value *> Args, FastMathFlags FMF, unsigned VF) const { 703 int Cost = TTIImpl->getIntrinsicInstrCost(ID, RetTy, Args, FMF, VF); 704 assert(Cost >= 0 && "TTI should not produce negative costs!"); 705 return Cost; 706} 707 708int TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy, 709 ArrayRef<Type *> Tys) const { 710 int Cost = TTIImpl->getCallInstrCost(F, RetTy, Tys); 711 assert(Cost >= 0 && "TTI should not produce negative costs!"); 712 return Cost; 713} 714 715unsigned TargetTransformInfo::getNumberOfParts(Type *Tp) const { 716 return TTIImpl->getNumberOfParts(Tp); 717} 718 719int TargetTransformInfo::getAddressComputationCost(Type *Tp, 720 ScalarEvolution *SE, 721 const SCEV *Ptr) const { 722 int Cost = TTIImpl->getAddressComputationCost(Tp, SE, Ptr); 723 assert(Cost >= 0 && "TTI should not produce negative costs!"); 724 return Cost; 725} 726 727int TargetTransformInfo::getMemcpyCost(const Instruction *I) const { 728 int Cost = TTIImpl->getMemcpyCost(I); 729 assert(Cost >= 0 && "TTI should not produce negative costs!"); 730 return Cost; 731} 732 733int TargetTransformInfo::getArithmeticReductionCost(unsigned Opcode, Type *Ty, 734 bool IsPairwiseForm) const { 735 int Cost = TTIImpl->getArithmeticReductionCost(Opcode, Ty, IsPairwiseForm); 736 assert(Cost >= 0 && "TTI should not produce negative costs!"); 737 return Cost; 738} 739 740int TargetTransformInfo::getMinMaxReductionCost(Type *Ty, Type *CondTy, 741 bool IsPairwiseForm, 742 bool IsUnsigned) const { 743 int Cost = 744 TTIImpl->getMinMaxReductionCost(Ty, CondTy, IsPairwiseForm, IsUnsigned); 745 assert(Cost >= 0 && "TTI should not produce negative costs!"); 746 return Cost; 747} 748 749unsigned 750TargetTransformInfo::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const { 751 return TTIImpl->getCostOfKeepingLiveOverCall(Tys); 752} 753 754bool TargetTransformInfo::getTgtMemIntrinsic(IntrinsicInst *Inst, 755 MemIntrinsicInfo &Info) const { 756 return TTIImpl->getTgtMemIntrinsic(Inst, Info); 757} 758 759unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const { 760 return TTIImpl->getAtomicMemIntrinsicMaxElementSize(); 761} 762 763Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic( 764 IntrinsicInst *Inst, Type *ExpectedType) const { 765 return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType); 766} 767 768Type *TargetTransformInfo::getMemcpyLoopLoweringType(LLVMContext &Context, 769 Value *Length, 770 unsigned SrcAlign, 771 unsigned DestAlign) const { 772 return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAlign, 773 DestAlign); 774} 775 776void TargetTransformInfo::getMemcpyLoopResidualLoweringType( 777 SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context, 778 unsigned RemainingBytes, unsigned SrcAlign, unsigned DestAlign) const { 779 TTIImpl->getMemcpyLoopResidualLoweringType(OpsOut, Context, RemainingBytes, 780 SrcAlign, DestAlign); 781} 782 783bool TargetTransformInfo::areInlineCompatible(const Function *Caller, 784 const Function *Callee) const { 785 return TTIImpl->areInlineCompatible(Caller, Callee); 786} 787 788bool TargetTransformInfo::areFunctionArgsABICompatible( 789 const Function *Caller, const Function *Callee, 790 SmallPtrSetImpl<Argument *> &Args) const { 791 return TTIImpl->areFunctionArgsABICompatible(Caller, Callee, Args); 792} 793 794bool TargetTransformInfo::isIndexedLoadLegal(MemIndexedMode Mode, 795 Type *Ty) const { 796 return TTIImpl->isIndexedLoadLegal(Mode, Ty); 797} 798 799bool TargetTransformInfo::isIndexedStoreLegal(MemIndexedMode Mode, 800 Type *Ty) const { 801 return TTIImpl->isIndexedStoreLegal(Mode, Ty); 802} 803 804unsigned TargetTransformInfo::getLoadStoreVecRegBitWidth(unsigned AS) const { 805 return TTIImpl->getLoadStoreVecRegBitWidth(AS); 806} 807 808bool TargetTransformInfo::isLegalToVectorizeLoad(LoadInst *LI) const { 809 return TTIImpl->isLegalToVectorizeLoad(LI); 810} 811 812bool TargetTransformInfo::isLegalToVectorizeStore(StoreInst *SI) const { 813 return TTIImpl->isLegalToVectorizeStore(SI); 814} 815 816bool TargetTransformInfo::isLegalToVectorizeLoadChain( 817 unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const { 818 return TTIImpl->isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment, 819 AddrSpace); 820} 821 822bool TargetTransformInfo::isLegalToVectorizeStoreChain( 823 unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const { 824 return TTIImpl->isLegalToVectorizeStoreChain(ChainSizeInBytes, Alignment, 825 AddrSpace); 826} 827 828unsigned TargetTransformInfo::getLoadVectorFactor(unsigned VF, 829 unsigned LoadSize, 830 unsigned ChainSizeInBytes, 831 VectorType *VecTy) const { 832 return TTIImpl->getLoadVectorFactor(VF, LoadSize, ChainSizeInBytes, VecTy); 833} 834 835unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF, 836 unsigned StoreSize, 837 unsigned ChainSizeInBytes, 838 VectorType *VecTy) const { 839 return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy); 840} 841 842bool TargetTransformInfo::useReductionIntrinsic(unsigned Opcode, 843 Type *Ty, ReductionFlags Flags) const { 844 return TTIImpl->useReductionIntrinsic(Opcode, Ty, Flags); 845} 846 847bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const { 848 return TTIImpl->shouldExpandReduction(II); 849} 850 851unsigned TargetTransformInfo::getGISelRematGlobalCost() const { 852 return TTIImpl->getGISelRematGlobalCost(); 853} 854 855int TargetTransformInfo::getInstructionLatency(const Instruction *I) const { 856 return TTIImpl->getInstructionLatency(I); 857} 858 859static bool matchPairwiseShuffleMask(ShuffleVectorInst *SI, bool IsLeft, 860 unsigned Level) { 861 // We don't need a shuffle if we just want to have element 0 in position 0 of 862 // the vector. 863 if (!SI && Level == 0 && IsLeft) 864 return true; 865 else if (!SI) 866 return false; 867 868 SmallVector<int, 32> Mask(SI->getType()->getVectorNumElements(), -1); 869 870 // Build a mask of 0, 2, ... (left) or 1, 3, ... (right) depending on whether 871 // we look at the left or right side. 872 for (unsigned i = 0, e = (1 << Level), val = !IsLeft; i != e; ++i, val += 2) 873 Mask[i] = val; 874 875 SmallVector<int, 16> ActualMask = SI->getShuffleMask(); 876 return Mask == ActualMask; 877} 878 879namespace { 880/// Kind of the reduction data. 881enum ReductionKind { 882 RK_None, /// Not a reduction. 883 RK_Arithmetic, /// Binary reduction data. 884 RK_MinMax, /// Min/max reduction data. 885 RK_UnsignedMinMax, /// Unsigned min/max reduction data. 886}; 887/// Contains opcode + LHS/RHS parts of the reduction operations. 888struct ReductionData { 889 ReductionData() = delete; 890 ReductionData(ReductionKind Kind, unsigned Opcode, Value *LHS, Value *RHS) 891 : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind) { 892 assert(Kind != RK_None && "expected binary or min/max reduction only."); 893 } 894 unsigned Opcode = 0; 895 Value *LHS = nullptr; 896 Value *RHS = nullptr; 897 ReductionKind Kind = RK_None; 898 bool hasSameData(ReductionData &RD) const { 899 return Kind == RD.Kind && Opcode == RD.Opcode; 900 } 901}; 902} // namespace 903 904static Optional<ReductionData> getReductionData(Instruction *I) { 905 Value *L, *R; 906 if (m_BinOp(m_Value(L), m_Value(R)).match(I)) 907 return ReductionData(RK_Arithmetic, I->getOpcode(), L, R); 908 if (auto *SI = dyn_cast<SelectInst>(I)) { 909 if (m_SMin(m_Value(L), m_Value(R)).match(SI) || 910 m_SMax(m_Value(L), m_Value(R)).match(SI) || 911 m_OrdFMin(m_Value(L), m_Value(R)).match(SI) || 912 m_OrdFMax(m_Value(L), m_Value(R)).match(SI) || 913 m_UnordFMin(m_Value(L), m_Value(R)).match(SI) || 914 m_UnordFMax(m_Value(L), m_Value(R)).match(SI)) { 915 auto *CI = cast<CmpInst>(SI->getCondition()); 916 return ReductionData(RK_MinMax, CI->getOpcode(), L, R); 917 } 918 if (m_UMin(m_Value(L), m_Value(R)).match(SI) || 919 m_UMax(m_Value(L), m_Value(R)).match(SI)) { 920 auto *CI = cast<CmpInst>(SI->getCondition()); 921 return ReductionData(RK_UnsignedMinMax, CI->getOpcode(), L, R); 922 } 923 } 924 return llvm::None; 925} 926 927static ReductionKind matchPairwiseReductionAtLevel(Instruction *I, 928 unsigned Level, 929 unsigned NumLevels) { 930 // Match one level of pairwise operations. 931 // %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, 932 // <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef> 933 // %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, 934 // <4 x i32> <i32 1, i32 3, i32 undef, i32 undef> 935 // %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1 936 if (!I) 937 return RK_None; 938 939 assert(I->getType()->isVectorTy() && "Expecting a vector type"); 940 941 Optional<ReductionData> RD = getReductionData(I); 942 if (!RD) 943 return RK_None; 944 945 ShuffleVectorInst *LS = dyn_cast<ShuffleVectorInst>(RD->LHS); 946 if (!LS && Level) 947 return RK_None; 948 ShuffleVectorInst *RS = dyn_cast<ShuffleVectorInst>(RD->RHS); 949 if (!RS && Level) 950 return RK_None; 951 952 // On level 0 we can omit one shufflevector instruction. 953 if (!Level && !RS && !LS) 954 return RK_None; 955 956 // Shuffle inputs must match. 957 Value *NextLevelOpL = LS ? LS->getOperand(0) : nullptr; 958 Value *NextLevelOpR = RS ? RS->getOperand(0) : nullptr; 959 Value *NextLevelOp = nullptr; 960 if (NextLevelOpR && NextLevelOpL) { 961 // If we have two shuffles their operands must match. 962 if (NextLevelOpL != NextLevelOpR) 963 return RK_None; 964 965 NextLevelOp = NextLevelOpL; 966 } else if (Level == 0 && (NextLevelOpR || NextLevelOpL)) { 967 // On the first level we can omit the shufflevector <0, undef,...>. So the 968 // input to the other shufflevector <1, undef> must match with one of the 969 // inputs to the current binary operation. 970 // Example: 971 // %NextLevelOpL = shufflevector %R, <1, undef ...> 972 // %BinOp = fadd %NextLevelOpL, %R 973 if (NextLevelOpL && NextLevelOpL != RD->RHS) 974 return RK_None; 975 else if (NextLevelOpR && NextLevelOpR != RD->LHS) 976 return RK_None; 977 978 NextLevelOp = NextLevelOpL ? RD->RHS : RD->LHS; 979 } else 980 return RK_None; 981 982 // Check that the next levels binary operation exists and matches with the 983 // current one. 984 if (Level + 1 != NumLevels) { 985 Optional<ReductionData> NextLevelRD = 986 getReductionData(cast<Instruction>(NextLevelOp)); 987 if (!NextLevelRD || !RD->hasSameData(*NextLevelRD)) 988 return RK_None; 989 } 990 991 // Shuffle mask for pairwise operation must match. 992 if (matchPairwiseShuffleMask(LS, /*IsLeft=*/true, Level)) { 993 if (!matchPairwiseShuffleMask(RS, /*IsLeft=*/false, Level)) 994 return RK_None; 995 } else if (matchPairwiseShuffleMask(RS, /*IsLeft=*/true, Level)) { 996 if (!matchPairwiseShuffleMask(LS, /*IsLeft=*/false, Level)) 997 return RK_None; 998 } else { 999 return RK_None; 1000 } 1001 1002 if (++Level == NumLevels) 1003 return RD->Kind; 1004 1005 // Match next level. 1006 return matchPairwiseReductionAtLevel(cast<Instruction>(NextLevelOp), Level, 1007 NumLevels); 1008} 1009 1010static ReductionKind matchPairwiseReduction(const ExtractElementInst *ReduxRoot, 1011 unsigned &Opcode, Type *&Ty) { 1012 if (!EnableReduxCost) 1013 return RK_None; 1014 1015 // Need to extract the first element. 1016 ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1)); 1017 unsigned Idx = ~0u; 1018 if (CI) 1019 Idx = CI->getZExtValue(); 1020 if (Idx != 0) 1021 return RK_None; 1022 1023 auto *RdxStart = dyn_cast<Instruction>(ReduxRoot->getOperand(0)); 1024 if (!RdxStart) 1025 return RK_None; 1026 Optional<ReductionData> RD = getReductionData(RdxStart); 1027 if (!RD) 1028 return RK_None; 1029 1030 Type *VecTy = RdxStart->getType(); 1031 unsigned NumVecElems = VecTy->getVectorNumElements(); 1032 if (!isPowerOf2_32(NumVecElems)) 1033 return RK_None; 1034 1035 // We look for a sequence of shuffle,shuffle,add triples like the following 1036 // that builds a pairwise reduction tree. 1037 // 1038 // (X0, X1, X2, X3) 1039 // (X0 + X1, X2 + X3, undef, undef) 1040 // ((X0 + X1) + (X2 + X3), undef, undef, undef) 1041 // 1042 // %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, 1043 // <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef> 1044 // %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, 1045 // <4 x i32> <i32 1, i32 3, i32 undef, i32 undef> 1046 // %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1 1047 // %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, 1048 // <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef> 1049 // %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, 1050 // <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> 1051 // %bin.rdx8 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1 1052 // %r = extractelement <4 x float> %bin.rdx8, i32 0 1053 if (matchPairwiseReductionAtLevel(RdxStart, 0, Log2_32(NumVecElems)) == 1054 RK_None) 1055 return RK_None; 1056 1057 Opcode = RD->Opcode; 1058 Ty = VecTy; 1059 1060 return RD->Kind; 1061} 1062 1063static std::pair<Value *, ShuffleVectorInst *> 1064getShuffleAndOtherOprd(Value *L, Value *R) { 1065 ShuffleVectorInst *S = nullptr; 1066 1067 if ((S = dyn_cast<ShuffleVectorInst>(L))) 1068 return std::make_pair(R, S); 1069 1070 S = dyn_cast<ShuffleVectorInst>(R); 1071 return std::make_pair(L, S); 1072} 1073 1074static ReductionKind 1075matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot, 1076 unsigned &Opcode, Type *&Ty) { 1077 if (!EnableReduxCost) 1078 return RK_None; 1079 1080 // Need to extract the first element. 1081 ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1)); 1082 unsigned Idx = ~0u; 1083 if (CI) 1084 Idx = CI->getZExtValue(); 1085 if (Idx != 0) 1086 return RK_None; 1087 1088 auto *RdxStart = dyn_cast<Instruction>(ReduxRoot->getOperand(0)); 1089 if (!RdxStart) 1090 return RK_None; 1091 Optional<ReductionData> RD = getReductionData(RdxStart); 1092 if (!RD) 1093 return RK_None; 1094 1095 Type *VecTy = ReduxRoot->getOperand(0)->getType(); 1096 unsigned NumVecElems = VecTy->getVectorNumElements(); 1097 if (!isPowerOf2_32(NumVecElems)) 1098 return RK_None; 1099 1100 // We look for a sequence of shuffles and adds like the following matching one 1101 // fadd, shuffle vector pair at a time. 1102 // 1103 // %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, 1104 // <4 x i32> <i32 2, i32 3, i32 undef, i32 undef> 1105 // %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf 1106 // %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, 1107 // <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> 1108 // %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7 1109 // %r = extractelement <4 x float> %bin.rdx8, i32 0 1110 1111 unsigned MaskStart = 1; 1112 Instruction *RdxOp = RdxStart; 1113 SmallVector<int, 32> ShuffleMask(NumVecElems, 0); 1114 unsigned NumVecElemsRemain = NumVecElems; 1115 while (NumVecElemsRemain - 1) { 1116 // Check for the right reduction operation. 1117 if (!RdxOp) 1118 return RK_None; 1119 Optional<ReductionData> RDLevel = getReductionData(RdxOp); 1120 if (!RDLevel || !RDLevel->hasSameData(*RD)) 1121 return RK_None; 1122 1123 Value *NextRdxOp; 1124 ShuffleVectorInst *Shuffle; 1125 std::tie(NextRdxOp, Shuffle) = 1126 getShuffleAndOtherOprd(RDLevel->LHS, RDLevel->RHS); 1127 1128 // Check the current reduction operation and the shuffle use the same value. 1129 if (Shuffle == nullptr) 1130 return RK_None; 1131 if (Shuffle->getOperand(0) != NextRdxOp) 1132 return RK_None; 1133 1134 // Check that shuffle masks matches. 1135 for (unsigned j = 0; j != MaskStart; ++j) 1136 ShuffleMask[j] = MaskStart + j; 1137 // Fill the rest of the mask with -1 for undef. 1138 std::fill(&ShuffleMask[MaskStart], ShuffleMask.end(), -1); 1139 1140 SmallVector<int, 16> Mask = Shuffle->getShuffleMask(); 1141 if (ShuffleMask != Mask) 1142 return RK_None; 1143 1144 RdxOp = dyn_cast<Instruction>(NextRdxOp); 1145 NumVecElemsRemain /= 2; 1146 MaskStart *= 2; 1147 } 1148 1149 Opcode = RD->Opcode; 1150 Ty = VecTy; 1151 return RD->Kind; 1152} 1153 1154int TargetTransformInfo::getInstructionThroughput(const Instruction *I) const { 1155 switch (I->getOpcode()) { 1156 case Instruction::GetElementPtr: 1157 return getUserCost(I); 1158 1159 case Instruction::Ret: 1160 case Instruction::PHI: 1161 case Instruction::Br: { 1162 return getCFInstrCost(I->getOpcode()); 1163 } 1164 case Instruction::Add: 1165 case Instruction::FAdd: 1166 case Instruction::Sub: 1167 case Instruction::FSub: 1168 case Instruction::Mul: 1169 case Instruction::FMul: 1170 case Instruction::UDiv: 1171 case Instruction::SDiv: 1172 case Instruction::FDiv: 1173 case Instruction::URem: 1174 case Instruction::SRem: 1175 case Instruction::FRem: 1176 case Instruction::Shl: 1177 case Instruction::LShr: 1178 case Instruction::AShr: 1179 case Instruction::And: 1180 case Instruction::Or: 1181 case Instruction::Xor: { 1182 TargetTransformInfo::OperandValueKind Op1VK, Op2VK; 1183 TargetTransformInfo::OperandValueProperties Op1VP, Op2VP; 1184 Op1VK = getOperandInfo(I->getOperand(0), Op1VP); 1185 Op2VK = getOperandInfo(I->getOperand(1), Op2VP); 1186 SmallVector<const Value *, 2> Operands(I->operand_values()); 1187 return getArithmeticInstrCost(I->getOpcode(), I->getType(), Op1VK, Op2VK, 1188 Op1VP, Op2VP, Operands, I); 1189 } 1190 case Instruction::FNeg: { 1191 TargetTransformInfo::OperandValueKind Op1VK, Op2VK; 1192 TargetTransformInfo::OperandValueProperties Op1VP, Op2VP; 1193 Op1VK = getOperandInfo(I->getOperand(0), Op1VP); 1194 Op2VK = OK_AnyValue; 1195 Op2VP = OP_None; 1196 SmallVector<const Value *, 2> Operands(I->operand_values()); 1197 return getArithmeticInstrCost(I->getOpcode(), I->getType(), Op1VK, Op2VK, 1198 Op1VP, Op2VP, Operands, I); 1199 } 1200 case Instruction::Select: { 1201 const SelectInst *SI = cast<SelectInst>(I); 1202 Type *CondTy = SI->getCondition()->getType(); 1203 return getCmpSelInstrCost(I->getOpcode(), I->getType(), CondTy, I); 1204 } 1205 case Instruction::ICmp: 1206 case Instruction::FCmp: { 1207 Type *ValTy = I->getOperand(0)->getType(); 1208 return getCmpSelInstrCost(I->getOpcode(), ValTy, I->getType(), I); 1209 } 1210 case Instruction::Store: { 1211 const StoreInst *SI = cast<StoreInst>(I); 1212 Type *ValTy = SI->getValueOperand()->getType(); 1213 return getMemoryOpCost(I->getOpcode(), ValTy, 1214 MaybeAlign(SI->getAlignment()), 1215 SI->getPointerAddressSpace(), I); 1216 } 1217 case Instruction::Load: { 1218 const LoadInst *LI = cast<LoadInst>(I); 1219 return getMemoryOpCost(I->getOpcode(), I->getType(), 1220 MaybeAlign(LI->getAlignment()), 1221 LI->getPointerAddressSpace(), I); 1222 } 1223 case Instruction::ZExt: 1224 case Instruction::SExt: 1225 case Instruction::FPToUI: 1226 case Instruction::FPToSI: 1227 case Instruction::FPExt: 1228 case Instruction::PtrToInt: 1229 case Instruction::IntToPtr: 1230 case Instruction::SIToFP: 1231 case Instruction::UIToFP: 1232 case Instruction::Trunc: 1233 case Instruction::FPTrunc: 1234 case Instruction::BitCast: 1235 case Instruction::AddrSpaceCast: { 1236 Type *SrcTy = I->getOperand(0)->getType(); 1237 return getCastInstrCost(I->getOpcode(), I->getType(), SrcTy, I); 1238 } 1239 case Instruction::ExtractElement: { 1240 const ExtractElementInst * EEI = cast<ExtractElementInst>(I); 1241 ConstantInt *CI = dyn_cast<ConstantInt>(I->getOperand(1)); 1242 unsigned Idx = -1; 1243 if (CI) 1244 Idx = CI->getZExtValue(); 1245 1246 // Try to match a reduction sequence (series of shufflevector and vector 1247 // adds followed by a extractelement). 1248 unsigned ReduxOpCode; 1249 Type *ReduxType; 1250 1251 switch (matchVectorSplittingReduction(EEI, ReduxOpCode, ReduxType)) { 1252 case RK_Arithmetic: 1253 return getArithmeticReductionCost(ReduxOpCode, ReduxType, 1254 /*IsPairwiseForm=*/false); 1255 case RK_MinMax: 1256 return getMinMaxReductionCost( 1257 ReduxType, CmpInst::makeCmpResultType(ReduxType), 1258 /*IsPairwiseForm=*/false, /*IsUnsigned=*/false); 1259 case RK_UnsignedMinMax: 1260 return getMinMaxReductionCost( 1261 ReduxType, CmpInst::makeCmpResultType(ReduxType), 1262 /*IsPairwiseForm=*/false, /*IsUnsigned=*/true); 1263 case RK_None: 1264 break; 1265 } 1266 1267 switch (matchPairwiseReduction(EEI, ReduxOpCode, ReduxType)) { 1268 case RK_Arithmetic: 1269 return getArithmeticReductionCost(ReduxOpCode, ReduxType, 1270 /*IsPairwiseForm=*/true); 1271 case RK_MinMax: 1272 return getMinMaxReductionCost( 1273 ReduxType, CmpInst::makeCmpResultType(ReduxType), 1274 /*IsPairwiseForm=*/true, /*IsUnsigned=*/false); 1275 case RK_UnsignedMinMax: 1276 return getMinMaxReductionCost( 1277 ReduxType, CmpInst::makeCmpResultType(ReduxType), 1278 /*IsPairwiseForm=*/true, /*IsUnsigned=*/true); 1279 case RK_None: 1280 break; 1281 } 1282 1283 return getVectorInstrCost(I->getOpcode(), 1284 EEI->getOperand(0)->getType(), Idx); 1285 } 1286 case Instruction::InsertElement: { 1287 const InsertElementInst * IE = cast<InsertElementInst>(I); 1288 ConstantInt *CI = dyn_cast<ConstantInt>(IE->getOperand(2)); 1289 unsigned Idx = -1; 1290 if (CI) 1291 Idx = CI->getZExtValue(); 1292 return getVectorInstrCost(I->getOpcode(), 1293 IE->getType(), Idx); 1294 } 1295 case Instruction::ExtractValue: 1296 return 0; // Model all ExtractValue nodes as free. 1297 case Instruction::ShuffleVector: { 1298 const ShuffleVectorInst *Shuffle = cast<ShuffleVectorInst>(I); 1299 Type *Ty = Shuffle->getType(); 1300 Type *SrcTy = Shuffle->getOperand(0)->getType(); 1301 1302 // TODO: Identify and add costs for insert subvector, etc. 1303 int SubIndex; 1304 if (Shuffle->isExtractSubvectorMask(SubIndex)) 1305 return TTIImpl->getShuffleCost(SK_ExtractSubvector, SrcTy, SubIndex, Ty); 1306 1307 if (Shuffle->changesLength()) 1308 return -1; 1309 1310 if (Shuffle->isIdentity()) 1311 return 0; 1312 1313 if (Shuffle->isReverse()) 1314 return TTIImpl->getShuffleCost(SK_Reverse, Ty, 0, nullptr); 1315 1316 if (Shuffle->isSelect()) 1317 return TTIImpl->getShuffleCost(SK_Select, Ty, 0, nullptr); 1318 1319 if (Shuffle->isTranspose()) 1320 return TTIImpl->getShuffleCost(SK_Transpose, Ty, 0, nullptr); 1321 1322 if (Shuffle->isZeroEltSplat()) 1323 return TTIImpl->getShuffleCost(SK_Broadcast, Ty, 0, nullptr); 1324 1325 if (Shuffle->isSingleSource()) 1326 return TTIImpl->getShuffleCost(SK_PermuteSingleSrc, Ty, 0, nullptr); 1327 1328 return TTIImpl->getShuffleCost(SK_PermuteTwoSrc, Ty, 0, nullptr); 1329 } 1330 case Instruction::Call: 1331 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) { 1332 SmallVector<Value *, 4> Args(II->arg_operands()); 1333 1334 FastMathFlags FMF; 1335 if (auto *FPMO = dyn_cast<FPMathOperator>(II)) 1336 FMF = FPMO->getFastMathFlags(); 1337 1338 return getIntrinsicInstrCost(II->getIntrinsicID(), II->getType(), 1339 Args, FMF); 1340 } 1341 return -1; 1342 default: 1343 // We don't have any information on this instruction. 1344 return -1; 1345 } 1346} 1347 1348TargetTransformInfo::Concept::~Concept() {} 1349 1350TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {} 1351 1352TargetIRAnalysis::TargetIRAnalysis( 1353 std::function<Result(const Function &)> TTICallback) 1354 : TTICallback(std::move(TTICallback)) {} 1355 1356TargetIRAnalysis::Result TargetIRAnalysis::run(const Function &F, 1357 FunctionAnalysisManager &) { 1358 return TTICallback(F); 1359} 1360 1361AnalysisKey TargetIRAnalysis::Key; 1362 1363TargetIRAnalysis::Result TargetIRAnalysis::getDefaultTTI(const Function &F) { 1364 return Result(F.getParent()->getDataLayout()); 1365} 1366 1367// Register the basic pass. 1368INITIALIZE_PASS(TargetTransformInfoWrapperPass, "tti", 1369 "Target Transform Information", false, true) 1370char TargetTransformInfoWrapperPass::ID = 0; 1371 1372void TargetTransformInfoWrapperPass::anchor() {} 1373 1374TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass() 1375 : ImmutablePass(ID) { 1376 initializeTargetTransformInfoWrapperPassPass( 1377 *PassRegistry::getPassRegistry()); 1378} 1379 1380TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass( 1381 TargetIRAnalysis TIRA) 1382 : ImmutablePass(ID), TIRA(std::move(TIRA)) { 1383 initializeTargetTransformInfoWrapperPassPass( 1384 *PassRegistry::getPassRegistry()); 1385} 1386 1387TargetTransformInfo &TargetTransformInfoWrapperPass::getTTI(const Function &F) { 1388 FunctionAnalysisManager DummyFAM; 1389 TTI = TIRA.run(F, DummyFAM); 1390 return *TTI; 1391} 1392 1393ImmutablePass * 1394llvm::createTargetTransformInfoWrapperPass(TargetIRAnalysis TIRA) { 1395 return new TargetTransformInfoWrapperPass(std::move(TIRA)); 1396} 1397