1//===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This pass implements the Bottom Up SLP vectorizer. It detects consecutive 10// stores that can be put together into vector-stores. Next, it attempts to 11// construct vectorizable tree using the use-def chains. If a profitable tree 12// was found, the SLP vectorizer performs vectorization on the tree. 13// 14// The pass is inspired by the work described in the paper: 15// "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks. 16// 17//===----------------------------------------------------------------------===// 18 19#include "llvm/Transforms/Vectorize/SLPVectorizer.h" 20#include "llvm/ADT/ArrayRef.h" 21#include "llvm/ADT/DenseMap.h" 22#include "llvm/ADT/DenseSet.h" 23#include "llvm/ADT/MapVector.h" 24#include "llvm/ADT/None.h" 25#include "llvm/ADT/Optional.h" 26#include "llvm/ADT/PostOrderIterator.h" 27#include "llvm/ADT/STLExtras.h" 28#include "llvm/ADT/SetVector.h" 29#include "llvm/ADT/SmallBitVector.h" 30#include "llvm/ADT/SmallPtrSet.h" 31#include "llvm/ADT/SmallSet.h" 32#include "llvm/ADT/SmallVector.h" 33#include "llvm/ADT/Statistic.h" 34#include "llvm/ADT/iterator.h" 35#include "llvm/ADT/iterator_range.h" 36#include "llvm/Analysis/AliasAnalysis.h" 37#include "llvm/Analysis/CodeMetrics.h" 38#include "llvm/Analysis/DemandedBits.h" 39#include "llvm/Analysis/GlobalsModRef.h" 40#include "llvm/Analysis/LoopAccessAnalysis.h" 41#include "llvm/Analysis/LoopInfo.h" 42#include "llvm/Analysis/MemoryLocation.h" 43#include "llvm/Analysis/OptimizationRemarkEmitter.h" 44#include "llvm/Analysis/ScalarEvolution.h" 45#include "llvm/Analysis/ScalarEvolutionExpressions.h" 46#include "llvm/Analysis/TargetLibraryInfo.h" 47#include "llvm/Analysis/TargetTransformInfo.h" 48#include "llvm/Analysis/ValueTracking.h" 49#include "llvm/Analysis/VectorUtils.h" 50#include "llvm/Analysis/AssumptionCache.h" 51#include "llvm/IR/Attributes.h" 52#include "llvm/IR/BasicBlock.h" 53#include "llvm/IR/Constant.h" 54#include "llvm/IR/Constants.h" 55#include "llvm/IR/DataLayout.h" 56#include "llvm/IR/DebugLoc.h" 57#include "llvm/IR/DerivedTypes.h" 58#include "llvm/IR/Dominators.h" 59#include "llvm/IR/Function.h" 60#include "llvm/IR/IRBuilder.h" 61#include "llvm/IR/InstrTypes.h" 62#include "llvm/IR/Instruction.h" 63#include "llvm/IR/Instructions.h" 64#include "llvm/IR/IntrinsicInst.h" 65#include "llvm/IR/Intrinsics.h" 66#include "llvm/IR/Module.h" 67#include "llvm/IR/NoFolder.h" 68#include "llvm/IR/Operator.h" 69#include "llvm/IR/PassManager.h" 70#include "llvm/IR/PatternMatch.h" 71#include "llvm/IR/Type.h" 72#include "llvm/IR/Use.h" 73#include "llvm/IR/User.h" 74#include "llvm/IR/Value.h" 75#include "llvm/IR/ValueHandle.h" 76#include "llvm/IR/Verifier.h" 77#include "llvm/InitializePasses.h" 78#include "llvm/Pass.h" 79#include "llvm/Support/Casting.h" 80#include "llvm/Support/CommandLine.h" 81#include "llvm/Support/Compiler.h" 82#include "llvm/Support/DOTGraphTraits.h" 83#include "llvm/Support/Debug.h" 84#include "llvm/Support/ErrorHandling.h" 85#include "llvm/Support/GraphWriter.h" 86#include "llvm/Support/KnownBits.h" 87#include "llvm/Support/MathExtras.h" 88#include "llvm/Support/raw_ostream.h" 89#include "llvm/Transforms/Utils/InjectTLIMappings.h" 90#include "llvm/Transforms/Utils/LoopUtils.h" 91#include "llvm/Transforms/Vectorize.h" 92#include <algorithm> 93#include <cassert> 94#include <cstdint> 95#include <iterator> 96#include <memory> 97#include <set> 98#include <string> 99#include <tuple> 100#include <utility> 101#include <vector> 102 103using namespace llvm; 104using namespace llvm::PatternMatch; 105using namespace slpvectorizer; 106 107#define SV_NAME "slp-vectorizer" 108#define DEBUG_TYPE "SLP" 109 110STATISTIC(NumVectorInstructions, "Number of vector instructions generated"); 111 112cl::opt<bool> RunSLPVectorization("vectorize-slp", cl::init(true), cl::Hidden, 113 cl::desc("Run the SLP vectorization passes")); 114 115static cl::opt<int> 116 SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden, 117 cl::desc("Only vectorize if you gain more than this " 118 "number ")); 119 120static cl::opt<bool> 121ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden, 122 cl::desc("Attempt to vectorize horizontal reductions")); 123 124static cl::opt<bool> ShouldStartVectorizeHorAtStore( 125 "slp-vectorize-hor-store", cl::init(false), cl::Hidden, 126 cl::desc( 127 "Attempt to vectorize horizontal reductions feeding into a store")); 128 129static cl::opt<int> 130MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden, 131 cl::desc("Attempt to vectorize for this register size in bits")); 132 133static cl::opt<int> 134MaxStoreLookup("slp-max-store-lookup", cl::init(32), cl::Hidden, 135 cl::desc("Maximum depth of the lookup for consecutive stores.")); 136 137/// Limits the size of scheduling regions in a block. 138/// It avoid long compile times for _very_ large blocks where vector 139/// instructions are spread over a wide range. 140/// This limit is way higher than needed by real-world functions. 141static cl::opt<int> 142ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden, 143 cl::desc("Limit the size of the SLP scheduling region per block")); 144 145static cl::opt<int> MinVectorRegSizeOption( 146 "slp-min-reg-size", cl::init(128), cl::Hidden, 147 cl::desc("Attempt to vectorize for this register size in bits")); 148 149static cl::opt<unsigned> RecursionMaxDepth( 150 "slp-recursion-max-depth", cl::init(12), cl::Hidden, 151 cl::desc("Limit the recursion depth when building a vectorizable tree")); 152 153static cl::opt<unsigned> MinTreeSize( 154 "slp-min-tree-size", cl::init(3), cl::Hidden, 155 cl::desc("Only vectorize small trees if they are fully vectorizable")); 156 157// The maximum depth that the look-ahead score heuristic will explore. 158// The higher this value, the higher the compilation time overhead. 159static cl::opt<int> LookAheadMaxDepth( 160 "slp-max-look-ahead-depth", cl::init(2), cl::Hidden, 161 cl::desc("The maximum look-ahead depth for operand reordering scores")); 162 163// The Look-ahead heuristic goes through the users of the bundle to calculate 164// the users cost in getExternalUsesCost(). To avoid compilation time increase 165// we limit the number of users visited to this value. 166static cl::opt<unsigned> LookAheadUsersBudget( 167 "slp-look-ahead-users-budget", cl::init(2), cl::Hidden, 168 cl::desc("The maximum number of users to visit while visiting the " 169 "predecessors. This prevents compilation time increase.")); 170 171static cl::opt<bool> 172 ViewSLPTree("view-slp-tree", cl::Hidden, 173 cl::desc("Display the SLP trees with Graphviz")); 174 175// Limit the number of alias checks. The limit is chosen so that 176// it has no negative effect on the llvm benchmarks. 177static const unsigned AliasedCheckLimit = 10; 178 179// Another limit for the alias checks: The maximum distance between load/store 180// instructions where alias checks are done. 181// This limit is useful for very large basic blocks. 182static const unsigned MaxMemDepDistance = 160; 183 184/// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling 185/// regions to be handled. 186static const int MinScheduleRegionSize = 16; 187 188/// Predicate for the element types that the SLP vectorizer supports. 189/// 190/// The most important thing to filter here are types which are invalid in LLVM 191/// vectors. We also filter target specific types which have absolutely no 192/// meaningful vectorization path such as x86_fp80 and ppc_f128. This just 193/// avoids spending time checking the cost model and realizing that they will 194/// be inevitably scalarized. 195static bool isValidElementType(Type *Ty) { 196 return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() && 197 !Ty->isPPC_FP128Ty(); 198} 199 200/// \returns true if all of the instructions in \p VL are in the same block or 201/// false otherwise. 202static bool allSameBlock(ArrayRef<Value *> VL) { 203 Instruction *I0 = dyn_cast<Instruction>(VL[0]); 204 if (!I0) 205 return false; 206 BasicBlock *BB = I0->getParent(); 207 for (int i = 1, e = VL.size(); i < e; i++) { 208 Instruction *I = dyn_cast<Instruction>(VL[i]); 209 if (!I) 210 return false; 211 212 if (BB != I->getParent()) 213 return false; 214 } 215 return true; 216} 217 218/// \returns True if all of the values in \p VL are constants (but not 219/// globals/constant expressions). 220static bool allConstant(ArrayRef<Value *> VL) { 221 // Constant expressions and globals can't be vectorized like normal integer/FP 222 // constants. 223 for (Value *i : VL) 224 if (!isa<Constant>(i) || isa<ConstantExpr>(i) || isa<GlobalValue>(i)) 225 return false; 226 return true; 227} 228 229/// \returns True if all of the values in \p VL are identical. 230static bool isSplat(ArrayRef<Value *> VL) { 231 for (unsigned i = 1, e = VL.size(); i < e; ++i) 232 if (VL[i] != VL[0]) 233 return false; 234 return true; 235} 236 237/// \returns True if \p I is commutative, handles CmpInst as well as Instruction. 238static bool isCommutative(Instruction *I) { 239 if (auto *IC = dyn_cast<CmpInst>(I)) 240 return IC->isCommutative(); 241 return I->isCommutative(); 242} 243 244/// Checks if the vector of instructions can be represented as a shuffle, like: 245/// %x0 = extractelement <4 x i8> %x, i32 0 246/// %x3 = extractelement <4 x i8> %x, i32 3 247/// %y1 = extractelement <4 x i8> %y, i32 1 248/// %y2 = extractelement <4 x i8> %y, i32 2 249/// %x0x0 = mul i8 %x0, %x0 250/// %x3x3 = mul i8 %x3, %x3 251/// %y1y1 = mul i8 %y1, %y1 252/// %y2y2 = mul i8 %y2, %y2 253/// %ins1 = insertelement <4 x i8> undef, i8 %x0x0, i32 0 254/// %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1 255/// %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2 256/// %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3 257/// ret <4 x i8> %ins4 258/// can be transformed into: 259/// %1 = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 3, i32 5, 260/// i32 6> 261/// %2 = mul <4 x i8> %1, %1 262/// ret <4 x i8> %2 263/// We convert this initially to something like: 264/// %x0 = extractelement <4 x i8> %x, i32 0 265/// %x3 = extractelement <4 x i8> %x, i32 3 266/// %y1 = extractelement <4 x i8> %y, i32 1 267/// %y2 = extractelement <4 x i8> %y, i32 2 268/// %1 = insertelement <4 x i8> undef, i8 %x0, i32 0 269/// %2 = insertelement <4 x i8> %1, i8 %x3, i32 1 270/// %3 = insertelement <4 x i8> %2, i8 %y1, i32 2 271/// %4 = insertelement <4 x i8> %3, i8 %y2, i32 3 272/// %5 = mul <4 x i8> %4, %4 273/// %6 = extractelement <4 x i8> %5, i32 0 274/// %ins1 = insertelement <4 x i8> undef, i8 %6, i32 0 275/// %7 = extractelement <4 x i8> %5, i32 1 276/// %ins2 = insertelement <4 x i8> %ins1, i8 %7, i32 1 277/// %8 = extractelement <4 x i8> %5, i32 2 278/// %ins3 = insertelement <4 x i8> %ins2, i8 %8, i32 2 279/// %9 = extractelement <4 x i8> %5, i32 3 280/// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3 281/// ret <4 x i8> %ins4 282/// InstCombiner transforms this into a shuffle and vector mul 283/// TODO: Can we split off and reuse the shuffle mask detection from 284/// TargetTransformInfo::getInstructionThroughput? 285static Optional<TargetTransformInfo::ShuffleKind> 286isShuffle(ArrayRef<Value *> VL) { 287 auto *EI0 = cast<ExtractElementInst>(VL[0]); 288 unsigned Size = EI0->getVectorOperandType()->getNumElements(); 289 Value *Vec1 = nullptr; 290 Value *Vec2 = nullptr; 291 enum ShuffleMode { Unknown, Select, Permute }; 292 ShuffleMode CommonShuffleMode = Unknown; 293 for (unsigned I = 0, E = VL.size(); I < E; ++I) { 294 auto *EI = cast<ExtractElementInst>(VL[I]); 295 auto *Vec = EI->getVectorOperand(); 296 // All vector operands must have the same number of vector elements. 297 if (cast<VectorType>(Vec->getType())->getNumElements() != Size) 298 return None; 299 auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand()); 300 if (!Idx) 301 return None; 302 // Undefined behavior if Idx is negative or >= Size. 303 if (Idx->getValue().uge(Size)) 304 continue; 305 unsigned IntIdx = Idx->getValue().getZExtValue(); 306 // We can extractelement from undef vector. 307 if (isa<UndefValue>(Vec)) 308 continue; 309 // For correct shuffling we have to have at most 2 different vector operands 310 // in all extractelement instructions. 311 if (!Vec1 || Vec1 == Vec) 312 Vec1 = Vec; 313 else if (!Vec2 || Vec2 == Vec) 314 Vec2 = Vec; 315 else 316 return None; 317 if (CommonShuffleMode == Permute) 318 continue; 319 // If the extract index is not the same as the operation number, it is a 320 // permutation. 321 if (IntIdx != I) { 322 CommonShuffleMode = Permute; 323 continue; 324 } 325 CommonShuffleMode = Select; 326 } 327 // If we're not crossing lanes in different vectors, consider it as blending. 328 if (CommonShuffleMode == Select && Vec2) 329 return TargetTransformInfo::SK_Select; 330 // If Vec2 was never used, we have a permutation of a single vector, otherwise 331 // we have permutation of 2 vectors. 332 return Vec2 ? TargetTransformInfo::SK_PermuteTwoSrc 333 : TargetTransformInfo::SK_PermuteSingleSrc; 334} 335 336namespace { 337 338/// Main data required for vectorization of instructions. 339struct InstructionsState { 340 /// The very first instruction in the list with the main opcode. 341 Value *OpValue = nullptr; 342 343 /// The main/alternate instruction. 344 Instruction *MainOp = nullptr; 345 Instruction *AltOp = nullptr; 346 347 /// The main/alternate opcodes for the list of instructions. 348 unsigned getOpcode() const { 349 return MainOp ? MainOp->getOpcode() : 0; 350 } 351 352 unsigned getAltOpcode() const { 353 return AltOp ? AltOp->getOpcode() : 0; 354 } 355 356 /// Some of the instructions in the list have alternate opcodes. 357 bool isAltShuffle() const { return getOpcode() != getAltOpcode(); } 358 359 bool isOpcodeOrAlt(Instruction *I) const { 360 unsigned CheckedOpcode = I->getOpcode(); 361 return getOpcode() == CheckedOpcode || getAltOpcode() == CheckedOpcode; 362 } 363 364 InstructionsState() = delete; 365 InstructionsState(Value *OpValue, Instruction *MainOp, Instruction *AltOp) 366 : OpValue(OpValue), MainOp(MainOp), AltOp(AltOp) {} 367}; 368 369} // end anonymous namespace 370 371/// Chooses the correct key for scheduling data. If \p Op has the same (or 372/// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p 373/// OpValue. 374static Value *isOneOf(const InstructionsState &S, Value *Op) { 375 auto *I = dyn_cast<Instruction>(Op); 376 if (I && S.isOpcodeOrAlt(I)) 377 return Op; 378 return S.OpValue; 379} 380 381/// \returns true if \p Opcode is allowed as part of of the main/alternate 382/// instruction for SLP vectorization. 383/// 384/// Example of unsupported opcode is SDIV that can potentially cause UB if the 385/// "shuffled out" lane would result in division by zero. 386static bool isValidForAlternation(unsigned Opcode) { 387 if (Instruction::isIntDivRem(Opcode)) 388 return false; 389 390 return true; 391} 392 393/// \returns analysis of the Instructions in \p VL described in 394/// InstructionsState, the Opcode that we suppose the whole list 395/// could be vectorized even if its structure is diverse. 396static InstructionsState getSameOpcode(ArrayRef<Value *> VL, 397 unsigned BaseIndex = 0) { 398 // Make sure these are all Instructions. 399 if (llvm::any_of(VL, [](Value *V) { return !isa<Instruction>(V); })) 400 return InstructionsState(VL[BaseIndex], nullptr, nullptr); 401 402 bool IsCastOp = isa<CastInst>(VL[BaseIndex]); 403 bool IsBinOp = isa<BinaryOperator>(VL[BaseIndex]); 404 unsigned Opcode = cast<Instruction>(VL[BaseIndex])->getOpcode(); 405 unsigned AltOpcode = Opcode; 406 unsigned AltIndex = BaseIndex; 407 408 // Check for one alternate opcode from another BinaryOperator. 409 // TODO - generalize to support all operators (types, calls etc.). 410 for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) { 411 unsigned InstOpcode = cast<Instruction>(VL[Cnt])->getOpcode(); 412 if (IsBinOp && isa<BinaryOperator>(VL[Cnt])) { 413 if (InstOpcode == Opcode || InstOpcode == AltOpcode) 414 continue; 415 if (Opcode == AltOpcode && isValidForAlternation(InstOpcode) && 416 isValidForAlternation(Opcode)) { 417 AltOpcode = InstOpcode; 418 AltIndex = Cnt; 419 continue; 420 } 421 } else if (IsCastOp && isa<CastInst>(VL[Cnt])) { 422 Type *Ty0 = cast<Instruction>(VL[BaseIndex])->getOperand(0)->getType(); 423 Type *Ty1 = cast<Instruction>(VL[Cnt])->getOperand(0)->getType(); 424 if (Ty0 == Ty1) { 425 if (InstOpcode == Opcode || InstOpcode == AltOpcode) 426 continue; 427 if (Opcode == AltOpcode) { 428 assert(isValidForAlternation(Opcode) && 429 isValidForAlternation(InstOpcode) && 430 "Cast isn't safe for alternation, logic needs to be updated!"); 431 AltOpcode = InstOpcode; 432 AltIndex = Cnt; 433 continue; 434 } 435 } 436 } else if (InstOpcode == Opcode || InstOpcode == AltOpcode) 437 continue; 438 return InstructionsState(VL[BaseIndex], nullptr, nullptr); 439 } 440 441 return InstructionsState(VL[BaseIndex], cast<Instruction>(VL[BaseIndex]), 442 cast<Instruction>(VL[AltIndex])); 443} 444 445/// \returns true if all of the values in \p VL have the same type or false 446/// otherwise. 447static bool allSameType(ArrayRef<Value *> VL) { 448 Type *Ty = VL[0]->getType(); 449 for (int i = 1, e = VL.size(); i < e; i++) 450 if (VL[i]->getType() != Ty) 451 return false; 452 453 return true; 454} 455 456/// \returns True if Extract{Value,Element} instruction extracts element Idx. 457static Optional<unsigned> getExtractIndex(Instruction *E) { 458 unsigned Opcode = E->getOpcode(); 459 assert((Opcode == Instruction::ExtractElement || 460 Opcode == Instruction::ExtractValue) && 461 "Expected extractelement or extractvalue instruction."); 462 if (Opcode == Instruction::ExtractElement) { 463 auto *CI = dyn_cast<ConstantInt>(E->getOperand(1)); 464 if (!CI) 465 return None; 466 return CI->getZExtValue(); 467 } 468 ExtractValueInst *EI = cast<ExtractValueInst>(E); 469 if (EI->getNumIndices() != 1) 470 return None; 471 return *EI->idx_begin(); 472} 473 474/// \returns True if in-tree use also needs extract. This refers to 475/// possible scalar operand in vectorized instruction. 476static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst, 477 TargetLibraryInfo *TLI) { 478 unsigned Opcode = UserInst->getOpcode(); 479 switch (Opcode) { 480 case Instruction::Load: { 481 LoadInst *LI = cast<LoadInst>(UserInst); 482 return (LI->getPointerOperand() == Scalar); 483 } 484 case Instruction::Store: { 485 StoreInst *SI = cast<StoreInst>(UserInst); 486 return (SI->getPointerOperand() == Scalar); 487 } 488 case Instruction::Call: { 489 CallInst *CI = cast<CallInst>(UserInst); 490 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI); 491 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) { 492 if (hasVectorInstrinsicScalarOpd(ID, i)) 493 return (CI->getArgOperand(i) == Scalar); 494 } 495 LLVM_FALLTHROUGH; 496 } 497 default: 498 return false; 499 } 500} 501 502/// \returns the AA location that is being access by the instruction. 503static MemoryLocation getLocation(Instruction *I, AliasAnalysis *AA) { 504 if (StoreInst *SI = dyn_cast<StoreInst>(I)) 505 return MemoryLocation::get(SI); 506 if (LoadInst *LI = dyn_cast<LoadInst>(I)) 507 return MemoryLocation::get(LI); 508 return MemoryLocation(); 509} 510 511/// \returns True if the instruction is not a volatile or atomic load/store. 512static bool isSimple(Instruction *I) { 513 if (LoadInst *LI = dyn_cast<LoadInst>(I)) 514 return LI->isSimple(); 515 if (StoreInst *SI = dyn_cast<StoreInst>(I)) 516 return SI->isSimple(); 517 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I)) 518 return !MI->isVolatile(); 519 return true; 520} 521 522namespace llvm { 523 524namespace slpvectorizer { 525 526/// Bottom Up SLP Vectorizer. 527class BoUpSLP { 528 struct TreeEntry; 529 struct ScheduleData; 530 531public: 532 using ValueList = SmallVector<Value *, 8>; 533 using InstrList = SmallVector<Instruction *, 16>; 534 using ValueSet = SmallPtrSet<Value *, 16>; 535 using StoreList = SmallVector<StoreInst *, 8>; 536 using ExtraValueToDebugLocsMap = 537 MapVector<Value *, SmallVector<Instruction *, 2>>; 538 539 BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti, 540 TargetLibraryInfo *TLi, AliasAnalysis *Aa, LoopInfo *Li, 541 DominatorTree *Dt, AssumptionCache *AC, DemandedBits *DB, 542 const DataLayout *DL, OptimizationRemarkEmitter *ORE) 543 : F(Func), SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC), 544 DB(DB), DL(DL), ORE(ORE), Builder(Se->getContext()) { 545 CodeMetrics::collectEphemeralValues(F, AC, EphValues); 546 // Use the vector register size specified by the target unless overridden 547 // by a command-line option. 548 // TODO: It would be better to limit the vectorization factor based on 549 // data type rather than just register size. For example, x86 AVX has 550 // 256-bit registers, but it does not support integer operations 551 // at that width (that requires AVX2). 552 if (MaxVectorRegSizeOption.getNumOccurrences()) 553 MaxVecRegSize = MaxVectorRegSizeOption; 554 else 555 MaxVecRegSize = TTI->getRegisterBitWidth(true); 556 557 if (MinVectorRegSizeOption.getNumOccurrences()) 558 MinVecRegSize = MinVectorRegSizeOption; 559 else 560 MinVecRegSize = TTI->getMinVectorRegisterBitWidth(); 561 } 562 563 /// Vectorize the tree that starts with the elements in \p VL. 564 /// Returns the vectorized root. 565 Value *vectorizeTree(); 566 567 /// Vectorize the tree but with the list of externally used values \p 568 /// ExternallyUsedValues. Values in this MapVector can be replaced but the 569 /// generated extractvalue instructions. 570 Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues); 571 572 /// \returns the cost incurred by unwanted spills and fills, caused by 573 /// holding live values over call sites. 574 int getSpillCost() const; 575 576 /// \returns the vectorization cost of the subtree that starts at \p VL. 577 /// A negative number means that this is profitable. 578 int getTreeCost(); 579 580 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for 581 /// the purpose of scheduling and extraction in the \p UserIgnoreLst. 582 void buildTree(ArrayRef<Value *> Roots, 583 ArrayRef<Value *> UserIgnoreLst = None); 584 585 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for 586 /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking 587 /// into account (and updating it, if required) list of externally used 588 /// values stored in \p ExternallyUsedValues. 589 void buildTree(ArrayRef<Value *> Roots, 590 ExtraValueToDebugLocsMap &ExternallyUsedValues, 591 ArrayRef<Value *> UserIgnoreLst = None); 592 593 /// Clear the internal data structures that are created by 'buildTree'. 594 void deleteTree() { 595 VectorizableTree.clear(); 596 ScalarToTreeEntry.clear(); 597 MustGather.clear(); 598 ExternalUses.clear(); 599 NumOpsWantToKeepOrder.clear(); 600 NumOpsWantToKeepOriginalOrder = 0; 601 for (auto &Iter : BlocksSchedules) { 602 BlockScheduling *BS = Iter.second.get(); 603 BS->clear(); 604 } 605 MinBWs.clear(); 606 } 607 608 unsigned getTreeSize() const { return VectorizableTree.size(); } 609 610 /// Perform LICM and CSE on the newly generated gather sequences. 611 void optimizeGatherSequence(); 612 613 /// \returns The best order of instructions for vectorization. 614 Optional<ArrayRef<unsigned>> bestOrder() const { 615 auto I = std::max_element( 616 NumOpsWantToKeepOrder.begin(), NumOpsWantToKeepOrder.end(), 617 [](const decltype(NumOpsWantToKeepOrder)::value_type &D1, 618 const decltype(NumOpsWantToKeepOrder)::value_type &D2) { 619 return D1.second < D2.second; 620 }); 621 if (I == NumOpsWantToKeepOrder.end() || 622 I->getSecond() <= NumOpsWantToKeepOriginalOrder) 623 return None; 624 625 return makeArrayRef(I->getFirst()); 626 } 627 628 /// \return The vector element size in bits to use when vectorizing the 629 /// expression tree ending at \p V. If V is a store, the size is the width of 630 /// the stored value. Otherwise, the size is the width of the largest loaded 631 /// value reaching V. This method is used by the vectorizer to calculate 632 /// vectorization factors. 633 unsigned getVectorElementSize(Value *V); 634 635 /// Compute the minimum type sizes required to represent the entries in a 636 /// vectorizable tree. 637 void computeMinimumValueSizes(); 638 639 // \returns maximum vector register size as set by TTI or overridden by cl::opt. 640 unsigned getMaxVecRegSize() const { 641 return MaxVecRegSize; 642 } 643 644 // \returns minimum vector register size as set by cl::opt. 645 unsigned getMinVecRegSize() const { 646 return MinVecRegSize; 647 } 648 649 /// Check if homogeneous aggregate is isomorphic to some VectorType. 650 /// Accepts homogeneous multidimensional aggregate of scalars/vectors like 651 /// {[4 x i16], [4 x i16]}, { <2 x float>, <2 x float> }, 652 /// {{{i16, i16}, {i16, i16}}, {{i16, i16}, {i16, i16}}} and so on. 653 /// 654 /// \returns number of elements in vector if isomorphism exists, 0 otherwise. 655 unsigned canMapToVector(Type *T, const DataLayout &DL) const; 656 657 /// \returns True if the VectorizableTree is both tiny and not fully 658 /// vectorizable. We do not vectorize such trees. 659 bool isTreeTinyAndNotFullyVectorizable() const; 660 661 /// Assume that a legal-sized 'or'-reduction of shifted/zexted loaded values 662 /// can be load combined in the backend. Load combining may not be allowed in 663 /// the IR optimizer, so we do not want to alter the pattern. For example, 664 /// partially transforming a scalar bswap() pattern into vector code is 665 /// effectively impossible for the backend to undo. 666 /// TODO: If load combining is allowed in the IR optimizer, this analysis 667 /// may not be necessary. 668 bool isLoadCombineReductionCandidate(unsigned ReductionOpcode) const; 669 670 /// Assume that a vector of stores of bitwise-or/shifted/zexted loaded values 671 /// can be load combined in the backend. Load combining may not be allowed in 672 /// the IR optimizer, so we do not want to alter the pattern. For example, 673 /// partially transforming a scalar bswap() pattern into vector code is 674 /// effectively impossible for the backend to undo. 675 /// TODO: If load combining is allowed in the IR optimizer, this analysis 676 /// may not be necessary. 677 bool isLoadCombineCandidate() const; 678 679 OptimizationRemarkEmitter *getORE() { return ORE; } 680 681 /// This structure holds any data we need about the edges being traversed 682 /// during buildTree_rec(). We keep track of: 683 /// (i) the user TreeEntry index, and 684 /// (ii) the index of the edge. 685 struct EdgeInfo { 686 EdgeInfo() = default; 687 EdgeInfo(TreeEntry *UserTE, unsigned EdgeIdx) 688 : UserTE(UserTE), EdgeIdx(EdgeIdx) {} 689 /// The user TreeEntry. 690 TreeEntry *UserTE = nullptr; 691 /// The operand index of the use. 692 unsigned EdgeIdx = UINT_MAX; 693#ifndef NDEBUG 694 friend inline raw_ostream &operator<<(raw_ostream &OS, 695 const BoUpSLP::EdgeInfo &EI) { 696 EI.dump(OS); 697 return OS; 698 } 699 /// Debug print. 700 void dump(raw_ostream &OS) const { 701 OS << "{User:" << (UserTE ? std::to_string(UserTE->Idx) : "null") 702 << " EdgeIdx:" << EdgeIdx << "}"; 703 } 704 LLVM_DUMP_METHOD void dump() const { dump(dbgs()); } 705#endif 706 }; 707 708 /// A helper data structure to hold the operands of a vector of instructions. 709 /// This supports a fixed vector length for all operand vectors. 710 class VLOperands { 711 /// For each operand we need (i) the value, and (ii) the opcode that it 712 /// would be attached to if the expression was in a left-linearized form. 713 /// This is required to avoid illegal operand reordering. 714 /// For example: 715 /// \verbatim 716 /// 0 Op1 717 /// |/ 718 /// Op1 Op2 Linearized + Op2 719 /// \ / ----------> |/ 720 /// - - 721 /// 722 /// Op1 - Op2 (0 + Op1) - Op2 723 /// \endverbatim 724 /// 725 /// Value Op1 is attached to a '+' operation, and Op2 to a '-'. 726 /// 727 /// Another way to think of this is to track all the operations across the 728 /// path from the operand all the way to the root of the tree and to 729 /// calculate the operation that corresponds to this path. For example, the 730 /// path from Op2 to the root crosses the RHS of the '-', therefore the 731 /// corresponding operation is a '-' (which matches the one in the 732 /// linearized tree, as shown above). 733 /// 734 /// For lack of a better term, we refer to this operation as Accumulated 735 /// Path Operation (APO). 736 struct OperandData { 737 OperandData() = default; 738 OperandData(Value *V, bool APO, bool IsUsed) 739 : V(V), APO(APO), IsUsed(IsUsed) {} 740 /// The operand value. 741 Value *V = nullptr; 742 /// TreeEntries only allow a single opcode, or an alternate sequence of 743 /// them (e.g, +, -). Therefore, we can safely use a boolean value for the 744 /// APO. It is set to 'true' if 'V' is attached to an inverse operation 745 /// in the left-linearized form (e.g., Sub/Div), and 'false' otherwise 746 /// (e.g., Add/Mul) 747 bool APO = false; 748 /// Helper data for the reordering function. 749 bool IsUsed = false; 750 }; 751 752 /// During operand reordering, we are trying to select the operand at lane 753 /// that matches best with the operand at the neighboring lane. Our 754 /// selection is based on the type of value we are looking for. For example, 755 /// if the neighboring lane has a load, we need to look for a load that is 756 /// accessing a consecutive address. These strategies are summarized in the 757 /// 'ReorderingMode' enumerator. 758 enum class ReorderingMode { 759 Load, ///< Matching loads to consecutive memory addresses 760 Opcode, ///< Matching instructions based on opcode (same or alternate) 761 Constant, ///< Matching constants 762 Splat, ///< Matching the same instruction multiple times (broadcast) 763 Failed, ///< We failed to create a vectorizable group 764 }; 765 766 using OperandDataVec = SmallVector<OperandData, 2>; 767 768 /// A vector of operand vectors. 769 SmallVector<OperandDataVec, 4> OpsVec; 770 771 const DataLayout &DL; 772 ScalarEvolution &SE; 773 const BoUpSLP &R; 774 775 /// \returns the operand data at \p OpIdx and \p Lane. 776 OperandData &getData(unsigned OpIdx, unsigned Lane) { 777 return OpsVec[OpIdx][Lane]; 778 } 779 780 /// \returns the operand data at \p OpIdx and \p Lane. Const version. 781 const OperandData &getData(unsigned OpIdx, unsigned Lane) const { 782 return OpsVec[OpIdx][Lane]; 783 } 784 785 /// Clears the used flag for all entries. 786 void clearUsed() { 787 for (unsigned OpIdx = 0, NumOperands = getNumOperands(); 788 OpIdx != NumOperands; ++OpIdx) 789 for (unsigned Lane = 0, NumLanes = getNumLanes(); Lane != NumLanes; 790 ++Lane) 791 OpsVec[OpIdx][Lane].IsUsed = false; 792 } 793 794 /// Swap the operand at \p OpIdx1 with that one at \p OpIdx2. 795 void swap(unsigned OpIdx1, unsigned OpIdx2, unsigned Lane) { 796 std::swap(OpsVec[OpIdx1][Lane], OpsVec[OpIdx2][Lane]); 797 } 798 799 // The hard-coded scores listed here are not very important. When computing 800 // the scores of matching one sub-tree with another, we are basically 801 // counting the number of values that are matching. So even if all scores 802 // are set to 1, we would still get a decent matching result. 803 // However, sometimes we have to break ties. For example we may have to 804 // choose between matching loads vs matching opcodes. This is what these 805 // scores are helping us with: they provide the order of preference. 806 807 /// Loads from consecutive memory addresses, e.g. load(A[i]), load(A[i+1]). 808 static const int ScoreConsecutiveLoads = 3; 809 /// ExtractElementInst from same vector and consecutive indexes. 810 static const int ScoreConsecutiveExtracts = 3; 811 /// Constants. 812 static const int ScoreConstants = 2; 813 /// Instructions with the same opcode. 814 static const int ScoreSameOpcode = 2; 815 /// Instructions with alt opcodes (e.g, add + sub). 816 static const int ScoreAltOpcodes = 1; 817 /// Identical instructions (a.k.a. splat or broadcast). 818 static const int ScoreSplat = 1; 819 /// Matching with an undef is preferable to failing. 820 static const int ScoreUndef = 1; 821 /// Score for failing to find a decent match. 822 static const int ScoreFail = 0; 823 /// User exteranl to the vectorized code. 824 static const int ExternalUseCost = 1; 825 /// The user is internal but in a different lane. 826 static const int UserInDiffLaneCost = ExternalUseCost; 827 828 /// \returns the score of placing \p V1 and \p V2 in consecutive lanes. 829 static int getShallowScore(Value *V1, Value *V2, const DataLayout &DL, 830 ScalarEvolution &SE) { 831 auto *LI1 = dyn_cast<LoadInst>(V1); 832 auto *LI2 = dyn_cast<LoadInst>(V2); 833 if (LI1 && LI2) 834 return isConsecutiveAccess(LI1, LI2, DL, SE) 835 ? VLOperands::ScoreConsecutiveLoads 836 : VLOperands::ScoreFail; 837 838 auto *C1 = dyn_cast<Constant>(V1); 839 auto *C2 = dyn_cast<Constant>(V2); 840 if (C1 && C2) 841 return VLOperands::ScoreConstants; 842 843 // Extracts from consecutive indexes of the same vector better score as 844 // the extracts could be optimized away. 845 Value *EV; 846 ConstantInt *Ex1Idx, *Ex2Idx; 847 if (match(V1, m_ExtractElt(m_Value(EV), m_ConstantInt(Ex1Idx))) && 848 match(V2, m_ExtractElt(m_Deferred(EV), m_ConstantInt(Ex2Idx))) && 849 Ex1Idx->getZExtValue() + 1 == Ex2Idx->getZExtValue()) 850 return VLOperands::ScoreConsecutiveExtracts; 851 852 auto *I1 = dyn_cast<Instruction>(V1); 853 auto *I2 = dyn_cast<Instruction>(V2); 854 if (I1 && I2) { 855 if (I1 == I2) 856 return VLOperands::ScoreSplat; 857 InstructionsState S = getSameOpcode({I1, I2}); 858 // Note: Only consider instructions with <= 2 operands to avoid 859 // complexity explosion. 860 if (S.getOpcode() && S.MainOp->getNumOperands() <= 2) 861 return S.isAltShuffle() ? VLOperands::ScoreAltOpcodes 862 : VLOperands::ScoreSameOpcode; 863 } 864 865 if (isa<UndefValue>(V2)) 866 return VLOperands::ScoreUndef; 867 868 return VLOperands::ScoreFail; 869 } 870 871 /// Holds the values and their lane that are taking part in the look-ahead 872 /// score calculation. This is used in the external uses cost calculation. 873 SmallDenseMap<Value *, int> InLookAheadValues; 874 875 /// \Returns the additinal cost due to uses of \p LHS and \p RHS that are 876 /// either external to the vectorized code, or require shuffling. 877 int getExternalUsesCost(const std::pair<Value *, int> &LHS, 878 const std::pair<Value *, int> &RHS) { 879 int Cost = 0; 880 std::array<std::pair<Value *, int>, 2> Values = {{LHS, RHS}}; 881 for (int Idx = 0, IdxE = Values.size(); Idx != IdxE; ++Idx) { 882 Value *V = Values[Idx].first; 883 // Calculate the absolute lane, using the minimum relative lane of LHS 884 // and RHS as base and Idx as the offset. 885 int Ln = std::min(LHS.second, RHS.second) + Idx; 886 assert(Ln >= 0 && "Bad lane calculation"); 887 unsigned UsersBudget = LookAheadUsersBudget; 888 for (User *U : V->users()) { 889 if (const TreeEntry *UserTE = R.getTreeEntry(U)) { 890 // The user is in the VectorizableTree. Check if we need to insert. 891 auto It = llvm::find(UserTE->Scalars, U); 892 assert(It != UserTE->Scalars.end() && "U is in UserTE"); 893 int UserLn = std::distance(UserTE->Scalars.begin(), It); 894 assert(UserLn >= 0 && "Bad lane"); 895 if (UserLn != Ln) 896 Cost += UserInDiffLaneCost; 897 } else { 898 // Check if the user is in the look-ahead code. 899 auto It2 = InLookAheadValues.find(U); 900 if (It2 != InLookAheadValues.end()) { 901 // The user is in the look-ahead code. Check the lane. 902 if (It2->second != Ln) 903 Cost += UserInDiffLaneCost; 904 } else { 905 // The user is neither in SLP tree nor in the look-ahead code. 906 Cost += ExternalUseCost; 907 } 908 } 909 // Limit the number of visited uses to cap compilation time. 910 if (--UsersBudget == 0) 911 break; 912 } 913 } 914 return Cost; 915 } 916 917 /// Go through the operands of \p LHS and \p RHS recursively until \p 918 /// MaxLevel, and return the cummulative score. For example: 919 /// \verbatim 920 /// A[0] B[0] A[1] B[1] C[0] D[0] B[1] A[1] 921 /// \ / \ / \ / \ / 922 /// + + + + 923 /// G1 G2 G3 G4 924 /// \endverbatim 925 /// The getScoreAtLevelRec(G1, G2) function will try to match the nodes at 926 /// each level recursively, accumulating the score. It starts from matching 927 /// the additions at level 0, then moves on to the loads (level 1). The 928 /// score of G1 and G2 is higher than G1 and G3, because {A[0],A[1]} and 929 /// {B[0],B[1]} match with VLOperands::ScoreConsecutiveLoads, while 930 /// {A[0],C[0]} has a score of VLOperands::ScoreFail. 931 /// Please note that the order of the operands does not matter, as we 932 /// evaluate the score of all profitable combinations of operands. In 933 /// other words the score of G1 and G4 is the same as G1 and G2. This 934 /// heuristic is based on ideas described in: 935 /// Look-ahead SLP: Auto-vectorization in the presence of commutative 936 /// operations, CGO 2018 by Vasileios Porpodas, Rodrigo C. O. Rocha, 937 /// Lu��s F. W. G��es 938 int getScoreAtLevelRec(const std::pair<Value *, int> &LHS, 939 const std::pair<Value *, int> &RHS, int CurrLevel, 940 int MaxLevel) { 941 942 Value *V1 = LHS.first; 943 Value *V2 = RHS.first; 944 // Get the shallow score of V1 and V2. 945 int ShallowScoreAtThisLevel = 946 std::max((int)ScoreFail, getShallowScore(V1, V2, DL, SE) - 947 getExternalUsesCost(LHS, RHS)); 948 int Lane1 = LHS.second; 949 int Lane2 = RHS.second; 950 951 // If reached MaxLevel, 952 // or if V1 and V2 are not instructions, 953 // or if they are SPLAT, 954 // or if they are not consecutive, early return the current cost. 955 auto *I1 = dyn_cast<Instruction>(V1); 956 auto *I2 = dyn_cast<Instruction>(V2); 957 if (CurrLevel == MaxLevel || !(I1 && I2) || I1 == I2 || 958 ShallowScoreAtThisLevel == VLOperands::ScoreFail || 959 (isa<LoadInst>(I1) && isa<LoadInst>(I2) && ShallowScoreAtThisLevel)) 960 return ShallowScoreAtThisLevel; 961 assert(I1 && I2 && "Should have early exited."); 962 963 // Keep track of in-tree values for determining the external-use cost. 964 InLookAheadValues[V1] = Lane1; 965 InLookAheadValues[V2] = Lane2; 966 967 // Contains the I2 operand indexes that got matched with I1 operands. 968 SmallSet<unsigned, 4> Op2Used; 969 970 // Recursion towards the operands of I1 and I2. We are trying all possbile 971 // operand pairs, and keeping track of the best score. 972 for (unsigned OpIdx1 = 0, NumOperands1 = I1->getNumOperands(); 973 OpIdx1 != NumOperands1; ++OpIdx1) { 974 // Try to pair op1I with the best operand of I2. 975 int MaxTmpScore = 0; 976 unsigned MaxOpIdx2 = 0; 977 bool FoundBest = false; 978 // If I2 is commutative try all combinations. 979 unsigned FromIdx = isCommutative(I2) ? 0 : OpIdx1; 980 unsigned ToIdx = isCommutative(I2) 981 ? I2->getNumOperands() 982 : std::min(I2->getNumOperands(), OpIdx1 + 1); 983 assert(FromIdx <= ToIdx && "Bad index"); 984 for (unsigned OpIdx2 = FromIdx; OpIdx2 != ToIdx; ++OpIdx2) { 985 // Skip operands already paired with OpIdx1. 986 if (Op2Used.count(OpIdx2)) 987 continue; 988 // Recursively calculate the cost at each level 989 int TmpScore = getScoreAtLevelRec({I1->getOperand(OpIdx1), Lane1}, 990 {I2->getOperand(OpIdx2), Lane2}, 991 CurrLevel + 1, MaxLevel); 992 // Look for the best score. 993 if (TmpScore > VLOperands::ScoreFail && TmpScore > MaxTmpScore) { 994 MaxTmpScore = TmpScore; 995 MaxOpIdx2 = OpIdx2; 996 FoundBest = true; 997 } 998 } 999 if (FoundBest) { 1000 // Pair {OpIdx1, MaxOpIdx2} was found to be best. Never revisit it. 1001 Op2Used.insert(MaxOpIdx2); 1002 ShallowScoreAtThisLevel += MaxTmpScore; 1003 } 1004 } 1005 return ShallowScoreAtThisLevel; 1006 } 1007 1008 /// \Returns the look-ahead score, which tells us how much the sub-trees 1009 /// rooted at \p LHS and \p RHS match, the more they match the higher the 1010 /// score. This helps break ties in an informed way when we cannot decide on 1011 /// the order of the operands by just considering the immediate 1012 /// predecessors. 1013 int getLookAheadScore(const std::pair<Value *, int> &LHS, 1014 const std::pair<Value *, int> &RHS) { 1015 InLookAheadValues.clear(); 1016 return getScoreAtLevelRec(LHS, RHS, 1, LookAheadMaxDepth); 1017 } 1018 1019 // Search all operands in Ops[*][Lane] for the one that matches best 1020 // Ops[OpIdx][LastLane] and return its opreand index. 1021 // If no good match can be found, return None. 1022 Optional<unsigned> 1023 getBestOperand(unsigned OpIdx, int Lane, int LastLane, 1024 ArrayRef<ReorderingMode> ReorderingModes) { 1025 unsigned NumOperands = getNumOperands(); 1026 1027 // The operand of the previous lane at OpIdx. 1028 Value *OpLastLane = getData(OpIdx, LastLane).V; 1029 1030 // Our strategy mode for OpIdx. 1031 ReorderingMode RMode = ReorderingModes[OpIdx]; 1032 1033 // The linearized opcode of the operand at OpIdx, Lane. 1034 bool OpIdxAPO = getData(OpIdx, Lane).APO; 1035 1036 // The best operand index and its score. 1037 // Sometimes we have more than one option (e.g., Opcode and Undefs), so we 1038 // are using the score to differentiate between the two. 1039 struct BestOpData { 1040 Optional<unsigned> Idx = None; 1041 unsigned Score = 0; 1042 } BestOp; 1043 1044 // Iterate through all unused operands and look for the best. 1045 for (unsigned Idx = 0; Idx != NumOperands; ++Idx) { 1046 // Get the operand at Idx and Lane. 1047 OperandData &OpData = getData(Idx, Lane); 1048 Value *Op = OpData.V; 1049 bool OpAPO = OpData.APO; 1050 1051 // Skip already selected operands. 1052 if (OpData.IsUsed) 1053 continue; 1054 1055 // Skip if we are trying to move the operand to a position with a 1056 // different opcode in the linearized tree form. This would break the 1057 // semantics. 1058 if (OpAPO != OpIdxAPO) 1059 continue; 1060 1061 // Look for an operand that matches the current mode. 1062 switch (RMode) { 1063 case ReorderingMode::Load: 1064 case ReorderingMode::Constant: 1065 case ReorderingMode::Opcode: { 1066 bool LeftToRight = Lane > LastLane; 1067 Value *OpLeft = (LeftToRight) ? OpLastLane : Op; 1068 Value *OpRight = (LeftToRight) ? Op : OpLastLane; 1069 unsigned Score = 1070 getLookAheadScore({OpLeft, LastLane}, {OpRight, Lane}); 1071 if (Score > BestOp.Score) { 1072 BestOp.Idx = Idx; 1073 BestOp.Score = Score; 1074 } 1075 break; 1076 } 1077 case ReorderingMode::Splat: 1078 if (Op == OpLastLane) 1079 BestOp.Idx = Idx; 1080 break; 1081 case ReorderingMode::Failed: 1082 return None; 1083 } 1084 } 1085 1086 if (BestOp.Idx) { 1087 getData(BestOp.Idx.getValue(), Lane).IsUsed = true; 1088 return BestOp.Idx; 1089 } 1090 // If we could not find a good match return None. 1091 return None; 1092 } 1093 1094 /// Helper for reorderOperandVecs. \Returns the lane that we should start 1095 /// reordering from. This is the one which has the least number of operands 1096 /// that can freely move about. 1097 unsigned getBestLaneToStartReordering() const { 1098 unsigned BestLane = 0; 1099 unsigned Min = UINT_MAX; 1100 for (unsigned Lane = 0, NumLanes = getNumLanes(); Lane != NumLanes; 1101 ++Lane) { 1102 unsigned NumFreeOps = getMaxNumOperandsThatCanBeReordered(Lane); 1103 if (NumFreeOps < Min) { 1104 Min = NumFreeOps; 1105 BestLane = Lane; 1106 } 1107 } 1108 return BestLane; 1109 } 1110 1111 /// \Returns the maximum number of operands that are allowed to be reordered 1112 /// for \p Lane. This is used as a heuristic for selecting the first lane to 1113 /// start operand reordering. 1114 unsigned getMaxNumOperandsThatCanBeReordered(unsigned Lane) const { 1115 unsigned CntTrue = 0; 1116 unsigned NumOperands = getNumOperands(); 1117 // Operands with the same APO can be reordered. We therefore need to count 1118 // how many of them we have for each APO, like this: Cnt[APO] = x. 1119 // Since we only have two APOs, namely true and false, we can avoid using 1120 // a map. Instead we can simply count the number of operands that 1121 // correspond to one of them (in this case the 'true' APO), and calculate 1122 // the other by subtracting it from the total number of operands. 1123 for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) 1124 if (getData(OpIdx, Lane).APO) 1125 ++CntTrue; 1126 unsigned CntFalse = NumOperands - CntTrue; 1127 return std::max(CntTrue, CntFalse); 1128 } 1129 1130 /// Go through the instructions in VL and append their operands. 1131 void appendOperandsOfVL(ArrayRef<Value *> VL) { 1132 assert(!VL.empty() && "Bad VL"); 1133 assert((empty() || VL.size() == getNumLanes()) && 1134 "Expected same number of lanes"); 1135 assert(isa<Instruction>(VL[0]) && "Expected instruction"); 1136 unsigned NumOperands = cast<Instruction>(VL[0])->getNumOperands(); 1137 OpsVec.resize(NumOperands); 1138 unsigned NumLanes = VL.size(); 1139 for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) { 1140 OpsVec[OpIdx].resize(NumLanes); 1141 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) { 1142 assert(isa<Instruction>(VL[Lane]) && "Expected instruction"); 1143 // Our tree has just 3 nodes: the root and two operands. 1144 // It is therefore trivial to get the APO. We only need to check the 1145 // opcode of VL[Lane] and whether the operand at OpIdx is the LHS or 1146 // RHS operand. The LHS operand of both add and sub is never attached 1147 // to an inversese operation in the linearized form, therefore its APO 1148 // is false. The RHS is true only if VL[Lane] is an inverse operation. 1149 1150 // Since operand reordering is performed on groups of commutative 1151 // operations or alternating sequences (e.g., +, -), we can safely 1152 // tell the inverse operations by checking commutativity. 1153 bool IsInverseOperation = !isCommutative(cast<Instruction>(VL[Lane])); 1154 bool APO = (OpIdx == 0) ? false : IsInverseOperation; 1155 OpsVec[OpIdx][Lane] = {cast<Instruction>(VL[Lane])->getOperand(OpIdx), 1156 APO, false}; 1157 } 1158 } 1159 } 1160 1161 /// \returns the number of operands. 1162 unsigned getNumOperands() const { return OpsVec.size(); } 1163 1164 /// \returns the number of lanes. 1165 unsigned getNumLanes() const { return OpsVec[0].size(); } 1166 1167 /// \returns the operand value at \p OpIdx and \p Lane. 1168 Value *getValue(unsigned OpIdx, unsigned Lane) const { 1169 return getData(OpIdx, Lane).V; 1170 } 1171 1172 /// \returns true if the data structure is empty. 1173 bool empty() const { return OpsVec.empty(); } 1174 1175 /// Clears the data. 1176 void clear() { OpsVec.clear(); } 1177 1178 /// \Returns true if there are enough operands identical to \p Op to fill 1179 /// the whole vector. 1180 /// Note: This modifies the 'IsUsed' flag, so a cleanUsed() must follow. 1181 bool shouldBroadcast(Value *Op, unsigned OpIdx, unsigned Lane) { 1182 bool OpAPO = getData(OpIdx, Lane).APO; 1183 for (unsigned Ln = 0, Lns = getNumLanes(); Ln != Lns; ++Ln) { 1184 if (Ln == Lane) 1185 continue; 1186 // This is set to true if we found a candidate for broadcast at Lane. 1187 bool FoundCandidate = false; 1188 for (unsigned OpI = 0, OpE = getNumOperands(); OpI != OpE; ++OpI) { 1189 OperandData &Data = getData(OpI, Ln); 1190 if (Data.APO != OpAPO || Data.IsUsed) 1191 continue; 1192 if (Data.V == Op) { 1193 FoundCandidate = true; 1194 Data.IsUsed = true; 1195 break; 1196 } 1197 } 1198 if (!FoundCandidate) 1199 return false; 1200 } 1201 return true; 1202 } 1203 1204 public: 1205 /// Initialize with all the operands of the instruction vector \p RootVL. 1206 VLOperands(ArrayRef<Value *> RootVL, const DataLayout &DL, 1207 ScalarEvolution &SE, const BoUpSLP &R) 1208 : DL(DL), SE(SE), R(R) { 1209 // Append all the operands of RootVL. 1210 appendOperandsOfVL(RootVL); 1211 } 1212 1213 /// \Returns a value vector with the operands across all lanes for the 1214 /// opearnd at \p OpIdx. 1215 ValueList getVL(unsigned OpIdx) const { 1216 ValueList OpVL(OpsVec[OpIdx].size()); 1217 assert(OpsVec[OpIdx].size() == getNumLanes() && 1218 "Expected same num of lanes across all operands"); 1219 for (unsigned Lane = 0, Lanes = getNumLanes(); Lane != Lanes; ++Lane) 1220 OpVL[Lane] = OpsVec[OpIdx][Lane].V; 1221 return OpVL; 1222 } 1223 1224 // Performs operand reordering for 2 or more operands. 1225 // The original operands are in OrigOps[OpIdx][Lane]. 1226 // The reordered operands are returned in 'SortedOps[OpIdx][Lane]'. 1227 void reorder() { 1228 unsigned NumOperands = getNumOperands(); 1229 unsigned NumLanes = getNumLanes(); 1230 // Each operand has its own mode. We are using this mode to help us select 1231 // the instructions for each lane, so that they match best with the ones 1232 // we have selected so far. 1233 SmallVector<ReorderingMode, 2> ReorderingModes(NumOperands); 1234 1235 // This is a greedy single-pass algorithm. We are going over each lane 1236 // once and deciding on the best order right away with no back-tracking. 1237 // However, in order to increase its effectiveness, we start with the lane 1238 // that has operands that can move the least. For example, given the 1239 // following lanes: 1240 // Lane 0 : A[0] = B[0] + C[0] // Visited 3rd 1241 // Lane 1 : A[1] = C[1] - B[1] // Visited 1st 1242 // Lane 2 : A[2] = B[2] + C[2] // Visited 2nd 1243 // Lane 3 : A[3] = C[3] - B[3] // Visited 4th 1244 // we will start at Lane 1, since the operands of the subtraction cannot 1245 // be reordered. Then we will visit the rest of the lanes in a circular 1246 // fashion. That is, Lanes 2, then Lane 0, and finally Lane 3. 1247 1248 // Find the first lane that we will start our search from. 1249 unsigned FirstLane = getBestLaneToStartReordering(); 1250 1251 // Initialize the modes. 1252 for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) { 1253 Value *OpLane0 = getValue(OpIdx, FirstLane); 1254 // Keep track if we have instructions with all the same opcode on one 1255 // side. 1256 if (isa<LoadInst>(OpLane0)) 1257 ReorderingModes[OpIdx] = ReorderingMode::Load; 1258 else if (isa<Instruction>(OpLane0)) { 1259 // Check if OpLane0 should be broadcast. 1260 if (shouldBroadcast(OpLane0, OpIdx, FirstLane)) 1261 ReorderingModes[OpIdx] = ReorderingMode::Splat; 1262 else 1263 ReorderingModes[OpIdx] = ReorderingMode::Opcode; 1264 } 1265 else if (isa<Constant>(OpLane0)) 1266 ReorderingModes[OpIdx] = ReorderingMode::Constant; 1267 else if (isa<Argument>(OpLane0)) 1268 // Our best hope is a Splat. It may save some cost in some cases. 1269 ReorderingModes[OpIdx] = ReorderingMode::Splat; 1270 else 1271 // NOTE: This should be unreachable. 1272 ReorderingModes[OpIdx] = ReorderingMode::Failed; 1273 } 1274 1275 // If the initial strategy fails for any of the operand indexes, then we 1276 // perform reordering again in a second pass. This helps avoid assigning 1277 // high priority to the failed strategy, and should improve reordering for 1278 // the non-failed operand indexes. 1279 for (int Pass = 0; Pass != 2; ++Pass) { 1280 // Skip the second pass if the first pass did not fail. 1281 bool StrategyFailed = false; 1282 // Mark all operand data as free to use. 1283 clearUsed(); 1284 // We keep the original operand order for the FirstLane, so reorder the 1285 // rest of the lanes. We are visiting the nodes in a circular fashion, 1286 // using FirstLane as the center point and increasing the radius 1287 // distance. 1288 for (unsigned Distance = 1; Distance != NumLanes; ++Distance) { 1289 // Visit the lane on the right and then the lane on the left. 1290 for (int Direction : {+1, -1}) { 1291 int Lane = FirstLane + Direction * Distance; 1292 if (Lane < 0 || Lane >= (int)NumLanes) 1293 continue; 1294 int LastLane = Lane - Direction; 1295 assert(LastLane >= 0 && LastLane < (int)NumLanes && 1296 "Out of bounds"); 1297 // Look for a good match for each operand. 1298 for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) { 1299 // Search for the operand that matches SortedOps[OpIdx][Lane-1]. 1300 Optional<unsigned> BestIdx = 1301 getBestOperand(OpIdx, Lane, LastLane, ReorderingModes); 1302 // By not selecting a value, we allow the operands that follow to 1303 // select a better matching value. We will get a non-null value in 1304 // the next run of getBestOperand(). 1305 if (BestIdx) { 1306 // Swap the current operand with the one returned by 1307 // getBestOperand(). 1308 swap(OpIdx, BestIdx.getValue(), Lane); 1309 } else { 1310 // We failed to find a best operand, set mode to 'Failed'. 1311 ReorderingModes[OpIdx] = ReorderingMode::Failed; 1312 // Enable the second pass. 1313 StrategyFailed = true; 1314 } 1315 } 1316 } 1317 } 1318 // Skip second pass if the strategy did not fail. 1319 if (!StrategyFailed) 1320 break; 1321 } 1322 } 1323 1324#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 1325 LLVM_DUMP_METHOD static StringRef getModeStr(ReorderingMode RMode) { 1326 switch (RMode) { 1327 case ReorderingMode::Load: 1328 return "Load"; 1329 case ReorderingMode::Opcode: 1330 return "Opcode"; 1331 case ReorderingMode::Constant: 1332 return "Constant"; 1333 case ReorderingMode::Splat: 1334 return "Splat"; 1335 case ReorderingMode::Failed: 1336 return "Failed"; 1337 } 1338 llvm_unreachable("Unimplemented Reordering Type"); 1339 } 1340 1341 LLVM_DUMP_METHOD static raw_ostream &printMode(ReorderingMode RMode, 1342 raw_ostream &OS) { 1343 return OS << getModeStr(RMode); 1344 } 1345 1346 /// Debug print. 1347 LLVM_DUMP_METHOD static void dumpMode(ReorderingMode RMode) { 1348 printMode(RMode, dbgs()); 1349 } 1350 1351 friend raw_ostream &operator<<(raw_ostream &OS, ReorderingMode RMode) { 1352 return printMode(RMode, OS); 1353 } 1354 1355 LLVM_DUMP_METHOD raw_ostream &print(raw_ostream &OS) const { 1356 const unsigned Indent = 2; 1357 unsigned Cnt = 0; 1358 for (const OperandDataVec &OpDataVec : OpsVec) { 1359 OS << "Operand " << Cnt++ << "\n"; 1360 for (const OperandData &OpData : OpDataVec) { 1361 OS.indent(Indent) << "{"; 1362 if (Value *V = OpData.V) 1363 OS << *V; 1364 else 1365 OS << "null"; 1366 OS << ", APO:" << OpData.APO << "}\n"; 1367 } 1368 OS << "\n"; 1369 } 1370 return OS; 1371 } 1372 1373 /// Debug print. 1374 LLVM_DUMP_METHOD void dump() const { print(dbgs()); } 1375#endif 1376 }; 1377 1378 /// Checks if the instruction is marked for deletion. 1379 bool isDeleted(Instruction *I) const { return DeletedInstructions.count(I); } 1380 1381 /// Marks values operands for later deletion by replacing them with Undefs. 1382 void eraseInstructions(ArrayRef<Value *> AV); 1383 1384 ~BoUpSLP(); 1385 1386private: 1387 /// Checks if all users of \p I are the part of the vectorization tree. 1388 bool areAllUsersVectorized(Instruction *I) const; 1389 1390 /// \returns the cost of the vectorizable entry. 1391 int getEntryCost(TreeEntry *E); 1392 1393 /// This is the recursive part of buildTree. 1394 void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth, 1395 const EdgeInfo &EI); 1396 1397 /// \returns true if the ExtractElement/ExtractValue instructions in \p VL can 1398 /// be vectorized to use the original vector (or aggregate "bitcast" to a 1399 /// vector) and sets \p CurrentOrder to the identity permutation; otherwise 1400 /// returns false, setting \p CurrentOrder to either an empty vector or a 1401 /// non-identity permutation that allows to reuse extract instructions. 1402 bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue, 1403 SmallVectorImpl<unsigned> &CurrentOrder) const; 1404 1405 /// Vectorize a single entry in the tree. 1406 Value *vectorizeTree(TreeEntry *E); 1407 1408 /// Vectorize a single entry in the tree, starting in \p VL. 1409 Value *vectorizeTree(ArrayRef<Value *> VL); 1410 1411 /// \returns the scalarization cost for this type. Scalarization in this 1412 /// context means the creation of vectors from a group of scalars. 1413 int getGatherCost(VectorType *Ty, 1414 const DenseSet<unsigned> &ShuffledIndices) const; 1415 1416 /// \returns the scalarization cost for this list of values. Assuming that 1417 /// this subtree gets vectorized, we may need to extract the values from the 1418 /// roots. This method calculates the cost of extracting the values. 1419 int getGatherCost(ArrayRef<Value *> VL) const; 1420 1421 /// Set the Builder insert point to one after the last instruction in 1422 /// the bundle 1423 void setInsertPointAfterBundle(TreeEntry *E); 1424 1425 /// \returns a vector from a collection of scalars in \p VL. 1426 Value *Gather(ArrayRef<Value *> VL, VectorType *Ty); 1427 1428 /// \returns whether the VectorizableTree is fully vectorizable and will 1429 /// be beneficial even the tree height is tiny. 1430 bool isFullyVectorizableTinyTree() const; 1431 1432 /// Reorder commutative or alt operands to get better probability of 1433 /// generating vectorized code. 1434 static void reorderInputsAccordingToOpcode(ArrayRef<Value *> VL, 1435 SmallVectorImpl<Value *> &Left, 1436 SmallVectorImpl<Value *> &Right, 1437 const DataLayout &DL, 1438 ScalarEvolution &SE, 1439 const BoUpSLP &R); 1440 struct TreeEntry { 1441 using VecTreeTy = SmallVector<std::unique_ptr<TreeEntry>, 8>; 1442 TreeEntry(VecTreeTy &Container) : Container(Container) {} 1443 1444 /// \returns true if the scalars in VL are equal to this entry. 1445 bool isSame(ArrayRef<Value *> VL) const { 1446 if (VL.size() == Scalars.size()) 1447 return std::equal(VL.begin(), VL.end(), Scalars.begin()); 1448 return VL.size() == ReuseShuffleIndices.size() && 1449 std::equal( 1450 VL.begin(), VL.end(), ReuseShuffleIndices.begin(), 1451 [this](Value *V, int Idx) { return V == Scalars[Idx]; }); 1452 } 1453 1454 /// A vector of scalars. 1455 ValueList Scalars; 1456 1457 /// The Scalars are vectorized into this value. It is initialized to Null. 1458 Value *VectorizedValue = nullptr; 1459 1460 /// Do we need to gather this sequence ? 1461 enum EntryState { Vectorize, NeedToGather }; 1462 EntryState State; 1463 1464 /// Does this sequence require some shuffling? 1465 SmallVector<int, 4> ReuseShuffleIndices; 1466 1467 /// Does this entry require reordering? 1468 ArrayRef<unsigned> ReorderIndices; 1469 1470 /// Points back to the VectorizableTree. 1471 /// 1472 /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has 1473 /// to be a pointer and needs to be able to initialize the child iterator. 1474 /// Thus we need a reference back to the container to translate the indices 1475 /// to entries. 1476 VecTreeTy &Container; 1477 1478 /// The TreeEntry index containing the user of this entry. We can actually 1479 /// have multiple users so the data structure is not truly a tree. 1480 SmallVector<EdgeInfo, 1> UserTreeIndices; 1481 1482 /// The index of this treeEntry in VectorizableTree. 1483 int Idx = -1; 1484 1485 private: 1486 /// The operands of each instruction in each lane Operands[op_index][lane]. 1487 /// Note: This helps avoid the replication of the code that performs the 1488 /// reordering of operands during buildTree_rec() and vectorizeTree(). 1489 SmallVector<ValueList, 2> Operands; 1490 1491 /// The main/alternate instruction. 1492 Instruction *MainOp = nullptr; 1493 Instruction *AltOp = nullptr; 1494 1495 public: 1496 /// Set this bundle's \p OpIdx'th operand to \p OpVL. 1497 void setOperand(unsigned OpIdx, ArrayRef<Value *> OpVL) { 1498 if (Operands.size() < OpIdx + 1) 1499 Operands.resize(OpIdx + 1); 1500 assert(Operands[OpIdx].size() == 0 && "Already resized?"); 1501 Operands[OpIdx].resize(Scalars.size()); 1502 for (unsigned Lane = 0, E = Scalars.size(); Lane != E; ++Lane) 1503 Operands[OpIdx][Lane] = OpVL[Lane]; 1504 } 1505 1506 /// Set the operands of this bundle in their original order. 1507 void setOperandsInOrder() { 1508 assert(Operands.empty() && "Already initialized?"); 1509 auto *I0 = cast<Instruction>(Scalars[0]); 1510 Operands.resize(I0->getNumOperands()); 1511 unsigned NumLanes = Scalars.size(); 1512 for (unsigned OpIdx = 0, NumOperands = I0->getNumOperands(); 1513 OpIdx != NumOperands; ++OpIdx) { 1514 Operands[OpIdx].resize(NumLanes); 1515 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) { 1516 auto *I = cast<Instruction>(Scalars[Lane]); 1517 assert(I->getNumOperands() == NumOperands && 1518 "Expected same number of operands"); 1519 Operands[OpIdx][Lane] = I->getOperand(OpIdx); 1520 } 1521 } 1522 } 1523 1524 /// \returns the \p OpIdx operand of this TreeEntry. 1525 ValueList &getOperand(unsigned OpIdx) { 1526 assert(OpIdx < Operands.size() && "Off bounds"); 1527 return Operands[OpIdx]; 1528 } 1529 1530 /// \returns the number of operands. 1531 unsigned getNumOperands() const { return Operands.size(); } 1532 1533 /// \return the single \p OpIdx operand. 1534 Value *getSingleOperand(unsigned OpIdx) const { 1535 assert(OpIdx < Operands.size() && "Off bounds"); 1536 assert(!Operands[OpIdx].empty() && "No operand available"); 1537 return Operands[OpIdx][0]; 1538 } 1539 1540 /// Some of the instructions in the list have alternate opcodes. 1541 bool isAltShuffle() const { 1542 return getOpcode() != getAltOpcode(); 1543 } 1544 1545 bool isOpcodeOrAlt(Instruction *I) const { 1546 unsigned CheckedOpcode = I->getOpcode(); 1547 return (getOpcode() == CheckedOpcode || 1548 getAltOpcode() == CheckedOpcode); 1549 } 1550 1551 /// Chooses the correct key for scheduling data. If \p Op has the same (or 1552 /// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is 1553 /// \p OpValue. 1554 Value *isOneOf(Value *Op) const { 1555 auto *I = dyn_cast<Instruction>(Op); 1556 if (I && isOpcodeOrAlt(I)) 1557 return Op; 1558 return MainOp; 1559 } 1560 1561 void setOperations(const InstructionsState &S) { 1562 MainOp = S.MainOp; 1563 AltOp = S.AltOp; 1564 } 1565 1566 Instruction *getMainOp() const { 1567 return MainOp; 1568 } 1569 1570 Instruction *getAltOp() const { 1571 return AltOp; 1572 } 1573 1574 /// The main/alternate opcodes for the list of instructions. 1575 unsigned getOpcode() const { 1576 return MainOp ? MainOp->getOpcode() : 0; 1577 } 1578 1579 unsigned getAltOpcode() const { 1580 return AltOp ? AltOp->getOpcode() : 0; 1581 } 1582 1583 /// Update operations state of this entry if reorder occurred. 1584 bool updateStateIfReorder() { 1585 if (ReorderIndices.empty()) 1586 return false; 1587 InstructionsState S = getSameOpcode(Scalars, ReorderIndices.front()); 1588 setOperations(S); 1589 return true; 1590 } 1591 1592#ifndef NDEBUG 1593 /// Debug printer. 1594 LLVM_DUMP_METHOD void dump() const { 1595 dbgs() << Idx << ".\n"; 1596 for (unsigned OpI = 0, OpE = Operands.size(); OpI != OpE; ++OpI) { 1597 dbgs() << "Operand " << OpI << ":\n"; 1598 for (const Value *V : Operands[OpI]) 1599 dbgs().indent(2) << *V << "\n"; 1600 } 1601 dbgs() << "Scalars: \n"; 1602 for (Value *V : Scalars) 1603 dbgs().indent(2) << *V << "\n"; 1604 dbgs() << "State: "; 1605 switch (State) { 1606 case Vectorize: 1607 dbgs() << "Vectorize\n"; 1608 break; 1609 case NeedToGather: 1610 dbgs() << "NeedToGather\n"; 1611 break; 1612 } 1613 dbgs() << "MainOp: "; 1614 if (MainOp) 1615 dbgs() << *MainOp << "\n"; 1616 else 1617 dbgs() << "NULL\n"; 1618 dbgs() << "AltOp: "; 1619 if (AltOp) 1620 dbgs() << *AltOp << "\n"; 1621 else 1622 dbgs() << "NULL\n"; 1623 dbgs() << "VectorizedValue: "; 1624 if (VectorizedValue) 1625 dbgs() << *VectorizedValue << "\n"; 1626 else 1627 dbgs() << "NULL\n"; 1628 dbgs() << "ReuseShuffleIndices: "; 1629 if (ReuseShuffleIndices.empty()) 1630 dbgs() << "Emtpy"; 1631 else 1632 for (unsigned ReuseIdx : ReuseShuffleIndices) 1633 dbgs() << ReuseIdx << ", "; 1634 dbgs() << "\n"; 1635 dbgs() << "ReorderIndices: "; 1636 for (unsigned ReorderIdx : ReorderIndices) 1637 dbgs() << ReorderIdx << ", "; 1638 dbgs() << "\n"; 1639 dbgs() << "UserTreeIndices: "; 1640 for (const auto &EInfo : UserTreeIndices) 1641 dbgs() << EInfo << ", "; 1642 dbgs() << "\n"; 1643 } 1644#endif 1645 }; 1646 1647 /// Create a new VectorizableTree entry. 1648 TreeEntry *newTreeEntry(ArrayRef<Value *> VL, Optional<ScheduleData *> Bundle, 1649 const InstructionsState &S, 1650 const EdgeInfo &UserTreeIdx, 1651 ArrayRef<unsigned> ReuseShuffleIndices = None, 1652 ArrayRef<unsigned> ReorderIndices = None) { 1653 bool Vectorized = (bool)Bundle; 1654 VectorizableTree.push_back(std::make_unique<TreeEntry>(VectorizableTree)); 1655 TreeEntry *Last = VectorizableTree.back().get(); 1656 Last->Idx = VectorizableTree.size() - 1; 1657 Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end()); 1658 Last->State = Vectorized ? TreeEntry::Vectorize : TreeEntry::NeedToGather; 1659 Last->ReuseShuffleIndices.append(ReuseShuffleIndices.begin(), 1660 ReuseShuffleIndices.end()); 1661 Last->ReorderIndices = ReorderIndices; 1662 Last->setOperations(S); 1663 if (Vectorized) { 1664 for (int i = 0, e = VL.size(); i != e; ++i) { 1665 assert(!getTreeEntry(VL[i]) && "Scalar already in tree!"); 1666 ScalarToTreeEntry[VL[i]] = Last; 1667 } 1668 // Update the scheduler bundle to point to this TreeEntry. 1669 unsigned Lane = 0; 1670 for (ScheduleData *BundleMember = Bundle.getValue(); BundleMember; 1671 BundleMember = BundleMember->NextInBundle) { 1672 BundleMember->TE = Last; 1673 BundleMember->Lane = Lane; 1674 ++Lane; 1675 } 1676 assert((!Bundle.getValue() || Lane == VL.size()) && 1677 "Bundle and VL out of sync"); 1678 } else { 1679 MustGather.insert(VL.begin(), VL.end()); 1680 } 1681 1682 if (UserTreeIdx.UserTE) 1683 Last->UserTreeIndices.push_back(UserTreeIdx); 1684 1685 return Last; 1686 } 1687 1688 /// -- Vectorization State -- 1689 /// Holds all of the tree entries. 1690 TreeEntry::VecTreeTy VectorizableTree; 1691 1692#ifndef NDEBUG 1693 /// Debug printer. 1694 LLVM_DUMP_METHOD void dumpVectorizableTree() const { 1695 for (unsigned Id = 0, IdE = VectorizableTree.size(); Id != IdE; ++Id) { 1696 VectorizableTree[Id]->dump(); 1697 dbgs() << "\n"; 1698 } 1699 } 1700#endif 1701 1702 TreeEntry *getTreeEntry(Value *V) { 1703 auto I = ScalarToTreeEntry.find(V); 1704 if (I != ScalarToTreeEntry.end()) 1705 return I->second; 1706 return nullptr; 1707 } 1708 1709 const TreeEntry *getTreeEntry(Value *V) const { 1710 auto I = ScalarToTreeEntry.find(V); 1711 if (I != ScalarToTreeEntry.end()) 1712 return I->second; 1713 return nullptr; 1714 } 1715 1716 /// Maps a specific scalar to its tree entry. 1717 SmallDenseMap<Value*, TreeEntry *> ScalarToTreeEntry; 1718 1719 /// Maps a value��to the proposed vectorizable size. 1720 SmallDenseMap<Value *, unsigned> InstrElementSize; 1721 1722 /// A list of scalars that we found that we need to keep as scalars. 1723 ValueSet MustGather; 1724 1725 /// This POD struct describes one external user in the vectorized tree. 1726 struct ExternalUser { 1727 ExternalUser(Value *S, llvm::User *U, int L) 1728 : Scalar(S), User(U), Lane(L) {} 1729 1730 // Which scalar in our function. 1731 Value *Scalar; 1732 1733 // Which user that uses the scalar. 1734 llvm::User *User; 1735 1736 // Which lane does the scalar belong to. 1737 int Lane; 1738 }; 1739 using UserList = SmallVector<ExternalUser, 16>; 1740 1741 /// Checks if two instructions may access the same memory. 1742 /// 1743 /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it 1744 /// is invariant in the calling loop. 1745 bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1, 1746 Instruction *Inst2) { 1747 // First check if the result is already in the cache. 1748 AliasCacheKey key = std::make_pair(Inst1, Inst2); 1749 Optional<bool> &result = AliasCache[key]; 1750 if (result.hasValue()) { 1751 return result.getValue(); 1752 } 1753 MemoryLocation Loc2 = getLocation(Inst2, AA); 1754 bool aliased = true; 1755 if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) { 1756 // Do the alias check. 1757 aliased = AA->alias(Loc1, Loc2); 1758 } 1759 // Store the result in the cache. 1760 result = aliased; 1761 return aliased; 1762 } 1763 1764 using AliasCacheKey = std::pair<Instruction *, Instruction *>; 1765 1766 /// Cache for alias results. 1767 /// TODO: consider moving this to the AliasAnalysis itself. 1768 DenseMap<AliasCacheKey, Optional<bool>> AliasCache; 1769 1770 /// Removes an instruction from its block and eventually deletes it. 1771 /// It's like Instruction::eraseFromParent() except that the actual deletion 1772 /// is delayed until BoUpSLP is destructed. 1773 /// This is required to ensure that there are no incorrect collisions in the 1774 /// AliasCache, which can happen if a new instruction is allocated at the 1775 /// same address as a previously deleted instruction. 1776 void eraseInstruction(Instruction *I, bool ReplaceOpsWithUndef = false) { 1777 auto It = DeletedInstructions.try_emplace(I, ReplaceOpsWithUndef).first; 1778 It->getSecond() = It->getSecond() && ReplaceOpsWithUndef; 1779 } 1780 1781 /// Temporary store for deleted instructions. Instructions will be deleted 1782 /// eventually when the BoUpSLP is destructed. 1783 DenseMap<Instruction *, bool> DeletedInstructions; 1784 1785 /// A list of values that need to extracted out of the tree. 1786 /// This list holds pairs of (Internal Scalar : External User). External User 1787 /// can be nullptr, it means that this Internal Scalar will be used later, 1788 /// after vectorization. 1789 UserList ExternalUses; 1790 1791 /// Values used only by @llvm.assume calls. 1792 SmallPtrSet<const Value *, 32> EphValues; 1793 1794 /// Holds all of the instructions that we gathered. 1795 SetVector<Instruction *> GatherSeq; 1796 1797 /// A list of blocks that we are going to CSE. 1798 SetVector<BasicBlock *> CSEBlocks; 1799 1800 /// Contains all scheduling relevant data for an instruction. 1801 /// A ScheduleData either represents a single instruction or a member of an 1802 /// instruction bundle (= a group of instructions which is combined into a 1803 /// vector instruction). 1804 struct ScheduleData { 1805 // The initial value for the dependency counters. It means that the 1806 // dependencies are not calculated yet. 1807 enum { InvalidDeps = -1 }; 1808 1809 ScheduleData() = default; 1810 1811 void init(int BlockSchedulingRegionID, Value *OpVal) { 1812 FirstInBundle = this; 1813 NextInBundle = nullptr; 1814 NextLoadStore = nullptr; 1815 IsScheduled = false; 1816 SchedulingRegionID = BlockSchedulingRegionID; 1817 UnscheduledDepsInBundle = UnscheduledDeps; 1818 clearDependencies(); 1819 OpValue = OpVal; 1820 TE = nullptr; 1821 Lane = -1; 1822 } 1823 1824 /// Returns true if the dependency information has been calculated. 1825 bool hasValidDependencies() const { return Dependencies != InvalidDeps; } 1826 1827 /// Returns true for single instructions and for bundle representatives 1828 /// (= the head of a bundle). 1829 bool isSchedulingEntity() const { return FirstInBundle == this; } 1830 1831 /// Returns true if it represents an instruction bundle and not only a 1832 /// single instruction. 1833 bool isPartOfBundle() const { 1834 return NextInBundle != nullptr || FirstInBundle != this; 1835 } 1836 1837 /// Returns true if it is ready for scheduling, i.e. it has no more 1838 /// unscheduled depending instructions/bundles. 1839 bool isReady() const { 1840 assert(isSchedulingEntity() && 1841 "can't consider non-scheduling entity for ready list"); 1842 return UnscheduledDepsInBundle == 0 && !IsScheduled; 1843 } 1844 1845 /// Modifies the number of unscheduled dependencies, also updating it for 1846 /// the whole bundle. 1847 int incrementUnscheduledDeps(int Incr) { 1848 UnscheduledDeps += Incr; 1849 return FirstInBundle->UnscheduledDepsInBundle += Incr; 1850 } 1851 1852 /// Sets the number of unscheduled dependencies to the number of 1853 /// dependencies. 1854 void resetUnscheduledDeps() { 1855 incrementUnscheduledDeps(Dependencies - UnscheduledDeps); 1856 } 1857 1858 /// Clears all dependency information. 1859 void clearDependencies() { 1860 Dependencies = InvalidDeps; 1861 resetUnscheduledDeps(); 1862 MemoryDependencies.clear(); 1863 } 1864 1865 void dump(raw_ostream &os) const { 1866 if (!isSchedulingEntity()) { 1867 os << "/ " << *Inst; 1868 } else if (NextInBundle) { 1869 os << '[' << *Inst; 1870 ScheduleData *SD = NextInBundle; 1871 while (SD) { 1872 os << ';' << *SD->Inst; 1873 SD = SD->NextInBundle; 1874 } 1875 os << ']'; 1876 } else { 1877 os << *Inst; 1878 } 1879 } 1880 1881 Instruction *Inst = nullptr; 1882 1883 /// Points to the head in an instruction bundle (and always to this for 1884 /// single instructions). 1885 ScheduleData *FirstInBundle = nullptr; 1886 1887 /// Single linked list of all instructions in a bundle. Null if it is a 1888 /// single instruction. 1889 ScheduleData *NextInBundle = nullptr; 1890 1891 /// Single linked list of all memory instructions (e.g. load, store, call) 1892 /// in the block - until the end of the scheduling region. 1893 ScheduleData *NextLoadStore = nullptr; 1894 1895 /// The dependent memory instructions. 1896 /// This list is derived on demand in calculateDependencies(). 1897 SmallVector<ScheduleData *, 4> MemoryDependencies; 1898 1899 /// This ScheduleData is in the current scheduling region if this matches 1900 /// the current SchedulingRegionID of BlockScheduling. 1901 int SchedulingRegionID = 0; 1902 1903 /// Used for getting a "good" final ordering of instructions. 1904 int SchedulingPriority = 0; 1905 1906 /// The number of dependencies. Constitutes of the number of users of the 1907 /// instruction plus the number of dependent memory instructions (if any). 1908 /// This value is calculated on demand. 1909 /// If InvalidDeps, the number of dependencies is not calculated yet. 1910 int Dependencies = InvalidDeps; 1911 1912 /// The number of dependencies minus the number of dependencies of scheduled 1913 /// instructions. As soon as this is zero, the instruction/bundle gets ready 1914 /// for scheduling. 1915 /// Note that this is negative as long as Dependencies is not calculated. 1916 int UnscheduledDeps = InvalidDeps; 1917 1918 /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for 1919 /// single instructions. 1920 int UnscheduledDepsInBundle = InvalidDeps; 1921 1922 /// True if this instruction is scheduled (or considered as scheduled in the 1923 /// dry-run). 1924 bool IsScheduled = false; 1925 1926 /// Opcode of the current instruction in the schedule data. 1927 Value *OpValue = nullptr; 1928 1929 /// The TreeEntry that this instruction corresponds to. 1930 TreeEntry *TE = nullptr; 1931 1932 /// The lane of this node in the TreeEntry. 1933 int Lane = -1; 1934 }; 1935 1936#ifndef NDEBUG 1937 friend inline raw_ostream &operator<<(raw_ostream &os, 1938 const BoUpSLP::ScheduleData &SD) { 1939 SD.dump(os); 1940 return os; 1941 } 1942#endif 1943 1944 friend struct GraphTraits<BoUpSLP *>; 1945 friend struct DOTGraphTraits<BoUpSLP *>; 1946 1947 /// Contains all scheduling data for a basic block. 1948 struct BlockScheduling { 1949 BlockScheduling(BasicBlock *BB) 1950 : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {} 1951 1952 void clear() { 1953 ReadyInsts.clear(); 1954 ScheduleStart = nullptr; 1955 ScheduleEnd = nullptr; 1956 FirstLoadStoreInRegion = nullptr; 1957 LastLoadStoreInRegion = nullptr; 1958 1959 // Reduce the maximum schedule region size by the size of the 1960 // previous scheduling run. 1961 ScheduleRegionSizeLimit -= ScheduleRegionSize; 1962 if (ScheduleRegionSizeLimit < MinScheduleRegionSize) 1963 ScheduleRegionSizeLimit = MinScheduleRegionSize; 1964 ScheduleRegionSize = 0; 1965 1966 // Make a new scheduling region, i.e. all existing ScheduleData is not 1967 // in the new region yet. 1968 ++SchedulingRegionID; 1969 } 1970 1971 ScheduleData *getScheduleData(Value *V) { 1972 ScheduleData *SD = ScheduleDataMap[V]; 1973 if (SD && SD->SchedulingRegionID == SchedulingRegionID) 1974 return SD; 1975 return nullptr; 1976 } 1977 1978 ScheduleData *getScheduleData(Value *V, Value *Key) { 1979 if (V == Key) 1980 return getScheduleData(V); 1981 auto I = ExtraScheduleDataMap.find(V); 1982 if (I != ExtraScheduleDataMap.end()) { 1983 ScheduleData *SD = I->second[Key]; 1984 if (SD && SD->SchedulingRegionID == SchedulingRegionID) 1985 return SD; 1986 } 1987 return nullptr; 1988 } 1989 1990 bool isInSchedulingRegion(ScheduleData *SD) const { 1991 return SD->SchedulingRegionID == SchedulingRegionID; 1992 } 1993 1994 /// Marks an instruction as scheduled and puts all dependent ready 1995 /// instructions into the ready-list. 1996 template <typename ReadyListType> 1997 void schedule(ScheduleData *SD, ReadyListType &ReadyList) { 1998 SD->IsScheduled = true; 1999 LLVM_DEBUG(dbgs() << "SLP: schedule " << *SD << "\n"); 2000 2001 ScheduleData *BundleMember = SD; 2002 while (BundleMember) { 2003 if (BundleMember->Inst != BundleMember->OpValue) { 2004 BundleMember = BundleMember->NextInBundle; 2005 continue; 2006 } 2007 // Handle the def-use chain dependencies. 2008 2009 // Decrement the unscheduled counter and insert to ready list if ready. 2010 auto &&DecrUnsched = [this, &ReadyList](Instruction *I) { 2011 doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) { 2012 if (OpDef && OpDef->hasValidDependencies() && 2013 OpDef->incrementUnscheduledDeps(-1) == 0) { 2014 // There are no more unscheduled dependencies after 2015 // decrementing, so we can put the dependent instruction 2016 // into the ready list. 2017 ScheduleData *DepBundle = OpDef->FirstInBundle; 2018 assert(!DepBundle->IsScheduled && 2019 "already scheduled bundle gets ready"); 2020 ReadyList.insert(DepBundle); 2021 LLVM_DEBUG(dbgs() 2022 << "SLP: gets ready (def): " << *DepBundle << "\n"); 2023 } 2024 }); 2025 }; 2026 2027 // If BundleMember is a vector bundle, its operands may have been 2028 // reordered duiring buildTree(). We therefore need to get its operands 2029 // through the TreeEntry. 2030 if (TreeEntry *TE = BundleMember->TE) { 2031 int Lane = BundleMember->Lane; 2032 assert(Lane >= 0 && "Lane not set"); 2033 2034 // Since vectorization tree is being built recursively this assertion 2035 // ensures that the tree entry has all operands set before reaching 2036 // this code. Couple of exceptions known at the moment are extracts 2037 // where their second (immediate) operand is not added. Since 2038 // immediates do not affect scheduler behavior this is considered 2039 // okay. 2040 auto *In = TE->getMainOp(); 2041 assert(In && 2042 (isa<ExtractValueInst>(In) || isa<ExtractElementInst>(In) || 2043 In->getNumOperands() == TE->getNumOperands()) && 2044 "Missed TreeEntry operands?"); 2045 (void)In; // fake use to avoid build failure when assertions disabled 2046 2047 for (unsigned OpIdx = 0, NumOperands = TE->getNumOperands(); 2048 OpIdx != NumOperands; ++OpIdx) 2049 if (auto *I = dyn_cast<Instruction>(TE->getOperand(OpIdx)[Lane])) 2050 DecrUnsched(I); 2051 } else { 2052 // If BundleMember is a stand-alone instruction, no operand reordering 2053 // has taken place, so we directly access its operands. 2054 for (Use &U : BundleMember->Inst->operands()) 2055 if (auto *I = dyn_cast<Instruction>(U.get())) 2056 DecrUnsched(I); 2057 } 2058 // Handle the memory dependencies. 2059 for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) { 2060 if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) { 2061 // There are no more unscheduled dependencies after decrementing, 2062 // so we can put the dependent instruction into the ready list. 2063 ScheduleData *DepBundle = MemoryDepSD->FirstInBundle; 2064 assert(!DepBundle->IsScheduled && 2065 "already scheduled bundle gets ready"); 2066 ReadyList.insert(DepBundle); 2067 LLVM_DEBUG(dbgs() 2068 << "SLP: gets ready (mem): " << *DepBundle << "\n"); 2069 } 2070 } 2071 BundleMember = BundleMember->NextInBundle; 2072 } 2073 } 2074 2075 void doForAllOpcodes(Value *V, 2076 function_ref<void(ScheduleData *SD)> Action) { 2077 if (ScheduleData *SD = getScheduleData(V)) 2078 Action(SD); 2079 auto I = ExtraScheduleDataMap.find(V); 2080 if (I != ExtraScheduleDataMap.end()) 2081 for (auto &P : I->second) 2082 if (P.second->SchedulingRegionID == SchedulingRegionID) 2083 Action(P.second); 2084 } 2085 2086 /// Put all instructions into the ReadyList which are ready for scheduling. 2087 template <typename ReadyListType> 2088 void initialFillReadyList(ReadyListType &ReadyList) { 2089 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) { 2090 doForAllOpcodes(I, [&](ScheduleData *SD) { 2091 if (SD->isSchedulingEntity() && SD->isReady()) { 2092 ReadyList.insert(SD); 2093 LLVM_DEBUG(dbgs() 2094 << "SLP: initially in ready list: " << *I << "\n"); 2095 } 2096 }); 2097 } 2098 } 2099 2100 /// Checks if a bundle of instructions can be scheduled, i.e. has no 2101 /// cyclic dependencies. This is only a dry-run, no instructions are 2102 /// actually moved at this stage. 2103 /// \returns the scheduling bundle. The returned Optional value is non-None 2104 /// if \p VL is allowed to be scheduled. 2105 Optional<ScheduleData *> 2106 tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP, 2107 const InstructionsState &S); 2108 2109 /// Un-bundles a group of instructions. 2110 void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue); 2111 2112 /// Allocates schedule data chunk. 2113 ScheduleData *allocateScheduleDataChunks(); 2114 2115 /// Extends the scheduling region so that V is inside the region. 2116 /// \returns true if the region size is within the limit. 2117 bool extendSchedulingRegion(Value *V, const InstructionsState &S); 2118 2119 /// Initialize the ScheduleData structures for new instructions in the 2120 /// scheduling region. 2121 void initScheduleData(Instruction *FromI, Instruction *ToI, 2122 ScheduleData *PrevLoadStore, 2123 ScheduleData *NextLoadStore); 2124 2125 /// Updates the dependency information of a bundle and of all instructions/ 2126 /// bundles which depend on the original bundle. 2127 void calculateDependencies(ScheduleData *SD, bool InsertInReadyList, 2128 BoUpSLP *SLP); 2129 2130 /// Sets all instruction in the scheduling region to un-scheduled. 2131 void resetSchedule(); 2132 2133 BasicBlock *BB; 2134 2135 /// Simple memory allocation for ScheduleData. 2136 std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks; 2137 2138 /// The size of a ScheduleData array in ScheduleDataChunks. 2139 int ChunkSize; 2140 2141 /// The allocator position in the current chunk, which is the last entry 2142 /// of ScheduleDataChunks. 2143 int ChunkPos; 2144 2145 /// Attaches ScheduleData to Instruction. 2146 /// Note that the mapping survives during all vectorization iterations, i.e. 2147 /// ScheduleData structures are recycled. 2148 DenseMap<Value *, ScheduleData *> ScheduleDataMap; 2149 2150 /// Attaches ScheduleData to Instruction with the leading key. 2151 DenseMap<Value *, SmallDenseMap<Value *, ScheduleData *>> 2152 ExtraScheduleDataMap; 2153 2154 struct ReadyList : SmallVector<ScheduleData *, 8> { 2155 void insert(ScheduleData *SD) { push_back(SD); } 2156 }; 2157 2158 /// The ready-list for scheduling (only used for the dry-run). 2159 ReadyList ReadyInsts; 2160 2161 /// The first instruction of the scheduling region. 2162 Instruction *ScheduleStart = nullptr; 2163 2164 /// The first instruction _after_ the scheduling region. 2165 Instruction *ScheduleEnd = nullptr; 2166 2167 /// The first memory accessing instruction in the scheduling region 2168 /// (can be null). 2169 ScheduleData *FirstLoadStoreInRegion = nullptr; 2170 2171 /// The last memory accessing instruction in the scheduling region 2172 /// (can be null). 2173 ScheduleData *LastLoadStoreInRegion = nullptr; 2174 2175 /// The current size of the scheduling region. 2176 int ScheduleRegionSize = 0; 2177 2178 /// The maximum size allowed for the scheduling region. 2179 int ScheduleRegionSizeLimit = ScheduleRegionSizeBudget; 2180 2181 /// The ID of the scheduling region. For a new vectorization iteration this 2182 /// is incremented which "removes" all ScheduleData from the region. 2183 // Make sure that the initial SchedulingRegionID is greater than the 2184 // initial SchedulingRegionID in ScheduleData (which is 0). 2185 int SchedulingRegionID = 1; 2186 }; 2187 2188 /// Attaches the BlockScheduling structures to basic blocks. 2189 MapVector<BasicBlock *, std::unique_ptr<BlockScheduling>> BlocksSchedules; 2190 2191 /// Performs the "real" scheduling. Done before vectorization is actually 2192 /// performed in a basic block. 2193 void scheduleBlock(BlockScheduling *BS); 2194 2195 /// List of users to ignore during scheduling and that don't need extracting. 2196 ArrayRef<Value *> UserIgnoreList; 2197 2198 using OrdersType = SmallVector<unsigned, 4>; 2199 /// A DenseMapInfo implementation for holding DenseMaps and DenseSets of 2200 /// sorted SmallVectors of unsigned. 2201 struct OrdersTypeDenseMapInfo { 2202 static OrdersType getEmptyKey() { 2203 OrdersType V; 2204 V.push_back(~1U); 2205 return V; 2206 } 2207 2208 static OrdersType getTombstoneKey() { 2209 OrdersType V; 2210 V.push_back(~2U); 2211 return V; 2212 } 2213 2214 static unsigned getHashValue(const OrdersType &V) { 2215 return static_cast<unsigned>(hash_combine_range(V.begin(), V.end())); 2216 } 2217 2218 static bool isEqual(const OrdersType &LHS, const OrdersType &RHS) { 2219 return LHS == RHS; 2220 } 2221 }; 2222 2223 /// Contains orders of operations along with the number of bundles that have 2224 /// operations in this order. It stores only those orders that require 2225 /// reordering, if reordering is not required it is counted using \a 2226 /// NumOpsWantToKeepOriginalOrder. 2227 DenseMap<OrdersType, unsigned, OrdersTypeDenseMapInfo> NumOpsWantToKeepOrder; 2228 /// Number of bundles that do not require reordering. 2229 unsigned NumOpsWantToKeepOriginalOrder = 0; 2230 2231 // Analysis and block reference. 2232 Function *F; 2233 ScalarEvolution *SE; 2234 TargetTransformInfo *TTI; 2235 TargetLibraryInfo *TLI; 2236 AliasAnalysis *AA; 2237 LoopInfo *LI; 2238 DominatorTree *DT; 2239 AssumptionCache *AC; 2240 DemandedBits *DB; 2241 const DataLayout *DL; 2242 OptimizationRemarkEmitter *ORE; 2243 2244 unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt. 2245 unsigned MinVecRegSize; // Set by cl::opt (default: 128). 2246 2247 /// Instruction builder to construct the vectorized tree. 2248 IRBuilder<> Builder; 2249 2250 /// A map of scalar integer values to the smallest bit width with which they 2251 /// can legally be represented. The values map to (width, signed) pairs, 2252 /// where "width" indicates the minimum bit width and "signed" is True if the 2253 /// value must be signed-extended, rather than zero-extended, back to its 2254 /// original width. 2255 MapVector<Value *, std::pair<uint64_t, bool>> MinBWs; 2256}; 2257 2258} // end namespace slpvectorizer 2259 2260template <> struct GraphTraits<BoUpSLP *> { 2261 using TreeEntry = BoUpSLP::TreeEntry; 2262 2263 /// NodeRef has to be a pointer per the GraphWriter. 2264 using NodeRef = TreeEntry *; 2265 2266 using ContainerTy = BoUpSLP::TreeEntry::VecTreeTy; 2267 2268 /// Add the VectorizableTree to the index iterator to be able to return 2269 /// TreeEntry pointers. 2270 struct ChildIteratorType 2271 : public iterator_adaptor_base< 2272 ChildIteratorType, SmallVector<BoUpSLP::EdgeInfo, 1>::iterator> { 2273 ContainerTy &VectorizableTree; 2274 2275 ChildIteratorType(SmallVector<BoUpSLP::EdgeInfo, 1>::iterator W, 2276 ContainerTy &VT) 2277 : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {} 2278 2279 NodeRef operator*() { return I->UserTE; } 2280 }; 2281 2282 static NodeRef getEntryNode(BoUpSLP &R) { 2283 return R.VectorizableTree[0].get(); 2284 } 2285 2286 static ChildIteratorType child_begin(NodeRef N) { 2287 return {N->UserTreeIndices.begin(), N->Container}; 2288 } 2289 2290 static ChildIteratorType child_end(NodeRef N) { 2291 return {N->UserTreeIndices.end(), N->Container}; 2292 } 2293 2294 /// For the node iterator we just need to turn the TreeEntry iterator into a 2295 /// TreeEntry* iterator so that it dereferences to NodeRef. 2296 class nodes_iterator { 2297 using ItTy = ContainerTy::iterator; 2298 ItTy It; 2299 2300 public: 2301 nodes_iterator(const ItTy &It2) : It(It2) {} 2302 NodeRef operator*() { return It->get(); } 2303 nodes_iterator operator++() { 2304 ++It; 2305 return *this; 2306 } 2307 bool operator!=(const nodes_iterator &N2) const { return N2.It != It; } 2308 }; 2309 2310 static nodes_iterator nodes_begin(BoUpSLP *R) { 2311 return nodes_iterator(R->VectorizableTree.begin()); 2312 } 2313 2314 static nodes_iterator nodes_end(BoUpSLP *R) { 2315 return nodes_iterator(R->VectorizableTree.end()); 2316 } 2317 2318 static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); } 2319}; 2320 2321template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits { 2322 using TreeEntry = BoUpSLP::TreeEntry; 2323 2324 DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {} 2325 2326 std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) { 2327 std::string Str; 2328 raw_string_ostream OS(Str); 2329 if (isSplat(Entry->Scalars)) { 2330 OS << "<splat> " << *Entry->Scalars[0]; 2331 return Str; 2332 } 2333 for (auto V : Entry->Scalars) { 2334 OS << *V; 2335 if (std::any_of( 2336 R->ExternalUses.begin(), R->ExternalUses.end(), 2337 [&](const BoUpSLP::ExternalUser &EU) { return EU.Scalar == V; })) 2338 OS << " <extract>"; 2339 OS << "\n"; 2340 } 2341 return Str; 2342 } 2343 2344 static std::string getNodeAttributes(const TreeEntry *Entry, 2345 const BoUpSLP *) { 2346 if (Entry->State == TreeEntry::NeedToGather) 2347 return "color=red"; 2348 return ""; 2349 } 2350}; 2351 2352} // end namespace llvm 2353 2354BoUpSLP::~BoUpSLP() { 2355 for (const auto &Pair : DeletedInstructions) { 2356 // Replace operands of ignored instructions with Undefs in case if they were 2357 // marked for deletion. 2358 if (Pair.getSecond()) { 2359 Value *Undef = UndefValue::get(Pair.getFirst()->getType()); 2360 Pair.getFirst()->replaceAllUsesWith(Undef); 2361 } 2362 Pair.getFirst()->dropAllReferences(); 2363 } 2364 for (const auto &Pair : DeletedInstructions) { 2365 assert(Pair.getFirst()->use_empty() && 2366 "trying to erase instruction with users."); 2367 Pair.getFirst()->eraseFromParent(); 2368 } 2369 assert(!verifyFunction(*F, &dbgs())); 2370} 2371 2372void BoUpSLP::eraseInstructions(ArrayRef<Value *> AV) { 2373 for (auto *V : AV) { 2374 if (auto *I = dyn_cast<Instruction>(V)) 2375 eraseInstruction(I, /*ReplaceWithUndef=*/true); 2376 }; 2377} 2378 2379void BoUpSLP::buildTree(ArrayRef<Value *> Roots, 2380 ArrayRef<Value *> UserIgnoreLst) { 2381 ExtraValueToDebugLocsMap ExternallyUsedValues; 2382 buildTree(Roots, ExternallyUsedValues, UserIgnoreLst); 2383} 2384 2385void BoUpSLP::buildTree(ArrayRef<Value *> Roots, 2386 ExtraValueToDebugLocsMap &ExternallyUsedValues, 2387 ArrayRef<Value *> UserIgnoreLst) { 2388 deleteTree(); 2389 UserIgnoreList = UserIgnoreLst; 2390 if (!allSameType(Roots)) 2391 return; 2392 buildTree_rec(Roots, 0, EdgeInfo()); 2393 2394 // Collect the values that we need to extract from the tree. 2395 for (auto &TEPtr : VectorizableTree) { 2396 TreeEntry *Entry = TEPtr.get(); 2397 2398 // No need to handle users of gathered values. 2399 if (Entry->State == TreeEntry::NeedToGather) 2400 continue; 2401 2402 // For each lane: 2403 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) { 2404 Value *Scalar = Entry->Scalars[Lane]; 2405 int FoundLane = Lane; 2406 if (!Entry->ReuseShuffleIndices.empty()) { 2407 FoundLane = 2408 std::distance(Entry->ReuseShuffleIndices.begin(), 2409 llvm::find(Entry->ReuseShuffleIndices, FoundLane)); 2410 } 2411 2412 // Check if the scalar is externally used as an extra arg. 2413 auto ExtI = ExternallyUsedValues.find(Scalar); 2414 if (ExtI != ExternallyUsedValues.end()) { 2415 LLVM_DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane " 2416 << Lane << " from " << *Scalar << ".\n"); 2417 ExternalUses.emplace_back(Scalar, nullptr, FoundLane); 2418 } 2419 for (User *U : Scalar->users()) { 2420 LLVM_DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n"); 2421 2422 Instruction *UserInst = dyn_cast<Instruction>(U); 2423 if (!UserInst) 2424 continue; 2425 2426 // Skip in-tree scalars that become vectors 2427 if (TreeEntry *UseEntry = getTreeEntry(U)) { 2428 Value *UseScalar = UseEntry->Scalars[0]; 2429 // Some in-tree scalars will remain as scalar in vectorized 2430 // instructions. If that is the case, the one in Lane 0 will 2431 // be used. 2432 if (UseScalar != U || 2433 !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) { 2434 LLVM_DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *U 2435 << ".\n"); 2436 assert(UseEntry->State != TreeEntry::NeedToGather && "Bad state"); 2437 continue; 2438 } 2439 } 2440 2441 // Ignore users in the user ignore list. 2442 if (is_contained(UserIgnoreList, UserInst)) 2443 continue; 2444 2445 LLVM_DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane " 2446 << Lane << " from " << *Scalar << ".\n"); 2447 ExternalUses.push_back(ExternalUser(Scalar, U, FoundLane)); 2448 } 2449 } 2450 } 2451} 2452 2453void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth, 2454 const EdgeInfo &UserTreeIdx) { 2455 assert((allConstant(VL) || allSameType(VL)) && "Invalid types!"); 2456 2457 InstructionsState S = getSameOpcode(VL); 2458 if (Depth == RecursionMaxDepth) { 2459 LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n"); 2460 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2461 return; 2462 } 2463 2464 // Don't handle vectors. 2465 if (S.OpValue->getType()->isVectorTy()) { 2466 LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n"); 2467 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2468 return; 2469 } 2470 2471 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue)) 2472 if (SI->getValueOperand()->getType()->isVectorTy()) { 2473 LLVM_DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n"); 2474 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2475 return; 2476 } 2477 2478 // If all of the operands are identical or constant we have a simple solution. 2479 if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.getOpcode()) { 2480 LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n"); 2481 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2482 return; 2483 } 2484 2485 // We now know that this is a vector of instructions of the same type from 2486 // the same block. 2487 2488 // Don't vectorize ephemeral values. 2489 for (Value *V : VL) { 2490 if (EphValues.count(V)) { 2491 LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V 2492 << ") is ephemeral.\n"); 2493 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2494 return; 2495 } 2496 } 2497 2498 // Check if this is a duplicate of another entry. 2499 if (TreeEntry *E = getTreeEntry(S.OpValue)) { 2500 LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.OpValue << ".\n"); 2501 if (!E->isSame(VL)) { 2502 LLVM_DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n"); 2503 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2504 return; 2505 } 2506 // Record the reuse of the tree node. FIXME, currently this is only used to 2507 // properly draw the graph rather than for the actual vectorization. 2508 E->UserTreeIndices.push_back(UserTreeIdx); 2509 LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValue 2510 << ".\n"); 2511 return; 2512 } 2513 2514 // Check that none of the instructions in the bundle are already in the tree. 2515 for (Value *V : VL) { 2516 auto *I = dyn_cast<Instruction>(V); 2517 if (!I) 2518 continue; 2519 if (getTreeEntry(I)) { 2520 LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V 2521 << ") is already in tree.\n"); 2522 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2523 return; 2524 } 2525 } 2526 2527 // If any of the scalars is marked as a value that needs to stay scalar, then 2528 // we need to gather the scalars. 2529 // The reduction nodes (stored in UserIgnoreList) also should stay scalar. 2530 for (Value *V : VL) { 2531 if (MustGather.count(V) || is_contained(UserIgnoreList, V)) { 2532 LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n"); 2533 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2534 return; 2535 } 2536 } 2537 2538 // Check that all of the users of the scalars that we want to vectorize are 2539 // schedulable. 2540 auto *VL0 = cast<Instruction>(S.OpValue); 2541 BasicBlock *BB = VL0->getParent(); 2542 2543 if (!DT->isReachableFromEntry(BB)) { 2544 // Don't go into unreachable blocks. They may contain instructions with 2545 // dependency cycles which confuse the final scheduling. 2546 LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n"); 2547 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2548 return; 2549 } 2550 2551 // Check that every instruction appears once in this bundle. 2552 SmallVector<unsigned, 4> ReuseShuffleIndicies; 2553 SmallVector<Value *, 4> UniqueValues; 2554 DenseMap<Value *, unsigned> UniquePositions; 2555 for (Value *V : VL) { 2556 auto Res = UniquePositions.try_emplace(V, UniqueValues.size()); 2557 ReuseShuffleIndicies.emplace_back(Res.first->second); 2558 if (Res.second) 2559 UniqueValues.emplace_back(V); 2560 } 2561 size_t NumUniqueScalarValues = UniqueValues.size(); 2562 if (NumUniqueScalarValues == VL.size()) { 2563 ReuseShuffleIndicies.clear(); 2564 } else { 2565 LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n"); 2566 if (NumUniqueScalarValues <= 1 || 2567 !llvm::isPowerOf2_32(NumUniqueScalarValues)) { 2568 LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n"); 2569 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2570 return; 2571 } 2572 VL = UniqueValues; 2573 } 2574 2575 auto &BSRef = BlocksSchedules[BB]; 2576 if (!BSRef) 2577 BSRef = std::make_unique<BlockScheduling>(BB); 2578 2579 BlockScheduling &BS = *BSRef.get(); 2580 2581 Optional<ScheduleData *> Bundle = BS.tryScheduleBundle(VL, this, S); 2582 if (!Bundle) { 2583 LLVM_DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n"); 2584 assert((!BS.getScheduleData(VL0) || 2585 !BS.getScheduleData(VL0)->isPartOfBundle()) && 2586 "tryScheduleBundle should cancelScheduling on failure"); 2587 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2588 ReuseShuffleIndicies); 2589 return; 2590 } 2591 LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n"); 2592 2593 unsigned ShuffleOrOp = S.isAltShuffle() ? 2594 (unsigned) Instruction::ShuffleVector : S.getOpcode(); 2595 switch (ShuffleOrOp) { 2596 case Instruction::PHI: { 2597 auto *PH = cast<PHINode>(VL0); 2598 2599 // Check for terminator values (e.g. invoke). 2600 for (unsigned j = 0; j < VL.size(); ++j) 2601 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) { 2602 Instruction *Term = dyn_cast<Instruction>( 2603 cast<PHINode>(VL[j])->getIncomingValueForBlock( 2604 PH->getIncomingBlock(i))); 2605 if (Term && Term->isTerminator()) { 2606 LLVM_DEBUG(dbgs() 2607 << "SLP: Need to swizzle PHINodes (terminator use).\n"); 2608 BS.cancelScheduling(VL, VL0); 2609 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2610 ReuseShuffleIndicies); 2611 return; 2612 } 2613 } 2614 2615 TreeEntry *TE = 2616 newTreeEntry(VL, Bundle, S, UserTreeIdx, ReuseShuffleIndicies); 2617 LLVM_DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n"); 2618 2619 // Keeps the reordered operands to avoid code duplication. 2620 SmallVector<ValueList, 2> OperandsVec; 2621 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) { 2622 ValueList Operands; 2623 // Prepare the operand vector. 2624 for (Value *j : VL) 2625 Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock( 2626 PH->getIncomingBlock(i))); 2627 TE->setOperand(i, Operands); 2628 OperandsVec.push_back(Operands); 2629 } 2630 for (unsigned OpIdx = 0, OpE = OperandsVec.size(); OpIdx != OpE; ++OpIdx) 2631 buildTree_rec(OperandsVec[OpIdx], Depth + 1, {TE, OpIdx}); 2632 return; 2633 } 2634 case Instruction::ExtractValue: 2635 case Instruction::ExtractElement: { 2636 OrdersType CurrentOrder; 2637 bool Reuse = canReuseExtract(VL, VL0, CurrentOrder); 2638 if (Reuse) { 2639 LLVM_DEBUG(dbgs() << "SLP: Reusing or shuffling extract sequence.\n"); 2640 ++NumOpsWantToKeepOriginalOrder; 2641 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 2642 ReuseShuffleIndicies); 2643 // This is a special case, as it does not gather, but at the same time 2644 // we are not extending buildTree_rec() towards the operands. 2645 ValueList Op0; 2646 Op0.assign(VL.size(), VL0->getOperand(0)); 2647 VectorizableTree.back()->setOperand(0, Op0); 2648 return; 2649 } 2650 if (!CurrentOrder.empty()) { 2651 LLVM_DEBUG({ 2652 dbgs() << "SLP: Reusing or shuffling of reordered extract sequence " 2653 "with order"; 2654 for (unsigned Idx : CurrentOrder) 2655 dbgs() << " " << Idx; 2656 dbgs() << "\n"; 2657 }); 2658 // Insert new order with initial value 0, if it does not exist, 2659 // otherwise return the iterator to the existing one. 2660 auto StoredCurrentOrderAndNum = 2661 NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first; 2662 ++StoredCurrentOrderAndNum->getSecond(); 2663 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 2664 ReuseShuffleIndicies, 2665 StoredCurrentOrderAndNum->getFirst()); 2666 // This is a special case, as it does not gather, but at the same time 2667 // we are not extending buildTree_rec() towards the operands. 2668 ValueList Op0; 2669 Op0.assign(VL.size(), VL0->getOperand(0)); 2670 VectorizableTree.back()->setOperand(0, Op0); 2671 return; 2672 } 2673 LLVM_DEBUG(dbgs() << "SLP: Gather extract sequence.\n"); 2674 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2675 ReuseShuffleIndicies); 2676 BS.cancelScheduling(VL, VL0); 2677 return; 2678 } 2679 case Instruction::Load: { 2680 // Check that a vectorized load would load the same memory as a scalar 2681 // load. For example, we don't want to vectorize loads that are smaller 2682 // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM 2683 // treats loading/storing it as an i8 struct. If we vectorize loads/stores 2684 // from such a struct, we read/write packed bits disagreeing with the 2685 // unvectorized version. 2686 Type *ScalarTy = VL0->getType(); 2687 2688 if (DL->getTypeSizeInBits(ScalarTy) != 2689 DL->getTypeAllocSizeInBits(ScalarTy)) { 2690 BS.cancelScheduling(VL, VL0); 2691 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2692 ReuseShuffleIndicies); 2693 LLVM_DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n"); 2694 return; 2695 } 2696 2697 // Make sure all loads in the bundle are simple - we can't vectorize 2698 // atomic or volatile loads. 2699 SmallVector<Value *, 4> PointerOps(VL.size()); 2700 auto POIter = PointerOps.begin(); 2701 for (Value *V : VL) { 2702 auto *L = cast<LoadInst>(V); 2703 if (!L->isSimple()) { 2704 BS.cancelScheduling(VL, VL0); 2705 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2706 ReuseShuffleIndicies); 2707 LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n"); 2708 return; 2709 } 2710 *POIter = L->getPointerOperand(); 2711 ++POIter; 2712 } 2713 2714 OrdersType CurrentOrder; 2715 // Check the order of pointer operands. 2716 if (llvm::sortPtrAccesses(PointerOps, *DL, *SE, CurrentOrder)) { 2717 Value *Ptr0; 2718 Value *PtrN; 2719 if (CurrentOrder.empty()) { 2720 Ptr0 = PointerOps.front(); 2721 PtrN = PointerOps.back(); 2722 } else { 2723 Ptr0 = PointerOps[CurrentOrder.front()]; 2724 PtrN = PointerOps[CurrentOrder.back()]; 2725 } 2726 const SCEV *Scev0 = SE->getSCEV(Ptr0); 2727 const SCEV *ScevN = SE->getSCEV(PtrN); 2728 const auto *Diff = 2729 dyn_cast<SCEVConstant>(SE->getMinusSCEV(ScevN, Scev0)); 2730 uint64_t Size = DL->getTypeAllocSize(ScalarTy); 2731 // Check that the sorted loads are consecutive. 2732 if (Diff && Diff->getAPInt() == (VL.size() - 1) * Size) { 2733 if (CurrentOrder.empty()) { 2734 // Original loads are consecutive and does not require reordering. 2735 ++NumOpsWantToKeepOriginalOrder; 2736 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, 2737 UserTreeIdx, ReuseShuffleIndicies); 2738 TE->setOperandsInOrder(); 2739 LLVM_DEBUG(dbgs() << "SLP: added a vector of loads.\n"); 2740 } else { 2741 // Need to reorder. 2742 auto I = NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first; 2743 ++I->getSecond(); 2744 TreeEntry *TE = 2745 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 2746 ReuseShuffleIndicies, I->getFirst()); 2747 TE->setOperandsInOrder(); 2748 LLVM_DEBUG(dbgs() << "SLP: added a vector of jumbled loads.\n"); 2749 } 2750 return; 2751 } 2752 } 2753 2754 LLVM_DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n"); 2755 BS.cancelScheduling(VL, VL0); 2756 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2757 ReuseShuffleIndicies); 2758 return; 2759 } 2760 case Instruction::ZExt: 2761 case Instruction::SExt: 2762 case Instruction::FPToUI: 2763 case Instruction::FPToSI: 2764 case Instruction::FPExt: 2765 case Instruction::PtrToInt: 2766 case Instruction::IntToPtr: 2767 case Instruction::SIToFP: 2768 case Instruction::UIToFP: 2769 case Instruction::Trunc: 2770 case Instruction::FPTrunc: 2771 case Instruction::BitCast: { 2772 Type *SrcTy = VL0->getOperand(0)->getType(); 2773 for (Value *V : VL) { 2774 Type *Ty = cast<Instruction>(V)->getOperand(0)->getType(); 2775 if (Ty != SrcTy || !isValidElementType(Ty)) { 2776 BS.cancelScheduling(VL, VL0); 2777 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2778 ReuseShuffleIndicies); 2779 LLVM_DEBUG(dbgs() 2780 << "SLP: Gathering casts with different src types.\n"); 2781 return; 2782 } 2783 } 2784 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 2785 ReuseShuffleIndicies); 2786 LLVM_DEBUG(dbgs() << "SLP: added a vector of casts.\n"); 2787 2788 TE->setOperandsInOrder(); 2789 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) { 2790 ValueList Operands; 2791 // Prepare the operand vector. 2792 for (Value *V : VL) 2793 Operands.push_back(cast<Instruction>(V)->getOperand(i)); 2794 2795 buildTree_rec(Operands, Depth + 1, {TE, i}); 2796 } 2797 return; 2798 } 2799 case Instruction::ICmp: 2800 case Instruction::FCmp: { 2801 // Check that all of the compares have the same predicate. 2802 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate(); 2803 CmpInst::Predicate SwapP0 = CmpInst::getSwappedPredicate(P0); 2804 Type *ComparedTy = VL0->getOperand(0)->getType(); 2805 for (Value *V : VL) { 2806 CmpInst *Cmp = cast<CmpInst>(V); 2807 if ((Cmp->getPredicate() != P0 && Cmp->getPredicate() != SwapP0) || 2808 Cmp->getOperand(0)->getType() != ComparedTy) { 2809 BS.cancelScheduling(VL, VL0); 2810 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2811 ReuseShuffleIndicies); 2812 LLVM_DEBUG(dbgs() 2813 << "SLP: Gathering cmp with different predicate.\n"); 2814 return; 2815 } 2816 } 2817 2818 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 2819 ReuseShuffleIndicies); 2820 LLVM_DEBUG(dbgs() << "SLP: added a vector of compares.\n"); 2821 2822 ValueList Left, Right; 2823 if (cast<CmpInst>(VL0)->isCommutative()) { 2824 // Commutative predicate - collect + sort operands of the instructions 2825 // so that each side is more likely to have the same opcode. 2826 assert(P0 == SwapP0 && "Commutative Predicate mismatch"); 2827 reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this); 2828 } else { 2829 // Collect operands - commute if it uses the swapped predicate. 2830 for (Value *V : VL) { 2831 auto *Cmp = cast<CmpInst>(V); 2832 Value *LHS = Cmp->getOperand(0); 2833 Value *RHS = Cmp->getOperand(1); 2834 if (Cmp->getPredicate() != P0) 2835 std::swap(LHS, RHS); 2836 Left.push_back(LHS); 2837 Right.push_back(RHS); 2838 } 2839 } 2840 TE->setOperand(0, Left); 2841 TE->setOperand(1, Right); 2842 buildTree_rec(Left, Depth + 1, {TE, 0}); 2843 buildTree_rec(Right, Depth + 1, {TE, 1}); 2844 return; 2845 } 2846 case Instruction::Select: 2847 case Instruction::FNeg: 2848 case Instruction::Add: 2849 case Instruction::FAdd: 2850 case Instruction::Sub: 2851 case Instruction::FSub: 2852 case Instruction::Mul: 2853 case Instruction::FMul: 2854 case Instruction::UDiv: 2855 case Instruction::SDiv: 2856 case Instruction::FDiv: 2857 case Instruction::URem: 2858 case Instruction::SRem: 2859 case Instruction::FRem: 2860 case Instruction::Shl: 2861 case Instruction::LShr: 2862 case Instruction::AShr: 2863 case Instruction::And: 2864 case Instruction::Or: 2865 case Instruction::Xor: { 2866 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 2867 ReuseShuffleIndicies); 2868 LLVM_DEBUG(dbgs() << "SLP: added a vector of un/bin op.\n"); 2869 2870 // Sort operands of the instructions so that each side is more likely to 2871 // have the same opcode. 2872 if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) { 2873 ValueList Left, Right; 2874 reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this); 2875 TE->setOperand(0, Left); 2876 TE->setOperand(1, Right); 2877 buildTree_rec(Left, Depth + 1, {TE, 0}); 2878 buildTree_rec(Right, Depth + 1, {TE, 1}); 2879 return; 2880 } 2881 2882 TE->setOperandsInOrder(); 2883 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) { 2884 ValueList Operands; 2885 // Prepare the operand vector. 2886 for (Value *j : VL) 2887 Operands.push_back(cast<Instruction>(j)->getOperand(i)); 2888 2889 buildTree_rec(Operands, Depth + 1, {TE, i}); 2890 } 2891 return; 2892 } 2893 case Instruction::GetElementPtr: { 2894 // We don't combine GEPs with complicated (nested) indexing. 2895 for (Value *V : VL) { 2896 if (cast<Instruction>(V)->getNumOperands() != 2) { 2897 LLVM_DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n"); 2898 BS.cancelScheduling(VL, VL0); 2899 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2900 ReuseShuffleIndicies); 2901 return; 2902 } 2903 } 2904 2905 // We can't combine several GEPs into one vector if they operate on 2906 // different types. 2907 Type *Ty0 = VL0->getOperand(0)->getType(); 2908 for (Value *V : VL) { 2909 Type *CurTy = cast<Instruction>(V)->getOperand(0)->getType(); 2910 if (Ty0 != CurTy) { 2911 LLVM_DEBUG(dbgs() 2912 << "SLP: not-vectorizable GEP (different types).\n"); 2913 BS.cancelScheduling(VL, VL0); 2914 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2915 ReuseShuffleIndicies); 2916 return; 2917 } 2918 } 2919 2920 // We don't combine GEPs with non-constant indexes. 2921 Type *Ty1 = VL0->getOperand(1)->getType(); 2922 for (Value *V : VL) { 2923 auto Op = cast<Instruction>(V)->getOperand(1); 2924 if (!isa<ConstantInt>(Op) || 2925 (Op->getType() != Ty1 && 2926 Op->getType()->getScalarSizeInBits() > 2927 DL->getIndexSizeInBits( 2928 V->getType()->getPointerAddressSpace()))) { 2929 LLVM_DEBUG(dbgs() 2930 << "SLP: not-vectorizable GEP (non-constant indexes).\n"); 2931 BS.cancelScheduling(VL, VL0); 2932 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2933 ReuseShuffleIndicies); 2934 return; 2935 } 2936 } 2937 2938 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 2939 ReuseShuffleIndicies); 2940 LLVM_DEBUG(dbgs() << "SLP: added a vector of GEPs.\n"); 2941 TE->setOperandsInOrder(); 2942 for (unsigned i = 0, e = 2; i < e; ++i) { 2943 ValueList Operands; 2944 // Prepare the operand vector. 2945 for (Value *V : VL) 2946 Operands.push_back(cast<Instruction>(V)->getOperand(i)); 2947 2948 buildTree_rec(Operands, Depth + 1, {TE, i}); 2949 } 2950 return; 2951 } 2952 case Instruction::Store: { 2953 // Check if the stores are consecutive or if we need to swizzle them. 2954 llvm::Type *ScalarTy = cast<StoreInst>(VL0)->getValueOperand()->getType(); 2955 // Make sure all stores in the bundle are simple - we can't vectorize 2956 // atomic or volatile stores. 2957 SmallVector<Value *, 4> PointerOps(VL.size()); 2958 ValueList Operands(VL.size()); 2959 auto POIter = PointerOps.begin(); 2960 auto OIter = Operands.begin(); 2961 for (Value *V : VL) { 2962 auto *SI = cast<StoreInst>(V); 2963 if (!SI->isSimple()) { 2964 BS.cancelScheduling(VL, VL0); 2965 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2966 ReuseShuffleIndicies); 2967 LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple stores.\n"); 2968 return; 2969 } 2970 *POIter = SI->getPointerOperand(); 2971 *OIter = SI->getValueOperand(); 2972 ++POIter; 2973 ++OIter; 2974 } 2975 2976 OrdersType CurrentOrder; 2977 // Check the order of pointer operands. 2978 if (llvm::sortPtrAccesses(PointerOps, *DL, *SE, CurrentOrder)) { 2979 Value *Ptr0; 2980 Value *PtrN; 2981 if (CurrentOrder.empty()) { 2982 Ptr0 = PointerOps.front(); 2983 PtrN = PointerOps.back(); 2984 } else { 2985 Ptr0 = PointerOps[CurrentOrder.front()]; 2986 PtrN = PointerOps[CurrentOrder.back()]; 2987 } 2988 const SCEV *Scev0 = SE->getSCEV(Ptr0); 2989 const SCEV *ScevN = SE->getSCEV(PtrN); 2990 const auto *Diff = 2991 dyn_cast<SCEVConstant>(SE->getMinusSCEV(ScevN, Scev0)); 2992 uint64_t Size = DL->getTypeAllocSize(ScalarTy); 2993 // Check that the sorted pointer operands are consecutive. 2994 if (Diff && Diff->getAPInt() == (VL.size() - 1) * Size) { 2995 if (CurrentOrder.empty()) { 2996 // Original stores are consecutive and does not require reordering. 2997 ++NumOpsWantToKeepOriginalOrder; 2998 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, 2999 UserTreeIdx, ReuseShuffleIndicies); 3000 TE->setOperandsInOrder(); 3001 buildTree_rec(Operands, Depth + 1, {TE, 0}); 3002 LLVM_DEBUG(dbgs() << "SLP: added a vector of stores.\n"); 3003 } else { 3004 // Need to reorder. 3005 auto I = NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first; 3006 ++(I->getSecond()); 3007 TreeEntry *TE = 3008 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 3009 ReuseShuffleIndicies, I->getFirst()); 3010 TE->setOperandsInOrder(); 3011 buildTree_rec(Operands, Depth + 1, {TE, 0}); 3012 LLVM_DEBUG(dbgs() << "SLP: added a vector of jumbled stores.\n"); 3013 } 3014 return; 3015 } 3016 } 3017 3018 BS.cancelScheduling(VL, VL0); 3019 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 3020 ReuseShuffleIndicies); 3021 LLVM_DEBUG(dbgs() << "SLP: Non-consecutive store.\n"); 3022 return; 3023 } 3024 case Instruction::Call: { 3025 // Check if the calls are all to the same vectorizable intrinsic or 3026 // library function. 3027 CallInst *CI = cast<CallInst>(VL0); 3028 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI); 3029 3030 VFShape Shape = VFShape::get( 3031 *CI, {static_cast<unsigned int>(VL.size()), false /*Scalable*/}, 3032 false /*HasGlobalPred*/); 3033 Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape); 3034 3035 if (!VecFunc && !isTriviallyVectorizable(ID)) { 3036 BS.cancelScheduling(VL, VL0); 3037 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 3038 ReuseShuffleIndicies); 3039 LLVM_DEBUG(dbgs() << "SLP: Non-vectorizable call.\n"); 3040 return; 3041 } 3042 Function *F = CI->getCalledFunction(); 3043 unsigned NumArgs = CI->getNumArgOperands(); 3044 SmallVector<Value*, 4> ScalarArgs(NumArgs, nullptr); 3045 for (unsigned j = 0; j != NumArgs; ++j) 3046 if (hasVectorInstrinsicScalarOpd(ID, j)) 3047 ScalarArgs[j] = CI->getArgOperand(j); 3048 for (Value *V : VL) { 3049 CallInst *CI2 = dyn_cast<CallInst>(V); 3050 if (!CI2 || CI2->getCalledFunction() != F || 3051 getVectorIntrinsicIDForCall(CI2, TLI) != ID || 3052 (VecFunc && 3053 VecFunc != VFDatabase(*CI2).getVectorizedFunction(Shape)) || 3054 !CI->hasIdenticalOperandBundleSchema(*CI2)) { 3055 BS.cancelScheduling(VL, VL0); 3056 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 3057 ReuseShuffleIndicies); 3058 LLVM_DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *V 3059 << "\n"); 3060 return; 3061 } 3062 // Some intrinsics have scalar arguments and should be same in order for 3063 // them to be vectorized. 3064 for (unsigned j = 0; j != NumArgs; ++j) { 3065 if (hasVectorInstrinsicScalarOpd(ID, j)) { 3066 Value *A1J = CI2->getArgOperand(j); 3067 if (ScalarArgs[j] != A1J) { 3068 BS.cancelScheduling(VL, VL0); 3069 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 3070 ReuseShuffleIndicies); 3071 LLVM_DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CI 3072 << " argument " << ScalarArgs[j] << "!=" << A1J 3073 << "\n"); 3074 return; 3075 } 3076 } 3077 } 3078 // Verify that the bundle operands are identical between the two calls. 3079 if (CI->hasOperandBundles() && 3080 !std::equal(CI->op_begin() + CI->getBundleOperandsStartIndex(), 3081 CI->op_begin() + CI->getBundleOperandsEndIndex(), 3082 CI2->op_begin() + CI2->getBundleOperandsStartIndex())) { 3083 BS.cancelScheduling(VL, VL0); 3084 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 3085 ReuseShuffleIndicies); 3086 LLVM_DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" 3087 << *CI << "!=" << *V << '\n'); 3088 return; 3089 } 3090 } 3091 3092 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 3093 ReuseShuffleIndicies); 3094 TE->setOperandsInOrder(); 3095 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) { 3096 ValueList Operands; 3097 // Prepare the operand vector. 3098 for (Value *V : VL) { 3099 auto *CI2 = cast<CallInst>(V); 3100 Operands.push_back(CI2->getArgOperand(i)); 3101 } 3102 buildTree_rec(Operands, Depth + 1, {TE, i}); 3103 } 3104 return; 3105 } 3106 case Instruction::ShuffleVector: { 3107 // If this is not an alternate sequence of opcode like add-sub 3108 // then do not vectorize this instruction. 3109 if (!S.isAltShuffle()) { 3110 BS.cancelScheduling(VL, VL0); 3111 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 3112 ReuseShuffleIndicies); 3113 LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n"); 3114 return; 3115 } 3116 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 3117 ReuseShuffleIndicies); 3118 LLVM_DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n"); 3119 3120 // Reorder operands if reordering would enable vectorization. 3121 if (isa<BinaryOperator>(VL0)) { 3122 ValueList Left, Right; 3123 reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this); 3124 TE->setOperand(0, Left); 3125 TE->setOperand(1, Right); 3126 buildTree_rec(Left, Depth + 1, {TE, 0}); 3127 buildTree_rec(Right, Depth + 1, {TE, 1}); 3128 return; 3129 } 3130 3131 TE->setOperandsInOrder(); 3132 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) { 3133 ValueList Operands; 3134 // Prepare the operand vector. 3135 for (Value *V : VL) 3136 Operands.push_back(cast<Instruction>(V)->getOperand(i)); 3137 3138 buildTree_rec(Operands, Depth + 1, {TE, i}); 3139 } 3140 return; 3141 } 3142 default: 3143 BS.cancelScheduling(VL, VL0); 3144 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 3145 ReuseShuffleIndicies); 3146 LLVM_DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n"); 3147 return; 3148 } 3149} 3150 3151unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const { 3152 unsigned N = 1; 3153 Type *EltTy = T; 3154 3155 while (isa<StructType>(EltTy) || isa<ArrayType>(EltTy) || 3156 isa<VectorType>(EltTy)) { 3157 if (auto *ST = dyn_cast<StructType>(EltTy)) { 3158 // Check that struct is homogeneous. 3159 for (const auto *Ty : ST->elements()) 3160 if (Ty != *ST->element_begin()) 3161 return 0; 3162 N *= ST->getNumElements(); 3163 EltTy = *ST->element_begin(); 3164 } else if (auto *AT = dyn_cast<ArrayType>(EltTy)) { 3165 N *= AT->getNumElements(); 3166 EltTy = AT->getElementType(); 3167 } else { 3168 auto *VT = cast<VectorType>(EltTy); 3169 N *= VT->getNumElements(); 3170 EltTy = VT->getElementType(); 3171 } 3172 } 3173 3174 if (!isValidElementType(EltTy)) 3175 return 0; 3176 uint64_t VTSize = DL.getTypeStoreSizeInBits(FixedVectorType::get(EltTy, N)); 3177 if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T)) 3178 return 0; 3179 return N; 3180} 3181 3182bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue, 3183 SmallVectorImpl<unsigned> &CurrentOrder) const { 3184 Instruction *E0 = cast<Instruction>(OpValue); 3185 assert(E0->getOpcode() == Instruction::ExtractElement || 3186 E0->getOpcode() == Instruction::ExtractValue); 3187 assert(E0->getOpcode() == getSameOpcode(VL).getOpcode() && "Invalid opcode"); 3188 // Check if all of the extracts come from the same vector and from the 3189 // correct offset. 3190 Value *Vec = E0->getOperand(0); 3191 3192 CurrentOrder.clear(); 3193 3194 // We have to extract from a vector/aggregate with the same number of elements. 3195 unsigned NElts; 3196 if (E0->getOpcode() == Instruction::ExtractValue) { 3197 const DataLayout &DL = E0->getModule()->getDataLayout(); 3198 NElts = canMapToVector(Vec->getType(), DL); 3199 if (!NElts) 3200 return false; 3201 // Check if load can be rewritten as load of vector. 3202 LoadInst *LI = dyn_cast<LoadInst>(Vec); 3203 if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size())) 3204 return false; 3205 } else { 3206 NElts = cast<VectorType>(Vec->getType())->getNumElements(); 3207 } 3208 3209 if (NElts != VL.size()) 3210 return false; 3211 3212 // Check that all of the indices extract from the correct offset. 3213 bool ShouldKeepOrder = true; 3214 unsigned E = VL.size(); 3215 // Assign to all items the initial value E + 1 so we can check if the extract 3216 // instruction index was used already. 3217 // Also, later we can check that all the indices are used and we have a 3218 // consecutive access in the extract instructions, by checking that no 3219 // element of CurrentOrder still has value E + 1. 3220 CurrentOrder.assign(E, E + 1); 3221 unsigned I = 0; 3222 for (; I < E; ++I) { 3223 auto *Inst = cast<Instruction>(VL[I]); 3224 if (Inst->getOperand(0) != Vec) 3225 break; 3226 Optional<unsigned> Idx = getExtractIndex(Inst); 3227 if (!Idx) 3228 break; 3229 const unsigned ExtIdx = *Idx; 3230 if (ExtIdx != I) { 3231 if (ExtIdx >= E || CurrentOrder[ExtIdx] != E + 1) 3232 break; 3233 ShouldKeepOrder = false; 3234 CurrentOrder[ExtIdx] = I; 3235 } else { 3236 if (CurrentOrder[I] != E + 1) 3237 break; 3238 CurrentOrder[I] = I; 3239 } 3240 } 3241 if (I < E) { 3242 CurrentOrder.clear(); 3243 return false; 3244 } 3245 3246 return ShouldKeepOrder; 3247} 3248 3249bool BoUpSLP::areAllUsersVectorized(Instruction *I) const { 3250 return I->hasOneUse() || 3251 std::all_of(I->user_begin(), I->user_end(), [this](User *U) { 3252 return ScalarToTreeEntry.count(U) > 0; 3253 }); 3254} 3255 3256static std::pair<unsigned, unsigned> 3257getVectorCallCosts(CallInst *CI, VectorType *VecTy, TargetTransformInfo *TTI, 3258 TargetLibraryInfo *TLI) { 3259 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI); 3260 3261 // Calculate the cost of the scalar and vector calls. 3262 IntrinsicCostAttributes CostAttrs(ID, *CI, VecTy->getNumElements()); 3263 int IntrinsicCost = 3264 TTI->getIntrinsicInstrCost(CostAttrs, TTI::TCK_RecipThroughput); 3265 3266 auto Shape = 3267 VFShape::get(*CI, {static_cast<unsigned>(VecTy->getNumElements()), false}, 3268 false /*HasGlobalPred*/); 3269 Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape); 3270 int LibCost = IntrinsicCost; 3271 if (!CI->isNoBuiltin() && VecFunc) { 3272 // Calculate the cost of the vector library call. 3273 SmallVector<Type *, 4> VecTys; 3274 for (Use &Arg : CI->args()) 3275 VecTys.push_back( 3276 FixedVectorType::get(Arg->getType(), VecTy->getNumElements())); 3277 3278 // If the corresponding vector call is cheaper, return its cost. 3279 LibCost = TTI->getCallInstrCost(nullptr, VecTy, VecTys, 3280 TTI::TCK_RecipThroughput); 3281 } 3282 return {IntrinsicCost, LibCost}; 3283} 3284 3285int BoUpSLP::getEntryCost(TreeEntry *E) { 3286 ArrayRef<Value*> VL = E->Scalars; 3287 3288 Type *ScalarTy = VL[0]->getType(); 3289 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0])) 3290 ScalarTy = SI->getValueOperand()->getType(); 3291 else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0])) 3292 ScalarTy = CI->getOperand(0)->getType(); 3293 auto *VecTy = FixedVectorType::get(ScalarTy, VL.size()); 3294 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; 3295 3296 // If we have computed a smaller type for the expression, update VecTy so 3297 // that the costs will be accurate. 3298 if (MinBWs.count(VL[0])) 3299 VecTy = FixedVectorType::get( 3300 IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size()); 3301 3302 unsigned ReuseShuffleNumbers = E->ReuseShuffleIndices.size(); 3303 bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty(); 3304 int ReuseShuffleCost = 0; 3305 if (NeedToShuffleReuses) { 3306 ReuseShuffleCost = 3307 TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, VecTy); 3308 } 3309 if (E->State == TreeEntry::NeedToGather) { 3310 if (allConstant(VL)) 3311 return 0; 3312 if (isSplat(VL)) { 3313 return ReuseShuffleCost + 3314 TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0); 3315 } 3316 if (E->getOpcode() == Instruction::ExtractElement && 3317 allSameType(VL) && allSameBlock(VL)) { 3318 Optional<TargetTransformInfo::ShuffleKind> ShuffleKind = isShuffle(VL); 3319 if (ShuffleKind.hasValue()) { 3320 int Cost = TTI->getShuffleCost(ShuffleKind.getValue(), VecTy); 3321 for (auto *V : VL) { 3322 // If all users of instruction are going to be vectorized and this 3323 // instruction itself is not going to be vectorized, consider this 3324 // instruction as dead and remove its cost from the final cost of the 3325 // vectorized tree. 3326 if (areAllUsersVectorized(cast<Instruction>(V)) && 3327 !ScalarToTreeEntry.count(V)) { 3328 auto *IO = cast<ConstantInt>( 3329 cast<ExtractElementInst>(V)->getIndexOperand()); 3330 Cost -= TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, 3331 IO->getZExtValue()); 3332 } 3333 } 3334 return ReuseShuffleCost + Cost; 3335 } 3336 } 3337 return ReuseShuffleCost + getGatherCost(VL); 3338 } 3339 assert(E->State == TreeEntry::Vectorize && "Unhandled state"); 3340 assert(E->getOpcode() && allSameType(VL) && allSameBlock(VL) && "Invalid VL"); 3341 Instruction *VL0 = E->getMainOp(); 3342 unsigned ShuffleOrOp = 3343 E->isAltShuffle() ? (unsigned)Instruction::ShuffleVector : E->getOpcode(); 3344 switch (ShuffleOrOp) { 3345 case Instruction::PHI: 3346 return 0; 3347 3348 case Instruction::ExtractValue: 3349 case Instruction::ExtractElement: { 3350 if (NeedToShuffleReuses) { 3351 unsigned Idx = 0; 3352 for (unsigned I : E->ReuseShuffleIndices) { 3353 if (ShuffleOrOp == Instruction::ExtractElement) { 3354 auto *IO = cast<ConstantInt>( 3355 cast<ExtractElementInst>(VL[I])->getIndexOperand()); 3356 Idx = IO->getZExtValue(); 3357 ReuseShuffleCost -= TTI->getVectorInstrCost( 3358 Instruction::ExtractElement, VecTy, Idx); 3359 } else { 3360 ReuseShuffleCost -= TTI->getVectorInstrCost( 3361 Instruction::ExtractElement, VecTy, Idx); 3362 ++Idx; 3363 } 3364 } 3365 Idx = ReuseShuffleNumbers; 3366 for (Value *V : VL) { 3367 if (ShuffleOrOp == Instruction::ExtractElement) { 3368 auto *IO = cast<ConstantInt>( 3369 cast<ExtractElementInst>(V)->getIndexOperand()); 3370 Idx = IO->getZExtValue(); 3371 } else { 3372 --Idx; 3373 } 3374 ReuseShuffleCost += 3375 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, Idx); 3376 } 3377 } 3378 int DeadCost = ReuseShuffleCost; 3379 if (!E->ReorderIndices.empty()) { 3380 // TODO: Merge this shuffle with the ReuseShuffleCost. 3381 DeadCost += TTI->getShuffleCost( 3382 TargetTransformInfo::SK_PermuteSingleSrc, VecTy); 3383 } 3384 for (unsigned i = 0, e = VL.size(); i < e; ++i) { 3385 Instruction *E = cast<Instruction>(VL[i]); 3386 // If all users are going to be vectorized, instruction can be 3387 // considered as dead. 3388 // The same, if have only one user, it will be vectorized for sure. 3389 if (areAllUsersVectorized(E)) { 3390 // Take credit for instruction that will become dead. 3391 if (E->hasOneUse()) { 3392 Instruction *Ext = E->user_back(); 3393 if ((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) && 3394 all_of(Ext->users(), 3395 [](User *U) { return isa<GetElementPtrInst>(U); })) { 3396 // Use getExtractWithExtendCost() to calculate the cost of 3397 // extractelement/ext pair. 3398 DeadCost -= TTI->getExtractWithExtendCost( 3399 Ext->getOpcode(), Ext->getType(), VecTy, i); 3400 // Add back the cost of s|zext which is subtracted separately. 3401 DeadCost += TTI->getCastInstrCost( 3402 Ext->getOpcode(), Ext->getType(), E->getType(), CostKind, 3403 Ext); 3404 continue; 3405 } 3406 } 3407 DeadCost -= 3408 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i); 3409 } 3410 } 3411 return DeadCost; 3412 } 3413 case Instruction::ZExt: 3414 case Instruction::SExt: 3415 case Instruction::FPToUI: 3416 case Instruction::FPToSI: 3417 case Instruction::FPExt: 3418 case Instruction::PtrToInt: 3419 case Instruction::IntToPtr: 3420 case Instruction::SIToFP: 3421 case Instruction::UIToFP: 3422 case Instruction::Trunc: 3423 case Instruction::FPTrunc: 3424 case Instruction::BitCast: { 3425 Type *SrcTy = VL0->getOperand(0)->getType(); 3426 int ScalarEltCost = 3427 TTI->getCastInstrCost(E->getOpcode(), ScalarTy, SrcTy, CostKind, 3428 VL0); 3429 if (NeedToShuffleReuses) { 3430 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost; 3431 } 3432 3433 // Calculate the cost of this instruction. 3434 int ScalarCost = VL.size() * ScalarEltCost; 3435 3436 auto *SrcVecTy = FixedVectorType::get(SrcTy, VL.size()); 3437 int VecCost = 0; 3438 // Check if the values are candidates to demote. 3439 if (!MinBWs.count(VL0) || VecTy != SrcVecTy) { 3440 VecCost = ReuseShuffleCost + 3441 TTI->getCastInstrCost(E->getOpcode(), VecTy, SrcVecTy, 3442 CostKind, VL0); 3443 } 3444 return VecCost - ScalarCost; 3445 } 3446 case Instruction::FCmp: 3447 case Instruction::ICmp: 3448 case Instruction::Select: { 3449 // Calculate the cost of this instruction. 3450 int ScalarEltCost = TTI->getCmpSelInstrCost(E->getOpcode(), ScalarTy, 3451 Builder.getInt1Ty(), 3452 CostKind, VL0); 3453 if (NeedToShuffleReuses) { 3454 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost; 3455 } 3456 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(), VL.size()); 3457 int ScalarCost = VecTy->getNumElements() * ScalarEltCost; 3458 int VecCost = TTI->getCmpSelInstrCost(E->getOpcode(), VecTy, MaskTy, 3459 CostKind, VL0); 3460 return ReuseShuffleCost + VecCost - ScalarCost; 3461 } 3462 case Instruction::FNeg: 3463 case Instruction::Add: 3464 case Instruction::FAdd: 3465 case Instruction::Sub: 3466 case Instruction::FSub: 3467 case Instruction::Mul: 3468 case Instruction::FMul: 3469 case Instruction::UDiv: 3470 case Instruction::SDiv: 3471 case Instruction::FDiv: 3472 case Instruction::URem: 3473 case Instruction::SRem: 3474 case Instruction::FRem: 3475 case Instruction::Shl: 3476 case Instruction::LShr: 3477 case Instruction::AShr: 3478 case Instruction::And: 3479 case Instruction::Or: 3480 case Instruction::Xor: { 3481 // Certain instructions can be cheaper to vectorize if they have a 3482 // constant second vector operand. 3483 TargetTransformInfo::OperandValueKind Op1VK = 3484 TargetTransformInfo::OK_AnyValue; 3485 TargetTransformInfo::OperandValueKind Op2VK = 3486 TargetTransformInfo::OK_UniformConstantValue; 3487 TargetTransformInfo::OperandValueProperties Op1VP = 3488 TargetTransformInfo::OP_None; 3489 TargetTransformInfo::OperandValueProperties Op2VP = 3490 TargetTransformInfo::OP_PowerOf2; 3491 3492 // If all operands are exactly the same ConstantInt then set the 3493 // operand kind to OK_UniformConstantValue. 3494 // If instead not all operands are constants, then set the operand kind 3495 // to OK_AnyValue. If all operands are constants but not the same, 3496 // then set the operand kind to OK_NonUniformConstantValue. 3497 ConstantInt *CInt0 = nullptr; 3498 for (unsigned i = 0, e = VL.size(); i < e; ++i) { 3499 const Instruction *I = cast<Instruction>(VL[i]); 3500 unsigned OpIdx = isa<BinaryOperator>(I) ? 1 : 0; 3501 ConstantInt *CInt = dyn_cast<ConstantInt>(I->getOperand(OpIdx)); 3502 if (!CInt) { 3503 Op2VK = TargetTransformInfo::OK_AnyValue; 3504 Op2VP = TargetTransformInfo::OP_None; 3505 break; 3506 } 3507 if (Op2VP == TargetTransformInfo::OP_PowerOf2 && 3508 !CInt->getValue().isPowerOf2()) 3509 Op2VP = TargetTransformInfo::OP_None; 3510 if (i == 0) { 3511 CInt0 = CInt; 3512 continue; 3513 } 3514 if (CInt0 != CInt) 3515 Op2VK = TargetTransformInfo::OK_NonUniformConstantValue; 3516 } 3517 3518 SmallVector<const Value *, 4> Operands(VL0->operand_values()); 3519 int ScalarEltCost = TTI->getArithmeticInstrCost( 3520 E->getOpcode(), ScalarTy, CostKind, Op1VK, Op2VK, Op1VP, Op2VP, 3521 Operands, VL0); 3522 if (NeedToShuffleReuses) { 3523 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost; 3524 } 3525 int ScalarCost = VecTy->getNumElements() * ScalarEltCost; 3526 int VecCost = TTI->getArithmeticInstrCost( 3527 E->getOpcode(), VecTy, CostKind, Op1VK, Op2VK, Op1VP, Op2VP, 3528 Operands, VL0); 3529 return ReuseShuffleCost + VecCost - ScalarCost; 3530 } 3531 case Instruction::GetElementPtr: { 3532 TargetTransformInfo::OperandValueKind Op1VK = 3533 TargetTransformInfo::OK_AnyValue; 3534 TargetTransformInfo::OperandValueKind Op2VK = 3535 TargetTransformInfo::OK_UniformConstantValue; 3536 3537 int ScalarEltCost = 3538 TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, CostKind, 3539 Op1VK, Op2VK); 3540 if (NeedToShuffleReuses) { 3541 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost; 3542 } 3543 int ScalarCost = VecTy->getNumElements() * ScalarEltCost; 3544 int VecCost = 3545 TTI->getArithmeticInstrCost(Instruction::Add, VecTy, CostKind, 3546 Op1VK, Op2VK); 3547 return ReuseShuffleCost + VecCost - ScalarCost; 3548 } 3549 case Instruction::Load: { 3550 // Cost of wide load - cost of scalar loads. 3551 Align alignment = cast<LoadInst>(VL0)->getAlign(); 3552 int ScalarEltCost = 3553 TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0, 3554 CostKind, VL0); 3555 if (NeedToShuffleReuses) { 3556 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost; 3557 } 3558 int ScalarLdCost = VecTy->getNumElements() * ScalarEltCost; 3559 int VecLdCost = 3560 TTI->getMemoryOpCost(Instruction::Load, VecTy, alignment, 0, 3561 CostKind, VL0); 3562 if (!E->ReorderIndices.empty()) { 3563 // TODO: Merge this shuffle with the ReuseShuffleCost. 3564 VecLdCost += TTI->getShuffleCost( 3565 TargetTransformInfo::SK_PermuteSingleSrc, VecTy); 3566 } 3567 return ReuseShuffleCost + VecLdCost - ScalarLdCost; 3568 } 3569 case Instruction::Store: { 3570 // We know that we can merge the stores. Calculate the cost. 3571 bool IsReorder = !E->ReorderIndices.empty(); 3572 auto *SI = 3573 cast<StoreInst>(IsReorder ? VL[E->ReorderIndices.front()] : VL0); 3574 Align Alignment = SI->getAlign(); 3575 int ScalarEltCost = 3576 TTI->getMemoryOpCost(Instruction::Store, ScalarTy, Alignment, 0, 3577 CostKind, VL0); 3578 if (NeedToShuffleReuses) 3579 ReuseShuffleCost = -(ReuseShuffleNumbers - VL.size()) * ScalarEltCost; 3580 int ScalarStCost = VecTy->getNumElements() * ScalarEltCost; 3581 int VecStCost = TTI->getMemoryOpCost(Instruction::Store, 3582 VecTy, Alignment, 0, CostKind, VL0); 3583 if (IsReorder) { 3584 // TODO: Merge this shuffle with the ReuseShuffleCost. 3585 VecStCost += TTI->getShuffleCost( 3586 TargetTransformInfo::SK_PermuteSingleSrc, VecTy); 3587 } 3588 return ReuseShuffleCost + VecStCost - ScalarStCost; 3589 } 3590 case Instruction::Call: { 3591 CallInst *CI = cast<CallInst>(VL0); 3592 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI); 3593 3594 // Calculate the cost of the scalar and vector calls. 3595 IntrinsicCostAttributes CostAttrs(ID, *CI, 1, 1); 3596 int ScalarEltCost = TTI->getIntrinsicInstrCost(CostAttrs, CostKind); 3597 if (NeedToShuffleReuses) { 3598 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost; 3599 } 3600 int ScalarCallCost = VecTy->getNumElements() * ScalarEltCost; 3601 3602 auto VecCallCosts = getVectorCallCosts(CI, VecTy, TTI, TLI); 3603 int VecCallCost = std::min(VecCallCosts.first, VecCallCosts.second); 3604 3605 LLVM_DEBUG(dbgs() << "SLP: Call cost " << VecCallCost - ScalarCallCost 3606 << " (" << VecCallCost << "-" << ScalarCallCost << ")" 3607 << " for " << *CI << "\n"); 3608 3609 return ReuseShuffleCost + VecCallCost - ScalarCallCost; 3610 } 3611 case Instruction::ShuffleVector: { 3612 assert(E->isAltShuffle() && 3613 ((Instruction::isBinaryOp(E->getOpcode()) && 3614 Instruction::isBinaryOp(E->getAltOpcode())) || 3615 (Instruction::isCast(E->getOpcode()) && 3616 Instruction::isCast(E->getAltOpcode()))) && 3617 "Invalid Shuffle Vector Operand"); 3618 int ScalarCost = 0; 3619 if (NeedToShuffleReuses) { 3620 for (unsigned Idx : E->ReuseShuffleIndices) { 3621 Instruction *I = cast<Instruction>(VL[Idx]); 3622 ReuseShuffleCost -= TTI->getInstructionCost(I, CostKind); 3623 } 3624 for (Value *V : VL) { 3625 Instruction *I = cast<Instruction>(V); 3626 ReuseShuffleCost += TTI->getInstructionCost(I, CostKind); 3627 } 3628 } 3629 for (Value *V : VL) { 3630 Instruction *I = cast<Instruction>(V); 3631 assert(E->isOpcodeOrAlt(I) && "Unexpected main/alternate opcode"); 3632 ScalarCost += TTI->getInstructionCost(I, CostKind); 3633 } 3634 // VecCost is equal to sum of the cost of creating 2 vectors 3635 // and the cost of creating shuffle. 3636 int VecCost = 0; 3637 if (Instruction::isBinaryOp(E->getOpcode())) { 3638 VecCost = TTI->getArithmeticInstrCost(E->getOpcode(), VecTy, CostKind); 3639 VecCost += TTI->getArithmeticInstrCost(E->getAltOpcode(), VecTy, 3640 CostKind); 3641 } else { 3642 Type *Src0SclTy = E->getMainOp()->getOperand(0)->getType(); 3643 Type *Src1SclTy = E->getAltOp()->getOperand(0)->getType(); 3644 auto *Src0Ty = FixedVectorType::get(Src0SclTy, VL.size()); 3645 auto *Src1Ty = FixedVectorType::get(Src1SclTy, VL.size()); 3646 VecCost = TTI->getCastInstrCost(E->getOpcode(), VecTy, Src0Ty, 3647 CostKind); 3648 VecCost += TTI->getCastInstrCost(E->getAltOpcode(), VecTy, Src1Ty, 3649 CostKind); 3650 } 3651 VecCost += TTI->getShuffleCost(TargetTransformInfo::SK_Select, VecTy, 0); 3652 return ReuseShuffleCost + VecCost - ScalarCost; 3653 } 3654 default: 3655 llvm_unreachable("Unknown instruction"); 3656 } 3657} 3658 3659bool BoUpSLP::isFullyVectorizableTinyTree() const { 3660 LLVM_DEBUG(dbgs() << "SLP: Check whether the tree with height " 3661 << VectorizableTree.size() << " is fully vectorizable .\n"); 3662 3663 // We only handle trees of heights 1 and 2. 3664 if (VectorizableTree.size() == 1 && 3665 VectorizableTree[0]->State == TreeEntry::Vectorize) 3666 return true; 3667 3668 if (VectorizableTree.size() != 2) 3669 return false; 3670 3671 // Handle splat and all-constants stores. 3672 if (VectorizableTree[0]->State == TreeEntry::Vectorize && 3673 (allConstant(VectorizableTree[1]->Scalars) || 3674 isSplat(VectorizableTree[1]->Scalars))) 3675 return true; 3676 3677 // Gathering cost would be too much for tiny trees. 3678 if (VectorizableTree[0]->State == TreeEntry::NeedToGather || 3679 VectorizableTree[1]->State == TreeEntry::NeedToGather) 3680 return false; 3681 3682 return true; 3683} 3684 3685static bool isLoadCombineCandidateImpl(Value *Root, unsigned NumElts, 3686 TargetTransformInfo *TTI) { 3687 // Look past the root to find a source value. Arbitrarily follow the 3688 // path through operand 0 of any 'or'. Also, peek through optional 3689 // shift-left-by-constant. 3690 Value *ZextLoad = Root; 3691 while (!isa<ConstantExpr>(ZextLoad) && 3692 (match(ZextLoad, m_Or(m_Value(), m_Value())) || 3693 match(ZextLoad, m_Shl(m_Value(), m_Constant())))) 3694 ZextLoad = cast<BinaryOperator>(ZextLoad)->getOperand(0); 3695 3696 // Check if the input is an extended load of the required or/shift expression. 3697 Value *LoadPtr; 3698 if (ZextLoad == Root || !match(ZextLoad, m_ZExt(m_Load(m_Value(LoadPtr))))) 3699 return false; 3700 3701 // Require that the total load bit width is a legal integer type. 3702 // For example, <8 x i8> --> i64 is a legal integer on a 64-bit target. 3703 // But <16 x i8> --> i128 is not, so the backend probably can't reduce it. 3704 Type *SrcTy = LoadPtr->getType()->getPointerElementType(); 3705 unsigned LoadBitWidth = SrcTy->getIntegerBitWidth() * NumElts; 3706 if (!TTI->isTypeLegal(IntegerType::get(Root->getContext(), LoadBitWidth))) 3707 return false; 3708 3709 // Everything matched - assume that we can fold the whole sequence using 3710 // load combining. 3711 LLVM_DEBUG(dbgs() << "SLP: Assume load combining for tree starting at " 3712 << *(cast<Instruction>(Root)) << "\n"); 3713 3714 return true; 3715} 3716 3717bool BoUpSLP::isLoadCombineReductionCandidate(unsigned RdxOpcode) const { 3718 if (RdxOpcode != Instruction::Or) 3719 return false; 3720 3721 unsigned NumElts = VectorizableTree[0]->Scalars.size(); 3722 Value *FirstReduced = VectorizableTree[0]->Scalars[0]; 3723 return isLoadCombineCandidateImpl(FirstReduced, NumElts, TTI); 3724} 3725 3726bool BoUpSLP::isLoadCombineCandidate() const { 3727 // Peek through a final sequence of stores and check if all operations are 3728 // likely to be load-combined. 3729 unsigned NumElts = VectorizableTree[0]->Scalars.size(); 3730 for (Value *Scalar : VectorizableTree[0]->Scalars) { 3731 Value *X; 3732 if (!match(Scalar, m_Store(m_Value(X), m_Value())) || 3733 !isLoadCombineCandidateImpl(X, NumElts, TTI)) 3734 return false; 3735 } 3736 return true; 3737} 3738 3739bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() const { 3740 // We can vectorize the tree if its size is greater than or equal to the 3741 // minimum size specified by the MinTreeSize command line option. 3742 if (VectorizableTree.size() >= MinTreeSize) 3743 return false; 3744 3745 // If we have a tiny tree (a tree whose size is less than MinTreeSize), we 3746 // can vectorize it if we can prove it fully vectorizable. 3747 if (isFullyVectorizableTinyTree()) 3748 return false; 3749 3750 assert(VectorizableTree.empty() 3751 ? ExternalUses.empty() 3752 : true && "We shouldn't have any external users"); 3753 3754 // Otherwise, we can't vectorize the tree. It is both tiny and not fully 3755 // vectorizable. 3756 return true; 3757} 3758 3759int BoUpSLP::getSpillCost() const { 3760 // Walk from the bottom of the tree to the top, tracking which values are 3761 // live. When we see a call instruction that is not part of our tree, 3762 // query TTI to see if there is a cost to keeping values live over it 3763 // (for example, if spills and fills are required). 3764 unsigned BundleWidth = VectorizableTree.front()->Scalars.size(); 3765 int Cost = 0; 3766 3767 SmallPtrSet<Instruction*, 4> LiveValues; 3768 Instruction *PrevInst = nullptr; 3769 3770 for (const auto &TEPtr : VectorizableTree) { 3771 Instruction *Inst = dyn_cast<Instruction>(TEPtr->Scalars[0]); 3772 if (!Inst) 3773 continue; 3774 3775 if (!PrevInst) { 3776 PrevInst = Inst; 3777 continue; 3778 } 3779 3780 // Update LiveValues. 3781 LiveValues.erase(PrevInst); 3782 for (auto &J : PrevInst->operands()) { 3783 if (isa<Instruction>(&*J) && getTreeEntry(&*J)) 3784 LiveValues.insert(cast<Instruction>(&*J)); 3785 } 3786 3787 LLVM_DEBUG({ 3788 dbgs() << "SLP: #LV: " << LiveValues.size(); 3789 for (auto *X : LiveValues) 3790 dbgs() << " " << X->getName(); 3791 dbgs() << ", Looking at "; 3792 Inst->dump(); 3793 }); 3794 3795 // Now find the sequence of instructions between PrevInst and Inst. 3796 unsigned NumCalls = 0; 3797 BasicBlock::reverse_iterator InstIt = ++Inst->getIterator().getReverse(), 3798 PrevInstIt = 3799 PrevInst->getIterator().getReverse(); 3800 while (InstIt != PrevInstIt) { 3801 if (PrevInstIt == PrevInst->getParent()->rend()) { 3802 PrevInstIt = Inst->getParent()->rbegin(); 3803 continue; 3804 } 3805 3806 // Debug information does not impact spill cost. 3807 if ((isa<CallInst>(&*PrevInstIt) && 3808 !isa<DbgInfoIntrinsic>(&*PrevInstIt)) && 3809 &*PrevInstIt != PrevInst) 3810 NumCalls++; 3811 3812 ++PrevInstIt; 3813 } 3814 3815 if (NumCalls) { 3816 SmallVector<Type*, 4> V; 3817 for (auto *II : LiveValues) 3818 V.push_back(FixedVectorType::get(II->getType(), BundleWidth)); 3819 Cost += NumCalls * TTI->getCostOfKeepingLiveOverCall(V); 3820 } 3821 3822 PrevInst = Inst; 3823 } 3824 3825 return Cost; 3826} 3827 3828int BoUpSLP::getTreeCost() { 3829 int Cost = 0; 3830 LLVM_DEBUG(dbgs() << "SLP: Calculating cost for tree of size " 3831 << VectorizableTree.size() << ".\n"); 3832 3833 unsigned BundleWidth = VectorizableTree[0]->Scalars.size(); 3834 3835 for (unsigned I = 0, E = VectorizableTree.size(); I < E; ++I) { 3836 TreeEntry &TE = *VectorizableTree[I].get(); 3837 3838 // We create duplicate tree entries for gather sequences that have multiple 3839 // uses. However, we should not compute the cost of duplicate sequences. 3840 // For example, if we have a build vector (i.e., insertelement sequence) 3841 // that is used by more than one vector instruction, we only need to 3842 // compute the cost of the insertelement instructions once. The redundant 3843 // instructions will be eliminated by CSE. 3844 // 3845 // We should consider not creating duplicate tree entries for gather 3846 // sequences, and instead add additional edges to the tree representing 3847 // their uses. Since such an approach results in fewer total entries, 3848 // existing heuristics based on tree size may yield different results. 3849 // 3850 if (TE.State == TreeEntry::NeedToGather && 3851 std::any_of(std::next(VectorizableTree.begin(), I + 1), 3852 VectorizableTree.end(), 3853 [TE](const std::unique_ptr<TreeEntry> &EntryPtr) { 3854 return EntryPtr->State == TreeEntry::NeedToGather && 3855 EntryPtr->isSame(TE.Scalars); 3856 })) 3857 continue; 3858 3859 int C = getEntryCost(&TE); 3860 LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C 3861 << " for bundle that starts with " << *TE.Scalars[0] 3862 << ".\n"); 3863 Cost += C; 3864 } 3865 3866 SmallPtrSet<Value *, 16> ExtractCostCalculated; 3867 int ExtractCost = 0; 3868 for (ExternalUser &EU : ExternalUses) { 3869 // We only add extract cost once for the same scalar. 3870 if (!ExtractCostCalculated.insert(EU.Scalar).second) 3871 continue; 3872 3873 // Uses by ephemeral values are free (because the ephemeral value will be 3874 // removed prior to code generation, and so the extraction will be 3875 // removed as well). 3876 if (EphValues.count(EU.User)) 3877 continue; 3878 3879 // If we plan to rewrite the tree in a smaller type, we will need to sign 3880 // extend the extracted value back to the original type. Here, we account 3881 // for the extract and the added cost of the sign extend if needed. 3882 auto *VecTy = FixedVectorType::get(EU.Scalar->getType(), BundleWidth); 3883 auto *ScalarRoot = VectorizableTree[0]->Scalars[0]; 3884 if (MinBWs.count(ScalarRoot)) { 3885 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first); 3886 auto Extend = 3887 MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt; 3888 VecTy = FixedVectorType::get(MinTy, BundleWidth); 3889 ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(), 3890 VecTy, EU.Lane); 3891 } else { 3892 ExtractCost += 3893 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane); 3894 } 3895 } 3896 3897 int SpillCost = getSpillCost(); 3898 Cost += SpillCost + ExtractCost; 3899 3900 std::string Str; 3901 { 3902 raw_string_ostream OS(Str); 3903 OS << "SLP: Spill Cost = " << SpillCost << ".\n" 3904 << "SLP: Extract Cost = " << ExtractCost << ".\n" 3905 << "SLP: Total Cost = " << Cost << ".\n"; 3906 } 3907 LLVM_DEBUG(dbgs() << Str); 3908 3909 if (ViewSLPTree) 3910 ViewGraph(this, "SLP" + F->getName(), false, Str); 3911 3912 return Cost; 3913} 3914 3915int BoUpSLP::getGatherCost(VectorType *Ty, 3916 const DenseSet<unsigned> &ShuffledIndices) const { 3917 unsigned NumElts = Ty->getNumElements(); 3918 APInt DemandedElts = APInt::getNullValue(NumElts); 3919 for (unsigned i = 0; i < NumElts; ++i) 3920 if (!ShuffledIndices.count(i)) 3921 DemandedElts.setBit(i); 3922 int Cost = TTI->getScalarizationOverhead(Ty, DemandedElts, /*Insert*/ true, 3923 /*Extract*/ false); 3924 if (!ShuffledIndices.empty()) 3925 Cost += TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, Ty); 3926 return Cost; 3927} 3928 3929int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) const { 3930 // Find the type of the operands in VL. 3931 Type *ScalarTy = VL[0]->getType(); 3932 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0])) 3933 ScalarTy = SI->getValueOperand()->getType(); 3934 auto *VecTy = FixedVectorType::get(ScalarTy, VL.size()); 3935 // Find the cost of inserting/extracting values from the vector. 3936 // Check if the same elements are inserted several times and count them as 3937 // shuffle candidates. 3938 DenseSet<unsigned> ShuffledElements; 3939 DenseSet<Value *> UniqueElements; 3940 // Iterate in reverse order to consider insert elements with the high cost. 3941 for (unsigned I = VL.size(); I > 0; --I) { 3942 unsigned Idx = I - 1; 3943 if (!UniqueElements.insert(VL[Idx]).second) 3944 ShuffledElements.insert(Idx); 3945 } 3946 return getGatherCost(VecTy, ShuffledElements); 3947} 3948 3949// Perform operand reordering on the instructions in VL and return the reordered 3950// operands in Left and Right. 3951void BoUpSLP::reorderInputsAccordingToOpcode(ArrayRef<Value *> VL, 3952 SmallVectorImpl<Value *> &Left, 3953 SmallVectorImpl<Value *> &Right, 3954 const DataLayout &DL, 3955 ScalarEvolution &SE, 3956 const BoUpSLP &R) { 3957 if (VL.empty()) 3958 return; 3959 VLOperands Ops(VL, DL, SE, R); 3960 // Reorder the operands in place. 3961 Ops.reorder(); 3962 Left = Ops.getVL(0); 3963 Right = Ops.getVL(1); 3964} 3965 3966void BoUpSLP::setInsertPointAfterBundle(TreeEntry *E) { 3967 // Get the basic block this bundle is in. All instructions in the bundle 3968 // should be in this block. 3969 auto *Front = E->getMainOp(); 3970 auto *BB = Front->getParent(); 3971 assert(llvm::all_of(make_range(E->Scalars.begin(), E->Scalars.end()), 3972 [=](Value *V) -> bool { 3973 auto *I = cast<Instruction>(V); 3974 return !E->isOpcodeOrAlt(I) || I->getParent() == BB; 3975 })); 3976 3977 // The last instruction in the bundle in program order. 3978 Instruction *LastInst = nullptr; 3979 3980 // Find the last instruction. The common case should be that BB has been 3981 // scheduled, and the last instruction is VL.back(). So we start with 3982 // VL.back() and iterate over schedule data until we reach the end of the 3983 // bundle. The end of the bundle is marked by null ScheduleData. 3984 if (BlocksSchedules.count(BB)) { 3985 auto *Bundle = 3986 BlocksSchedules[BB]->getScheduleData(E->isOneOf(E->Scalars.back())); 3987 if (Bundle && Bundle->isPartOfBundle()) 3988 for (; Bundle; Bundle = Bundle->NextInBundle) 3989 if (Bundle->OpValue == Bundle->Inst) 3990 LastInst = Bundle->Inst; 3991 } 3992 3993 // LastInst can still be null at this point if there's either not an entry 3994 // for BB in BlocksSchedules or there's no ScheduleData available for 3995 // VL.back(). This can be the case if buildTree_rec aborts for various 3996 // reasons (e.g., the maximum recursion depth is reached, the maximum region 3997 // size is reached, etc.). ScheduleData is initialized in the scheduling 3998 // "dry-run". 3999 // 4000 // If this happens, we can still find the last instruction by brute force. We 4001 // iterate forwards from Front (inclusive) until we either see all 4002 // instructions in the bundle or reach the end of the block. If Front is the 4003 // last instruction in program order, LastInst will be set to Front, and we 4004 // will visit all the remaining instructions in the block. 4005 // 4006 // One of the reasons we exit early from buildTree_rec is to place an upper 4007 // bound on compile-time. Thus, taking an additional compile-time hit here is 4008 // not ideal. However, this should be exceedingly rare since it requires that 4009 // we both exit early from buildTree_rec and that the bundle be out-of-order 4010 // (causing us to iterate all the way to the end of the block). 4011 if (!LastInst) { 4012 SmallPtrSet<Value *, 16> Bundle(E->Scalars.begin(), E->Scalars.end()); 4013 for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) { 4014 if (Bundle.erase(&I) && E->isOpcodeOrAlt(&I)) 4015 LastInst = &I; 4016 if (Bundle.empty()) 4017 break; 4018 } 4019 } 4020 assert(LastInst && "Failed to find last instruction in bundle"); 4021 4022 // Set the insertion point after the last instruction in the bundle. Set the 4023 // debug location to Front. 4024 Builder.SetInsertPoint(BB, ++LastInst->getIterator()); 4025 Builder.SetCurrentDebugLocation(Front->getDebugLoc()); 4026} 4027 4028Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) { 4029 Value *Vec = UndefValue::get(Ty); 4030 // Generate the 'InsertElement' instruction. 4031 for (unsigned i = 0; i < Ty->getNumElements(); ++i) { 4032 Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i)); 4033 if (auto *Insrt = dyn_cast<InsertElementInst>(Vec)) { 4034 GatherSeq.insert(Insrt); 4035 CSEBlocks.insert(Insrt->getParent()); 4036 4037 // Add to our 'need-to-extract' list. 4038 if (TreeEntry *E = getTreeEntry(VL[i])) { 4039 // Find which lane we need to extract. 4040 int FoundLane = -1; 4041 for (unsigned Lane = 0, LE = E->Scalars.size(); Lane != LE; ++Lane) { 4042 // Is this the lane of the scalar that we are looking for ? 4043 if (E->Scalars[Lane] == VL[i]) { 4044 FoundLane = Lane; 4045 break; 4046 } 4047 } 4048 assert(FoundLane >= 0 && "Could not find the correct lane"); 4049 if (!E->ReuseShuffleIndices.empty()) { 4050 FoundLane = 4051 std::distance(E->ReuseShuffleIndices.begin(), 4052 llvm::find(E->ReuseShuffleIndices, FoundLane)); 4053 } 4054 ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane)); 4055 } 4056 } 4057 } 4058 4059 return Vec; 4060} 4061 4062Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) { 4063 InstructionsState S = getSameOpcode(VL); 4064 if (S.getOpcode()) { 4065 if (TreeEntry *E = getTreeEntry(S.OpValue)) { 4066 if (E->isSame(VL)) { 4067 Value *V = vectorizeTree(E); 4068 if (VL.size() == E->Scalars.size() && !E->ReuseShuffleIndices.empty()) { 4069 // We need to get the vectorized value but without shuffle. 4070 if (auto *SV = dyn_cast<ShuffleVectorInst>(V)) { 4071 V = SV->getOperand(0); 4072 } else { 4073 // Reshuffle to get only unique values. 4074 SmallVector<int, 4> UniqueIdxs; 4075 SmallSet<int, 4> UsedIdxs; 4076 for (int Idx : E->ReuseShuffleIndices) 4077 if (UsedIdxs.insert(Idx).second) 4078 UniqueIdxs.emplace_back(Idx); 4079 V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()), 4080 UniqueIdxs); 4081 } 4082 } 4083 return V; 4084 } 4085 } 4086 } 4087 4088 Type *ScalarTy = S.OpValue->getType(); 4089 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue)) 4090 ScalarTy = SI->getValueOperand()->getType(); 4091 4092 // Check that every instruction appears once in this bundle. 4093 SmallVector<int, 4> ReuseShuffleIndicies; 4094 SmallVector<Value *, 4> UniqueValues; 4095 if (VL.size() > 2) { 4096 DenseMap<Value *, unsigned> UniquePositions; 4097 for (Value *V : VL) { 4098 auto Res = UniquePositions.try_emplace(V, UniqueValues.size()); 4099 ReuseShuffleIndicies.emplace_back(Res.first->second); 4100 if (Res.second || isa<Constant>(V)) 4101 UniqueValues.emplace_back(V); 4102 } 4103 // Do not shuffle single element or if number of unique values is not power 4104 // of 2. 4105 if (UniqueValues.size() == VL.size() || UniqueValues.size() <= 1 || 4106 !llvm::isPowerOf2_32(UniqueValues.size())) 4107 ReuseShuffleIndicies.clear(); 4108 else 4109 VL = UniqueValues; 4110 } 4111 auto *VecTy = FixedVectorType::get(ScalarTy, VL.size()); 4112 4113 Value *V = Gather(VL, VecTy); 4114 if (!ReuseShuffleIndicies.empty()) { 4115 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4116 ReuseShuffleIndicies, "shuffle"); 4117 if (auto *I = dyn_cast<Instruction>(V)) { 4118 GatherSeq.insert(I); 4119 CSEBlocks.insert(I->getParent()); 4120 } 4121 } 4122 return V; 4123} 4124 4125static void inversePermutation(ArrayRef<unsigned> Indices, 4126 SmallVectorImpl<int> &Mask) { 4127 Mask.clear(); 4128 const unsigned E = Indices.size(); 4129 Mask.resize(E); 4130 for (unsigned I = 0; I < E; ++I) 4131 Mask[Indices[I]] = I; 4132} 4133 4134Value *BoUpSLP::vectorizeTree(TreeEntry *E) { 4135 IRBuilder<>::InsertPointGuard Guard(Builder); 4136 4137 if (E->VectorizedValue) { 4138 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n"); 4139 return E->VectorizedValue; 4140 } 4141 4142 Instruction *VL0 = E->getMainOp(); 4143 Type *ScalarTy = VL0->getType(); 4144 if (StoreInst *SI = dyn_cast<StoreInst>(VL0)) 4145 ScalarTy = SI->getValueOperand()->getType(); 4146 auto *VecTy = FixedVectorType::get(ScalarTy, E->Scalars.size()); 4147 4148 bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty(); 4149 4150 if (E->State == TreeEntry::NeedToGather) { 4151 setInsertPointAfterBundle(E); 4152 auto *V = Gather(E->Scalars, VecTy); 4153 if (NeedToShuffleReuses) { 4154 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4155 E->ReuseShuffleIndices, "shuffle"); 4156 if (auto *I = dyn_cast<Instruction>(V)) { 4157 GatherSeq.insert(I); 4158 CSEBlocks.insert(I->getParent()); 4159 } 4160 } 4161 E->VectorizedValue = V; 4162 return V; 4163 } 4164 4165 assert(E->State == TreeEntry::Vectorize && "Unhandled state"); 4166 unsigned ShuffleOrOp = 4167 E->isAltShuffle() ? (unsigned)Instruction::ShuffleVector : E->getOpcode(); 4168 switch (ShuffleOrOp) { 4169 case Instruction::PHI: { 4170 auto *PH = cast<PHINode>(VL0); 4171 Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI()); 4172 Builder.SetCurrentDebugLocation(PH->getDebugLoc()); 4173 PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues()); 4174 Value *V = NewPhi; 4175 if (NeedToShuffleReuses) { 4176 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4177 E->ReuseShuffleIndices, "shuffle"); 4178 } 4179 E->VectorizedValue = V; 4180 4181 // PHINodes may have multiple entries from the same block. We want to 4182 // visit every block once. 4183 SmallPtrSet<BasicBlock*, 4> VisitedBBs; 4184 4185 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) { 4186 ValueList Operands; 4187 BasicBlock *IBB = PH->getIncomingBlock(i); 4188 4189 if (!VisitedBBs.insert(IBB).second) { 4190 NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB); 4191 continue; 4192 } 4193 4194 Builder.SetInsertPoint(IBB->getTerminator()); 4195 Builder.SetCurrentDebugLocation(PH->getDebugLoc()); 4196 Value *Vec = vectorizeTree(E->getOperand(i)); 4197 NewPhi->addIncoming(Vec, IBB); 4198 } 4199 4200 assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && 4201 "Invalid number of incoming values"); 4202 return V; 4203 } 4204 4205 case Instruction::ExtractElement: { 4206 Value *V = E->getSingleOperand(0); 4207 if (!E->ReorderIndices.empty()) { 4208 SmallVector<int, 4> Mask; 4209 inversePermutation(E->ReorderIndices, Mask); 4210 Builder.SetInsertPoint(VL0); 4211 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), Mask, 4212 "reorder_shuffle"); 4213 } 4214 if (NeedToShuffleReuses) { 4215 // TODO: Merge this shuffle with the ReorderShuffleMask. 4216 if (E->ReorderIndices.empty()) 4217 Builder.SetInsertPoint(VL0); 4218 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4219 E->ReuseShuffleIndices, "shuffle"); 4220 } 4221 E->VectorizedValue = V; 4222 return V; 4223 } 4224 case Instruction::ExtractValue: { 4225 LoadInst *LI = cast<LoadInst>(E->getSingleOperand(0)); 4226 Builder.SetInsertPoint(LI); 4227 PointerType *PtrTy = 4228 PointerType::get(VecTy, LI->getPointerAddressSpace()); 4229 Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy); 4230 LoadInst *V = Builder.CreateAlignedLoad(VecTy, Ptr, LI->getAlign()); 4231 Value *NewV = propagateMetadata(V, E->Scalars); 4232 if (!E->ReorderIndices.empty()) { 4233 SmallVector<int, 4> Mask; 4234 inversePermutation(E->ReorderIndices, Mask); 4235 NewV = Builder.CreateShuffleVector(NewV, UndefValue::get(VecTy), Mask, 4236 "reorder_shuffle"); 4237 } 4238 if (NeedToShuffleReuses) { 4239 // TODO: Merge this shuffle with the ReorderShuffleMask. 4240 NewV = Builder.CreateShuffleVector(NewV, UndefValue::get(VecTy), 4241 E->ReuseShuffleIndices, "shuffle"); 4242 } 4243 E->VectorizedValue = NewV; 4244 return NewV; 4245 } 4246 case Instruction::ZExt: 4247 case Instruction::SExt: 4248 case Instruction::FPToUI: 4249 case Instruction::FPToSI: 4250 case Instruction::FPExt: 4251 case Instruction::PtrToInt: 4252 case Instruction::IntToPtr: 4253 case Instruction::SIToFP: 4254 case Instruction::UIToFP: 4255 case Instruction::Trunc: 4256 case Instruction::FPTrunc: 4257 case Instruction::BitCast: { 4258 setInsertPointAfterBundle(E); 4259 4260 Value *InVec = vectorizeTree(E->getOperand(0)); 4261 4262 if (E->VectorizedValue) { 4263 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n"); 4264 return E->VectorizedValue; 4265 } 4266 4267 auto *CI = cast<CastInst>(VL0); 4268 Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy); 4269 if (NeedToShuffleReuses) { 4270 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4271 E->ReuseShuffleIndices, "shuffle"); 4272 } 4273 E->VectorizedValue = V; 4274 ++NumVectorInstructions; 4275 return V; 4276 } 4277 case Instruction::FCmp: 4278 case Instruction::ICmp: { 4279 setInsertPointAfterBundle(E); 4280 4281 Value *L = vectorizeTree(E->getOperand(0)); 4282 Value *R = vectorizeTree(E->getOperand(1)); 4283 4284 if (E->VectorizedValue) { 4285 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n"); 4286 return E->VectorizedValue; 4287 } 4288 4289 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate(); 4290 Value *V = Builder.CreateCmp(P0, L, R); 4291 propagateIRFlags(V, E->Scalars, VL0); 4292 if (NeedToShuffleReuses) { 4293 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4294 E->ReuseShuffleIndices, "shuffle"); 4295 } 4296 E->VectorizedValue = V; 4297 ++NumVectorInstructions; 4298 return V; 4299 } 4300 case Instruction::Select: { 4301 setInsertPointAfterBundle(E); 4302 4303 Value *Cond = vectorizeTree(E->getOperand(0)); 4304 Value *True = vectorizeTree(E->getOperand(1)); 4305 Value *False = vectorizeTree(E->getOperand(2)); 4306 4307 if (E->VectorizedValue) { 4308 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n"); 4309 return E->VectorizedValue; 4310 } 4311 4312 Value *V = Builder.CreateSelect(Cond, True, False); 4313 if (NeedToShuffleReuses) { 4314 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4315 E->ReuseShuffleIndices, "shuffle"); 4316 } 4317 E->VectorizedValue = V; 4318 ++NumVectorInstructions; 4319 return V; 4320 } 4321 case Instruction::FNeg: { 4322 setInsertPointAfterBundle(E); 4323 4324 Value *Op = vectorizeTree(E->getOperand(0)); 4325 4326 if (E->VectorizedValue) { 4327 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n"); 4328 return E->VectorizedValue; 4329 } 4330 4331 Value *V = Builder.CreateUnOp( 4332 static_cast<Instruction::UnaryOps>(E->getOpcode()), Op); 4333 propagateIRFlags(V, E->Scalars, VL0); 4334 if (auto *I = dyn_cast<Instruction>(V)) 4335 V = propagateMetadata(I, E->Scalars); 4336 4337 if (NeedToShuffleReuses) { 4338 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4339 E->ReuseShuffleIndices, "shuffle"); 4340 } 4341 E->VectorizedValue = V; 4342 ++NumVectorInstructions; 4343 4344 return V; 4345 } 4346 case Instruction::Add: 4347 case Instruction::FAdd: 4348 case Instruction::Sub: 4349 case Instruction::FSub: 4350 case Instruction::Mul: 4351 case Instruction::FMul: 4352 case Instruction::UDiv: 4353 case Instruction::SDiv: 4354 case Instruction::FDiv: 4355 case Instruction::URem: 4356 case Instruction::SRem: 4357 case Instruction::FRem: 4358 case Instruction::Shl: 4359 case Instruction::LShr: 4360 case Instruction::AShr: 4361 case Instruction::And: 4362 case Instruction::Or: 4363 case Instruction::Xor: { 4364 setInsertPointAfterBundle(E); 4365 4366 Value *LHS = vectorizeTree(E->getOperand(0)); 4367 Value *RHS = vectorizeTree(E->getOperand(1)); 4368 4369 if (E->VectorizedValue) { 4370 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n"); 4371 return E->VectorizedValue; 4372 } 4373 4374 Value *V = Builder.CreateBinOp( 4375 static_cast<Instruction::BinaryOps>(E->getOpcode()), LHS, 4376 RHS); 4377 propagateIRFlags(V, E->Scalars, VL0); 4378 if (auto *I = dyn_cast<Instruction>(V)) 4379 V = propagateMetadata(I, E->Scalars); 4380 4381 if (NeedToShuffleReuses) { 4382 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4383 E->ReuseShuffleIndices, "shuffle"); 4384 } 4385 E->VectorizedValue = V; 4386 ++NumVectorInstructions; 4387 4388 return V; 4389 } 4390 case Instruction::Load: { 4391 // Loads are inserted at the head of the tree because we don't want to 4392 // sink them all the way down past store instructions. 4393 bool IsReorder = E->updateStateIfReorder(); 4394 if (IsReorder) 4395 VL0 = E->getMainOp(); 4396 setInsertPointAfterBundle(E); 4397 4398 LoadInst *LI = cast<LoadInst>(VL0); 4399 unsigned AS = LI->getPointerAddressSpace(); 4400 4401 Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(), 4402 VecTy->getPointerTo(AS)); 4403 4404 // The pointer operand uses an in-tree scalar so we add the new BitCast to 4405 // ExternalUses list to make sure that an extract will be generated in the 4406 // future. 4407 Value *PO = LI->getPointerOperand(); 4408 if (getTreeEntry(PO)) 4409 ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0)); 4410 4411 LI = Builder.CreateAlignedLoad(VecTy, VecPtr, LI->getAlign()); 4412 Value *V = propagateMetadata(LI, E->Scalars); 4413 if (IsReorder) { 4414 SmallVector<int, 4> Mask; 4415 inversePermutation(E->ReorderIndices, Mask); 4416 V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()), 4417 Mask, "reorder_shuffle"); 4418 } 4419 if (NeedToShuffleReuses) { 4420 // TODO: Merge this shuffle with the ReorderShuffleMask. 4421 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4422 E->ReuseShuffleIndices, "shuffle"); 4423 } 4424 E->VectorizedValue = V; 4425 ++NumVectorInstructions; 4426 return V; 4427 } 4428 case Instruction::Store: { 4429 bool IsReorder = !E->ReorderIndices.empty(); 4430 auto *SI = cast<StoreInst>( 4431 IsReorder ? E->Scalars[E->ReorderIndices.front()] : VL0); 4432 unsigned AS = SI->getPointerAddressSpace(); 4433 4434 setInsertPointAfterBundle(E); 4435 4436 Value *VecValue = vectorizeTree(E->getOperand(0)); 4437 if (IsReorder) { 4438 SmallVector<int, 4> Mask(E->ReorderIndices.begin(), 4439 E->ReorderIndices.end()); 4440 VecValue = Builder.CreateShuffleVector( 4441 VecValue, UndefValue::get(VecValue->getType()), Mask, 4442 "reorder_shuffle"); 4443 } 4444 Value *ScalarPtr = SI->getPointerOperand(); 4445 Value *VecPtr = Builder.CreateBitCast( 4446 ScalarPtr, VecValue->getType()->getPointerTo(AS)); 4447 StoreInst *ST = Builder.CreateAlignedStore(VecValue, VecPtr, 4448 SI->getAlign()); 4449 4450 // The pointer operand uses an in-tree scalar, so add the new BitCast to 4451 // ExternalUses to make sure that an extract will be generated in the 4452 // future. 4453 if (getTreeEntry(ScalarPtr)) 4454 ExternalUses.push_back(ExternalUser(ScalarPtr, cast<User>(VecPtr), 0)); 4455 4456 Value *V = propagateMetadata(ST, E->Scalars); 4457 if (NeedToShuffleReuses) { 4458 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4459 E->ReuseShuffleIndices, "shuffle"); 4460 } 4461 E->VectorizedValue = V; 4462 ++NumVectorInstructions; 4463 return V; 4464 } 4465 case Instruction::GetElementPtr: { 4466 setInsertPointAfterBundle(E); 4467 4468 Value *Op0 = vectorizeTree(E->getOperand(0)); 4469 4470 std::vector<Value *> OpVecs; 4471 for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e; 4472 ++j) { 4473 ValueList &VL = E->getOperand(j); 4474 // Need to cast all elements to the same type before vectorization to 4475 // avoid crash. 4476 Type *VL0Ty = VL0->getOperand(j)->getType(); 4477 Type *Ty = llvm::all_of( 4478 VL, [VL0Ty](Value *V) { return VL0Ty == V->getType(); }) 4479 ? VL0Ty 4480 : DL->getIndexType(cast<GetElementPtrInst>(VL0) 4481 ->getPointerOperandType() 4482 ->getScalarType()); 4483 for (Value *&V : VL) { 4484 auto *CI = cast<ConstantInt>(V); 4485 V = ConstantExpr::getIntegerCast(CI, Ty, 4486 CI->getValue().isSignBitSet()); 4487 } 4488 Value *OpVec = vectorizeTree(VL); 4489 OpVecs.push_back(OpVec); 4490 } 4491 4492 Value *V = Builder.CreateGEP( 4493 cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs); 4494 if (Instruction *I = dyn_cast<Instruction>(V)) 4495 V = propagateMetadata(I, E->Scalars); 4496 4497 if (NeedToShuffleReuses) { 4498 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4499 E->ReuseShuffleIndices, "shuffle"); 4500 } 4501 E->VectorizedValue = V; 4502 ++NumVectorInstructions; 4503 4504 return V; 4505 } 4506 case Instruction::Call: { 4507 CallInst *CI = cast<CallInst>(VL0); 4508 setInsertPointAfterBundle(E); 4509 4510 Intrinsic::ID IID = Intrinsic::not_intrinsic; 4511 if (Function *FI = CI->getCalledFunction()) 4512 IID = FI->getIntrinsicID(); 4513 4514 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI); 4515 4516 auto VecCallCosts = getVectorCallCosts(CI, VecTy, TTI, TLI); 4517 bool UseIntrinsic = ID != Intrinsic::not_intrinsic && 4518 VecCallCosts.first <= VecCallCosts.second; 4519 4520 Value *ScalarArg = nullptr; 4521 std::vector<Value *> OpVecs; 4522 for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) { 4523 ValueList OpVL; 4524 // Some intrinsics have scalar arguments. This argument should not be 4525 // vectorized. 4526 if (UseIntrinsic && hasVectorInstrinsicScalarOpd(IID, j)) { 4527 CallInst *CEI = cast<CallInst>(VL0); 4528 ScalarArg = CEI->getArgOperand(j); 4529 OpVecs.push_back(CEI->getArgOperand(j)); 4530 continue; 4531 } 4532 4533 Value *OpVec = vectorizeTree(E->getOperand(j)); 4534 LLVM_DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n"); 4535 OpVecs.push_back(OpVec); 4536 } 4537 4538 Function *CF; 4539 if (!UseIntrinsic) { 4540 VFShape Shape = VFShape::get( 4541 *CI, {static_cast<unsigned>(VecTy->getNumElements()), false}, 4542 false /*HasGlobalPred*/); 4543 CF = VFDatabase(*CI).getVectorizedFunction(Shape); 4544 } else { 4545 Type *Tys[] = {FixedVectorType::get(CI->getType(), E->Scalars.size())}; 4546 CF = Intrinsic::getDeclaration(F->getParent(), ID, Tys); 4547 } 4548 4549 SmallVector<OperandBundleDef, 1> OpBundles; 4550 CI->getOperandBundlesAsDefs(OpBundles); 4551 Value *V = Builder.CreateCall(CF, OpVecs, OpBundles); 4552 4553 // The scalar argument uses an in-tree scalar so we add the new vectorized 4554 // call to ExternalUses list to make sure that an extract will be 4555 // generated in the future. 4556 if (ScalarArg && getTreeEntry(ScalarArg)) 4557 ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0)); 4558 4559 propagateIRFlags(V, E->Scalars, VL0); 4560 if (NeedToShuffleReuses) { 4561 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4562 E->ReuseShuffleIndices, "shuffle"); 4563 } 4564 E->VectorizedValue = V; 4565 ++NumVectorInstructions; 4566 return V; 4567 } 4568 case Instruction::ShuffleVector: { 4569 assert(E->isAltShuffle() && 4570 ((Instruction::isBinaryOp(E->getOpcode()) && 4571 Instruction::isBinaryOp(E->getAltOpcode())) || 4572 (Instruction::isCast(E->getOpcode()) && 4573 Instruction::isCast(E->getAltOpcode()))) && 4574 "Invalid Shuffle Vector Operand"); 4575 4576 Value *LHS = nullptr, *RHS = nullptr; 4577 if (Instruction::isBinaryOp(E->getOpcode())) { 4578 setInsertPointAfterBundle(E); 4579 LHS = vectorizeTree(E->getOperand(0)); 4580 RHS = vectorizeTree(E->getOperand(1)); 4581 } else { 4582 setInsertPointAfterBundle(E); 4583 LHS = vectorizeTree(E->getOperand(0)); 4584 } 4585 4586 if (E->VectorizedValue) { 4587 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n"); 4588 return E->VectorizedValue; 4589 } 4590 4591 Value *V0, *V1; 4592 if (Instruction::isBinaryOp(E->getOpcode())) { 4593 V0 = Builder.CreateBinOp( 4594 static_cast<Instruction::BinaryOps>(E->getOpcode()), LHS, RHS); 4595 V1 = Builder.CreateBinOp( 4596 static_cast<Instruction::BinaryOps>(E->getAltOpcode()), LHS, RHS); 4597 } else { 4598 V0 = Builder.CreateCast( 4599 static_cast<Instruction::CastOps>(E->getOpcode()), LHS, VecTy); 4600 V1 = Builder.CreateCast( 4601 static_cast<Instruction::CastOps>(E->getAltOpcode()), LHS, VecTy); 4602 } 4603 4604 // Create shuffle to take alternate operations from the vector. 4605 // Also, gather up main and alt scalar ops to propagate IR flags to 4606 // each vector operation. 4607 ValueList OpScalars, AltScalars; 4608 unsigned e = E->Scalars.size(); 4609 SmallVector<int, 8> Mask(e); 4610 for (unsigned i = 0; i < e; ++i) { 4611 auto *OpInst = cast<Instruction>(E->Scalars[i]); 4612 assert(E->isOpcodeOrAlt(OpInst) && "Unexpected main/alternate opcode"); 4613 if (OpInst->getOpcode() == E->getAltOpcode()) { 4614 Mask[i] = e + i; 4615 AltScalars.push_back(E->Scalars[i]); 4616 } else { 4617 Mask[i] = i; 4618 OpScalars.push_back(E->Scalars[i]); 4619 } 4620 } 4621 4622 propagateIRFlags(V0, OpScalars); 4623 propagateIRFlags(V1, AltScalars); 4624 4625 Value *V = Builder.CreateShuffleVector(V0, V1, Mask); 4626 if (Instruction *I = dyn_cast<Instruction>(V)) 4627 V = propagateMetadata(I, E->Scalars); 4628 if (NeedToShuffleReuses) { 4629 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4630 E->ReuseShuffleIndices, "shuffle"); 4631 } 4632 E->VectorizedValue = V; 4633 ++NumVectorInstructions; 4634 4635 return V; 4636 } 4637 default: 4638 llvm_unreachable("unknown inst"); 4639 } 4640 return nullptr; 4641} 4642 4643Value *BoUpSLP::vectorizeTree() { 4644 ExtraValueToDebugLocsMap ExternallyUsedValues; 4645 return vectorizeTree(ExternallyUsedValues); 4646} 4647 4648Value * 4649BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) { 4650 // All blocks must be scheduled before any instructions are inserted. 4651 for (auto &BSIter : BlocksSchedules) { 4652 scheduleBlock(BSIter.second.get()); 4653 } 4654 4655 Builder.SetInsertPoint(&F->getEntryBlock().front()); 4656 auto *VectorRoot = vectorizeTree(VectorizableTree[0].get()); 4657 4658 // If the vectorized tree can be rewritten in a smaller type, we truncate the 4659 // vectorized root. InstCombine will then rewrite the entire expression. We 4660 // sign extend the extracted values below. 4661 auto *ScalarRoot = VectorizableTree[0]->Scalars[0]; 4662 if (MinBWs.count(ScalarRoot)) { 4663 if (auto *I = dyn_cast<Instruction>(VectorRoot)) 4664 Builder.SetInsertPoint(&*++BasicBlock::iterator(I)); 4665 auto BundleWidth = VectorizableTree[0]->Scalars.size(); 4666 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first); 4667 auto *VecTy = FixedVectorType::get(MinTy, BundleWidth); 4668 auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy); 4669 VectorizableTree[0]->VectorizedValue = Trunc; 4670 } 4671 4672 LLVM_DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size() 4673 << " values .\n"); 4674 4675 // If necessary, sign-extend or zero-extend ScalarRoot to the larger type 4676 // specified by ScalarType. 4677 auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) { 4678 if (!MinBWs.count(ScalarRoot)) 4679 return Ex; 4680 if (MinBWs[ScalarRoot].second) 4681 return Builder.CreateSExt(Ex, ScalarType); 4682 return Builder.CreateZExt(Ex, ScalarType); 4683 }; 4684 4685 // Extract all of the elements with the external uses. 4686 for (const auto &ExternalUse : ExternalUses) { 4687 Value *Scalar = ExternalUse.Scalar; 4688 llvm::User *User = ExternalUse.User; 4689 4690 // Skip users that we already RAUW. This happens when one instruction 4691 // has multiple uses of the same value. 4692 if (User && !is_contained(Scalar->users(), User)) 4693 continue; 4694 TreeEntry *E = getTreeEntry(Scalar); 4695 assert(E && "Invalid scalar"); 4696 assert(E->State == TreeEntry::Vectorize && "Extracting from a gather list"); 4697 4698 Value *Vec = E->VectorizedValue; 4699 assert(Vec && "Can't find vectorizable value"); 4700 4701 Value *Lane = Builder.getInt32(ExternalUse.Lane); 4702 // If User == nullptr, the Scalar is used as extra arg. Generate 4703 // ExtractElement instruction and update the record for this scalar in 4704 // ExternallyUsedValues. 4705 if (!User) { 4706 assert(ExternallyUsedValues.count(Scalar) && 4707 "Scalar with nullptr as an external user must be registered in " 4708 "ExternallyUsedValues map"); 4709 if (auto *VecI = dyn_cast<Instruction>(Vec)) { 4710 Builder.SetInsertPoint(VecI->getParent(), 4711 std::next(VecI->getIterator())); 4712 } else { 4713 Builder.SetInsertPoint(&F->getEntryBlock().front()); 4714 } 4715 Value *Ex = Builder.CreateExtractElement(Vec, Lane); 4716 Ex = extend(ScalarRoot, Ex, Scalar->getType()); 4717 CSEBlocks.insert(cast<Instruction>(Scalar)->getParent()); 4718 auto &Locs = ExternallyUsedValues[Scalar]; 4719 ExternallyUsedValues.insert({Ex, Locs}); 4720 ExternallyUsedValues.erase(Scalar); 4721 // Required to update internally referenced instructions. 4722 Scalar->replaceAllUsesWith(Ex); 4723 continue; 4724 } 4725 4726 // Generate extracts for out-of-tree users. 4727 // Find the insertion point for the extractelement lane. 4728 if (auto *VecI = dyn_cast<Instruction>(Vec)) { 4729 if (PHINode *PH = dyn_cast<PHINode>(User)) { 4730 for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) { 4731 if (PH->getIncomingValue(i) == Scalar) { 4732 Instruction *IncomingTerminator = 4733 PH->getIncomingBlock(i)->getTerminator(); 4734 if (isa<CatchSwitchInst>(IncomingTerminator)) { 4735 Builder.SetInsertPoint(VecI->getParent(), 4736 std::next(VecI->getIterator())); 4737 } else { 4738 Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator()); 4739 } 4740 Value *Ex = Builder.CreateExtractElement(Vec, Lane); 4741 Ex = extend(ScalarRoot, Ex, Scalar->getType()); 4742 CSEBlocks.insert(PH->getIncomingBlock(i)); 4743 PH->setOperand(i, Ex); 4744 } 4745 } 4746 } else { 4747 Builder.SetInsertPoint(cast<Instruction>(User)); 4748 Value *Ex = Builder.CreateExtractElement(Vec, Lane); 4749 Ex = extend(ScalarRoot, Ex, Scalar->getType()); 4750 CSEBlocks.insert(cast<Instruction>(User)->getParent()); 4751 User->replaceUsesOfWith(Scalar, Ex); 4752 } 4753 } else { 4754 Builder.SetInsertPoint(&F->getEntryBlock().front()); 4755 Value *Ex = Builder.CreateExtractElement(Vec, Lane); 4756 Ex = extend(ScalarRoot, Ex, Scalar->getType()); 4757 CSEBlocks.insert(&F->getEntryBlock()); 4758 User->replaceUsesOfWith(Scalar, Ex); 4759 } 4760 4761 LLVM_DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n"); 4762 } 4763 4764 // For each vectorized value: 4765 for (auto &TEPtr : VectorizableTree) { 4766 TreeEntry *Entry = TEPtr.get(); 4767 4768 // No need to handle users of gathered values. 4769 if (Entry->State == TreeEntry::NeedToGather) 4770 continue; 4771 4772 assert(Entry->VectorizedValue && "Can't find vectorizable value"); 4773 4774 // For each lane: 4775 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) { 4776 Value *Scalar = Entry->Scalars[Lane]; 4777 4778#ifndef NDEBUG 4779 Type *Ty = Scalar->getType(); 4780 if (!Ty->isVoidTy()) { 4781 for (User *U : Scalar->users()) { 4782 LLVM_DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n"); 4783 4784 // It is legal to delete users in the ignorelist. 4785 assert((getTreeEntry(U) || is_contained(UserIgnoreList, U)) && 4786 "Deleting out-of-tree value"); 4787 } 4788 } 4789#endif 4790 LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n"); 4791 eraseInstruction(cast<Instruction>(Scalar)); 4792 } 4793 } 4794 4795 Builder.ClearInsertionPoint(); 4796 InstrElementSize.clear(); 4797 4798 return VectorizableTree[0]->VectorizedValue; 4799} 4800 4801void BoUpSLP::optimizeGatherSequence() { 4802 LLVM_DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size() 4803 << " gather sequences instructions.\n"); 4804 // LICM InsertElementInst sequences. 4805 for (Instruction *I : GatherSeq) { 4806 if (isDeleted(I)) 4807 continue; 4808 4809 // Check if this block is inside a loop. 4810 Loop *L = LI->getLoopFor(I->getParent()); 4811 if (!L) 4812 continue; 4813 4814 // Check if it has a preheader. 4815 BasicBlock *PreHeader = L->getLoopPreheader(); 4816 if (!PreHeader) 4817 continue; 4818 4819 // If the vector or the element that we insert into it are 4820 // instructions that are defined in this basic block then we can't 4821 // hoist this instruction. 4822 auto *Op0 = dyn_cast<Instruction>(I->getOperand(0)); 4823 auto *Op1 = dyn_cast<Instruction>(I->getOperand(1)); 4824 if (Op0 && L->contains(Op0)) 4825 continue; 4826 if (Op1 && L->contains(Op1)) 4827 continue; 4828 4829 // We can hoist this instruction. Move it to the pre-header. 4830 I->moveBefore(PreHeader->getTerminator()); 4831 } 4832 4833 // Make a list of all reachable blocks in our CSE queue. 4834 SmallVector<const DomTreeNode *, 8> CSEWorkList; 4835 CSEWorkList.reserve(CSEBlocks.size()); 4836 for (BasicBlock *BB : CSEBlocks) 4837 if (DomTreeNode *N = DT->getNode(BB)) { 4838 assert(DT->isReachableFromEntry(N)); 4839 CSEWorkList.push_back(N); 4840 } 4841 4842 // Sort blocks by domination. This ensures we visit a block after all blocks 4843 // dominating it are visited. 4844 llvm::stable_sort(CSEWorkList, 4845 [this](const DomTreeNode *A, const DomTreeNode *B) { 4846 return DT->properlyDominates(A, B); 4847 }); 4848 4849 // Perform O(N^2) search over the gather sequences and merge identical 4850 // instructions. TODO: We can further optimize this scan if we split the 4851 // instructions into different buckets based on the insert lane. 4852 SmallVector<Instruction *, 16> Visited; 4853 for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) { 4854 assert((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && 4855 "Worklist not sorted properly!"); 4856 BasicBlock *BB = (*I)->getBlock(); 4857 // For all instructions in blocks containing gather sequences: 4858 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) { 4859 Instruction *In = &*it++; 4860 if (isDeleted(In)) 4861 continue; 4862 if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In)) 4863 continue; 4864 4865 // Check if we can replace this instruction with any of the 4866 // visited instructions. 4867 for (Instruction *v : Visited) { 4868 if (In->isIdenticalTo(v) && 4869 DT->dominates(v->getParent(), In->getParent())) { 4870 In->replaceAllUsesWith(v); 4871 eraseInstruction(In); 4872 In = nullptr; 4873 break; 4874 } 4875 } 4876 if (In) { 4877 assert(!is_contained(Visited, In)); 4878 Visited.push_back(In); 4879 } 4880 } 4881 } 4882 CSEBlocks.clear(); 4883 GatherSeq.clear(); 4884} 4885 4886// Groups the instructions to a bundle (which is then a single scheduling entity) 4887// and schedules instructions until the bundle gets ready. 4888Optional<BoUpSLP::ScheduleData *> 4889BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP, 4890 const InstructionsState &S) { 4891 if (isa<PHINode>(S.OpValue)) 4892 return nullptr; 4893 4894 // Initialize the instruction bundle. 4895 Instruction *OldScheduleEnd = ScheduleEnd; 4896 ScheduleData *PrevInBundle = nullptr; 4897 ScheduleData *Bundle = nullptr; 4898 bool ReSchedule = false; 4899 LLVM_DEBUG(dbgs() << "SLP: bundle: " << *S.OpValue << "\n"); 4900 4901 // Make sure that the scheduling region contains all 4902 // instructions of the bundle. 4903 for (Value *V : VL) { 4904 if (!extendSchedulingRegion(V, S)) 4905 return None; 4906 } 4907 4908 for (Value *V : VL) { 4909 ScheduleData *BundleMember = getScheduleData(V); 4910 assert(BundleMember && 4911 "no ScheduleData for bundle member (maybe not in same basic block)"); 4912 if (BundleMember->IsScheduled) { 4913 // A bundle member was scheduled as single instruction before and now 4914 // needs to be scheduled as part of the bundle. We just get rid of the 4915 // existing schedule. 4916 LLVM_DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMember 4917 << " was already scheduled\n"); 4918 ReSchedule = true; 4919 } 4920 assert(BundleMember->isSchedulingEntity() && 4921 "bundle member already part of other bundle"); 4922 if (PrevInBundle) { 4923 PrevInBundle->NextInBundle = BundleMember; 4924 } else { 4925 Bundle = BundleMember; 4926 } 4927 BundleMember->UnscheduledDepsInBundle = 0; 4928 Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps; 4929 4930 // Group the instructions to a bundle. 4931 BundleMember->FirstInBundle = Bundle; 4932 PrevInBundle = BundleMember; 4933 } 4934 if (ScheduleEnd != OldScheduleEnd) { 4935 // The scheduling region got new instructions at the lower end (or it is a 4936 // new region for the first bundle). This makes it necessary to 4937 // recalculate all dependencies. 4938 // It is seldom that this needs to be done a second time after adding the 4939 // initial bundle to the region. 4940 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) { 4941 doForAllOpcodes(I, [](ScheduleData *SD) { 4942 SD->clearDependencies(); 4943 }); 4944 } 4945 ReSchedule = true; 4946 } 4947 if (ReSchedule) { 4948 resetSchedule(); 4949 initialFillReadyList(ReadyInsts); 4950 } 4951 assert(Bundle && "Failed to find schedule bundle"); 4952 4953 LLVM_DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block " 4954 << BB->getName() << "\n"); 4955 4956 calculateDependencies(Bundle, true, SLP); 4957 4958 // Now try to schedule the new bundle. As soon as the bundle is "ready" it 4959 // means that there are no cyclic dependencies and we can schedule it. 4960 // Note that's important that we don't "schedule" the bundle yet (see 4961 // cancelScheduling). 4962 while (!Bundle->isReady() && !ReadyInsts.empty()) { 4963 4964 ScheduleData *pickedSD = ReadyInsts.back(); 4965 ReadyInsts.pop_back(); 4966 4967 if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) { 4968 schedule(pickedSD, ReadyInsts); 4969 } 4970 } 4971 if (!Bundle->isReady()) { 4972 cancelScheduling(VL, S.OpValue); 4973 return None; 4974 } 4975 return Bundle; 4976} 4977 4978void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL, 4979 Value *OpValue) { 4980 if (isa<PHINode>(OpValue)) 4981 return; 4982 4983 ScheduleData *Bundle = getScheduleData(OpValue); 4984 LLVM_DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n"); 4985 assert(!Bundle->IsScheduled && 4986 "Can't cancel bundle which is already scheduled"); 4987 assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && 4988 "tried to unbundle something which is not a bundle"); 4989 4990 // Un-bundle: make single instructions out of the bundle. 4991 ScheduleData *BundleMember = Bundle; 4992 while (BundleMember) { 4993 assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links"); 4994 BundleMember->FirstInBundle = BundleMember; 4995 ScheduleData *Next = BundleMember->NextInBundle; 4996 BundleMember->NextInBundle = nullptr; 4997 BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps; 4998 if (BundleMember->UnscheduledDepsInBundle == 0) { 4999 ReadyInsts.insert(BundleMember); 5000 } 5001 BundleMember = Next; 5002 } 5003} 5004 5005BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() { 5006 // Allocate a new ScheduleData for the instruction. 5007 if (ChunkPos >= ChunkSize) { 5008 ScheduleDataChunks.push_back(std::make_unique<ScheduleData[]>(ChunkSize)); 5009 ChunkPos = 0; 5010 } 5011 return &(ScheduleDataChunks.back()[ChunkPos++]); 5012} 5013 5014bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V, 5015 const InstructionsState &S) { 5016 if (getScheduleData(V, isOneOf(S, V))) 5017 return true; 5018 Instruction *I = dyn_cast<Instruction>(V); 5019 assert(I && "bundle member must be an instruction"); 5020 assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled"); 5021 auto &&CheckSheduleForI = [this, &S](Instruction *I) -> bool { 5022 ScheduleData *ISD = getScheduleData(I); 5023 if (!ISD) 5024 return false; 5025 assert(isInSchedulingRegion(ISD) && 5026 "ScheduleData not in scheduling region"); 5027 ScheduleData *SD = allocateScheduleDataChunks(); 5028 SD->Inst = I; 5029 SD->init(SchedulingRegionID, S.OpValue); 5030 ExtraScheduleDataMap[I][S.OpValue] = SD; 5031 return true; 5032 }; 5033 if (CheckSheduleForI(I)) 5034 return true; 5035 if (!ScheduleStart) { 5036 // It's the first instruction in the new region. 5037 initScheduleData(I, I->getNextNode(), nullptr, nullptr); 5038 ScheduleStart = I; 5039 ScheduleEnd = I->getNextNode(); 5040 if (isOneOf(S, I) != I) 5041 CheckSheduleForI(I); 5042 assert(ScheduleEnd && "tried to vectorize a terminator?"); 5043 LLVM_DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n"); 5044 return true; 5045 } 5046 // Search up and down at the same time, because we don't know if the new 5047 // instruction is above or below the existing scheduling region. 5048 BasicBlock::reverse_iterator UpIter = 5049 ++ScheduleStart->getIterator().getReverse(); 5050 BasicBlock::reverse_iterator UpperEnd = BB->rend(); 5051 BasicBlock::iterator DownIter = ScheduleEnd->getIterator(); 5052 BasicBlock::iterator LowerEnd = BB->end(); 5053 while (true) { 5054 if (++ScheduleRegionSize > ScheduleRegionSizeLimit) { 5055 LLVM_DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n"); 5056 return false; 5057 } 5058 5059 if (UpIter != UpperEnd) { 5060 if (&*UpIter == I) { 5061 initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion); 5062 ScheduleStart = I; 5063 if (isOneOf(S, I) != I) 5064 CheckSheduleForI(I); 5065 LLVM_DEBUG(dbgs() << "SLP: extend schedule region start to " << *I 5066 << "\n"); 5067 return true; 5068 } 5069 ++UpIter; 5070 } 5071 if (DownIter != LowerEnd) { 5072 if (&*DownIter == I) { 5073 initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion, 5074 nullptr); 5075 ScheduleEnd = I->getNextNode(); 5076 if (isOneOf(S, I) != I) 5077 CheckSheduleForI(I); 5078 assert(ScheduleEnd && "tried to vectorize a terminator?"); 5079 LLVM_DEBUG(dbgs() << "SLP: extend schedule region end to " << *I 5080 << "\n"); 5081 return true; 5082 } 5083 ++DownIter; 5084 } 5085 assert((UpIter != UpperEnd || DownIter != LowerEnd) && 5086 "instruction not found in block"); 5087 } 5088 return true; 5089} 5090 5091void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI, 5092 Instruction *ToI, 5093 ScheduleData *PrevLoadStore, 5094 ScheduleData *NextLoadStore) { 5095 ScheduleData *CurrentLoadStore = PrevLoadStore; 5096 for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) { 5097 ScheduleData *SD = ScheduleDataMap[I]; 5098 if (!SD) { 5099 SD = allocateScheduleDataChunks(); 5100 ScheduleDataMap[I] = SD; 5101 SD->Inst = I; 5102 } 5103 assert(!isInSchedulingRegion(SD) && 5104 "new ScheduleData already in scheduling region"); 5105 SD->init(SchedulingRegionID, I); 5106 5107 if (I->mayReadOrWriteMemory() && 5108 (!isa<IntrinsicInst>(I) || 5109 cast<IntrinsicInst>(I)->getIntrinsicID() != Intrinsic::sideeffect)) { 5110 // Update the linked list of memory accessing instructions. 5111 if (CurrentLoadStore) { 5112 CurrentLoadStore->NextLoadStore = SD; 5113 } else { 5114 FirstLoadStoreInRegion = SD; 5115 } 5116 CurrentLoadStore = SD; 5117 } 5118 } 5119 if (NextLoadStore) { 5120 if (CurrentLoadStore) 5121 CurrentLoadStore->NextLoadStore = NextLoadStore; 5122 } else { 5123 LastLoadStoreInRegion = CurrentLoadStore; 5124 } 5125} 5126 5127void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD, 5128 bool InsertInReadyList, 5129 BoUpSLP *SLP) { 5130 assert(SD->isSchedulingEntity()); 5131 5132 SmallVector<ScheduleData *, 10> WorkList; 5133 WorkList.push_back(SD); 5134 5135 while (!WorkList.empty()) { 5136 ScheduleData *SD = WorkList.back(); 5137 WorkList.pop_back(); 5138 5139 ScheduleData *BundleMember = SD; 5140 while (BundleMember) { 5141 assert(isInSchedulingRegion(BundleMember)); 5142 if (!BundleMember->hasValidDependencies()) { 5143 5144 LLVM_DEBUG(dbgs() << "SLP: update deps of " << *BundleMember 5145 << "\n"); 5146 BundleMember->Dependencies = 0; 5147 BundleMember->resetUnscheduledDeps(); 5148 5149 // Handle def-use chain dependencies. 5150 if (BundleMember->OpValue != BundleMember->Inst) { 5151 ScheduleData *UseSD = getScheduleData(BundleMember->Inst); 5152 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) { 5153 BundleMember->Dependencies++; 5154 ScheduleData *DestBundle = UseSD->FirstInBundle; 5155 if (!DestBundle->IsScheduled) 5156 BundleMember->incrementUnscheduledDeps(1); 5157 if (!DestBundle->hasValidDependencies()) 5158 WorkList.push_back(DestBundle); 5159 } 5160 } else { 5161 for (User *U : BundleMember->Inst->users()) { 5162 if (isa<Instruction>(U)) { 5163 ScheduleData *UseSD = getScheduleData(U); 5164 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) { 5165 BundleMember->Dependencies++; 5166 ScheduleData *DestBundle = UseSD->FirstInBundle; 5167 if (!DestBundle->IsScheduled) 5168 BundleMember->incrementUnscheduledDeps(1); 5169 if (!DestBundle->hasValidDependencies()) 5170 WorkList.push_back(DestBundle); 5171 } 5172 } else { 5173 // I'm not sure if this can ever happen. But we need to be safe. 5174 // This lets the instruction/bundle never be scheduled and 5175 // eventually disable vectorization. 5176 BundleMember->Dependencies++; 5177 BundleMember->incrementUnscheduledDeps(1); 5178 } 5179 } 5180 } 5181 5182 // Handle the memory dependencies. 5183 ScheduleData *DepDest = BundleMember->NextLoadStore; 5184 if (DepDest) { 5185 Instruction *SrcInst = BundleMember->Inst; 5186 MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA); 5187 bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory(); 5188 unsigned numAliased = 0; 5189 unsigned DistToSrc = 1; 5190 5191 while (DepDest) { 5192 assert(isInSchedulingRegion(DepDest)); 5193 5194 // We have two limits to reduce the complexity: 5195 // 1) AliasedCheckLimit: It's a small limit to reduce calls to 5196 // SLP->isAliased (which is the expensive part in this loop). 5197 // 2) MaxMemDepDistance: It's for very large blocks and it aborts 5198 // the whole loop (even if the loop is fast, it's quadratic). 5199 // It's important for the loop break condition (see below) to 5200 // check this limit even between two read-only instructions. 5201 if (DistToSrc >= MaxMemDepDistance || 5202 ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) && 5203 (numAliased >= AliasedCheckLimit || 5204 SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) { 5205 5206 // We increment the counter only if the locations are aliased 5207 // (instead of counting all alias checks). This gives a better 5208 // balance between reduced runtime and accurate dependencies. 5209 numAliased++; 5210 5211 DepDest->MemoryDependencies.push_back(BundleMember); 5212 BundleMember->Dependencies++; 5213 ScheduleData *DestBundle = DepDest->FirstInBundle; 5214 if (!DestBundle->IsScheduled) { 5215 BundleMember->incrementUnscheduledDeps(1); 5216 } 5217 if (!DestBundle->hasValidDependencies()) { 5218 WorkList.push_back(DestBundle); 5219 } 5220 } 5221 DepDest = DepDest->NextLoadStore; 5222 5223 // Example, explaining the loop break condition: Let's assume our 5224 // starting instruction is i0 and MaxMemDepDistance = 3. 5225 // 5226 // +--------v--v--v 5227 // i0,i1,i2,i3,i4,i5,i6,i7,i8 5228 // +--------^--^--^ 5229 // 5230 // MaxMemDepDistance let us stop alias-checking at i3 and we add 5231 // dependencies from i0 to i3,i4,.. (even if they are not aliased). 5232 // Previously we already added dependencies from i3 to i6,i7,i8 5233 // (because of MaxMemDepDistance). As we added a dependency from 5234 // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8 5235 // and we can abort this loop at i6. 5236 if (DistToSrc >= 2 * MaxMemDepDistance) 5237 break; 5238 DistToSrc++; 5239 } 5240 } 5241 } 5242 BundleMember = BundleMember->NextInBundle; 5243 } 5244 if (InsertInReadyList && SD->isReady()) { 5245 ReadyInsts.push_back(SD); 5246 LLVM_DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst 5247 << "\n"); 5248 } 5249 } 5250} 5251 5252void BoUpSLP::BlockScheduling::resetSchedule() { 5253 assert(ScheduleStart && 5254 "tried to reset schedule on block which has not been scheduled"); 5255 for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) { 5256 doForAllOpcodes(I, [&](ScheduleData *SD) { 5257 assert(isInSchedulingRegion(SD) && 5258 "ScheduleData not in scheduling region"); 5259 SD->IsScheduled = false; 5260 SD->resetUnscheduledDeps(); 5261 }); 5262 } 5263 ReadyInsts.clear(); 5264} 5265 5266void BoUpSLP::scheduleBlock(BlockScheduling *BS) { 5267 if (!BS->ScheduleStart) 5268 return; 5269 5270 LLVM_DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n"); 5271 5272 BS->resetSchedule(); 5273 5274 // For the real scheduling we use a more sophisticated ready-list: it is 5275 // sorted by the original instruction location. This lets the final schedule 5276 // be as close as possible to the original instruction order. 5277 struct ScheduleDataCompare { 5278 bool operator()(ScheduleData *SD1, ScheduleData *SD2) const { 5279 return SD2->SchedulingPriority < SD1->SchedulingPriority; 5280 } 5281 }; 5282 std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts; 5283 5284 // Ensure that all dependency data is updated and fill the ready-list with 5285 // initial instructions. 5286 int Idx = 0; 5287 int NumToSchedule = 0; 5288 for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd; 5289 I = I->getNextNode()) { 5290 BS->doForAllOpcodes(I, [this, &Idx, &NumToSchedule, BS](ScheduleData *SD) { 5291 assert(SD->isPartOfBundle() == 5292 (getTreeEntry(SD->Inst) != nullptr) && 5293 "scheduler and vectorizer bundle mismatch"); 5294 SD->FirstInBundle->SchedulingPriority = Idx++; 5295 if (SD->isSchedulingEntity()) { 5296 BS->calculateDependencies(SD, false, this); 5297 NumToSchedule++; 5298 } 5299 }); 5300 } 5301 BS->initialFillReadyList(ReadyInsts); 5302 5303 Instruction *LastScheduledInst = BS->ScheduleEnd; 5304 5305 // Do the "real" scheduling. 5306 while (!ReadyInsts.empty()) { 5307 ScheduleData *picked = *ReadyInsts.begin(); 5308 ReadyInsts.erase(ReadyInsts.begin()); 5309 5310 // Move the scheduled instruction(s) to their dedicated places, if not 5311 // there yet. 5312 ScheduleData *BundleMember = picked; 5313 while (BundleMember) { 5314 Instruction *pickedInst = BundleMember->Inst; 5315 if (LastScheduledInst->getNextNode() != pickedInst) { 5316 BS->BB->getInstList().remove(pickedInst); 5317 BS->BB->getInstList().insert(LastScheduledInst->getIterator(), 5318 pickedInst); 5319 } 5320 LastScheduledInst = pickedInst; 5321 BundleMember = BundleMember->NextInBundle; 5322 } 5323 5324 BS->schedule(picked, ReadyInsts); 5325 NumToSchedule--; 5326 } 5327 assert(NumToSchedule == 0 && "could not schedule all instructions"); 5328 5329 // Avoid duplicate scheduling of the block. 5330 BS->ScheduleStart = nullptr; 5331} 5332 5333unsigned BoUpSLP::getVectorElementSize(Value *V) { 5334 // If V is a store, just return the width of the stored value without 5335 // traversing the expression tree. This is the common case. 5336 if (auto *Store = dyn_cast<StoreInst>(V)) 5337 return DL->getTypeSizeInBits(Store->getValueOperand()->getType()); 5338 5339 auto E = InstrElementSize.find(V); 5340 if (E != InstrElementSize.end()) 5341 return E->second; 5342 5343 // If V is not a store, we can traverse the expression tree to find loads 5344 // that feed it. The type of the loaded value may indicate a more suitable 5345 // width than V's type. We want to base the vector element size on the width 5346 // of memory operations where possible. 5347 SmallVector<Instruction *, 16> Worklist; 5348 SmallPtrSet<Instruction *, 16> Visited; 5349 if (auto *I = dyn_cast<Instruction>(V)) { 5350 Worklist.push_back(I); 5351 Visited.insert(I); 5352 } 5353 5354 // Traverse the expression tree in bottom-up order looking for loads. If we 5355 // encounter an instruction we don't yet handle, we give up. 5356 auto MaxWidth = 0u; 5357 auto FoundUnknownInst = false; 5358 while (!Worklist.empty() && !FoundUnknownInst) { 5359 auto *I = Worklist.pop_back_val(); 5360 5361 // We should only be looking at scalar instructions here. If the current 5362 // instruction has a vector type, give up. 5363 auto *Ty = I->getType(); 5364 if (isa<VectorType>(Ty)) 5365 FoundUnknownInst = true; 5366 5367 // If the current instruction is a load, update MaxWidth to reflect the 5368 // width of the loaded value. 5369 else if (isa<LoadInst>(I)) 5370 MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty)); 5371 5372 // Otherwise, we need to visit the operands of the instruction. We only 5373 // handle the interesting cases from buildTree here. If an operand is an 5374 // instruction we haven't yet visited, we add it to the worklist. 5375 else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) || 5376 isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) { 5377 for (Use &U : I->operands()) 5378 if (auto *J = dyn_cast<Instruction>(U.get())) 5379 if (Visited.insert(J).second) 5380 Worklist.push_back(J); 5381 } 5382 5383 // If we don't yet handle the instruction, give up. 5384 else 5385 FoundUnknownInst = true; 5386 } 5387 5388 int Width = MaxWidth; 5389 // If we didn't encounter a memory access in the expression tree, or if we 5390 // gave up for some reason, just return the width of V. Otherwise, return the 5391 // maximum width we found. 5392 if (!MaxWidth || FoundUnknownInst) 5393 Width = DL->getTypeSizeInBits(V->getType()); 5394 5395 for (Instruction *I : Visited) 5396 InstrElementSize[I] = Width; 5397 5398 return Width; 5399} 5400 5401// Determine if a value V in a vectorizable expression Expr can be demoted to a 5402// smaller type with a truncation. We collect the values that will be demoted 5403// in ToDemote and additional roots that require investigating in Roots. 5404static bool collectValuesToDemote(Value *V, SmallPtrSetImpl<Value *> &Expr, 5405 SmallVectorImpl<Value *> &ToDemote, 5406 SmallVectorImpl<Value *> &Roots) { 5407 // We can always demote constants. 5408 if (isa<Constant>(V)) { 5409 ToDemote.push_back(V); 5410 return true; 5411 } 5412 5413 // If the value is not an instruction in the expression with only one use, it 5414 // cannot be demoted. 5415 auto *I = dyn_cast<Instruction>(V); 5416 if (!I || !I->hasOneUse() || !Expr.count(I)) 5417 return false; 5418 5419 switch (I->getOpcode()) { 5420 5421 // We can always demote truncations and extensions. Since truncations can 5422 // seed additional demotion, we save the truncated value. 5423 case Instruction::Trunc: 5424 Roots.push_back(I->getOperand(0)); 5425 break; 5426 case Instruction::ZExt: 5427 case Instruction::SExt: 5428 break; 5429 5430 // We can demote certain binary operations if we can demote both of their 5431 // operands. 5432 case Instruction::Add: 5433 case Instruction::Sub: 5434 case Instruction::Mul: 5435 case Instruction::And: 5436 case Instruction::Or: 5437 case Instruction::Xor: 5438 if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) || 5439 !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots)) 5440 return false; 5441 break; 5442 5443 // We can demote selects if we can demote their true and false values. 5444 case Instruction::Select: { 5445 SelectInst *SI = cast<SelectInst>(I); 5446 if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) || 5447 !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots)) 5448 return false; 5449 break; 5450 } 5451 5452 // We can demote phis if we can demote all their incoming operands. Note that 5453 // we don't need to worry about cycles since we ensure single use above. 5454 case Instruction::PHI: { 5455 PHINode *PN = cast<PHINode>(I); 5456 for (Value *IncValue : PN->incoming_values()) 5457 if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots)) 5458 return false; 5459 break; 5460 } 5461 5462 // Otherwise, conservatively give up. 5463 default: 5464 return false; 5465 } 5466 5467 // Record the value that we can demote. 5468 ToDemote.push_back(V); 5469 return true; 5470} 5471 5472void BoUpSLP::computeMinimumValueSizes() { 5473 // If there are no external uses, the expression tree must be rooted by a 5474 // store. We can't demote in-memory values, so there is nothing to do here. 5475 if (ExternalUses.empty()) 5476 return; 5477 5478 // We only attempt to truncate integer expressions. 5479 auto &TreeRoot = VectorizableTree[0]->Scalars; 5480 auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType()); 5481 if (!TreeRootIT) 5482 return; 5483 5484 // If the expression is not rooted by a store, these roots should have 5485 // external uses. We will rely on InstCombine to rewrite the expression in 5486 // the narrower type. However, InstCombine only rewrites single-use values. 5487 // This means that if a tree entry other than a root is used externally, it 5488 // must have multiple uses and InstCombine will not rewrite it. The code 5489 // below ensures that only the roots are used externally. 5490 SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end()); 5491 for (auto &EU : ExternalUses) 5492 if (!Expr.erase(EU.Scalar)) 5493 return; 5494 if (!Expr.empty()) 5495 return; 5496 5497 // Collect the scalar values of the vectorizable expression. We will use this 5498 // context to determine which values can be demoted. If we see a truncation, 5499 // we mark it as seeding another demotion. 5500 for (auto &EntryPtr : VectorizableTree) 5501 Expr.insert(EntryPtr->Scalars.begin(), EntryPtr->Scalars.end()); 5502 5503 // Ensure the roots of the vectorizable tree don't form a cycle. They must 5504 // have a single external user that is not in the vectorizable tree. 5505 for (auto *Root : TreeRoot) 5506 if (!Root->hasOneUse() || Expr.count(*Root->user_begin())) 5507 return; 5508 5509 // Conservatively determine if we can actually truncate the roots of the 5510 // expression. Collect the values that can be demoted in ToDemote and 5511 // additional roots that require investigating in Roots. 5512 SmallVector<Value *, 32> ToDemote; 5513 SmallVector<Value *, 4> Roots; 5514 for (auto *Root : TreeRoot) 5515 if (!collectValuesToDemote(Root, Expr, ToDemote, Roots)) 5516 return; 5517 5518 // The maximum bit width required to represent all the values that can be 5519 // demoted without loss of precision. It would be safe to truncate the roots 5520 // of the expression to this width. 5521 auto MaxBitWidth = 8u; 5522 5523 // We first check if all the bits of the roots are demanded. If they're not, 5524 // we can truncate the roots to this narrower type. 5525 for (auto *Root : TreeRoot) { 5526 auto Mask = DB->getDemandedBits(cast<Instruction>(Root)); 5527 MaxBitWidth = std::max<unsigned>( 5528 Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth); 5529 } 5530 5531 // True if the roots can be zero-extended back to their original type, rather 5532 // than sign-extended. We know that if the leading bits are not demanded, we 5533 // can safely zero-extend. So we initialize IsKnownPositive to True. 5534 bool IsKnownPositive = true; 5535 5536 // If all the bits of the roots are demanded, we can try a little harder to 5537 // compute a narrower type. This can happen, for example, if the roots are 5538 // getelementptr indices. InstCombine promotes these indices to the pointer 5539 // width. Thus, all their bits are technically demanded even though the 5540 // address computation might be vectorized in a smaller type. 5541 // 5542 // We start by looking at each entry that can be demoted. We compute the 5543 // maximum bit width required to store the scalar by using ValueTracking to 5544 // compute the number of high-order bits we can truncate. 5545 if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType()) && 5546 llvm::all_of(TreeRoot, [](Value *R) { 5547 assert(R->hasOneUse() && "Root should have only one use!"); 5548 return isa<GetElementPtrInst>(R->user_back()); 5549 })) { 5550 MaxBitWidth = 8u; 5551 5552 // Determine if the sign bit of all the roots is known to be zero. If not, 5553 // IsKnownPositive is set to False. 5554 IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) { 5555 KnownBits Known = computeKnownBits(R, *DL); 5556 return Known.isNonNegative(); 5557 }); 5558 5559 // Determine the maximum number of bits required to store the scalar 5560 // values. 5561 for (auto *Scalar : ToDemote) { 5562 auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT); 5563 auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType()); 5564 MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth); 5565 } 5566 5567 // If we can't prove that the sign bit is zero, we must add one to the 5568 // maximum bit width to account for the unknown sign bit. This preserves 5569 // the existing sign bit so we can safely sign-extend the root back to the 5570 // original type. Otherwise, if we know the sign bit is zero, we will 5571 // zero-extend the root instead. 5572 // 5573 // FIXME: This is somewhat suboptimal, as there will be cases where adding 5574 // one to the maximum bit width will yield a larger-than-necessary 5575 // type. In general, we need to add an extra bit only if we can't 5576 // prove that the upper bit of the original type is equal to the 5577 // upper bit of the proposed smaller type. If these two bits are the 5578 // same (either zero or one) we know that sign-extending from the 5579 // smaller type will result in the same value. Here, since we can't 5580 // yet prove this, we are just making the proposed smaller type 5581 // larger to ensure correctness. 5582 if (!IsKnownPositive) 5583 ++MaxBitWidth; 5584 } 5585 5586 // Round MaxBitWidth up to the next power-of-two. 5587 if (!isPowerOf2_64(MaxBitWidth)) 5588 MaxBitWidth = NextPowerOf2(MaxBitWidth); 5589 5590 // If the maximum bit width we compute is less than the with of the roots' 5591 // type, we can proceed with the narrowing. Otherwise, do nothing. 5592 if (MaxBitWidth >= TreeRootIT->getBitWidth()) 5593 return; 5594 5595 // If we can truncate the root, we must collect additional values that might 5596 // be demoted as a result. That is, those seeded by truncations we will 5597 // modify. 5598 while (!Roots.empty()) 5599 collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots); 5600 5601 // Finally, map the values we can demote to the maximum bit with we computed. 5602 for (auto *Scalar : ToDemote) 5603 MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive); 5604} 5605 5606namespace { 5607 5608/// The SLPVectorizer Pass. 5609struct SLPVectorizer : public FunctionPass { 5610 SLPVectorizerPass Impl; 5611 5612 /// Pass identification, replacement for typeid 5613 static char ID; 5614 5615 explicit SLPVectorizer() : FunctionPass(ID) { 5616 initializeSLPVectorizerPass(*PassRegistry::getPassRegistry()); 5617 } 5618 5619 bool doInitialization(Module &M) override { 5620 return false; 5621 } 5622 5623 bool runOnFunction(Function &F) override { 5624 if (skipFunction(F)) 5625 return false; 5626 5627 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE(); 5628 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); 5629 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>(); 5630 auto *TLI = TLIP ? &TLIP->getTLI(F) : nullptr; 5631 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); 5632 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); 5633 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree(); 5634 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F); 5635 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits(); 5636 auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE(); 5637 5638 return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE); 5639 } 5640 5641 void getAnalysisUsage(AnalysisUsage &AU) const override { 5642 FunctionPass::getAnalysisUsage(AU); 5643 AU.addRequired<AssumptionCacheTracker>(); 5644 AU.addRequired<ScalarEvolutionWrapperPass>(); 5645 AU.addRequired<AAResultsWrapperPass>(); 5646 AU.addRequired<TargetTransformInfoWrapperPass>(); 5647 AU.addRequired<LoopInfoWrapperPass>(); 5648 AU.addRequired<DominatorTreeWrapperPass>(); 5649 AU.addRequired<DemandedBitsWrapperPass>(); 5650 AU.addRequired<OptimizationRemarkEmitterWrapperPass>(); 5651 AU.addRequired<InjectTLIMappingsLegacy>(); 5652 AU.addPreserved<LoopInfoWrapperPass>(); 5653 AU.addPreserved<DominatorTreeWrapperPass>(); 5654 AU.addPreserved<AAResultsWrapperPass>(); 5655 AU.addPreserved<GlobalsAAWrapperPass>(); 5656 AU.setPreservesCFG(); 5657 } 5658}; 5659 5660} // end anonymous namespace 5661 5662PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &AM) { 5663 auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F); 5664 auto *TTI = &AM.getResult<TargetIRAnalysis>(F); 5665 auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F); 5666 auto *AA = &AM.getResult<AAManager>(F); 5667 auto *LI = &AM.getResult<LoopAnalysis>(F); 5668 auto *DT = &AM.getResult<DominatorTreeAnalysis>(F); 5669 auto *AC = &AM.getResult<AssumptionAnalysis>(F); 5670 auto *DB = &AM.getResult<DemandedBitsAnalysis>(F); 5671 auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F); 5672 5673 bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE); 5674 if (!Changed) 5675 return PreservedAnalyses::all(); 5676 5677 PreservedAnalyses PA; 5678 PA.preserveSet<CFGAnalyses>(); 5679 PA.preserve<AAManager>(); 5680 PA.preserve<GlobalsAA>(); 5681 return PA; 5682} 5683 5684bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_, 5685 TargetTransformInfo *TTI_, 5686 TargetLibraryInfo *TLI_, AliasAnalysis *AA_, 5687 LoopInfo *LI_, DominatorTree *DT_, 5688 AssumptionCache *AC_, DemandedBits *DB_, 5689 OptimizationRemarkEmitter *ORE_) { 5690 if (!RunSLPVectorization) 5691 return false; 5692 SE = SE_; 5693 TTI = TTI_; 5694 TLI = TLI_; 5695 AA = AA_; 5696 LI = LI_; 5697 DT = DT_; 5698 AC = AC_; 5699 DB = DB_; 5700 DL = &F.getParent()->getDataLayout(); 5701 5702 Stores.clear(); 5703 GEPs.clear(); 5704 bool Changed = false; 5705 5706 // If the target claims to have no vector registers don't attempt 5707 // vectorization. 5708 if (!TTI->getNumberOfRegisters(TTI->getRegisterClassForType(true))) 5709 return false; 5710 5711 // Don't vectorize when the attribute NoImplicitFloat is used. 5712 if (F.hasFnAttribute(Attribute::NoImplicitFloat)) 5713 return false; 5714 5715 LLVM_DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n"); 5716 5717 // Use the bottom up slp vectorizer to construct chains that start with 5718 // store instructions. 5719 BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_); 5720 5721 // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to 5722 // delete instructions. 5723 5724 // Scan the blocks in the function in post order. 5725 for (auto BB : post_order(&F.getEntryBlock())) { 5726 collectSeedInstructions(BB); 5727 5728 // Vectorize trees that end at stores. 5729 if (!Stores.empty()) { 5730 LLVM_DEBUG(dbgs() << "SLP: Found stores for " << Stores.size() 5731 << " underlying objects.\n"); 5732 Changed |= vectorizeStoreChains(R); 5733 } 5734 5735 // Vectorize trees that end at reductions. 5736 Changed |= vectorizeChainsInBlock(BB, R); 5737 5738 // Vectorize the index computations of getelementptr instructions. This 5739 // is primarily intended to catch gather-like idioms ending at 5740 // non-consecutive loads. 5741 if (!GEPs.empty()) { 5742 LLVM_DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size() 5743 << " underlying objects.\n"); 5744 Changed |= vectorizeGEPIndices(BB, R); 5745 } 5746 } 5747 5748 if (Changed) { 5749 R.optimizeGatherSequence(); 5750 LLVM_DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n"); 5751 } 5752 return Changed; 5753} 5754 5755bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R, 5756 unsigned Idx) { 5757 LLVM_DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << Chain.size() 5758 << "\n"); 5759 const unsigned Sz = R.getVectorElementSize(Chain[0]); 5760 const unsigned MinVF = R.getMinVecRegSize() / Sz; 5761 unsigned VF = Chain.size(); 5762 5763 if (!isPowerOf2_32(Sz) || !isPowerOf2_32(VF) || VF < 2 || VF < MinVF) 5764 return false; 5765 5766 LLVM_DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << Idx 5767 << "\n"); 5768 5769 R.buildTree(Chain); 5770 Optional<ArrayRef<unsigned>> Order = R.bestOrder(); 5771 // TODO: Handle orders of size less than number of elements in the vector. 5772 if (Order && Order->size() == Chain.size()) { 5773 // TODO: reorder tree nodes without tree rebuilding. 5774 SmallVector<Value *, 4> ReorderedOps(Chain.rbegin(), Chain.rend()); 5775 llvm::transform(*Order, ReorderedOps.begin(), 5776 [Chain](const unsigned Idx) { return Chain[Idx]; }); 5777 R.buildTree(ReorderedOps); 5778 } 5779 if (R.isTreeTinyAndNotFullyVectorizable()) 5780 return false; 5781 if (R.isLoadCombineCandidate()) 5782 return false; 5783 5784 R.computeMinimumValueSizes(); 5785 5786 int Cost = R.getTreeCost(); 5787 5788 LLVM_DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF << "\n"); 5789 if (Cost < -SLPCostThreshold) { 5790 LLVM_DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n"); 5791 5792 using namespace ore; 5793 5794 R.getORE()->emit(OptimizationRemark(SV_NAME, "StoresVectorized", 5795 cast<StoreInst>(Chain[0])) 5796 << "Stores SLP vectorized with cost " << NV("Cost", Cost) 5797 << " and with tree size " 5798 << NV("TreeSize", R.getTreeSize())); 5799 5800 R.vectorizeTree(); 5801 return true; 5802 } 5803 5804 return false; 5805} 5806 5807bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores, 5808 BoUpSLP &R) { 5809 // We may run into multiple chains that merge into a single chain. We mark the 5810 // stores that we vectorized so that we don't visit the same store twice. 5811 BoUpSLP::ValueSet VectorizedStores; 5812 bool Changed = false; 5813 5814 int E = Stores.size(); 5815 SmallBitVector Tails(E, false); 5816 SmallVector<int, 16> ConsecutiveChain(E, E + 1); 5817 int MaxIter = MaxStoreLookup.getValue(); 5818 int IterCnt; 5819 auto &&FindConsecutiveAccess = [this, &Stores, &Tails, &IterCnt, MaxIter, 5820 &ConsecutiveChain](int K, int Idx) { 5821 if (IterCnt >= MaxIter) 5822 return true; 5823 ++IterCnt; 5824 if (!isConsecutiveAccess(Stores[K], Stores[Idx], *DL, *SE)) 5825 return false; 5826 5827 Tails.set(Idx); 5828 ConsecutiveChain[K] = Idx; 5829 return true; 5830 }; 5831 // Do a quadratic search on all of the given stores in reverse order and find 5832 // all of the pairs of stores that follow each other. 5833 for (int Idx = E - 1; Idx >= 0; --Idx) { 5834 // If a store has multiple consecutive store candidates, search according 5835 // to the sequence: Idx-1, Idx+1, Idx-2, Idx+2, ... 5836 // This is because usually pairing with immediate succeeding or preceding 5837 // candidate create the best chance to find slp vectorization opportunity. 5838 const int MaxLookDepth = std::max(E - Idx, Idx + 1); 5839 IterCnt = 0; 5840 for (int Offset = 1, F = MaxLookDepth; Offset < F; ++Offset) 5841 if ((Idx >= Offset && FindConsecutiveAccess(Idx - Offset, Idx)) || 5842 (Idx + Offset < E && FindConsecutiveAccess(Idx + Offset, Idx))) 5843 break; 5844 } 5845 5846 // For stores that start but don't end a link in the chain: 5847 for (int Cnt = E; Cnt > 0; --Cnt) { 5848 int I = Cnt - 1; 5849 if (ConsecutiveChain[I] == E + 1 || Tails.test(I)) 5850 continue; 5851 // We found a store instr that starts a chain. Now follow the chain and try 5852 // to vectorize it. 5853 BoUpSLP::ValueList Operands; 5854 // Collect the chain into a list. 5855 while (I != E + 1 && !VectorizedStores.count(Stores[I])) { 5856 Operands.push_back(Stores[I]); 5857 // Move to the next value in the chain. 5858 I = ConsecutiveChain[I]; 5859 } 5860 5861 // If a vector register can't hold 1 element, we are done. 5862 unsigned MaxVecRegSize = R.getMaxVecRegSize(); 5863 unsigned EltSize = R.getVectorElementSize(Stores[0]); 5864 if (MaxVecRegSize % EltSize != 0) 5865 continue; 5866 5867 unsigned MaxElts = MaxVecRegSize / EltSize; 5868 // FIXME: Is division-by-2 the correct step? Should we assert that the 5869 // register size is a power-of-2? 5870 unsigned StartIdx = 0; 5871 for (unsigned Size = llvm::PowerOf2Ceil(MaxElts); Size >= 2; Size /= 2) { 5872 for (unsigned Cnt = StartIdx, E = Operands.size(); Cnt + Size <= E;) { 5873 ArrayRef<Value *> Slice = makeArrayRef(Operands).slice(Cnt, Size); 5874 if (!VectorizedStores.count(Slice.front()) && 5875 !VectorizedStores.count(Slice.back()) && 5876 vectorizeStoreChain(Slice, R, Cnt)) { 5877 // Mark the vectorized stores so that we don't vectorize them again. 5878 VectorizedStores.insert(Slice.begin(), Slice.end()); 5879 Changed = true; 5880 // If we vectorized initial block, no need to try to vectorize it 5881 // again. 5882 if (Cnt == StartIdx) 5883 StartIdx += Size; 5884 Cnt += Size; 5885 continue; 5886 } 5887 ++Cnt; 5888 } 5889 // Check if the whole array was vectorized already - exit. 5890 if (StartIdx >= Operands.size()) 5891 break; 5892 } 5893 } 5894 5895 return Changed; 5896} 5897 5898void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) { 5899 // Initialize the collections. We will make a single pass over the block. 5900 Stores.clear(); 5901 GEPs.clear(); 5902 5903 // Visit the store and getelementptr instructions in BB and organize them in 5904 // Stores and GEPs according to the underlying objects of their pointer 5905 // operands. 5906 for (Instruction &I : *BB) { 5907 // Ignore store instructions that are volatile or have a pointer operand 5908 // that doesn't point to a scalar type. 5909 if (auto *SI = dyn_cast<StoreInst>(&I)) { 5910 if (!SI->isSimple()) 5911 continue; 5912 if (!isValidElementType(SI->getValueOperand()->getType())) 5913 continue; 5914 Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI); 5915 } 5916 5917 // Ignore getelementptr instructions that have more than one index, a 5918 // constant index, or a pointer operand that doesn't point to a scalar 5919 // type. 5920 else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) { 5921 auto Idx = GEP->idx_begin()->get(); 5922 if (GEP->getNumIndices() > 1 || isa<Constant>(Idx)) 5923 continue; 5924 if (!isValidElementType(Idx->getType())) 5925 continue; 5926 if (GEP->getType()->isVectorTy()) 5927 continue; 5928 GEPs[GEP->getPointerOperand()].push_back(GEP); 5929 } 5930 } 5931} 5932 5933bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) { 5934 if (!A || !B) 5935 return false; 5936 Value *VL[] = {A, B}; 5937 return tryToVectorizeList(VL, R, /*AllowReorder=*/true); 5938} 5939 5940bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R, 5941 bool AllowReorder, 5942 ArrayRef<Value *> InsertUses) { 5943 if (VL.size() < 2) 5944 return false; 5945 5946 LLVM_DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = " 5947 << VL.size() << ".\n"); 5948 5949 // Check that all of the parts are instructions of the same type, 5950 // we permit an alternate opcode via InstructionsState. 5951 InstructionsState S = getSameOpcode(VL); 5952 if (!S.getOpcode()) 5953 return false; 5954 5955 Instruction *I0 = cast<Instruction>(S.OpValue); 5956 // Make sure invalid types (including vector type) are rejected before 5957 // determining vectorization factor for scalar instructions. 5958 for (Value *V : VL) { 5959 Type *Ty = V->getType(); 5960 if (!isValidElementType(Ty)) { 5961 // NOTE: the following will give user internal llvm type name, which may 5962 // not be useful. 5963 R.getORE()->emit([&]() { 5964 std::string type_str; 5965 llvm::raw_string_ostream rso(type_str); 5966 Ty->print(rso); 5967 return OptimizationRemarkMissed(SV_NAME, "UnsupportedType", I0) 5968 << "Cannot SLP vectorize list: type " 5969 << rso.str() + " is unsupported by vectorizer"; 5970 }); 5971 return false; 5972 } 5973 } 5974 5975 unsigned Sz = R.getVectorElementSize(I0); 5976 unsigned MinVF = std::max(2U, R.getMinVecRegSize() / Sz); 5977 unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF); 5978 if (MaxVF < 2) { 5979 R.getORE()->emit([&]() { 5980 return OptimizationRemarkMissed(SV_NAME, "SmallVF", I0) 5981 << "Cannot SLP vectorize list: vectorization factor " 5982 << "less than 2 is not supported"; 5983 }); 5984 return false; 5985 } 5986 5987 bool Changed = false; 5988 bool CandidateFound = false; 5989 int MinCost = SLPCostThreshold; 5990 5991 bool CompensateUseCost = 5992 !InsertUses.empty() && llvm::all_of(InsertUses, [](const Value *V) { 5993 return V && isa<InsertElementInst>(V); 5994 }); 5995 assert((!CompensateUseCost || InsertUses.size() == VL.size()) && 5996 "Each scalar expected to have an associated InsertElement user."); 5997 5998 unsigned NextInst = 0, MaxInst = VL.size(); 5999 for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF; VF /= 2) { 6000 // No actual vectorization should happen, if number of parts is the same as 6001 // provided vectorization factor (i.e. the scalar type is used for vector 6002 // code during codegen). 6003 auto *VecTy = FixedVectorType::get(VL[0]->getType(), VF); 6004 if (TTI->getNumberOfParts(VecTy) == VF) 6005 continue; 6006 for (unsigned I = NextInst; I < MaxInst; ++I) { 6007 unsigned OpsWidth = 0; 6008 6009 if (I + VF > MaxInst) 6010 OpsWidth = MaxInst - I; 6011 else 6012 OpsWidth = VF; 6013 6014 if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2) 6015 break; 6016 6017 ArrayRef<Value *> Ops = VL.slice(I, OpsWidth); 6018 // Check that a previous iteration of this loop did not delete the Value. 6019 if (llvm::any_of(Ops, [&R](Value *V) { 6020 auto *I = dyn_cast<Instruction>(V); 6021 return I && R.isDeleted(I); 6022 })) 6023 continue; 6024 6025 LLVM_DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations " 6026 << "\n"); 6027 6028 R.buildTree(Ops); 6029 Optional<ArrayRef<unsigned>> Order = R.bestOrder(); 6030 // TODO: check if we can allow reordering for more cases. 6031 if (AllowReorder && Order) { 6032 // TODO: reorder tree nodes without tree rebuilding. 6033 // Conceptually, there is nothing actually preventing us from trying to 6034 // reorder a larger list. In fact, we do exactly this when vectorizing 6035 // reductions. However, at this point, we only expect to get here when 6036 // there are exactly two operations. 6037 assert(Ops.size() == 2); 6038 Value *ReorderedOps[] = {Ops[1], Ops[0]}; 6039 R.buildTree(ReorderedOps, None); 6040 } 6041 if (R.isTreeTinyAndNotFullyVectorizable()) 6042 continue; 6043 6044 R.computeMinimumValueSizes(); 6045 int Cost = R.getTreeCost(); 6046 CandidateFound = true; 6047 if (CompensateUseCost) { 6048 // TODO: Use TTI's getScalarizationOverhead for sequence of inserts 6049 // rather than sum of single inserts as the latter may overestimate 6050 // cost. This work should imply improving cost estimation for extracts 6051 // that added in for external (for vectorization tree) users,i.e. that 6052 // part should also switch to same interface. 6053 // For example, the following case is projected code after SLP: 6054 // %4 = extractelement <4 x i64> %3, i32 0 6055 // %v0 = insertelement <4 x i64> undef, i64 %4, i32 0 6056 // %5 = extractelement <4 x i64> %3, i32 1 6057 // %v1 = insertelement <4 x i64> %v0, i64 %5, i32 1 6058 // %6 = extractelement <4 x i64> %3, i32 2 6059 // %v2 = insertelement <4 x i64> %v1, i64 %6, i32 2 6060 // %7 = extractelement <4 x i64> %3, i32 3 6061 // %v3 = insertelement <4 x i64> %v2, i64 %7, i32 3 6062 // 6063 // Extracts here added by SLP in order to feed users (the inserts) of 6064 // original scalars and contribute to "ExtractCost" at cost evaluation. 6065 // The inserts in turn form sequence to build an aggregate that 6066 // detected by findBuildAggregate routine. 6067 // SLP makes an assumption that such sequence will be optimized away 6068 // later (instcombine) so it tries to compensate ExctractCost with 6069 // cost of insert sequence. 6070 // Current per element cost calculation approach is not quite accurate 6071 // and tends to create bias toward favoring vectorization. 6072 // Switching to the TTI interface might help a bit. 6073 // Alternative solution could be pattern-match to detect a no-op or 6074 // shuffle. 6075 unsigned UserCost = 0; 6076 for (unsigned Lane = 0; Lane < OpsWidth; Lane++) { 6077 auto *IE = cast<InsertElementInst>(InsertUses[I + Lane]); 6078 if (auto *CI = dyn_cast<ConstantInt>(IE->getOperand(2))) 6079 UserCost += TTI->getVectorInstrCost( 6080 Instruction::InsertElement, IE->getType(), CI->getZExtValue()); 6081 } 6082 LLVM_DEBUG(dbgs() << "SLP: Compensate cost of users by: " << UserCost 6083 << ".\n"); 6084 Cost -= UserCost; 6085 } 6086 6087 MinCost = std::min(MinCost, Cost); 6088 6089 if (Cost < -SLPCostThreshold) { 6090 LLVM_DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n"); 6091 R.getORE()->emit(OptimizationRemark(SV_NAME, "VectorizedList", 6092 cast<Instruction>(Ops[0])) 6093 << "SLP vectorized with cost " << ore::NV("Cost", Cost) 6094 << " and with tree size " 6095 << ore::NV("TreeSize", R.getTreeSize())); 6096 6097 R.vectorizeTree(); 6098 // Move to the next bundle. 6099 I += VF - 1; 6100 NextInst = I + 1; 6101 Changed = true; 6102 } 6103 } 6104 } 6105 6106 if (!Changed && CandidateFound) { 6107 R.getORE()->emit([&]() { 6108 return OptimizationRemarkMissed(SV_NAME, "NotBeneficial", I0) 6109 << "List vectorization was possible but not beneficial with cost " 6110 << ore::NV("Cost", MinCost) << " >= " 6111 << ore::NV("Treshold", -SLPCostThreshold); 6112 }); 6113 } else if (!Changed) { 6114 R.getORE()->emit([&]() { 6115 return OptimizationRemarkMissed(SV_NAME, "NotPossible", I0) 6116 << "Cannot SLP vectorize list: vectorization was impossible" 6117 << " with available vectorization factors"; 6118 }); 6119 } 6120 return Changed; 6121} 6122 6123bool SLPVectorizerPass::tryToVectorize(Instruction *I, BoUpSLP &R) { 6124 if (!I) 6125 return false; 6126 6127 if (!isa<BinaryOperator>(I) && !isa<CmpInst>(I)) 6128 return false; 6129 6130 Value *P = I->getParent(); 6131 6132 // Vectorize in current basic block only. 6133 auto *Op0 = dyn_cast<Instruction>(I->getOperand(0)); 6134 auto *Op1 = dyn_cast<Instruction>(I->getOperand(1)); 6135 if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P) 6136 return false; 6137 6138 // Try to vectorize V. 6139 if (tryToVectorizePair(Op0, Op1, R)) 6140 return true; 6141 6142 auto *A = dyn_cast<BinaryOperator>(Op0); 6143 auto *B = dyn_cast<BinaryOperator>(Op1); 6144 // Try to skip B. 6145 if (B && B->hasOneUse()) { 6146 auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0)); 6147 auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1)); 6148 if (B0 && B0->getParent() == P && tryToVectorizePair(A, B0, R)) 6149 return true; 6150 if (B1 && B1->getParent() == P && tryToVectorizePair(A, B1, R)) 6151 return true; 6152 } 6153 6154 // Try to skip A. 6155 if (A && A->hasOneUse()) { 6156 auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0)); 6157 auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1)); 6158 if (A0 && A0->getParent() == P && tryToVectorizePair(A0, B, R)) 6159 return true; 6160 if (A1 && A1->getParent() == P && tryToVectorizePair(A1, B, R)) 6161 return true; 6162 } 6163 return false; 6164} 6165 6166/// Generate a shuffle mask to be used in a reduction tree. 6167/// 6168/// \param VecLen The length of the vector to be reduced. 6169/// \param NumEltsToRdx The number of elements that should be reduced in the 6170/// vector. 6171/// \param IsPairwise Whether the reduction is a pairwise or splitting 6172/// reduction. A pairwise reduction will generate a mask of 6173/// <0,2,...> or <1,3,..> while a splitting reduction will generate 6174/// <2,3, undef,undef> for a vector of 4 and NumElts = 2. 6175/// \param IsLeft True will generate a mask of even elements, odd otherwise. 6176static SmallVector<int, 32> createRdxShuffleMask(unsigned VecLen, 6177 unsigned NumEltsToRdx, 6178 bool IsPairwise, bool IsLeft) { 6179 assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask"); 6180 6181 SmallVector<int, 32> ShuffleMask(VecLen, -1); 6182 6183 if (IsPairwise) 6184 // Build a mask of 0, 2, ... (left) or 1, 3, ... (right). 6185 for (unsigned i = 0; i != NumEltsToRdx; ++i) 6186 ShuffleMask[i] = 2 * i + !IsLeft; 6187 else 6188 // Move the upper half of the vector to the lower half. 6189 for (unsigned i = 0; i != NumEltsToRdx; ++i) 6190 ShuffleMask[i] = NumEltsToRdx + i; 6191 6192 return ShuffleMask; 6193} 6194 6195namespace { 6196 6197/// Model horizontal reductions. 6198/// 6199/// A horizontal reduction is a tree of reduction operations (currently add and 6200/// fadd) that has operations that can be put into a vector as its leaf. 6201/// For example, this tree: 6202/// 6203/// mul mul mul mul 6204/// \ / \ / 6205/// + + 6206/// \ / 6207/// + 6208/// This tree has "mul" as its reduced values and "+" as its reduction 6209/// operations. A reduction might be feeding into a store or a binary operation 6210/// feeding a phi. 6211/// ... 6212/// \ / 6213/// + 6214/// | 6215/// phi += 6216/// 6217/// Or: 6218/// ... 6219/// \ / 6220/// + 6221/// | 6222/// *p = 6223/// 6224class HorizontalReduction { 6225 using ReductionOpsType = SmallVector<Value *, 16>; 6226 using ReductionOpsListType = SmallVector<ReductionOpsType, 2>; 6227 ReductionOpsListType ReductionOps; 6228 SmallVector<Value *, 32> ReducedVals; 6229 // Use map vector to make stable output. 6230 MapVector<Instruction *, Value *> ExtraArgs; 6231 6232 /// Kind of the reduction data. 6233 enum ReductionKind { 6234 RK_None, /// Not a reduction. 6235 RK_Arithmetic, /// Binary reduction data. 6236 RK_Min, /// Minimum reduction data. 6237 RK_UMin, /// Unsigned minimum reduction data. 6238 RK_Max, /// Maximum reduction data. 6239 RK_UMax, /// Unsigned maximum reduction data. 6240 }; 6241 6242 /// Contains info about operation, like its opcode, left and right operands. 6243 class OperationData { 6244 /// Opcode of the instruction. 6245 unsigned Opcode = 0; 6246 6247 /// Left operand of the reduction operation. 6248 Value *LHS = nullptr; 6249 6250 /// Right operand of the reduction operation. 6251 Value *RHS = nullptr; 6252 6253 /// Kind of the reduction operation. 6254 ReductionKind Kind = RK_None; 6255 6256 /// True if float point min/max reduction has no NaNs. 6257 bool NoNaN = false; 6258 6259 /// Checks if the reduction operation can be vectorized. 6260 bool isVectorizable() const { 6261 return LHS && RHS && 6262 // We currently only support add/mul/logical && min/max reductions. 6263 ((Kind == RK_Arithmetic && 6264 (Opcode == Instruction::Add || Opcode == Instruction::FAdd || 6265 Opcode == Instruction::Mul || Opcode == Instruction::FMul || 6266 Opcode == Instruction::And || Opcode == Instruction::Or || 6267 Opcode == Instruction::Xor)) || 6268 ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) && 6269 (Kind == RK_Min || Kind == RK_Max)) || 6270 (Opcode == Instruction::ICmp && 6271 (Kind == RK_UMin || Kind == RK_UMax))); 6272 } 6273 6274 /// Creates reduction operation with the current opcode. 6275 Value *createOp(IRBuilder<> &Builder, const Twine &Name) const { 6276 assert(isVectorizable() && 6277 "Expected add|fadd or min/max reduction operation."); 6278 Value *Cmp = nullptr; 6279 switch (Kind) { 6280 case RK_Arithmetic: 6281 return Builder.CreateBinOp((Instruction::BinaryOps)Opcode, LHS, RHS, 6282 Name); 6283 case RK_Min: 6284 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSLT(LHS, RHS) 6285 : Builder.CreateFCmpOLT(LHS, RHS); 6286 return Builder.CreateSelect(Cmp, LHS, RHS, Name); 6287 case RK_Max: 6288 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSGT(LHS, RHS) 6289 : Builder.CreateFCmpOGT(LHS, RHS); 6290 return Builder.CreateSelect(Cmp, LHS, RHS, Name); 6291 case RK_UMin: 6292 assert(Opcode == Instruction::ICmp && "Expected integer types."); 6293 Cmp = Builder.CreateICmpULT(LHS, RHS); 6294 return Builder.CreateSelect(Cmp, LHS, RHS, Name); 6295 case RK_UMax: 6296 assert(Opcode == Instruction::ICmp && "Expected integer types."); 6297 Cmp = Builder.CreateICmpUGT(LHS, RHS); 6298 return Builder.CreateSelect(Cmp, LHS, RHS, Name); 6299 case RK_None: 6300 break; 6301 } 6302 llvm_unreachable("Unknown reduction operation."); 6303 } 6304 6305 public: 6306 explicit OperationData() = default; 6307 6308 /// Construction for reduced values. They are identified by opcode only and 6309 /// don't have associated LHS/RHS values. 6310 explicit OperationData(Value *V) { 6311 if (auto *I = dyn_cast<Instruction>(V)) 6312 Opcode = I->getOpcode(); 6313 } 6314 6315 /// Constructor for reduction operations with opcode and its left and 6316 /// right operands. 6317 OperationData(unsigned Opcode, Value *LHS, Value *RHS, ReductionKind Kind, 6318 bool NoNaN = false) 6319 : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind), NoNaN(NoNaN) { 6320 assert(Kind != RK_None && "One of the reduction operations is expected."); 6321 } 6322 6323 explicit operator bool() const { return Opcode; } 6324 6325 /// Return true if this operation is any kind of minimum or maximum. 6326 bool isMinMax() const { 6327 switch (Kind) { 6328 case RK_Arithmetic: 6329 return false; 6330 case RK_Min: 6331 case RK_Max: 6332 case RK_UMin: 6333 case RK_UMax: 6334 return true; 6335 case RK_None: 6336 break; 6337 } 6338 llvm_unreachable("Reduction kind is not set"); 6339 } 6340 6341 /// Get the index of the first operand. 6342 unsigned getFirstOperandIndex() const { 6343 assert(!!*this && "The opcode is not set."); 6344 // We allow calling this before 'Kind' is set, so handle that specially. 6345 if (Kind == RK_None) 6346 return 0; 6347 return isMinMax() ? 1 : 0; 6348 } 6349 6350 /// Total number of operands in the reduction operation. 6351 unsigned getNumberOfOperands() const { 6352 assert(Kind != RK_None && !!*this && LHS && RHS && 6353 "Expected reduction operation."); 6354 return isMinMax() ? 3 : 2; 6355 } 6356 6357 /// Checks if the operation has the same parent as \p P. 6358 bool hasSameParent(Instruction *I, Value *P, bool IsRedOp) const { 6359 assert(Kind != RK_None && !!*this && LHS && RHS && 6360 "Expected reduction operation."); 6361 if (!IsRedOp) 6362 return I->getParent() == P; 6363 if (isMinMax()) { 6364 // SelectInst must be used twice while the condition op must have single 6365 // use only. 6366 auto *Cmp = cast<Instruction>(cast<SelectInst>(I)->getCondition()); 6367 return I->getParent() == P && Cmp && Cmp->getParent() == P; 6368 } 6369 // Arithmetic reduction operation must be used once only. 6370 return I->getParent() == P; 6371 } 6372 6373 /// Expected number of uses for reduction operations/reduced values. 6374 bool hasRequiredNumberOfUses(Instruction *I, bool IsReductionOp) const { 6375 assert(Kind != RK_None && !!*this && LHS && RHS && 6376 "Expected reduction operation."); 6377 if (isMinMax()) 6378 return I->hasNUses(2) && 6379 (!IsReductionOp || 6380 cast<SelectInst>(I)->getCondition()->hasOneUse()); 6381 return I->hasOneUse(); 6382 } 6383 6384 /// Initializes the list of reduction operations. 6385 void initReductionOps(ReductionOpsListType &ReductionOps) { 6386 assert(Kind != RK_None && !!*this && LHS && RHS && 6387 "Expected reduction operation."); 6388 if (isMinMax()) 6389 ReductionOps.assign(2, ReductionOpsType()); 6390 else 6391 ReductionOps.assign(1, ReductionOpsType()); 6392 } 6393 6394 /// Add all reduction operations for the reduction instruction \p I. 6395 void addReductionOps(Instruction *I, ReductionOpsListType &ReductionOps) { 6396 assert(Kind != RK_None && !!*this && LHS && RHS && 6397 "Expected reduction operation."); 6398 if (isMinMax()) { 6399 ReductionOps[0].emplace_back(cast<SelectInst>(I)->getCondition()); 6400 ReductionOps[1].emplace_back(I); 6401 } else { 6402 ReductionOps[0].emplace_back(I); 6403 } 6404 } 6405 6406 /// Checks if instruction is associative and can be vectorized. 6407 bool isAssociative(Instruction *I) const { 6408 assert(Kind != RK_None && *this && LHS && RHS && 6409 "Expected reduction operation."); 6410 switch (Kind) { 6411 case RK_Arithmetic: 6412 return I->isAssociative(); 6413 case RK_Min: 6414 case RK_Max: 6415 return Opcode == Instruction::ICmp || 6416 cast<Instruction>(I->getOperand(0))->isFast(); 6417 case RK_UMin: 6418 case RK_UMax: 6419 assert(Opcode == Instruction::ICmp && 6420 "Only integer compare operation is expected."); 6421 return true; 6422 case RK_None: 6423 break; 6424 } 6425 llvm_unreachable("Reduction kind is not set"); 6426 } 6427 6428 /// Checks if the reduction operation can be vectorized. 6429 bool isVectorizable(Instruction *I) const { 6430 return isVectorizable() && isAssociative(I); 6431 } 6432 6433 /// Checks if two operation data are both a reduction op or both a reduced 6434 /// value. 6435 bool operator==(const OperationData &OD) const { 6436 assert(((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) && 6437 "One of the comparing operations is incorrect."); 6438 return this == &OD || (Kind == OD.Kind && Opcode == OD.Opcode); 6439 } 6440 bool operator!=(const OperationData &OD) const { return !(*this == OD); } 6441 void clear() { 6442 Opcode = 0; 6443 LHS = nullptr; 6444 RHS = nullptr; 6445 Kind = RK_None; 6446 NoNaN = false; 6447 } 6448 6449 /// Get the opcode of the reduction operation. 6450 unsigned getOpcode() const { 6451 assert(isVectorizable() && "Expected vectorizable operation."); 6452 return Opcode; 6453 } 6454 6455 /// Get kind of reduction data. 6456 ReductionKind getKind() const { return Kind; } 6457 Value *getLHS() const { return LHS; } 6458 Value *getRHS() const { return RHS; } 6459 Type *getConditionType() const { 6460 return isMinMax() ? CmpInst::makeCmpResultType(LHS->getType()) : nullptr; 6461 } 6462 6463 /// Creates reduction operation with the current opcode with the IR flags 6464 /// from \p ReductionOps. 6465 Value *createOp(IRBuilder<> &Builder, const Twine &Name, 6466 const ReductionOpsListType &ReductionOps) const { 6467 assert(isVectorizable() && 6468 "Expected add|fadd or min/max reduction operation."); 6469 auto *Op = createOp(Builder, Name); 6470 switch (Kind) { 6471 case RK_Arithmetic: 6472 propagateIRFlags(Op, ReductionOps[0]); 6473 return Op; 6474 case RK_Min: 6475 case RK_Max: 6476 case RK_UMin: 6477 case RK_UMax: 6478 if (auto *SI = dyn_cast<SelectInst>(Op)) 6479 propagateIRFlags(SI->getCondition(), ReductionOps[0]); 6480 propagateIRFlags(Op, ReductionOps[1]); 6481 return Op; 6482 case RK_None: 6483 break; 6484 } 6485 llvm_unreachable("Unknown reduction operation."); 6486 } 6487 /// Creates reduction operation with the current opcode with the IR flags 6488 /// from \p I. 6489 Value *createOp(IRBuilder<> &Builder, const Twine &Name, 6490 Instruction *I) const { 6491 assert(isVectorizable() && 6492 "Expected add|fadd or min/max reduction operation."); 6493 auto *Op = createOp(Builder, Name); 6494 switch (Kind) { 6495 case RK_Arithmetic: 6496 propagateIRFlags(Op, I); 6497 return Op; 6498 case RK_Min: 6499 case RK_Max: 6500 case RK_UMin: 6501 case RK_UMax: 6502 if (auto *SI = dyn_cast<SelectInst>(Op)) { 6503 propagateIRFlags(SI->getCondition(), 6504 cast<SelectInst>(I)->getCondition()); 6505 } 6506 propagateIRFlags(Op, I); 6507 return Op; 6508 case RK_None: 6509 break; 6510 } 6511 llvm_unreachable("Unknown reduction operation."); 6512 } 6513 6514 TargetTransformInfo::ReductionFlags getFlags() const { 6515 TargetTransformInfo::ReductionFlags Flags; 6516 Flags.NoNaN = NoNaN; 6517 switch (Kind) { 6518 case RK_Arithmetic: 6519 break; 6520 case RK_Min: 6521 Flags.IsSigned = Opcode == Instruction::ICmp; 6522 Flags.IsMaxOp = false; 6523 break; 6524 case RK_Max: 6525 Flags.IsSigned = Opcode == Instruction::ICmp; 6526 Flags.IsMaxOp = true; 6527 break; 6528 case RK_UMin: 6529 Flags.IsSigned = false; 6530 Flags.IsMaxOp = false; 6531 break; 6532 case RK_UMax: 6533 Flags.IsSigned = false; 6534 Flags.IsMaxOp = true; 6535 break; 6536 case RK_None: 6537 llvm_unreachable("Reduction kind is not set"); 6538 } 6539 return Flags; 6540 } 6541 }; 6542 6543 WeakTrackingVH ReductionRoot; 6544 6545 /// The operation data of the reduction operation. 6546 OperationData ReductionData; 6547 6548 /// The operation data of the values we perform a reduction on. 6549 OperationData ReducedValueData; 6550 6551 /// Should we model this reduction as a pairwise reduction tree or a tree that 6552 /// splits the vector in halves and adds those halves. 6553 bool IsPairwiseReduction = false; 6554 6555 /// Checks if the ParentStackElem.first should be marked as a reduction 6556 /// operation with an extra argument or as extra argument itself. 6557 void markExtraArg(std::pair<Instruction *, unsigned> &ParentStackElem, 6558 Value *ExtraArg) { 6559 if (ExtraArgs.count(ParentStackElem.first)) { 6560 ExtraArgs[ParentStackElem.first] = nullptr; 6561 // We ran into something like: 6562 // ParentStackElem.first = ExtraArgs[ParentStackElem.first] + ExtraArg. 6563 // The whole ParentStackElem.first should be considered as an extra value 6564 // in this case. 6565 // Do not perform analysis of remaining operands of ParentStackElem.first 6566 // instruction, this whole instruction is an extra argument. 6567 ParentStackElem.second = ParentStackElem.first->getNumOperands(); 6568 } else { 6569 // We ran into something like: 6570 // ParentStackElem.first += ... + ExtraArg + ... 6571 ExtraArgs[ParentStackElem.first] = ExtraArg; 6572 } 6573 } 6574 6575 static OperationData getOperationData(Value *V) { 6576 if (!V) 6577 return OperationData(); 6578 6579 Value *LHS; 6580 Value *RHS; 6581 if (m_BinOp(m_Value(LHS), m_Value(RHS)).match(V)) { 6582 return OperationData(cast<BinaryOperator>(V)->getOpcode(), LHS, RHS, 6583 RK_Arithmetic); 6584 } 6585 if (auto *Select = dyn_cast<SelectInst>(V)) { 6586 // Look for a min/max pattern. 6587 if (m_UMin(m_Value(LHS), m_Value(RHS)).match(Select)) { 6588 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin); 6589 } else if (m_SMin(m_Value(LHS), m_Value(RHS)).match(Select)) { 6590 return OperationData(Instruction::ICmp, LHS, RHS, RK_Min); 6591 } else if (m_OrdFMin(m_Value(LHS), m_Value(RHS)).match(Select) || 6592 m_UnordFMin(m_Value(LHS), m_Value(RHS)).match(Select)) { 6593 return OperationData( 6594 Instruction::FCmp, LHS, RHS, RK_Min, 6595 cast<Instruction>(Select->getCondition())->hasNoNaNs()); 6596 } else if (m_UMax(m_Value(LHS), m_Value(RHS)).match(Select)) { 6597 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax); 6598 } else if (m_SMax(m_Value(LHS), m_Value(RHS)).match(Select)) { 6599 return OperationData(Instruction::ICmp, LHS, RHS, RK_Max); 6600 } else if (m_OrdFMax(m_Value(LHS), m_Value(RHS)).match(Select) || 6601 m_UnordFMax(m_Value(LHS), m_Value(RHS)).match(Select)) { 6602 return OperationData( 6603 Instruction::FCmp, LHS, RHS, RK_Max, 6604 cast<Instruction>(Select->getCondition())->hasNoNaNs()); 6605 } else { 6606 // Try harder: look for min/max pattern based on instructions producing 6607 // same values such as: select ((cmp Inst1, Inst2), Inst1, Inst2). 6608 // During the intermediate stages of SLP, it's very common to have 6609 // pattern like this (since optimizeGatherSequence is run only once 6610 // at the end): 6611 // %1 = extractelement <2 x i32> %a, i32 0 6612 // %2 = extractelement <2 x i32> %a, i32 1 6613 // %cond = icmp sgt i32 %1, %2 6614 // %3 = extractelement <2 x i32> %a, i32 0 6615 // %4 = extractelement <2 x i32> %a, i32 1 6616 // %select = select i1 %cond, i32 %3, i32 %4 6617 CmpInst::Predicate Pred; 6618 Instruction *L1; 6619 Instruction *L2; 6620 6621 LHS = Select->getTrueValue(); 6622 RHS = Select->getFalseValue(); 6623 Value *Cond = Select->getCondition(); 6624 6625 // TODO: Support inverse predicates. 6626 if (match(Cond, m_Cmp(Pred, m_Specific(LHS), m_Instruction(L2)))) { 6627 if (!isa<ExtractElementInst>(RHS) || 6628 !L2->isIdenticalTo(cast<Instruction>(RHS))) 6629 return OperationData(V); 6630 } else if (match(Cond, m_Cmp(Pred, m_Instruction(L1), m_Specific(RHS)))) { 6631 if (!isa<ExtractElementInst>(LHS) || 6632 !L1->isIdenticalTo(cast<Instruction>(LHS))) 6633 return OperationData(V); 6634 } else { 6635 if (!isa<ExtractElementInst>(LHS) || !isa<ExtractElementInst>(RHS)) 6636 return OperationData(V); 6637 if (!match(Cond, m_Cmp(Pred, m_Instruction(L1), m_Instruction(L2))) || 6638 !L1->isIdenticalTo(cast<Instruction>(LHS)) || 6639 !L2->isIdenticalTo(cast<Instruction>(RHS))) 6640 return OperationData(V); 6641 } 6642 switch (Pred) { 6643 default: 6644 return OperationData(V); 6645 6646 case CmpInst::ICMP_ULT: 6647 case CmpInst::ICMP_ULE: 6648 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin); 6649 6650 case CmpInst::ICMP_SLT: 6651 case CmpInst::ICMP_SLE: 6652 return OperationData(Instruction::ICmp, LHS, RHS, RK_Min); 6653 6654 case CmpInst::FCMP_OLT: 6655 case CmpInst::FCMP_OLE: 6656 case CmpInst::FCMP_ULT: 6657 case CmpInst::FCMP_ULE: 6658 return OperationData(Instruction::FCmp, LHS, RHS, RK_Min, 6659 cast<Instruction>(Cond)->hasNoNaNs()); 6660 6661 case CmpInst::ICMP_UGT: 6662 case CmpInst::ICMP_UGE: 6663 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax); 6664 6665 case CmpInst::ICMP_SGT: 6666 case CmpInst::ICMP_SGE: 6667 return OperationData(Instruction::ICmp, LHS, RHS, RK_Max); 6668 6669 case CmpInst::FCMP_OGT: 6670 case CmpInst::FCMP_OGE: 6671 case CmpInst::FCMP_UGT: 6672 case CmpInst::FCMP_UGE: 6673 return OperationData(Instruction::FCmp, LHS, RHS, RK_Max, 6674 cast<Instruction>(Cond)->hasNoNaNs()); 6675 } 6676 } 6677 } 6678 return OperationData(V); 6679 } 6680 6681public: 6682 HorizontalReduction() = default; 6683 6684 /// Try to find a reduction tree. 6685 bool matchAssociativeReduction(PHINode *Phi, Instruction *B) { 6686 assert((!Phi || is_contained(Phi->operands(), B)) && 6687 "Thi phi needs to use the binary operator"); 6688 6689 ReductionData = getOperationData(B); 6690 6691 // We could have a initial reductions that is not an add. 6692 // r *= v1 + v2 + v3 + v4 6693 // In such a case start looking for a tree rooted in the first '+'. 6694 if (Phi) { 6695 if (ReductionData.getLHS() == Phi) { 6696 Phi = nullptr; 6697 B = dyn_cast<Instruction>(ReductionData.getRHS()); 6698 ReductionData = getOperationData(B); 6699 } else if (ReductionData.getRHS() == Phi) { 6700 Phi = nullptr; 6701 B = dyn_cast<Instruction>(ReductionData.getLHS()); 6702 ReductionData = getOperationData(B); 6703 } 6704 } 6705 6706 if (!ReductionData.isVectorizable(B)) 6707 return false; 6708 6709 Type *Ty = B->getType(); 6710 if (!isValidElementType(Ty)) 6711 return false; 6712 if (!Ty->isIntOrIntVectorTy() && !Ty->isFPOrFPVectorTy()) 6713 return false; 6714 6715 ReducedValueData.clear(); 6716 ReductionRoot = B; 6717 6718 // Post order traverse the reduction tree starting at B. We only handle true 6719 // trees containing only binary operators. 6720 SmallVector<std::pair<Instruction *, unsigned>, 32> Stack; 6721 Stack.push_back(std::make_pair(B, ReductionData.getFirstOperandIndex())); 6722 ReductionData.initReductionOps(ReductionOps); 6723 while (!Stack.empty()) { 6724 Instruction *TreeN = Stack.back().first; 6725 unsigned EdgeToVist = Stack.back().second++; 6726 OperationData OpData = getOperationData(TreeN); 6727 bool IsReducedValue = OpData != ReductionData; 6728 6729 // Postorder vist. 6730 if (IsReducedValue || EdgeToVist == OpData.getNumberOfOperands()) { 6731 if (IsReducedValue) 6732 ReducedVals.push_back(TreeN); 6733 else { 6734 auto I = ExtraArgs.find(TreeN); 6735 if (I != ExtraArgs.end() && !I->second) { 6736 // Check if TreeN is an extra argument of its parent operation. 6737 if (Stack.size() <= 1) { 6738 // TreeN can't be an extra argument as it is a root reduction 6739 // operation. 6740 return false; 6741 } 6742 // Yes, TreeN is an extra argument, do not add it to a list of 6743 // reduction operations. 6744 // Stack[Stack.size() - 2] always points to the parent operation. 6745 markExtraArg(Stack[Stack.size() - 2], TreeN); 6746 ExtraArgs.erase(TreeN); 6747 } else 6748 ReductionData.addReductionOps(TreeN, ReductionOps); 6749 } 6750 // Retract. 6751 Stack.pop_back(); 6752 continue; 6753 } 6754 6755 // Visit left or right. 6756 Value *NextV = TreeN->getOperand(EdgeToVist); 6757 if (NextV != Phi) { 6758 auto *I = dyn_cast<Instruction>(NextV); 6759 OpData = getOperationData(I); 6760 // Continue analysis if the next operand is a reduction operation or 6761 // (possibly) a reduced value. If the reduced value opcode is not set, 6762 // the first met operation != reduction operation is considered as the 6763 // reduced value class. 6764 if (I && (!ReducedValueData || OpData == ReducedValueData || 6765 OpData == ReductionData)) { 6766 const bool IsReductionOperation = OpData == ReductionData; 6767 // Only handle trees in the current basic block. 6768 if (!ReductionData.hasSameParent(I, B->getParent(), 6769 IsReductionOperation)) { 6770 // I is an extra argument for TreeN (its parent operation). 6771 markExtraArg(Stack.back(), I); 6772 continue; 6773 } 6774 6775 // Each tree node needs to have minimal number of users except for the 6776 // ultimate reduction. 6777 if (!ReductionData.hasRequiredNumberOfUses(I, 6778 OpData == ReductionData) && 6779 I != B) { 6780 // I is an extra argument for TreeN (its parent operation). 6781 markExtraArg(Stack.back(), I); 6782 continue; 6783 } 6784 6785 if (IsReductionOperation) { 6786 // We need to be able to reassociate the reduction operations. 6787 if (!OpData.isAssociative(I)) { 6788 // I is an extra argument for TreeN (its parent operation). 6789 markExtraArg(Stack.back(), I); 6790 continue; 6791 } 6792 } else if (ReducedValueData && 6793 ReducedValueData != OpData) { 6794 // Make sure that the opcodes of the operations that we are going to 6795 // reduce match. 6796 // I is an extra argument for TreeN (its parent operation). 6797 markExtraArg(Stack.back(), I); 6798 continue; 6799 } else if (!ReducedValueData) 6800 ReducedValueData = OpData; 6801 6802 Stack.push_back(std::make_pair(I, OpData.getFirstOperandIndex())); 6803 continue; 6804 } 6805 } 6806 // NextV is an extra argument for TreeN (its parent operation). 6807 markExtraArg(Stack.back(), NextV); 6808 } 6809 return true; 6810 } 6811 6812 /// Attempt to vectorize the tree found by 6813 /// matchAssociativeReduction. 6814 bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) { 6815 if (ReducedVals.empty()) 6816 return false; 6817 6818 // If there is a sufficient number of reduction values, reduce 6819 // to a nearby power-of-2. Can safely generate oversized 6820 // vectors and rely on the backend to split them to legal sizes. 6821 unsigned NumReducedVals = ReducedVals.size(); 6822 if (NumReducedVals < 4) 6823 return false; 6824 6825 unsigned ReduxWidth = PowerOf2Floor(NumReducedVals); 6826 6827 Value *VectorizedTree = nullptr; 6828 6829 // FIXME: Fast-math-flags should be set based on the instructions in the 6830 // reduction (not all of 'fast' are required). 6831 IRBuilder<> Builder(cast<Instruction>(ReductionRoot)); 6832 FastMathFlags Unsafe; 6833 Unsafe.setFast(); 6834 Builder.setFastMathFlags(Unsafe); 6835 unsigned i = 0; 6836 6837 BoUpSLP::ExtraValueToDebugLocsMap ExternallyUsedValues; 6838 // The same extra argument may be used several time, so log each attempt 6839 // to use it. 6840 for (auto &Pair : ExtraArgs) { 6841 assert(Pair.first && "DebugLoc must be set."); 6842 ExternallyUsedValues[Pair.second].push_back(Pair.first); 6843 } 6844 6845 // The compare instruction of a min/max is the insertion point for new 6846 // instructions and may be replaced with a new compare instruction. 6847 auto getCmpForMinMaxReduction = [](Instruction *RdxRootInst) { 6848 assert(isa<SelectInst>(RdxRootInst) && 6849 "Expected min/max reduction to have select root instruction"); 6850 Value *ScalarCond = cast<SelectInst>(RdxRootInst)->getCondition(); 6851 assert(isa<Instruction>(ScalarCond) && 6852 "Expected min/max reduction to have compare condition"); 6853 return cast<Instruction>(ScalarCond); 6854 }; 6855 6856 // The reduction root is used as the insertion point for new instructions, 6857 // so set it as externally used to prevent it from being deleted. 6858 ExternallyUsedValues[ReductionRoot]; 6859 SmallVector<Value *, 16> IgnoreList; 6860 for (auto &V : ReductionOps) 6861 IgnoreList.append(V.begin(), V.end()); 6862 while (i < NumReducedVals - ReduxWidth + 1 && ReduxWidth > 2) { 6863 auto VL = makeArrayRef(&ReducedVals[i], ReduxWidth); 6864 V.buildTree(VL, ExternallyUsedValues, IgnoreList); 6865 Optional<ArrayRef<unsigned>> Order = V.bestOrder(); 6866 // TODO: Handle orders of size less than number of elements in the vector. 6867 if (Order && Order->size() == VL.size()) { 6868 // TODO: reorder tree nodes without tree rebuilding. 6869 SmallVector<Value *, 4> ReorderedOps(VL.size()); 6870 llvm::transform(*Order, ReorderedOps.begin(), 6871 [VL](const unsigned Idx) { return VL[Idx]; }); 6872 V.buildTree(ReorderedOps, ExternallyUsedValues, IgnoreList); 6873 } 6874 if (V.isTreeTinyAndNotFullyVectorizable()) 6875 break; 6876 if (V.isLoadCombineReductionCandidate(ReductionData.getOpcode())) 6877 break; 6878 6879 V.computeMinimumValueSizes(); 6880 6881 // Estimate cost. 6882 int TreeCost = V.getTreeCost(); 6883 int ReductionCost = getReductionCost(TTI, ReducedVals[i], ReduxWidth); 6884 int Cost = TreeCost + ReductionCost; 6885 if (Cost >= -SLPCostThreshold) { 6886 V.getORE()->emit([&]() { 6887 return OptimizationRemarkMissed( 6888 SV_NAME, "HorSLPNotBeneficial", cast<Instruction>(VL[0])) 6889 << "Vectorizing horizontal reduction is possible" 6890 << "but not beneficial with cost " 6891 << ore::NV("Cost", Cost) << " and threshold " 6892 << ore::NV("Threshold", -SLPCostThreshold); 6893 }); 6894 break; 6895 } 6896 6897 LLVM_DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:" 6898 << Cost << ". (HorRdx)\n"); 6899 V.getORE()->emit([&]() { 6900 return OptimizationRemark( 6901 SV_NAME, "VectorizedHorizontalReduction", cast<Instruction>(VL[0])) 6902 << "Vectorized horizontal reduction with cost " 6903 << ore::NV("Cost", Cost) << " and with tree size " 6904 << ore::NV("TreeSize", V.getTreeSize()); 6905 }); 6906 6907 // Vectorize a tree. 6908 DebugLoc Loc = cast<Instruction>(ReducedVals[i])->getDebugLoc(); 6909 Value *VectorizedRoot = V.vectorizeTree(ExternallyUsedValues); 6910 6911 // Emit a reduction. For min/max, the root is a select, but the insertion 6912 // point is the compare condition of that select. 6913 Instruction *RdxRootInst = cast<Instruction>(ReductionRoot); 6914 if (ReductionData.isMinMax()) 6915 Builder.SetInsertPoint(getCmpForMinMaxReduction(RdxRootInst)); 6916 else 6917 Builder.SetInsertPoint(RdxRootInst); 6918 6919 Value *ReducedSubTree = 6920 emitReduction(VectorizedRoot, Builder, ReduxWidth, TTI); 6921 if (VectorizedTree) { 6922 Builder.SetCurrentDebugLocation(Loc); 6923 OperationData VectReductionData(ReductionData.getOpcode(), 6924 VectorizedTree, ReducedSubTree, 6925 ReductionData.getKind()); 6926 VectorizedTree = 6927 VectReductionData.createOp(Builder, "op.rdx", ReductionOps); 6928 } else 6929 VectorizedTree = ReducedSubTree; 6930 i += ReduxWidth; 6931 ReduxWidth = PowerOf2Floor(NumReducedVals - i); 6932 } 6933 6934 if (VectorizedTree) { 6935 // Finish the reduction. 6936 for (; i < NumReducedVals; ++i) { 6937 auto *I = cast<Instruction>(ReducedVals[i]); 6938 Builder.SetCurrentDebugLocation(I->getDebugLoc()); 6939 OperationData VectReductionData(ReductionData.getOpcode(), 6940 VectorizedTree, I, 6941 ReductionData.getKind()); 6942 VectorizedTree = VectReductionData.createOp(Builder, "", ReductionOps); 6943 } 6944 for (auto &Pair : ExternallyUsedValues) { 6945 // Add each externally used value to the final reduction. 6946 for (auto *I : Pair.second) { 6947 Builder.SetCurrentDebugLocation(I->getDebugLoc()); 6948 OperationData VectReductionData(ReductionData.getOpcode(), 6949 VectorizedTree, Pair.first, 6950 ReductionData.getKind()); 6951 VectorizedTree = VectReductionData.createOp(Builder, "op.extra", I); 6952 } 6953 } 6954 6955 // Update users. For a min/max reduction that ends with a compare and 6956 // select, we also have to RAUW for the compare instruction feeding the 6957 // reduction root. That's because the original compare may have extra uses 6958 // besides the final select of the reduction. 6959 if (ReductionData.isMinMax()) { 6960 if (auto *VecSelect = dyn_cast<SelectInst>(VectorizedTree)) { 6961 Instruction *ScalarCmp = 6962 getCmpForMinMaxReduction(cast<Instruction>(ReductionRoot)); 6963 ScalarCmp->replaceAllUsesWith(VecSelect->getCondition()); 6964 } 6965 } 6966 ReductionRoot->replaceAllUsesWith(VectorizedTree); 6967 6968 // Mark all scalar reduction ops for deletion, they are replaced by the 6969 // vector reductions. 6970 V.eraseInstructions(IgnoreList); 6971 } 6972 return VectorizedTree != nullptr; 6973 } 6974 6975 unsigned numReductionValues() const { 6976 return ReducedVals.size(); 6977 } 6978 6979private: 6980 /// Calculate the cost of a reduction. 6981 int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal, 6982 unsigned ReduxWidth) { 6983 Type *ScalarTy = FirstReducedVal->getType(); 6984 auto *VecTy = FixedVectorType::get(ScalarTy, ReduxWidth); 6985 6986 int PairwiseRdxCost; 6987 int SplittingRdxCost; 6988 switch (ReductionData.getKind()) { 6989 case RK_Arithmetic: 6990 PairwiseRdxCost = 6991 TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy, 6992 /*IsPairwiseForm=*/true); 6993 SplittingRdxCost = 6994 TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy, 6995 /*IsPairwiseForm=*/false); 6996 break; 6997 case RK_Min: 6998 case RK_Max: 6999 case RK_UMin: 7000 case RK_UMax: { 7001 auto *VecCondTy = cast<VectorType>(CmpInst::makeCmpResultType(VecTy)); 7002 bool IsUnsigned = ReductionData.getKind() == RK_UMin || 7003 ReductionData.getKind() == RK_UMax; 7004 PairwiseRdxCost = 7005 TTI->getMinMaxReductionCost(VecTy, VecCondTy, 7006 /*IsPairwiseForm=*/true, IsUnsigned); 7007 SplittingRdxCost = 7008 TTI->getMinMaxReductionCost(VecTy, VecCondTy, 7009 /*IsPairwiseForm=*/false, IsUnsigned); 7010 break; 7011 } 7012 case RK_None: 7013 llvm_unreachable("Expected arithmetic or min/max reduction operation"); 7014 } 7015 7016 IsPairwiseReduction = PairwiseRdxCost < SplittingRdxCost; 7017 int VecReduxCost = IsPairwiseReduction ? PairwiseRdxCost : SplittingRdxCost; 7018 7019 int ScalarReduxCost = 0; 7020 switch (ReductionData.getKind()) { 7021 case RK_Arithmetic: 7022 ScalarReduxCost = 7023 TTI->getArithmeticInstrCost(ReductionData.getOpcode(), ScalarTy); 7024 break; 7025 case RK_Min: 7026 case RK_Max: 7027 case RK_UMin: 7028 case RK_UMax: 7029 ScalarReduxCost = 7030 TTI->getCmpSelInstrCost(ReductionData.getOpcode(), ScalarTy) + 7031 TTI->getCmpSelInstrCost(Instruction::Select, ScalarTy, 7032 CmpInst::makeCmpResultType(ScalarTy)); 7033 break; 7034 case RK_None: 7035 llvm_unreachable("Expected arithmetic or min/max reduction operation"); 7036 } 7037 ScalarReduxCost *= (ReduxWidth - 1); 7038 7039 LLVM_DEBUG(dbgs() << "SLP: Adding cost " << VecReduxCost - ScalarReduxCost 7040 << " for reduction that starts with " << *FirstReducedVal 7041 << " (It is a " 7042 << (IsPairwiseReduction ? "pairwise" : "splitting") 7043 << " reduction)\n"); 7044 7045 return VecReduxCost - ScalarReduxCost; 7046 } 7047 7048 /// Emit a horizontal reduction of the vectorized value. 7049 Value *emitReduction(Value *VectorizedValue, IRBuilder<> &Builder, 7050 unsigned ReduxWidth, const TargetTransformInfo *TTI) { 7051 assert(VectorizedValue && "Need to have a vectorized tree node"); 7052 assert(isPowerOf2_32(ReduxWidth) && 7053 "We only handle power-of-two reductions for now"); 7054 7055 if (!IsPairwiseReduction) { 7056 // FIXME: The builder should use an FMF guard. It should not be hard-coded 7057 // to 'fast'. 7058 assert(Builder.getFastMathFlags().isFast() && "Expected 'fast' FMF"); 7059 return createSimpleTargetReduction( 7060 Builder, TTI, ReductionData.getOpcode(), VectorizedValue, 7061 ReductionData.getFlags(), ReductionOps.back()); 7062 } 7063 7064 Value *TmpVec = VectorizedValue; 7065 for (unsigned i = ReduxWidth / 2; i != 0; i >>= 1) { 7066 auto LeftMask = createRdxShuffleMask(ReduxWidth, i, true, true); 7067 auto RightMask = createRdxShuffleMask(ReduxWidth, i, true, false); 7068 7069 Value *LeftShuf = Builder.CreateShuffleVector( 7070 TmpVec, UndefValue::get(TmpVec->getType()), LeftMask, "rdx.shuf.l"); 7071 Value *RightShuf = Builder.CreateShuffleVector( 7072 TmpVec, UndefValue::get(TmpVec->getType()), (RightMask), 7073 "rdx.shuf.r"); 7074 OperationData VectReductionData(ReductionData.getOpcode(), LeftShuf, 7075 RightShuf, ReductionData.getKind()); 7076 TmpVec = VectReductionData.createOp(Builder, "op.rdx", ReductionOps); 7077 } 7078 7079 // The result is in the first element of the vector. 7080 return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0)); 7081 } 7082}; 7083 7084} // end anonymous namespace 7085 7086/// Recognize construction of vectors like 7087/// %ra = insertelement <4 x float> undef, float %s0, i32 0 7088/// %rb = insertelement <4 x float> %ra, float %s1, i32 1 7089/// %rc = insertelement <4 x float> %rb, float %s2, i32 2 7090/// %rd = insertelement <4 x float> %rc, float %s3, i32 3 7091/// starting from the last insertelement or insertvalue instruction. 7092/// 7093/// Also recognize aggregates like {<2 x float>, <2 x float>}, 7094/// {{float, float}, {float, float}}, [2 x {float, float}] and so on. 7095/// See llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll for examples. 7096/// 7097/// Assume LastInsertInst is of InsertElementInst or InsertValueInst type. 7098/// 7099/// \return true if it matches. 7100static bool findBuildAggregate(Value *LastInsertInst, TargetTransformInfo *TTI, 7101 SmallVectorImpl<Value *> &BuildVectorOpds, 7102 SmallVectorImpl<Value *> &InsertElts) { 7103 assert((isa<InsertElementInst>(LastInsertInst) || 7104 isa<InsertValueInst>(LastInsertInst)) && 7105 "Expected insertelement or insertvalue instruction!"); 7106 do { 7107 Value *InsertedOperand; 7108 auto *IE = dyn_cast<InsertElementInst>(LastInsertInst); 7109 if (IE) { 7110 InsertedOperand = IE->getOperand(1); 7111 LastInsertInst = IE->getOperand(0); 7112 } else { 7113 auto *IV = cast<InsertValueInst>(LastInsertInst); 7114 InsertedOperand = IV->getInsertedValueOperand(); 7115 LastInsertInst = IV->getAggregateOperand(); 7116 } 7117 if (isa<InsertElementInst>(InsertedOperand) || 7118 isa<InsertValueInst>(InsertedOperand)) { 7119 SmallVector<Value *, 8> TmpBuildVectorOpds; 7120 SmallVector<Value *, 8> TmpInsertElts; 7121 if (!findBuildAggregate(InsertedOperand, TTI, TmpBuildVectorOpds, 7122 TmpInsertElts)) 7123 return false; 7124 BuildVectorOpds.append(TmpBuildVectorOpds.rbegin(), 7125 TmpBuildVectorOpds.rend()); 7126 InsertElts.append(TmpInsertElts.rbegin(), TmpInsertElts.rend()); 7127 } else { 7128 BuildVectorOpds.push_back(InsertedOperand); 7129 InsertElts.push_back(IE); 7130 } 7131 if (isa<UndefValue>(LastInsertInst)) 7132 break; 7133 if ((!isa<InsertValueInst>(LastInsertInst) && 7134 !isa<InsertElementInst>(LastInsertInst)) || 7135 !LastInsertInst->hasOneUse()) 7136 return false; 7137 } while (true); 7138 std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end()); 7139 std::reverse(InsertElts.begin(), InsertElts.end()); 7140 return true; 7141} 7142 7143static bool PhiTypeSorterFunc(Value *V, Value *V2) { 7144 return V->getType() < V2->getType(); 7145} 7146 7147/// Try and get a reduction value from a phi node. 7148/// 7149/// Given a phi node \p P in a block \p ParentBB, consider possible reductions 7150/// if they come from either \p ParentBB or a containing loop latch. 7151/// 7152/// \returns A candidate reduction value if possible, or \code nullptr \endcode 7153/// if not possible. 7154static Value *getReductionValue(const DominatorTree *DT, PHINode *P, 7155 BasicBlock *ParentBB, LoopInfo *LI) { 7156 // There are situations where the reduction value is not dominated by the 7157 // reduction phi. Vectorizing such cases has been reported to cause 7158 // miscompiles. See PR25787. 7159 auto DominatedReduxValue = [&](Value *R) { 7160 return isa<Instruction>(R) && 7161 DT->dominates(P->getParent(), cast<Instruction>(R)->getParent()); 7162 }; 7163 7164 Value *Rdx = nullptr; 7165 7166 // Return the incoming value if it comes from the same BB as the phi node. 7167 if (P->getIncomingBlock(0) == ParentBB) { 7168 Rdx = P->getIncomingValue(0); 7169 } else if (P->getIncomingBlock(1) == ParentBB) { 7170 Rdx = P->getIncomingValue(1); 7171 } 7172 7173 if (Rdx && DominatedReduxValue(Rdx)) 7174 return Rdx; 7175 7176 // Otherwise, check whether we have a loop latch to look at. 7177 Loop *BBL = LI->getLoopFor(ParentBB); 7178 if (!BBL) 7179 return nullptr; 7180 BasicBlock *BBLatch = BBL->getLoopLatch(); 7181 if (!BBLatch) 7182 return nullptr; 7183 7184 // There is a loop latch, return the incoming value if it comes from 7185 // that. This reduction pattern occasionally turns up. 7186 if (P->getIncomingBlock(0) == BBLatch) { 7187 Rdx = P->getIncomingValue(0); 7188 } else if (P->getIncomingBlock(1) == BBLatch) { 7189 Rdx = P->getIncomingValue(1); 7190 } 7191 7192 if (Rdx && DominatedReduxValue(Rdx)) 7193 return Rdx; 7194 7195 return nullptr; 7196} 7197 7198/// Attempt to reduce a horizontal reduction. 7199/// If it is legal to match a horizontal reduction feeding the phi node \a P 7200/// with reduction operators \a Root (or one of its operands) in a basic block 7201/// \a BB, then check if it can be done. If horizontal reduction is not found 7202/// and root instruction is a binary operation, vectorization of the operands is 7203/// attempted. 7204/// \returns true if a horizontal reduction was matched and reduced or operands 7205/// of one of the binary instruction were vectorized. 7206/// \returns false if a horizontal reduction was not matched (or not possible) 7207/// or no vectorization of any binary operation feeding \a Root instruction was 7208/// performed. 7209static bool tryToVectorizeHorReductionOrInstOperands( 7210 PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R, 7211 TargetTransformInfo *TTI, 7212 const function_ref<bool(Instruction *, BoUpSLP &)> Vectorize) { 7213 if (!ShouldVectorizeHor) 7214 return false; 7215 7216 if (!Root) 7217 return false; 7218 7219 if (Root->getParent() != BB || isa<PHINode>(Root)) 7220 return false; 7221 // Start analysis starting from Root instruction. If horizontal reduction is 7222 // found, try to vectorize it. If it is not a horizontal reduction or 7223 // vectorization is not possible or not effective, and currently analyzed 7224 // instruction is a binary operation, try to vectorize the operands, using 7225 // pre-order DFS traversal order. If the operands were not vectorized, repeat 7226 // the same procedure considering each operand as a possible root of the 7227 // horizontal reduction. 7228 // Interrupt the process if the Root instruction itself was vectorized or all 7229 // sub-trees not higher that RecursionMaxDepth were analyzed/vectorized. 7230 SmallVector<std::pair<Instruction *, unsigned>, 8> Stack(1, {Root, 0}); 7231 SmallPtrSet<Value *, 8> VisitedInstrs; 7232 bool Res = false; 7233 while (!Stack.empty()) { 7234 Instruction *Inst; 7235 unsigned Level; 7236 std::tie(Inst, Level) = Stack.pop_back_val(); 7237 auto *BI = dyn_cast<BinaryOperator>(Inst); 7238 auto *SI = dyn_cast<SelectInst>(Inst); 7239 if (BI || SI) { 7240 HorizontalReduction HorRdx; 7241 if (HorRdx.matchAssociativeReduction(P, Inst)) { 7242 if (HorRdx.tryToReduce(R, TTI)) { 7243 Res = true; 7244 // Set P to nullptr to avoid re-analysis of phi node in 7245 // matchAssociativeReduction function unless this is the root node. 7246 P = nullptr; 7247 continue; 7248 } 7249 } 7250 if (P && BI) { 7251 Inst = dyn_cast<Instruction>(BI->getOperand(0)); 7252 if (Inst == P) 7253 Inst = dyn_cast<Instruction>(BI->getOperand(1)); 7254 if (!Inst) { 7255 // Set P to nullptr to avoid re-analysis of phi node in 7256 // matchAssociativeReduction function unless this is the root node. 7257 P = nullptr; 7258 continue; 7259 } 7260 } 7261 } 7262 // Set P to nullptr to avoid re-analysis of phi node in 7263 // matchAssociativeReduction function unless this is the root node. 7264 P = nullptr; 7265 if (Vectorize(Inst, R)) { 7266 Res = true; 7267 continue; 7268 } 7269 7270 // Try to vectorize operands. 7271 // Continue analysis for the instruction from the same basic block only to 7272 // save compile time. 7273 if (++Level < RecursionMaxDepth) 7274 for (auto *Op : Inst->operand_values()) 7275 if (VisitedInstrs.insert(Op).second) 7276 if (auto *I = dyn_cast<Instruction>(Op)) 7277 if (!isa<PHINode>(I) && !R.isDeleted(I) && I->getParent() == BB) 7278 Stack.emplace_back(I, Level); 7279 } 7280 return Res; 7281} 7282 7283bool SLPVectorizerPass::vectorizeRootInstruction(PHINode *P, Value *V, 7284 BasicBlock *BB, BoUpSLP &R, 7285 TargetTransformInfo *TTI) { 7286 if (!V) 7287 return false; 7288 auto *I = dyn_cast<Instruction>(V); 7289 if (!I) 7290 return false; 7291 7292 if (!isa<BinaryOperator>(I)) 7293 P = nullptr; 7294 // Try to match and vectorize a horizontal reduction. 7295 auto &&ExtraVectorization = [this](Instruction *I, BoUpSLP &R) -> bool { 7296 return tryToVectorize(I, R); 7297 }; 7298 return tryToVectorizeHorReductionOrInstOperands(P, I, BB, R, TTI, 7299 ExtraVectorization); 7300} 7301 7302bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI, 7303 BasicBlock *BB, BoUpSLP &R) { 7304 const DataLayout &DL = BB->getModule()->getDataLayout(); 7305 if (!R.canMapToVector(IVI->getType(), DL)) 7306 return false; 7307 7308 SmallVector<Value *, 16> BuildVectorOpds; 7309 SmallVector<Value *, 16> BuildVectorInsts; 7310 if (!findBuildAggregate(IVI, TTI, BuildVectorOpds, BuildVectorInsts) || 7311 BuildVectorOpds.size() < 2) 7312 return false; 7313 7314 LLVM_DEBUG(dbgs() << "SLP: array mappable to vector: " << *IVI << "\n"); 7315 // Aggregate value is unlikely to be processed in vector register, we need to 7316 // extract scalars into scalar registers, so NeedExtraction is set true. 7317 return tryToVectorizeList(BuildVectorOpds, R, /*AllowReorder=*/false, 7318 BuildVectorInsts); 7319} 7320 7321bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI, 7322 BasicBlock *BB, BoUpSLP &R) { 7323 SmallVector<Value *, 16> BuildVectorInsts; 7324 SmallVector<Value *, 16> BuildVectorOpds; 7325 if (!findBuildAggregate(IEI, TTI, BuildVectorOpds, BuildVectorInsts) || 7326 BuildVectorOpds.size() < 2 || 7327 (llvm::all_of(BuildVectorOpds, 7328 [](Value *V) { return isa<ExtractElementInst>(V); }) && 7329 isShuffle(BuildVectorOpds))) 7330 return false; 7331 7332 // Vectorize starting with the build vector operands ignoring the BuildVector 7333 // instructions for the purpose of scheduling and user extraction. 7334 return tryToVectorizeList(BuildVectorOpds, R, /*AllowReorder=*/false, 7335 BuildVectorInsts); 7336} 7337 7338bool SLPVectorizerPass::vectorizeCmpInst(CmpInst *CI, BasicBlock *BB, 7339 BoUpSLP &R) { 7340 if (tryToVectorizePair(CI->getOperand(0), CI->getOperand(1), R)) 7341 return true; 7342 7343 bool OpsChanged = false; 7344 for (int Idx = 0; Idx < 2; ++Idx) { 7345 OpsChanged |= 7346 vectorizeRootInstruction(nullptr, CI->getOperand(Idx), BB, R, TTI); 7347 } 7348 return OpsChanged; 7349} 7350 7351bool SLPVectorizerPass::vectorizeSimpleInstructions( 7352 SmallVectorImpl<Instruction *> &Instructions, BasicBlock *BB, BoUpSLP &R) { 7353 bool OpsChanged = false; 7354 for (auto *I : reverse(Instructions)) { 7355 if (R.isDeleted(I)) 7356 continue; 7357 if (auto *LastInsertValue = dyn_cast<InsertValueInst>(I)) 7358 OpsChanged |= vectorizeInsertValueInst(LastInsertValue, BB, R); 7359 else if (auto *LastInsertElem = dyn_cast<InsertElementInst>(I)) 7360 OpsChanged |= vectorizeInsertElementInst(LastInsertElem, BB, R); 7361 else if (auto *CI = dyn_cast<CmpInst>(I)) 7362 OpsChanged |= vectorizeCmpInst(CI, BB, R); 7363 } 7364 Instructions.clear(); 7365 return OpsChanged; 7366} 7367 7368bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) { 7369 bool Changed = false; 7370 SmallVector<Value *, 4> Incoming; 7371 SmallPtrSet<Value *, 16> VisitedInstrs; 7372 unsigned MaxVecRegSize = R.getMaxVecRegSize(); 7373 7374 bool HaveVectorizedPhiNodes = true; 7375 while (HaveVectorizedPhiNodes) { 7376 HaveVectorizedPhiNodes = false; 7377 7378 // Collect the incoming values from the PHIs. 7379 Incoming.clear(); 7380 for (Instruction &I : *BB) { 7381 PHINode *P = dyn_cast<PHINode>(&I); 7382 if (!P) 7383 break; 7384 7385 if (!VisitedInstrs.count(P) && !R.isDeleted(P)) 7386 Incoming.push_back(P); 7387 } 7388 7389 // Sort by type. 7390 llvm::stable_sort(Incoming, PhiTypeSorterFunc); 7391 7392 // Try to vectorize elements base on their type. 7393 for (SmallVector<Value *, 4>::iterator IncIt = Incoming.begin(), 7394 E = Incoming.end(); 7395 IncIt != E;) { 7396 7397 // Look for the next elements with the same type. 7398 SmallVector<Value *, 4>::iterator SameTypeIt = IncIt; 7399 Type *EltTy = (*IncIt)->getType(); 7400 7401 assert(EltTy->isSized() && 7402 "Instructions should all be sized at this point"); 7403 TypeSize EltTS = DL->getTypeSizeInBits(EltTy); 7404 if (EltTS.isScalable()) { 7405 // For now, just ignore vectorizing scalable types. 7406 ++IncIt; 7407 continue; 7408 } 7409 7410 unsigned EltSize = EltTS.getFixedSize(); 7411 unsigned MaxNumElts = MaxVecRegSize / EltSize; 7412 if (MaxNumElts < 2) { 7413 ++IncIt; 7414 continue; 7415 } 7416 7417 while (SameTypeIt != E && 7418 (*SameTypeIt)->getType() == EltTy && 7419 (SameTypeIt - IncIt) < MaxNumElts) { 7420 VisitedInstrs.insert(*SameTypeIt); 7421 ++SameTypeIt; 7422 } 7423 7424 // Try to vectorize them. 7425 unsigned NumElts = (SameTypeIt - IncIt); 7426 LLVM_DEBUG(dbgs() << "SLP: Trying to vectorize starting at PHIs (" 7427 << NumElts << ")\n"); 7428 // The order in which the phi nodes appear in the program does not matter. 7429 // So allow tryToVectorizeList to reorder them if it is beneficial. This 7430 // is done when there are exactly two elements since tryToVectorizeList 7431 // asserts that there are only two values when AllowReorder is true. 7432 bool AllowReorder = NumElts == 2; 7433 if (NumElts > 1 && 7434 tryToVectorizeList(makeArrayRef(IncIt, NumElts), R, AllowReorder)) { 7435 // Success start over because instructions might have been changed. 7436 HaveVectorizedPhiNodes = true; 7437 Changed = true; 7438 break; 7439 } 7440 7441 // Start over at the next instruction of a different type (or the end). 7442 IncIt = SameTypeIt; 7443 } 7444 } 7445 7446 VisitedInstrs.clear(); 7447 7448 SmallVector<Instruction *, 8> PostProcessInstructions; 7449 SmallDenseSet<Instruction *, 4> KeyNodes; 7450 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; ++it) { 7451 // Skip instructions marked for the deletion. 7452 if (R.isDeleted(&*it)) 7453 continue; 7454 // We may go through BB multiple times so skip the one we have checked. 7455 if (!VisitedInstrs.insert(&*it).second) { 7456 if (it->use_empty() && KeyNodes.count(&*it) > 0 && 7457 vectorizeSimpleInstructions(PostProcessInstructions, BB, R)) { 7458 // We would like to start over since some instructions are deleted 7459 // and the iterator may become invalid value. 7460 Changed = true; 7461 it = BB->begin(); 7462 e = BB->end(); 7463 } 7464 continue; 7465 } 7466 7467 if (isa<DbgInfoIntrinsic>(it)) 7468 continue; 7469 7470 // Try to vectorize reductions that use PHINodes. 7471 if (PHINode *P = dyn_cast<PHINode>(it)) { 7472 // Check that the PHI is a reduction PHI. 7473 if (P->getNumIncomingValues() != 2) 7474 return Changed; 7475 7476 // Try to match and vectorize a horizontal reduction. 7477 if (vectorizeRootInstruction(P, getReductionValue(DT, P, BB, LI), BB, R, 7478 TTI)) { 7479 Changed = true; 7480 it = BB->begin(); 7481 e = BB->end(); 7482 continue; 7483 } 7484 continue; 7485 } 7486 7487 // Ran into an instruction without users, like terminator, or function call 7488 // with ignored return value, store. Ignore unused instructions (basing on 7489 // instruction type, except for CallInst and InvokeInst). 7490 if (it->use_empty() && (it->getType()->isVoidTy() || isa<CallInst>(it) || 7491 isa<InvokeInst>(it))) { 7492 KeyNodes.insert(&*it); 7493 bool OpsChanged = false; 7494 if (ShouldStartVectorizeHorAtStore || !isa<StoreInst>(it)) { 7495 for (auto *V : it->operand_values()) { 7496 // Try to match and vectorize a horizontal reduction. 7497 OpsChanged |= vectorizeRootInstruction(nullptr, V, BB, R, TTI); 7498 } 7499 } 7500 // Start vectorization of post-process list of instructions from the 7501 // top-tree instructions to try to vectorize as many instructions as 7502 // possible. 7503 OpsChanged |= vectorizeSimpleInstructions(PostProcessInstructions, BB, R); 7504 if (OpsChanged) { 7505 // We would like to start over since some instructions are deleted 7506 // and the iterator may become invalid value. 7507 Changed = true; 7508 it = BB->begin(); 7509 e = BB->end(); 7510 continue; 7511 } 7512 } 7513 7514 if (isa<InsertElementInst>(it) || isa<CmpInst>(it) || 7515 isa<InsertValueInst>(it)) 7516 PostProcessInstructions.push_back(&*it); 7517 } 7518 7519 return Changed; 7520} 7521 7522bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) { 7523 auto Changed = false; 7524 for (auto &Entry : GEPs) { 7525 // If the getelementptr list has fewer than two elements, there's nothing 7526 // to do. 7527 if (Entry.second.size() < 2) 7528 continue; 7529 7530 LLVM_DEBUG(dbgs() << "SLP: Analyzing a getelementptr list of length " 7531 << Entry.second.size() << ".\n"); 7532 7533 // Process the GEP list in chunks suitable for the target's supported 7534 // vector size. If a vector register can't hold 1 element, we are done. We 7535 // are trying to vectorize the index computations, so the maximum number of 7536 // elements is based on the size of the index expression, rather than the 7537 // size of the GEP itself (the target's pointer size). 7538 unsigned MaxVecRegSize = R.getMaxVecRegSize(); 7539 unsigned EltSize = R.getVectorElementSize(*Entry.second[0]->idx_begin()); 7540 if (MaxVecRegSize < EltSize) 7541 continue; 7542 7543 unsigned MaxElts = MaxVecRegSize / EltSize; 7544 for (unsigned BI = 0, BE = Entry.second.size(); BI < BE; BI += MaxElts) { 7545 auto Len = std::min<unsigned>(BE - BI, MaxElts); 7546 auto GEPList = makeArrayRef(&Entry.second[BI], Len); 7547 7548 // Initialize a set a candidate getelementptrs. Note that we use a 7549 // SetVector here to preserve program order. If the index computations 7550 // are vectorizable and begin with loads, we want to minimize the chance 7551 // of having to reorder them later. 7552 SetVector<Value *> Candidates(GEPList.begin(), GEPList.end()); 7553 7554 // Some of the candidates may have already been vectorized after we 7555 // initially collected them. If so, they are marked as deleted, so remove 7556 // them from the set of candidates. 7557 Candidates.remove_if( 7558 [&R](Value *I) { return R.isDeleted(cast<Instruction>(I)); }); 7559 7560 // Remove from the set of candidates all pairs of getelementptrs with 7561 // constant differences. Such getelementptrs are likely not good 7562 // candidates for vectorization in a bottom-up phase since one can be 7563 // computed from the other. We also ensure all candidate getelementptr 7564 // indices are unique. 7565 for (int I = 0, E = GEPList.size(); I < E && Candidates.size() > 1; ++I) { 7566 auto *GEPI = GEPList[I]; 7567 if (!Candidates.count(GEPI)) 7568 continue; 7569 auto *SCEVI = SE->getSCEV(GEPList[I]); 7570 for (int J = I + 1; J < E && Candidates.size() > 1; ++J) { 7571 auto *GEPJ = GEPList[J]; 7572 auto *SCEVJ = SE->getSCEV(GEPList[J]); 7573 if (isa<SCEVConstant>(SE->getMinusSCEV(SCEVI, SCEVJ))) { 7574 Candidates.remove(GEPI); 7575 Candidates.remove(GEPJ); 7576 } else if (GEPI->idx_begin()->get() == GEPJ->idx_begin()->get()) { 7577 Candidates.remove(GEPJ); 7578 } 7579 } 7580 } 7581 7582 // We break out of the above computation as soon as we know there are 7583 // fewer than two candidates remaining. 7584 if (Candidates.size() < 2) 7585 continue; 7586 7587 // Add the single, non-constant index of each candidate to the bundle. We 7588 // ensured the indices met these constraints when we originally collected 7589 // the getelementptrs. 7590 SmallVector<Value *, 16> Bundle(Candidates.size()); 7591 auto BundleIndex = 0u; 7592 for (auto *V : Candidates) { 7593 auto *GEP = cast<GetElementPtrInst>(V); 7594 auto *GEPIdx = GEP->idx_begin()->get(); 7595 assert(GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx)); 7596 Bundle[BundleIndex++] = GEPIdx; 7597 } 7598 7599 // Try and vectorize the indices. We are currently only interested in 7600 // gather-like cases of the form: 7601 // 7602 // ... = g[a[0] - b[0]] + g[a[1] - b[1]] + ... 7603 // 7604 // where the loads of "a", the loads of "b", and the subtractions can be 7605 // performed in parallel. It's likely that detecting this pattern in a 7606 // bottom-up phase will be simpler and less costly than building a 7607 // full-blown top-down phase beginning at the consecutive loads. 7608 Changed |= tryToVectorizeList(Bundle, R); 7609 } 7610 } 7611 return Changed; 7612} 7613 7614bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) { 7615 bool Changed = false; 7616 // Attempt to sort and vectorize each of the store-groups. 7617 for (StoreListMap::iterator it = Stores.begin(), e = Stores.end(); it != e; 7618 ++it) { 7619 if (it->second.size() < 2) 7620 continue; 7621 7622 LLVM_DEBUG(dbgs() << "SLP: Analyzing a store chain of length " 7623 << it->second.size() << ".\n"); 7624 7625 Changed |= vectorizeStores(it->second, R); 7626 } 7627 return Changed; 7628} 7629 7630char SLPVectorizer::ID = 0; 7631 7632static const char lv_name[] = "SLP Vectorizer"; 7633 7634INITIALIZE_PASS_BEGIN(SLPVectorizer, SV_NAME, lv_name, false, false) 7635INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) 7636INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) 7637INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) 7638INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass) 7639INITIALIZE_PASS_DEPENDENCY(LoopSimplify) 7640INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass) 7641INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass) 7642INITIALIZE_PASS_DEPENDENCY(InjectTLIMappingsLegacy) 7643INITIALIZE_PASS_END(SLPVectorizer, SV_NAME, lv_name, false, false) 7644 7645Pass *llvm::createSLPVectorizerPass() { return new SLPVectorizer(); } 7646