Lines Matching defs:LU
989 /// folded in \p LU at isel time.
991 /// This function returns true if \p LU can accommodate what \p F
998 const LSRUse &LU, const Formula &F);
1000 // Get the cost of the scaling factor used in F for LU.
1002 const LSRUse &LU, const Formula &F,
1050 const LSRUse &LU,
1316 const LSRUse &LU,
1348 NumBaseParts - (1 + (F.Scale && isAMCompletelyFolded(*TTI, LU, F)));
1352 C.ScaleCost += getScalingFactorCost(*TTI, LU, F, *L);
1355 for (const LSRFixup &Fixup : LU.Fixups) {
1366 if (LU.Kind == LSRUse::Address && Offset != 0 &&
1367 !isAMCompletelyFolded(*TTI, LSRUse::Address, LU.AccessTy, F.BaseGV,
1402 if (LU.Kind == LSRUse::ICmpZero && !F.hasZeroEnd() &&
1409 if (LU.Kind != LSRUse::ICmpZero)
1733 const LSRUse &LU, const Formula &F) {
1735 if (LU.Kind == LSRUse::Address && TTI.LSRWithInstrQueries()) {
1736 for (const LSRFixup &Fixup : LU.Fixups)
1737 if (!isAMCompletelyFolded(TTI, LSRUse::Address, LU.AccessTy, F.BaseGV,
1744 return isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
1745 LU.AccessTy, F.BaseGV, F.BaseOffset, F.HasBaseReg,
1750 const LSRUse &LU, const Formula &F,
1757 if (!isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
1758 LU.AccessTy, F, L))
1761 switch (LU.Kind) {
1765 LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MinOffset, F.HasBaseReg,
1766 F.Scale, LU.AccessTy.AddrSpace);
1768 LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MaxOffset, F.HasBaseReg,
1769 F.Scale, LU.AccessTy.AddrSpace);
1966 bool reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg,
1972 void DeleteUse(LSRUse &LU, size_t LUIdx);
1976 void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
1977 void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
1979 bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F);
1983 void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base,
1986 void GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
1989 void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base);
1990 void GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
1993 void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
1994 void GenerateConstantOffsetsImpl(LSRUse &LU, unsigned LUIdx,
1998 void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
1999 void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2000 void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2001 void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base);
2031 const LSRUse &LU,
2034 Value *Expand(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2037 void RewriteForPHI(PHINode *PN, const LSRUse &LU, const LSRFixup &LF,
2040 void Rewrite(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2511 bool LSRInstance::reconcileNewOffset(LSRUse &LU, int64_t NewOffset,
2514 int64_t NewMinOffset = LU.MinOffset;
2515 int64_t NewMaxOffset = LU.MaxOffset;
2521 if (LU.Kind != Kind)
2528 if (AccessTy.MemTy != LU.AccessTy.MemTy) {
2535 if (NewOffset < LU.MinOffset) {
2537 LU.MaxOffset - NewOffset, HasBaseReg))
2540 } else if (NewOffset > LU.MaxOffset) {
2542 NewOffset - LU.MinOffset, HasBaseReg))
2548 LU.MinOffset = NewMinOffset;
2549 LU.MaxOffset = NewMaxOffset;
2550 LU.AccessTy = NewAccessTy;
2575 LSRUse &LU = Uses[LUIdx];
2576 if (reconcileNewOffset(LU, Offset, /*HasBaseReg=*/true, Kind, AccessTy))
2585 LSRUse &LU = Uses[LUIdx];
2587 LU.MinOffset = Offset;
2588 LU.MaxOffset = Offset;
2593 void LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) {
2594 if (&LU != &Uses.back())
2595 std::swap(LU, Uses.back());
2609 LSRUse &LU = Uses[LUIdx];
2615 if (&LU != &OrigLU &&
2616 LU.Kind != LSRUse::ICmpZero &&
2617 LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy &&
2618 LU.WidestFixupType == OrigLU.WidestFixupType &&
2619 LU.HasFormulaWithSameRegs(OrigF)) {
2621 for (const Formula &F : LU.Formulae) {
2630 return &LU;
3312 LSRUse &LU = Uses[LUIdx];
3315 LSRFixup &LF = LU.getNewFixup();
3320 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3322 if (!LU.WidestFixupType ||
3323 SE.getTypeSizeInBits(LU.WidestFixupType) <
3325 LU.WidestFixupType = LF.OperandValToReplace->getType();
3328 if (LU.Formulae.empty()) {
3329 InsertInitialFormula(S, LU, LUIdx);
3330 CountRegisters(LU.Formulae.back(), LUIdx);
3340 LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx) {
3343 LU.RigidFormula = true;
3347 bool Inserted = InsertFormula(LU, LUIdx, F);
3355 LSRUse &LU, size_t LUIdx) {
3359 bool Inserted = InsertFormula(LU, LUIdx, F);
3373 bool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) {
3375 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) &&
3378 if (!LU.InsertFormula(F, *L))
3464 LSRUse &LU = Uses[LUIdx];
3465 LSRFixup &LF = LU.getNewFixup();
3469 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3470 if (!LU.WidestFixupType ||
3471 SE.getTypeSizeInBits(LU.WidestFixupType) <
3473 LU.WidestFixupType = LF.OperandValToReplace->getType();
3474 InsertSupplementalFormula(US, LU, LUIdx);
3475 CountRegisters(LU.Formulae.back(), Uses.size() - 1);
3546 LSRUse &LU, const SCEV *S, const Loop *L,
3548 if (LU.Kind != LSRUse::Address ||
3549 !LU.AccessTy.getType()->isIntOrIntVectorTy())
3568 void LSRInstance::GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
3577 if (TTI.shouldFavorPostInc() && mayUsePostIncMode(TTI, LU, BaseReg, L, SE))
3597 if (isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
3598 LU.AccessTy, *J, Base.getNumRegs() > 1))
3610 isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
3611 LU.AccessTy, InnerAddOps[0], Base.getNumRegs() > 1))
3648 if (InsertFormula(LU, LUIdx, F))
3655 GenerateReassociations(LU, LUIdx, LU.Formulae.back(),
3661 void LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx,
3669 GenerateReassociationsImpl(LU, LUIdx, Base, Depth, i);
3672 GenerateReassociationsImpl(LU, LUIdx, Base, Depth,
3678 void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx,
3720 (void)InsertFormula(LU, LUIdx, F);
3741 void LSRInstance::GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
3750 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
3756 (void)InsertFormula(LU, LUIdx, F);
3760 void LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx,
3766 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, i);
3768 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, /* Idx */ -1,
3774 LSRUse &LU, unsigned LUIdx, const Formula &Base,
3781 if (isLegalUse(TTI, LU.MinOffset - Offset, LU.MaxOffset - Offset, LU.Kind,
3782 LU.AccessTy, F)) {
3798 (void)InsertFormula(LU, LUIdx, F);
3812 if (FavorBackedgeIndex && LU.Kind == LSRUse::Address) {
3835 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
3841 (void)InsertFormula(LU, LUIdx, F);
3845 void LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx,
3850 Worklist.push_back(LU.MinOffset);
3851 if (LU.MaxOffset != LU.MinOffset)
3852 Worklist.push_back(LU.MaxOffset);
3855 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, i);
3857 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, /* Idx */ -1,
3863 void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx,
3865 if (LU.Kind != LSRUse::ICmpZero) return;
3873 if (LU.MinOffset != LU.MaxOffset) return;
3897 int64_t Offset = LU.MinOffset;
3901 if (Offset / Factor != LU.MinOffset)
3912 if (!isLegalUse(TTI, Offset, Offset, LU.Kind, LU.AccessTy, F))
3916 F.BaseOffset = (uint64_t)F.BaseOffset + Offset - LU.MinOffset;
3949 (void)InsertFormula(LU, LUIdx, F);
3956 void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) {
3973 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
3977 if (LU.Kind == LSRUse::Basic &&
3978 isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LSRUse::Special,
3979 LU.AccessTy, Base) &&
3980 LU.AllFixupsOutsideLoop)
3981 LU.Kind = LSRUse::Special;
3987 if (LU.Kind == LSRUse::ICmpZero &&
3993 if (AR && (AR->getLoop() == L || LU.AllFixupsOutsideLoop)) {
4008 (AR->getLoop() != L && LU.AllFixupsOutsideLoop)))
4012 if (F.Scale == 1 && LU.AllFixupsOutsideLoop)
4014 (void)InsertFormula(LU, LUIdx, F);
4022 void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
4063 (void)InsertFormula(LU, LUIdx, F);
4185 LSRUse &LU = Uses[LUIdx];
4194 for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) {
4195 Formula F = LU.Formulae[L];
4210 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
4226 (void)InsertFormula(LU, LUIdx, NewF);
4235 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset,
4236 LU.Kind, LU.AccessTy, NewF)) {
4238 mayUsePostIncMode(TTI, LU, OrigReg, this->L, SE))
4261 (void)InsertFormula(LU, LUIdx, NewF);
4276 LSRUse &LU = Uses[LUIdx];
4277 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4278 GenerateReassociations(LU, LUIdx, LU.Formulae[i]);
4279 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4280 GenerateCombinations(LU, LUIdx, LU.Formulae[i]);
4283 LSRUse &LU = Uses[LUIdx];
4284 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4285 GenerateSymbolicOffsets(LU, LUIdx, LU.Formulae[i]);
4286 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4287 GenerateConstantOffsets(LU, LUIdx, LU.Formulae[i]);
4288 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4289 GenerateICmpZeroScales(LU, LUIdx, LU.Formulae[i]);
4290 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4291 GenerateScales(LU, LUIdx, LU.Formulae[i]);
4294 LSRUse &LU = Uses[LUIdx];
4295 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4296 GenerateTruncates(LU, LUIdx, LU.Formulae[i]);
4324 LSRUse &LU = Uses[LUIdx];
4325 LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
4329 for (size_t FIdx = 0, NumForms = LU.Formulae.size();
4331 Formula &F = LU.Formulae[FIdx];
4342 CostF.RateFormula(F, Regs, VisitedRegs, LU, &LoserRegs);
4371 Formula &Best = LU.Formulae[P.first->second];
4375 CostBest.RateFormula(Best, Regs, VisitedRegs, LU);
4386 LU.DeleteFormula(F);
4394 LU.RecomputeRegs(LUIdx, RegUses);
4412 for (const LSRUse &LU : Uses) {
4413 size_t FSize = LU.Formulae.size();
4437 LSRUse &LU = Uses[LUIdx];
4439 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
4440 Formula &F = LU.Formulae[i];
4453 if (LU.HasFormulaWithSameRegs(NewF)) {
4456 LU.DeleteFormula(F);
4469 if (LU.HasFormulaWithSameRegs(NewF)) {
4472 LU.DeleteFormula(F);
4483 LU.RecomputeRegs(LUIdx, RegUses);
4504 LSRUse &LU = Uses[LUIdx];
4505 for (const Formula &F : LU.Formulae) {
4509 LSRUse *LUThatHas = FindUseWithSimilarFormula(F, LU);
4514 LU.Kind, LU.AccessTy))
4517 LLVM_DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); dbgs() << '\n');
4519 LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop;
4521 // Transfer the fixups of LU to LUThatHas.
4522 for (LSRFixup &Fixup : LU.Fixups) {
4546 DeleteUse(LU, LUIdx);
4601 LSRUse &LU = Uses[LUIdx];
4602 LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
4629 CostFA.RateFormula(FA, Regs, VisitedRegs, LU);
4631 CostFB.RateFormula(FB, Regs, VisitedRegs, LU);
4636 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms;
4638 Formula &F = LU.Formulae[FIdx];
4645 Formula &Best = LU.Formulae[P.first->second];
4655 LU.DeleteFormula(F);
4661 LU.RecomputeRegs(LUIdx, RegUses);
4687 LSRUse &LU = Uses[LUIdx];
4689 if (LU.Kind != LSRUse::Address)
4691 if (!TTI.isIndexedLoadLegal(TTI.MIM_PostInc, LU.AccessTy.getType()) &&
4692 !TTI.isIndexedStoreLegal(TTI.MIM_PostInc, LU.AccessTy.getType()))
4696 for (const Formula &F : LU.Formulae)
4700 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms;
4702 Formula &F = LU.Formulae[FIdx];
4706 LU.DeleteFormula(F);
4713 LU.RecomputeRegs(LUIdx, RegUses);
4782 for (const LSRUse &LU : Uses) {
4783 if (!LU.Regs.count(Reg))
4785 float P = LU.getNotSelectedProbability(Reg);
4799 LSRUse &LU = Uses[LUIdx];
4801 if (LU.Formulae.size() < 2)
4806 float FMinRegNum = LU.Formulae[0].getNumRegs();
4807 float FMinARegNum = LU.Formulae[0].getNumRegs();
4809 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
4810 Formula &F = LU.Formulae[i];
4816 FRegNum += RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg);
4819 RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg);
4824 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg);
4827 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg);
4837 LLVM_DEBUG(dbgs() << " The formula "; LU.Formulae[MinIdx].print(dbgs());
4840 std::swap(LU.Formulae[MinIdx], LU.Formulae[0]);
4841 while (LU.Formulae.size() != 1) {
4842 LLVM_DEBUG(dbgs() << " Deleting "; LU.Formulae.back().print(dbgs());
4844 LU.Formulae.pop_back();
4846 LU.RecomputeRegs(LUIdx, RegUses);
4847 assert(LU.Formulae.size() == 1 && "Should be exactly 1 min regs formula");
4848 Formula &F = LU.Formulae[0];
4897 LSRUse &LU = Uses[LUIdx];
4898 if (!LU.Regs.count(Best)) continue;
4901 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
4902 Formula &F = LU.Formulae[i];
4905 LU.DeleteFormula(F);
4915 LU.RecomputeRegs(LUIdx, RegUses);
4956 const LSRUse &LU = Uses[Workspace.size()];
4964 if (LU.Regs.count(S))
4969 for (const Formula &F : LU.Formulae) {
4976 if (!TTI.shouldFavorPostInc() || LU.Kind != LSRUse::Address) {
4997 NewCost.RateFormula(F, NewRegs, VisitedRegs, LU);
5118 const LSRUse &LU,
5126 if (LU.Kind == LSRUse::ICmpZero)
5180 Value *LSRInstance::Expand(const LSRUse &LU, const LSRFixup &LF,
5184 if (LU.RigidFormula)
5189 IP = AdjustInsertPositionForExpand(IP, LF, LU, Rewriter);
5230 if (LU.Kind == LSRUse::ICmpZero) {
5249 if (!Ops.empty() && LU.Kind == LSRUse::Address &&
5250 isAMCompletelyFolded(TTI, LU, F)) {
5285 if (LU.Kind == LSRUse::ICmpZero) {
5321 if (LU.Kind == LSRUse::ICmpZero) {
5360 PHINode *PN, const LSRUse &LU, const LSRFixup &LF, const Formula &F,
5415 Value *FullV = Expand(LU, LF, F, BB->getTerminator()->getIterator(),
5474 void LSRInstance::Rewrite(const LSRUse &LU, const LSRFixup &LF,
5480 RewriteForPHI(PN, LU, LF, F, Rewriter, DeadInsts);
5483 Expand(LU, LF, F, LF.UserInst->getIterator(), Rewriter, DeadInsts);
5499 if (LU.Kind == LSRUse::ICmpZero)
5654 for (const LSRUse &LU : Uses) {
5655 for (const Formula &F : LU.Formulae)
5656 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
5688 for (const LSRUse &LU : Uses)
5689 for (const LSRFixup &LF : LU.Fixups) {
5698 for (const LSRUse &LU : Uses) {
5700 LU.print(OS);
5702 for (const Formula &F : LU.Formulae) {