Lines Matching defs:LU
1000 /// folded in \p LU at isel time.
1002 /// This function returns true if \p LU can accommodate what \p F
1009 const LSRUse &LU, const Formula &F);
1011 // Get the cost of the scaling factor used in F for LU.
1013 const LSRUse &LU, const Formula &F,
1063 const LSRUse &LU,
1327 const LSRUse &LU,
1359 NumBaseParts - (1 + (F.Scale && isAMCompletelyFolded(*TTI, LU, F)));
1363 C.ScaleCost += *getScalingFactorCost(*TTI, LU, F, *L).getValue();
1366 for (const LSRFixup &Fixup : LU.Fixups) {
1377 if (LU.Kind == LSRUse::Address && Offset != 0 &&
1378 !isAMCompletelyFolded(*TTI, LSRUse::Address, LU.AccessTy, F.BaseGV,
1413 if (LU.Kind == LSRUse::ICmpZero && !F.hasZeroEnd() &&
1420 if (LU.Kind != LSRUse::ICmpZero)
1744 const LSRUse &LU, const Formula &F) {
1746 if (LU.Kind == LSRUse::Address && TTI.LSRWithInstrQueries()) {
1747 for (const LSRFixup &Fixup : LU.Fixups)
1748 if (!isAMCompletelyFolded(TTI, LSRUse::Address, LU.AccessTy, F.BaseGV,
1755 return isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
1756 LU.AccessTy, F.BaseGV, F.BaseOffset, F.HasBaseReg,
1761 const LSRUse &LU, const Formula &F,
1768 if (!isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
1769 LU.AccessTy, F, L))
1772 switch (LU.Kind) {
1776 LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MinOffset, F.HasBaseReg,
1777 F.Scale, LU.AccessTy.AddrSpace);
1779 LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MaxOffset, F.HasBaseReg,
1780 F.Scale, LU.AccessTy.AddrSpace);
1977 bool reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg,
1983 void DeleteUse(LSRUse &LU, size_t LUIdx);
1987 void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
1988 void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
1990 bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F);
1994 void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base,
1997 void GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
2000 void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base);
2001 void GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2004 void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2005 void GenerateConstantOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2009 void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2010 void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2011 void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2012 void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base);
2042 const LSRUse &LU,
2045 Value *Expand(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2048 void RewriteForPHI(PHINode *PN, const LSRUse &LU, const LSRFixup &LF,
2051 void Rewrite(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2523 bool LSRInstance::reconcileNewOffset(LSRUse &LU, int64_t NewOffset,
2526 int64_t NewMinOffset = LU.MinOffset;
2527 int64_t NewMaxOffset = LU.MaxOffset;
2533 if (LU.Kind != Kind)
2540 if (AccessTy.MemTy != LU.AccessTy.MemTy) {
2547 if (NewOffset < LU.MinOffset) {
2549 LU.MaxOffset - NewOffset, HasBaseReg))
2552 } else if (NewOffset > LU.MaxOffset) {
2554 NewOffset - LU.MinOffset, HasBaseReg))
2560 LU.MinOffset = NewMinOffset;
2561 LU.MaxOffset = NewMaxOffset;
2562 LU.AccessTy = NewAccessTy;
2587 LSRUse &LU = Uses[LUIdx];
2588 if (reconcileNewOffset(LU, Offset, /*HasBaseReg=*/true, Kind, AccessTy))
2597 LSRUse &LU = Uses[LUIdx];
2599 LU.MinOffset = Offset;
2600 LU.MaxOffset = Offset;
2605 void LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) {
2606 if (&LU != &Uses.back())
2607 std::swap(LU, Uses.back());
2621 LSRUse &LU = Uses[LUIdx];
2627 if (&LU != &OrigLU &&
2628 LU.Kind != LSRUse::ICmpZero &&
2629 LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy &&
2630 LU.WidestFixupType == OrigLU.WidestFixupType &&
2631 LU.HasFormulaWithSameRegs(OrigF)) {
2633 for (const Formula &F : LU.Formulae) {
2642 return &LU;
3324 LSRUse &LU = Uses[LUIdx];
3327 LSRFixup &LF = LU.getNewFixup();
3332 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3334 if (!LU.WidestFixupType ||
3335 SE.getTypeSizeInBits(LU.WidestFixupType) <
3337 LU.WidestFixupType = LF.OperandValToReplace->getType();
3340 if (LU.Formulae.empty()) {
3341 InsertInitialFormula(S, LU, LUIdx);
3342 CountRegisters(LU.Formulae.back(), LUIdx);
3352 LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx) {
3355 LU.RigidFormula = true;
3359 bool Inserted = InsertFormula(LU, LUIdx, F);
3367 LSRUse &LU, size_t LUIdx) {
3371 bool Inserted = InsertFormula(LU, LUIdx, F);
3385 bool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) {
3387 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) &&
3390 if (!LU.InsertFormula(F, *L))
3479 LSRUse &LU = Uses[LUIdx];
3480 LSRFixup &LF = LU.getNewFixup();
3484 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3485 if (!LU.WidestFixupType ||
3486 SE.getTypeSizeInBits(LU.WidestFixupType) <
3488 LU.WidestFixupType = LF.OperandValToReplace->getType();
3489 InsertSupplementalFormula(US, LU, LUIdx);
3490 CountRegisters(LU.Formulae.back(), Uses.size() - 1);
3561 LSRUse &LU, const SCEV *S, const Loop *L,
3563 if (LU.Kind != LSRUse::Address ||
3564 !LU.AccessTy.getType()->isIntOrIntVectorTy())
3583 void LSRInstance::GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
3592 if (AMK == TTI::AMK_PostIndexed && mayUsePostIncMode(TTI, LU, BaseReg, L, SE))
3612 if (isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
3613 LU.AccessTy, *J, Base.getNumRegs() > 1))
3625 isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
3626 LU.AccessTy, InnerAddOps[0], Base.getNumRegs() > 1))
3663 if (InsertFormula(LU, LUIdx, F))
3670 GenerateReassociations(LU, LUIdx, LU.Formulae.back(),
3676 void LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx,
3684 GenerateReassociationsImpl(LU, LUIdx, Base, Depth, i);
3687 GenerateReassociationsImpl(LU, LUIdx, Base, Depth,
3693 void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx,
3735 (void)InsertFormula(LU, LUIdx, F);
3756 void LSRInstance::GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
3765 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
3771 (void)InsertFormula(LU, LUIdx, F);
3775 void LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx,
3781 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, i);
3783 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, /* Idx */ -1,
3789 LSRUse &LU, unsigned LUIdx, const Formula &Base,
3796 if (isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F)) {
3812 (void)InsertFormula(LU, LUIdx, F);
3826 if (AMK == TTI::AMK_PreIndexed && LU.Kind == LSRUse::Address) {
3849 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
3859 (void)InsertFormula(LU, LUIdx, F);
3863 void LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx,
3868 Worklist.push_back(LU.MinOffset);
3869 if (LU.MaxOffset != LU.MinOffset)
3870 Worklist.push_back(LU.MaxOffset);
3873 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, i);
3875 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, /* Idx */ -1,
3881 void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx,
3883 if (LU.Kind != LSRUse::ICmpZero) return;
3891 if (LU.MinOffset != LU.MaxOffset) return;
3915 int64_t Offset = LU.MinOffset;
3919 if (Offset / Factor != LU.MinOffset)
3930 if (!isLegalUse(TTI, Offset, Offset, LU.Kind, LU.AccessTy, F))
3934 F.BaseOffset = (uint64_t)F.BaseOffset + Offset - LU.MinOffset;
3967 (void)InsertFormula(LU, LUIdx, F);
3974 void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) {
3991 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
3995 if (LU.Kind == LSRUse::Basic &&
3996 isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LSRUse::Special,
3997 LU.AccessTy, Base) &&
3998 LU.AllFixupsOutsideLoop)
3999 LU.Kind = LSRUse::Special;
4005 if (LU.Kind == LSRUse::ICmpZero &&
4011 if (AR && (AR->getLoop() == L || LU.AllFixupsOutsideLoop)) {
4026 (AR->getLoop() != L && LU.AllFixupsOutsideLoop)))
4030 if (F.Scale == 1 && LU.AllFixupsOutsideLoop)
4032 (void)InsertFormula(LU, LUIdx, F);
4040 void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
4081 (void)InsertFormula(LU, LUIdx, F);
4203 LSRUse &LU = Uses[LUIdx];
4212 for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) {
4213 Formula F = LU.Formulae[L];
4228 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
4244 (void)InsertFormula(LU, LUIdx, NewF);
4253 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset,
4254 LU.Kind, LU.AccessTy, NewF)) {
4256 mayUsePostIncMode(TTI, LU, OrigReg, this->L, SE))
4279 (void)InsertFormula(LU, LUIdx, NewF);
4294 LSRUse &LU = Uses[LUIdx];
4295 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4296 GenerateReassociations(LU, LUIdx, LU.Formulae[i]);
4297 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4298 GenerateCombinations(LU, LUIdx, LU.Formulae[i]);
4301 LSRUse &LU = Uses[LUIdx];
4302 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4303 GenerateSymbolicOffsets(LU, LUIdx, LU.Formulae[i]);
4304 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4305 GenerateConstantOffsets(LU, LUIdx, LU.Formulae[i]);
4306 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4307 GenerateICmpZeroScales(LU, LUIdx, LU.Formulae[i]);
4308 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4309 GenerateScales(LU, LUIdx, LU.Formulae[i]);
4312 LSRUse &LU = Uses[LUIdx];
4313 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4314 GenerateTruncates(LU, LUIdx, LU.Formulae[i]);
4342 LSRUse &LU = Uses[LUIdx];
4343 LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
4347 for (size_t FIdx = 0, NumForms = LU.Formulae.size();
4349 Formula &F = LU.Formulae[FIdx];
4360 CostF.RateFormula(F, Regs, VisitedRegs, LU, &LoserRegs);
4389 Formula &Best = LU.Formulae[P.first->second];
4393 CostBest.RateFormula(Best, Regs, VisitedRegs, LU);
4404 LU.DeleteFormula(F);
4412 LU.RecomputeRegs(LUIdx, RegUses);
4430 for (const LSRUse &LU : Uses) {
4431 size_t FSize = LU.Formulae.size();
4455 LSRUse &LU = Uses[LUIdx];
4457 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
4458 Formula &F = LU.Formulae[i];
4471 if (LU.HasFormulaWithSameRegs(NewF)) {
4474 LU.DeleteFormula(F);
4487 if (LU.HasFormulaWithSameRegs(NewF)) {
4490 LU.DeleteFormula(F);
4501 LU.RecomputeRegs(LUIdx, RegUses);
4522 LSRUse &LU = Uses[LUIdx];
4523 for (const Formula &F : LU.Formulae) {
4527 LSRUse *LUThatHas = FindUseWithSimilarFormula(F, LU);
4532 LU.Kind, LU.AccessTy))
4535 LLVM_DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); dbgs() << '\n');
4537 LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop;
4539 // Transfer the fixups of LU to LUThatHas.
4540 for (LSRFixup &Fixup : LU.Fixups) {
4564 DeleteUse(LU, LUIdx);
4619 LSRUse &LU = Uses[LUIdx];
4620 LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
4647 CostFA.RateFormula(FA, Regs, VisitedRegs, LU);
4649 CostFB.RateFormula(FB, Regs, VisitedRegs, LU);
4654 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms;
4656 Formula &F = LU.Formulae[FIdx];
4663 Formula &Best = LU.Formulae[P.first->second];
4673 LU.DeleteFormula(F);
4679 LU.RecomputeRegs(LUIdx, RegUses);
4705 LSRUse &LU = Uses[LUIdx];
4707 if (LU.Kind != LSRUse::Address)
4709 if (!TTI.isIndexedLoadLegal(TTI.MIM_PostInc, LU.AccessTy.getType()) &&
4710 !TTI.isIndexedStoreLegal(TTI.MIM_PostInc, LU.AccessTy.getType()))
4714 for (const Formula &F : LU.Formulae)
4718 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms;
4720 Formula &F = LU.Formulae[FIdx];
4724 LU.DeleteFormula(F);
4731 LU.RecomputeRegs(LUIdx, RegUses);
4800 for (const LSRUse &LU : Uses) {
4801 if (!LU.Regs.count(Reg))
4803 float P = LU.getNotSelectedProbability(Reg);
4817 LSRUse &LU = Uses[LUIdx];
4819 if (LU.Formulae.size() < 2)
4824 float FMinRegNum = LU.Formulae[0].getNumRegs();
4825 float FMinARegNum = LU.Formulae[0].getNumRegs();
4827 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
4828 Formula &F = LU.Formulae[i];
4834 FRegNum += RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg);
4837 RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg);
4842 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg);
4845 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg);
4855 LLVM_DEBUG(dbgs() << " The formula "; LU.Formulae[MinIdx].print(dbgs());
4858 std::swap(LU.Formulae[MinIdx], LU.Formulae[0]);
4859 while (LU.Formulae.size() != 1) {
4860 LLVM_DEBUG(dbgs() << " Deleting "; LU.Formulae.back().print(dbgs());
4862 LU.Formulae.pop_back();
4864 LU.RecomputeRegs(LUIdx, RegUses);
4865 assert(LU.Formulae.size() == 1 && "Should be exactly 1 min regs formula");
4866 Formula &F = LU.Formulae[0];
4915 LSRUse &LU = Uses[LUIdx];
4916 if (!LU.Regs.count(Best)) continue;
4919 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
4920 Formula &F = LU.Formulae[i];
4923 LU.DeleteFormula(F);
4933 LU.RecomputeRegs(LUIdx, RegUses);
4974 const LSRUse &LU = Uses[Workspace.size()];
4982 if (LU.Regs.count(S))
4987 for (const Formula &F : LU.Formulae) {
4994 if (AMK != TTI::AMK_PostIndexed || LU.Kind != LSRUse::Address) {
5015 NewCost.RateFormula(F, NewRegs, VisitedRegs, LU);
5136 const LSRUse &LU,
5144 if (LU.Kind == LSRUse::ICmpZero)
5198 Value *LSRInstance::Expand(const LSRUse &LU, const LSRFixup &LF,
5202 if (LU.RigidFormula)
5207 IP = AdjustInsertPositionForExpand(IP, LF, LU, Rewriter);
5248 if (LU.Kind == LSRUse::ICmpZero) {
5267 if (!Ops.empty() && LU.Kind == LSRUse::Address &&
5268 isAMCompletelyFolded(TTI, LU, F)) {
5303 if (LU.Kind == LSRUse::ICmpZero) {
5339 if (LU.Kind == LSRUse::ICmpZero) {
5378 PHINode *PN, const LSRUse &LU, const LSRFixup &LF, const Formula &F,
5434 Value *FullV = Expand(LU, LF, F, BB->getTerminator()->getIterator(),
5493 void LSRInstance::Rewrite(const LSRUse &LU, const LSRFixup &LF,
5499 RewriteForPHI(PN, LU, LF, F, Rewriter, DeadInsts);
5502 Expand(LU, LF, F, LF.UserInst->getIterator(), Rewriter, DeadInsts);
5518 if (LU.Kind == LSRUse::ICmpZero)
5723 for (const LSRUse &LU : Uses) {
5724 for (const Formula &F : LU.Formulae)
5725 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
5757 for (const LSRUse &LU : Uses)
5758 for (const LSRFixup &LF : LU.Fixups) {
5767 for (const LSRUse &LU : Uses) {
5769 LU.print(OS);
5771 for (const Formula &F : LU.Formulae) {