1//===- GCNIterativeScheduler.cpp ------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the class GCNIterativeScheduler.
11///
12//===----------------------------------------------------------------------===//
13
14#include "GCNIterativeScheduler.h"
15#include "GCNSchedStrategy.h"
16#include "SIMachineFunctionInfo.h"
17
18using namespace llvm;
19
20#define DEBUG_TYPE "machine-scheduler"
21
22namespace llvm {
23
24std::vector<const SUnit *> makeMinRegSchedule(ArrayRef<const SUnit *> TopRoots,
25                                              const ScheduleDAG &DAG);
26
27  std::vector<const SUnit*> makeGCNILPScheduler(ArrayRef<const SUnit*> BotRoots,
28    const ScheduleDAG &DAG);
29}
30
31// shim accessors for different order containers
32static inline MachineInstr *getMachineInstr(MachineInstr *MI) {
33  return MI;
34}
35static inline MachineInstr *getMachineInstr(const SUnit *SU) {
36  return SU->getInstr();
37}
38static inline MachineInstr *getMachineInstr(const SUnit &SU) {
39  return SU.getInstr();
40}
41
42#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
43LLVM_DUMP_METHOD
44static void printRegion(raw_ostream &OS,
45                        MachineBasicBlock::iterator Begin,
46                        MachineBasicBlock::iterator End,
47                        const LiveIntervals *LIS,
48                        unsigned MaxInstNum =
49                          std::numeric_limits<unsigned>::max()) {
50  auto BB = Begin->getParent();
51  OS << BB->getParent()->getName() << ":" << printMBBReference(*BB) << ' '
52     << BB->getName() << ":\n";
53  auto I = Begin;
54  MaxInstNum = std::max(MaxInstNum, 1u);
55  for (; I != End && MaxInstNum; ++I, --MaxInstNum) {
56    if (!I->isDebugInstr() && LIS)
57      OS << LIS->getInstructionIndex(*I);
58    OS << '\t' << *I;
59  }
60  if (I != End) {
61    OS << "\t...\n";
62    I = std::prev(End);
63    if (!I->isDebugInstr() && LIS)
64      OS << LIS->getInstructionIndex(*I);
65    OS << '\t' << *I;
66  }
67  if (End != BB->end()) { // print boundary inst if present
68    OS << "----\n";
69    if (LIS) OS << LIS->getInstructionIndex(*End) << '\t';
70    OS << *End;
71  }
72}
73
74LLVM_DUMP_METHOD
75static void printLivenessInfo(raw_ostream &OS,
76                              MachineBasicBlock::iterator Begin,
77                              MachineBasicBlock::iterator End,
78                              const LiveIntervals *LIS) {
79  const auto BB = Begin->getParent();
80  const auto &MRI = BB->getParent()->getRegInfo();
81
82  const auto LiveIns = getLiveRegsBefore(*Begin, *LIS);
83  OS << "LIn RP: " << print(getRegPressure(MRI, LiveIns));
84
85  const auto BottomMI = End == BB->end() ? std::prev(End) : End;
86  const auto LiveOuts = getLiveRegsAfter(*BottomMI, *LIS);
87  OS << "LOt RP: " << print(getRegPressure(MRI, LiveOuts));
88}
89
90LLVM_DUMP_METHOD
91void GCNIterativeScheduler::printRegions(raw_ostream &OS) const {
92  const auto &ST = MF.getSubtarget<GCNSubtarget>();
93  for (const auto R : Regions) {
94    OS << "Region to schedule ";
95    printRegion(OS, R->Begin, R->End, LIS, 1);
96    printLivenessInfo(OS, R->Begin, R->End, LIS);
97    OS << "Max RP: " << print(R->MaxPressure, &ST);
98  }
99}
100
101LLVM_DUMP_METHOD
102void GCNIterativeScheduler::printSchedResult(raw_ostream &OS,
103                                             const Region *R,
104                                             const GCNRegPressure &RP) const {
105  OS << "\nAfter scheduling ";
106  printRegion(OS, R->Begin, R->End, LIS);
107  printSchedRP(OS, R->MaxPressure, RP);
108  OS << '\n';
109}
110
111LLVM_DUMP_METHOD
112void GCNIterativeScheduler::printSchedRP(raw_ostream &OS,
113                                         const GCNRegPressure &Before,
114                                         const GCNRegPressure &After) const {
115  const auto &ST = MF.getSubtarget<GCNSubtarget>();
116  OS << "RP before: " << print(Before, &ST)
117     << "RP after:  " << print(After, &ST);
118}
119#endif
120
121// DAG builder helper
122class GCNIterativeScheduler::BuildDAG {
123  GCNIterativeScheduler &Sch;
124  SmallVector<SUnit *, 8> TopRoots;
125
126  SmallVector<SUnit*, 8> BotRoots;
127public:
128  BuildDAG(const Region &R, GCNIterativeScheduler &_Sch)
129    : Sch(_Sch) {
130    auto BB = R.Begin->getParent();
131    Sch.BaseClass::startBlock(BB);
132    Sch.BaseClass::enterRegion(BB, R.Begin, R.End, R.NumRegionInstrs);
133
134    Sch.buildSchedGraph(Sch.AA, nullptr, nullptr, nullptr,
135                        /*TrackLaneMask*/true);
136    Sch.Topo.InitDAGTopologicalSorting();
137    Sch.findRootsAndBiasEdges(TopRoots, BotRoots);
138  }
139
140  ~BuildDAG() {
141    Sch.BaseClass::exitRegion();
142    Sch.BaseClass::finishBlock();
143  }
144
145  ArrayRef<const SUnit *> getTopRoots() const {
146    return TopRoots;
147  }
148  ArrayRef<SUnit*> getBottomRoots() const {
149    return BotRoots;
150  }
151};
152
153class GCNIterativeScheduler::OverrideLegacyStrategy {
154  GCNIterativeScheduler &Sch;
155  Region &Rgn;
156  std::unique_ptr<MachineSchedStrategy> SaveSchedImpl;
157  GCNRegPressure SaveMaxRP;
158
159public:
160  OverrideLegacyStrategy(Region &R,
161                         MachineSchedStrategy &OverrideStrategy,
162                         GCNIterativeScheduler &_Sch)
163    : Sch(_Sch)
164    , Rgn(R)
165    , SaveSchedImpl(std::move(_Sch.SchedImpl))
166    , SaveMaxRP(R.MaxPressure) {
167    Sch.SchedImpl.reset(&OverrideStrategy);
168    auto BB = R.Begin->getParent();
169    Sch.BaseClass::startBlock(BB);
170    Sch.BaseClass::enterRegion(BB, R.Begin, R.End, R.NumRegionInstrs);
171  }
172
173  ~OverrideLegacyStrategy() {
174    Sch.BaseClass::exitRegion();
175    Sch.BaseClass::finishBlock();
176    Sch.SchedImpl.release();
177    Sch.SchedImpl = std::move(SaveSchedImpl);
178  }
179
180  void schedule() {
181    assert(Sch.RegionBegin == Rgn.Begin && Sch.RegionEnd == Rgn.End);
182    LLVM_DEBUG(dbgs() << "\nScheduling ";
183               printRegion(dbgs(), Rgn.Begin, Rgn.End, Sch.LIS, 2));
184    Sch.BaseClass::schedule();
185
186    // Unfortunately placeDebugValues incorrectly modifies RegionEnd, restore
187    Sch.RegionEnd = Rgn.End;
188    //assert(Rgn.End == Sch.RegionEnd);
189    Rgn.Begin = Sch.RegionBegin;
190    Rgn.MaxPressure.clear();
191  }
192
193  void restoreOrder() {
194    assert(Sch.RegionBegin == Rgn.Begin && Sch.RegionEnd == Rgn.End);
195    // DAG SUnits are stored using original region's order
196    // so just use SUnits as the restoring schedule
197    Sch.scheduleRegion(Rgn, Sch.SUnits, SaveMaxRP);
198  }
199};
200
201namespace {
202
203// just a stub to make base class happy
204class SchedStrategyStub : public MachineSchedStrategy {
205public:
206  bool shouldTrackPressure() const override { return false; }
207  bool shouldTrackLaneMasks() const override { return false; }
208  void initialize(ScheduleDAGMI *DAG) override {}
209  SUnit *pickNode(bool &IsTopNode) override { return nullptr; }
210  void schedNode(SUnit *SU, bool IsTopNode) override {}
211  void releaseTopNode(SUnit *SU) override {}
212  void releaseBottomNode(SUnit *SU) override {}
213};
214
215} // end anonymous namespace
216
217GCNIterativeScheduler::GCNIterativeScheduler(MachineSchedContext *C,
218                                             StrategyKind S)
219  : BaseClass(C, std::make_unique<SchedStrategyStub>())
220  , Context(C)
221  , Strategy(S)
222  , UPTracker(*LIS) {
223}
224
225// returns max pressure for a region
226GCNRegPressure
227GCNIterativeScheduler::getRegionPressure(MachineBasicBlock::iterator Begin,
228                                         MachineBasicBlock::iterator End)
229  const {
230  // For the purpose of pressure tracking bottom inst of the region should
231  // be also processed. End is either BB end, BB terminator inst or sched
232  // boundary inst.
233  auto const BBEnd = Begin->getParent()->end();
234  auto const BottomMI = End == BBEnd ? std::prev(End) : End;
235
236  // scheduleRegions walks bottom to top, so its likely we just get next
237  // instruction to track
238  auto AfterBottomMI = std::next(BottomMI);
239  if (AfterBottomMI == BBEnd ||
240      &*AfterBottomMI != UPTracker.getLastTrackedMI()) {
241    UPTracker.reset(*BottomMI);
242  } else {
243    assert(UPTracker.isValid());
244  }
245
246  for (auto I = BottomMI; I != Begin; --I)
247    UPTracker.recede(*I);
248
249  UPTracker.recede(*Begin);
250
251  assert(UPTracker.isValid() ||
252         (dbgs() << "Tracked region ",
253          printRegion(dbgs(), Begin, End, LIS), false));
254  return UPTracker.moveMaxPressure();
255}
256
257// returns max pressure for a tentative schedule
258template <typename Range> GCNRegPressure
259GCNIterativeScheduler::getSchedulePressure(const Region &R,
260                                           Range &&Schedule) const {
261  auto const BBEnd = R.Begin->getParent()->end();
262  GCNUpwardRPTracker RPTracker(*LIS);
263  if (R.End != BBEnd) {
264    // R.End points to the boundary instruction but the
265    // schedule doesn't include it
266    RPTracker.reset(*R.End);
267    RPTracker.recede(*R.End);
268  } else {
269    // R.End doesn't point to the boundary instruction
270    RPTracker.reset(*std::prev(BBEnd));
271  }
272  for (auto I = Schedule.end(), B = Schedule.begin(); I != B;) {
273    RPTracker.recede(*getMachineInstr(*--I));
274  }
275  return RPTracker.moveMaxPressure();
276}
277
278void GCNIterativeScheduler::enterRegion(MachineBasicBlock *BB, // overridden
279                                        MachineBasicBlock::iterator Begin,
280                                        MachineBasicBlock::iterator End,
281                                        unsigned NumRegionInstrs) {
282  BaseClass::enterRegion(BB, Begin, End, NumRegionInstrs);
283  if (NumRegionInstrs > 2) {
284    Regions.push_back(
285      new (Alloc.Allocate())
286      Region { Begin, End, NumRegionInstrs,
287               getRegionPressure(Begin, End), nullptr });
288  }
289}
290
291void GCNIterativeScheduler::schedule() { // overridden
292  // do nothing
293  LLVM_DEBUG(printLivenessInfo(dbgs(), RegionBegin, RegionEnd, LIS);
294             if (!Regions.empty() && Regions.back()->Begin == RegionBegin) {
295               dbgs() << "Max RP: "
296                      << print(Regions.back()->MaxPressure,
297                               &MF.getSubtarget<GCNSubtarget>());
298             } dbgs()
299             << '\n';);
300}
301
302void GCNIterativeScheduler::finalizeSchedule() { // overridden
303  if (Regions.empty())
304    return;
305  switch (Strategy) {
306  case SCHEDULE_MINREGONLY: scheduleMinReg(); break;
307  case SCHEDULE_MINREGFORCED: scheduleMinReg(true); break;
308  case SCHEDULE_LEGACYMAXOCCUPANCY: scheduleLegacyMaxOccupancy(); break;
309  case SCHEDULE_ILP: scheduleILP(false); break;
310  }
311}
312
313// Detach schedule from SUnits and interleave it with debug values.
314// Returned schedule becomes independent of DAG state.
315std::vector<MachineInstr*>
316GCNIterativeScheduler::detachSchedule(ScheduleRef Schedule) const {
317  std::vector<MachineInstr*> Res;
318  Res.reserve(Schedule.size() * 2);
319
320  if (FirstDbgValue)
321    Res.push_back(FirstDbgValue);
322
323  const auto DbgB = DbgValues.begin(), DbgE = DbgValues.end();
324  for (const auto *SU : Schedule) {
325    Res.push_back(SU->getInstr());
326    const auto &D = std::find_if(DbgB, DbgE, [SU](decltype(*DbgB) &P) {
327      return P.second == SU->getInstr();
328    });
329    if (D != DbgE)
330      Res.push_back(D->first);
331  }
332  return Res;
333}
334
335void GCNIterativeScheduler::setBestSchedule(Region &R,
336                                            ScheduleRef Schedule,
337                                            const GCNRegPressure &MaxRP) {
338  R.BestSchedule.reset(
339    new TentativeSchedule{ detachSchedule(Schedule), MaxRP });
340}
341
342void GCNIterativeScheduler::scheduleBest(Region &R) {
343  assert(R.BestSchedule.get() && "No schedule specified");
344  scheduleRegion(R, R.BestSchedule->Schedule, R.BestSchedule->MaxPressure);
345  R.BestSchedule.reset();
346}
347
348// minimal required region scheduler, works for ranges of SUnits*,
349// SUnits or MachineIntrs*
350template <typename Range>
351void GCNIterativeScheduler::scheduleRegion(Region &R, Range &&Schedule,
352                                           const GCNRegPressure &MaxRP) {
353  assert(RegionBegin == R.Begin && RegionEnd == R.End);
354  assert(LIS != nullptr);
355#ifndef NDEBUG
356  const auto SchedMaxRP = getSchedulePressure(R, Schedule);
357#endif
358  auto BB = R.Begin->getParent();
359  auto Top = R.Begin;
360  for (const auto &I : Schedule) {
361    auto MI = getMachineInstr(I);
362    if (MI != &*Top) {
363      BB->remove(MI);
364      BB->insert(Top, MI);
365      if (!MI->isDebugInstr())
366        LIS->handleMove(*MI, true);
367    }
368    if (!MI->isDebugInstr()) {
369      // Reset read - undef flags and update them later.
370      for (auto &Op : MI->operands())
371        if (Op.isReg() && Op.isDef())
372          Op.setIsUndef(false);
373
374      RegisterOperands RegOpers;
375      RegOpers.collect(*MI, *TRI, MRI, /*ShouldTrackLaneMasks*/true,
376                                       /*IgnoreDead*/false);
377      // Adjust liveness and add missing dead+read-undef flags.
378      auto SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
379      RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
380    }
381    Top = std::next(MI->getIterator());
382  }
383  RegionBegin = getMachineInstr(Schedule.front());
384
385  // Schedule consisting of MachineInstr* is considered 'detached'
386  // and already interleaved with debug values
387  if (!std::is_same<decltype(*Schedule.begin()), MachineInstr*>::value) {
388    placeDebugValues();
389    // Unfortunately placeDebugValues incorrectly modifies RegionEnd, restore
390    // assert(R.End == RegionEnd);
391    RegionEnd = R.End;
392  }
393
394  R.Begin = RegionBegin;
395  R.MaxPressure = MaxRP;
396
397#ifndef NDEBUG
398  const auto RegionMaxRP = getRegionPressure(R);
399  const auto &ST = MF.getSubtarget<GCNSubtarget>();
400#endif
401  assert(
402      (SchedMaxRP == RegionMaxRP && (MaxRP.empty() || SchedMaxRP == MaxRP)) ||
403      (dbgs() << "Max RP mismatch!!!\n"
404                 "RP for schedule (calculated): "
405              << print(SchedMaxRP, &ST)
406              << "RP for schedule (reported): " << print(MaxRP, &ST)
407              << "RP after scheduling: " << print(RegionMaxRP, &ST),
408       false));
409}
410
411// Sort recorded regions by pressure - highest at the front
412void GCNIterativeScheduler::sortRegionsByPressure(unsigned TargetOcc) {
413  const auto &ST = MF.getSubtarget<GCNSubtarget>();
414  llvm::sort(Regions, [&ST, TargetOcc](const Region *R1, const Region *R2) {
415    return R2->MaxPressure.less(ST, R1->MaxPressure, TargetOcc);
416  });
417}
418
419///////////////////////////////////////////////////////////////////////////////
420// Legacy MaxOccupancy Strategy
421
422// Tries to increase occupancy applying minreg scheduler for a sequence of
423// most demanding regions. Obtained schedules are saved as BestSchedule for a
424// region.
425// TargetOcc is the best achievable occupancy for a kernel.
426// Returns better occupancy on success or current occupancy on fail.
427// BestSchedules aren't deleted on fail.
428unsigned GCNIterativeScheduler::tryMaximizeOccupancy(unsigned TargetOcc) {
429  // TODO: assert Regions are sorted descending by pressure
430  const auto &ST = MF.getSubtarget<GCNSubtarget>();
431  const auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
432  LLVM_DEBUG(dbgs() << "Trying to improve occupancy, target = " << TargetOcc
433                    << ", current = " << Occ << '\n');
434
435  auto NewOcc = TargetOcc;
436  for (auto *R : Regions) {
437    if (R->MaxPressure.getOccupancy(ST) >= NewOcc)
438      break;
439
440    LLVM_DEBUG(printRegion(dbgs(), R->Begin, R->End, LIS, 3);
441               printLivenessInfo(dbgs(), R->Begin, R->End, LIS));
442
443    BuildDAG DAG(*R, *this);
444    const auto MinSchedule = makeMinRegSchedule(DAG.getTopRoots(), *this);
445    const auto MaxRP = getSchedulePressure(*R, MinSchedule);
446    LLVM_DEBUG(dbgs() << "Occupancy improvement attempt:\n";
447               printSchedRP(dbgs(), R->MaxPressure, MaxRP));
448
449    NewOcc = std::min(NewOcc, MaxRP.getOccupancy(ST));
450    if (NewOcc <= Occ)
451      break;
452
453    setBestSchedule(*R, MinSchedule, MaxRP);
454  }
455  LLVM_DEBUG(dbgs() << "New occupancy = " << NewOcc
456                    << ", prev occupancy = " << Occ << '\n');
457  if (NewOcc > Occ) {
458    SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
459    MFI->increaseOccupancy(MF, NewOcc);
460  }
461
462  return std::max(NewOcc, Occ);
463}
464
465void GCNIterativeScheduler::scheduleLegacyMaxOccupancy(
466  bool TryMaximizeOccupancy) {
467  const auto &ST = MF.getSubtarget<GCNSubtarget>();
468  SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
469  auto TgtOcc = MFI->getMinAllowedOccupancy();
470
471  sortRegionsByPressure(TgtOcc);
472  auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
473
474  if (TryMaximizeOccupancy && Occ < TgtOcc)
475    Occ = tryMaximizeOccupancy(TgtOcc);
476
477  // This is really weird but for some magic scheduling regions twice
478  // gives performance improvement
479  const int NumPasses = Occ < TgtOcc ? 2 : 1;
480
481  TgtOcc = std::min(Occ, TgtOcc);
482  LLVM_DEBUG(dbgs() << "Scheduling using default scheduler, "
483                       "target occupancy = "
484                    << TgtOcc << '\n');
485  GCNMaxOccupancySchedStrategy LStrgy(Context);
486  unsigned FinalOccupancy = std::min(Occ, MFI->getOccupancy());
487
488  for (int I = 0; I < NumPasses; ++I) {
489    // running first pass with TargetOccupancy = 0 mimics previous scheduling
490    // approach and is a performance magic
491    LStrgy.setTargetOccupancy(I == 0 ? 0 : TgtOcc);
492    for (auto *R : Regions) {
493      OverrideLegacyStrategy Ovr(*R, LStrgy, *this);
494
495      Ovr.schedule();
496      const auto RP = getRegionPressure(*R);
497      LLVM_DEBUG(printSchedRP(dbgs(), R->MaxPressure, RP));
498
499      if (RP.getOccupancy(ST) < TgtOcc) {
500        LLVM_DEBUG(dbgs() << "Didn't fit into target occupancy O" << TgtOcc);
501        if (R->BestSchedule.get() &&
502            R->BestSchedule->MaxPressure.getOccupancy(ST) >= TgtOcc) {
503          LLVM_DEBUG(dbgs() << ", scheduling minimal register\n");
504          scheduleBest(*R);
505        } else {
506          LLVM_DEBUG(dbgs() << ", restoring\n");
507          Ovr.restoreOrder();
508          assert(R->MaxPressure.getOccupancy(ST) >= TgtOcc);
509        }
510      }
511      FinalOccupancy = std::min(FinalOccupancy, RP.getOccupancy(ST));
512    }
513  }
514  MFI->limitOccupancy(FinalOccupancy);
515}
516
517///////////////////////////////////////////////////////////////////////////////
518// Minimal Register Strategy
519
520void GCNIterativeScheduler::scheduleMinReg(bool force) {
521  const auto &ST = MF.getSubtarget<GCNSubtarget>();
522  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
523  const auto TgtOcc = MFI->getOccupancy();
524  sortRegionsByPressure(TgtOcc);
525
526  auto MaxPressure = Regions.front()->MaxPressure;
527  for (auto *R : Regions) {
528    if (!force && R->MaxPressure.less(ST, MaxPressure, TgtOcc))
529      break;
530
531    BuildDAG DAG(*R, *this);
532    const auto MinSchedule = makeMinRegSchedule(DAG.getTopRoots(), *this);
533
534    const auto RP = getSchedulePressure(*R, MinSchedule);
535    LLVM_DEBUG(if (R->MaxPressure.less(ST, RP, TgtOcc)) {
536      dbgs() << "\nWarning: Pressure becomes worse after minreg!";
537      printSchedRP(dbgs(), R->MaxPressure, RP);
538    });
539
540    if (!force && MaxPressure.less(ST, RP, TgtOcc))
541      break;
542
543    scheduleRegion(*R, MinSchedule, RP);
544    LLVM_DEBUG(printSchedResult(dbgs(), R, RP));
545
546    MaxPressure = RP;
547  }
548}
549
550///////////////////////////////////////////////////////////////////////////////
551// ILP scheduler port
552
553void GCNIterativeScheduler::scheduleILP(
554  bool TryMaximizeOccupancy) {
555  const auto &ST = MF.getSubtarget<GCNSubtarget>();
556  SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
557  auto TgtOcc = MFI->getMinAllowedOccupancy();
558
559  sortRegionsByPressure(TgtOcc);
560  auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
561
562  if (TryMaximizeOccupancy && Occ < TgtOcc)
563    Occ = tryMaximizeOccupancy(TgtOcc);
564
565  TgtOcc = std::min(Occ, TgtOcc);
566  LLVM_DEBUG(dbgs() << "Scheduling using default scheduler, "
567                       "target occupancy = "
568                    << TgtOcc << '\n');
569
570  unsigned FinalOccupancy = std::min(Occ, MFI->getOccupancy());
571  for (auto *R : Regions) {
572    BuildDAG DAG(*R, *this);
573    const auto ILPSchedule = makeGCNILPScheduler(DAG.getBottomRoots(), *this);
574
575    const auto RP = getSchedulePressure(*R, ILPSchedule);
576    LLVM_DEBUG(printSchedRP(dbgs(), R->MaxPressure, RP));
577
578    if (RP.getOccupancy(ST) < TgtOcc) {
579      LLVM_DEBUG(dbgs() << "Didn't fit into target occupancy O" << TgtOcc);
580      if (R->BestSchedule.get() &&
581        R->BestSchedule->MaxPressure.getOccupancy(ST) >= TgtOcc) {
582        LLVM_DEBUG(dbgs() << ", scheduling minimal register\n");
583        scheduleBest(*R);
584      }
585    } else {
586      scheduleRegion(*R, ILPSchedule, RP);
587      LLVM_DEBUG(printSchedResult(dbgs(), R, RP));
588      FinalOccupancy = std::min(FinalOccupancy, RP.getOccupancy(ST));
589    }
590  }
591  MFI->limitOccupancy(FinalOccupancy);
592}
593