1//===- GCNRegPressure.cpp -------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the GCNRegPressure class.
11///
12//===----------------------------------------------------------------------===//
13
14#include "GCNRegPressure.h"
15#include "AMDGPU.h"
16#include "llvm/CodeGen/RegisterPressure.h"
17
18using namespace llvm;
19
20#define DEBUG_TYPE "machine-scheduler"
21
22bool llvm::isEqual(const GCNRPTracker::LiveRegSet &S1,
23                   const GCNRPTracker::LiveRegSet &S2) {
24  if (S1.size() != S2.size())
25    return false;
26
27  for (const auto &P : S1) {
28    auto I = S2.find(P.first);
29    if (I == S2.end() || I->second != P.second)
30      return false;
31  }
32  return true;
33}
34
35///////////////////////////////////////////////////////////////////////////////
36// GCNRegPressure
37
38unsigned GCNRegPressure::getRegKind(Register Reg,
39                                    const MachineRegisterInfo &MRI) {
40  assert(Reg.isVirtual());
41  const auto RC = MRI.getRegClass(Reg);
42  auto STI = static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
43  return STI->isSGPRClass(RC)
44             ? (STI->getRegSizeInBits(*RC) == 32 ? SGPR32 : SGPR_TUPLE)
45         : STI->isAGPRClass(RC)
46             ? (STI->getRegSizeInBits(*RC) == 32 ? AGPR32 : AGPR_TUPLE)
47             : (STI->getRegSizeInBits(*RC) == 32 ? VGPR32 : VGPR_TUPLE);
48}
49
50void GCNRegPressure::inc(unsigned Reg,
51                         LaneBitmask PrevMask,
52                         LaneBitmask NewMask,
53                         const MachineRegisterInfo &MRI) {
54  if (SIRegisterInfo::getNumCoveredRegs(NewMask) ==
55      SIRegisterInfo::getNumCoveredRegs(PrevMask))
56    return;
57
58  int Sign = 1;
59  if (NewMask < PrevMask) {
60    std::swap(NewMask, PrevMask);
61    Sign = -1;
62  }
63
64  switch (auto Kind = getRegKind(Reg, MRI)) {
65  case SGPR32:
66  case VGPR32:
67  case AGPR32:
68    Value[Kind] += Sign;
69    break;
70
71  case SGPR_TUPLE:
72  case VGPR_TUPLE:
73  case AGPR_TUPLE:
74    assert(PrevMask < NewMask);
75
76    Value[Kind == SGPR_TUPLE ? SGPR32 : Kind == AGPR_TUPLE ? AGPR32 : VGPR32] +=
77      Sign * SIRegisterInfo::getNumCoveredRegs(~PrevMask & NewMask);
78
79    if (PrevMask.none()) {
80      assert(NewMask.any());
81      const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
82      Value[Kind] +=
83          Sign * TRI->getRegClassWeight(MRI.getRegClass(Reg)).RegWeight;
84    }
85    break;
86
87  default: llvm_unreachable("Unknown register kind");
88  }
89}
90
91bool GCNRegPressure::less(const GCNSubtarget &ST,
92                          const GCNRegPressure& O,
93                          unsigned MaxOccupancy) const {
94  const auto SGPROcc = std::min(MaxOccupancy,
95                                ST.getOccupancyWithNumSGPRs(getSGPRNum()));
96  const auto VGPROcc =
97    std::min(MaxOccupancy,
98             ST.getOccupancyWithNumVGPRs(getVGPRNum(ST.hasGFX90AInsts())));
99  const auto OtherSGPROcc = std::min(MaxOccupancy,
100                                ST.getOccupancyWithNumSGPRs(O.getSGPRNum()));
101  const auto OtherVGPROcc =
102    std::min(MaxOccupancy,
103             ST.getOccupancyWithNumVGPRs(O.getVGPRNum(ST.hasGFX90AInsts())));
104
105  const auto Occ = std::min(SGPROcc, VGPROcc);
106  const auto OtherOcc = std::min(OtherSGPROcc, OtherVGPROcc);
107  if (Occ != OtherOcc)
108    return Occ > OtherOcc;
109
110  bool SGPRImportant = SGPROcc < VGPROcc;
111  const bool OtherSGPRImportant = OtherSGPROcc < OtherVGPROcc;
112
113  // if both pressures disagree on what is more important compare vgprs
114  if (SGPRImportant != OtherSGPRImportant) {
115    SGPRImportant = false;
116  }
117
118  // compare large regs pressure
119  bool SGPRFirst = SGPRImportant;
120  for (int I = 2; I > 0; --I, SGPRFirst = !SGPRFirst) {
121    if (SGPRFirst) {
122      auto SW = getSGPRTuplesWeight();
123      auto OtherSW = O.getSGPRTuplesWeight();
124      if (SW != OtherSW)
125        return SW < OtherSW;
126    } else {
127      auto VW = getVGPRTuplesWeight();
128      auto OtherVW = O.getVGPRTuplesWeight();
129      if (VW != OtherVW)
130        return VW < OtherVW;
131    }
132  }
133  return SGPRImportant ? (getSGPRNum() < O.getSGPRNum()):
134                         (getVGPRNum(ST.hasGFX90AInsts()) <
135                          O.getVGPRNum(ST.hasGFX90AInsts()));
136}
137
138Printable llvm::print(const GCNRegPressure &RP, const GCNSubtarget *ST) {
139  return Printable([&RP, ST](raw_ostream &OS) {
140    OS << "VGPRs: " << RP.Value[GCNRegPressure::VGPR32] << ' '
141       << "AGPRs: " << RP.getAGPRNum();
142    if (ST)
143      OS << "(O"
144         << ST->getOccupancyWithNumVGPRs(RP.getVGPRNum(ST->hasGFX90AInsts()))
145         << ')';
146    OS << ", SGPRs: " << RP.getSGPRNum();
147    if (ST)
148      OS << "(O" << ST->getOccupancyWithNumSGPRs(RP.getSGPRNum()) << ')';
149    OS << ", LVGPR WT: " << RP.getVGPRTuplesWeight()
150       << ", LSGPR WT: " << RP.getSGPRTuplesWeight();
151    if (ST)
152      OS << " -> Occ: " << RP.getOccupancy(*ST);
153    OS << '\n';
154  });
155}
156
157static LaneBitmask getDefRegMask(const MachineOperand &MO,
158                                 const MachineRegisterInfo &MRI) {
159  assert(MO.isDef() && MO.isReg() && MO.getReg().isVirtual());
160
161  // We don't rely on read-undef flag because in case of tentative schedule
162  // tracking it isn't set correctly yet. This works correctly however since
163  // use mask has been tracked before using LIS.
164  return MO.getSubReg() == 0 ?
165    MRI.getMaxLaneMaskForVReg(MO.getReg()) :
166    MRI.getTargetRegisterInfo()->getSubRegIndexLaneMask(MO.getSubReg());
167}
168
169static void
170collectVirtualRegUses(SmallVectorImpl<RegisterMaskPair> &RegMaskPairs,
171                      const MachineInstr &MI, const LiveIntervals &LIS,
172                      const MachineRegisterInfo &MRI) {
173  SlotIndex InstrSI;
174  for (const auto &MO : MI.operands()) {
175    if (!MO.isReg() || !MO.getReg().isVirtual())
176      continue;
177    if (!MO.isUse() || !MO.readsReg())
178      continue;
179
180    Register Reg = MO.getReg();
181    if (llvm::any_of(RegMaskPairs, [Reg](const RegisterMaskPair &RM) {
182          return RM.RegUnit == Reg;
183        }))
184      continue;
185
186    LaneBitmask UseMask;
187    auto &LI = LIS.getInterval(Reg);
188    if (!LI.hasSubRanges())
189      UseMask = MRI.getMaxLaneMaskForVReg(Reg);
190    else {
191      // For a tentative schedule LIS isn't updated yet but livemask should
192      // remain the same on any schedule. Subreg defs can be reordered but they
193      // all must dominate uses anyway.
194      if (!InstrSI)
195        InstrSI = LIS.getInstructionIndex(*MO.getParent()).getBaseIndex();
196      UseMask = getLiveLaneMask(LI, InstrSI, MRI);
197    }
198
199    RegMaskPairs.emplace_back(Reg, UseMask);
200  }
201}
202
203///////////////////////////////////////////////////////////////////////////////
204// GCNRPTracker
205
206LaneBitmask llvm::getLiveLaneMask(unsigned Reg, SlotIndex SI,
207                                  const LiveIntervals &LIS,
208                                  const MachineRegisterInfo &MRI) {
209  return getLiveLaneMask(LIS.getInterval(Reg), SI, MRI);
210}
211
212LaneBitmask llvm::getLiveLaneMask(const LiveInterval &LI, SlotIndex SI,
213                                  const MachineRegisterInfo &MRI) {
214  LaneBitmask LiveMask;
215  if (LI.hasSubRanges()) {
216    for (const auto &S : LI.subranges())
217      if (S.liveAt(SI)) {
218        LiveMask |= S.LaneMask;
219        assert(LiveMask == (LiveMask & MRI.getMaxLaneMaskForVReg(LI.reg())));
220      }
221  } else if (LI.liveAt(SI)) {
222    LiveMask = MRI.getMaxLaneMaskForVReg(LI.reg());
223  }
224  return LiveMask;
225}
226
227GCNRPTracker::LiveRegSet llvm::getLiveRegs(SlotIndex SI,
228                                           const LiveIntervals &LIS,
229                                           const MachineRegisterInfo &MRI) {
230  GCNRPTracker::LiveRegSet LiveRegs;
231  for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
232    auto Reg = Register::index2VirtReg(I);
233    if (!LIS.hasInterval(Reg))
234      continue;
235    auto LiveMask = getLiveLaneMask(Reg, SI, LIS, MRI);
236    if (LiveMask.any())
237      LiveRegs[Reg] = LiveMask;
238  }
239  return LiveRegs;
240}
241
242void GCNRPTracker::reset(const MachineInstr &MI,
243                         const LiveRegSet *LiveRegsCopy,
244                         bool After) {
245  const MachineFunction &MF = *MI.getMF();
246  MRI = &MF.getRegInfo();
247  if (LiveRegsCopy) {
248    if (&LiveRegs != LiveRegsCopy)
249      LiveRegs = *LiveRegsCopy;
250  } else {
251    LiveRegs = After ? getLiveRegsAfter(MI, LIS)
252                     : getLiveRegsBefore(MI, LIS);
253  }
254
255  MaxPressure = CurPressure = getRegPressure(*MRI, LiveRegs);
256}
257
258////////////////////////////////////////////////////////////////////////////////
259// GCNUpwardRPTracker
260
261void GCNUpwardRPTracker::reset(const MachineRegisterInfo &MRI_,
262                               const LiveRegSet &LiveRegs_) {
263  MRI = &MRI_;
264  LiveRegs = LiveRegs_;
265  LastTrackedMI = nullptr;
266  MaxPressure = CurPressure = getRegPressure(MRI_, LiveRegs_);
267}
268
269void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
270  assert(MRI && "call reset first");
271
272  LastTrackedMI = &MI;
273
274  if (MI.isDebugInstr())
275    return;
276
277  // Kill all defs.
278  GCNRegPressure DefPressure, ECDefPressure;
279  bool HasECDefs = false;
280  for (const MachineOperand &MO : MI.all_defs()) {
281    if (!MO.getReg().isVirtual())
282      continue;
283
284    Register Reg = MO.getReg();
285    LaneBitmask DefMask = getDefRegMask(MO, *MRI);
286
287    // Treat a def as fully live at the moment of definition: keep a record.
288    if (MO.isEarlyClobber()) {
289      ECDefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI);
290      HasECDefs = true;
291    } else
292      DefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI);
293
294    auto I = LiveRegs.find(Reg);
295    if (I == LiveRegs.end())
296      continue;
297
298    LaneBitmask &LiveMask = I->second;
299    LaneBitmask PrevMask = LiveMask;
300    LiveMask &= ~DefMask;
301    CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
302    if (LiveMask.none())
303      LiveRegs.erase(I);
304  }
305
306  // Update MaxPressure with defs pressure.
307  DefPressure += CurPressure;
308  if (HasECDefs)
309    DefPressure += ECDefPressure;
310  MaxPressure = max(DefPressure, MaxPressure);
311
312  // Make uses alive.
313  SmallVector<RegisterMaskPair, 8> RegUses;
314  collectVirtualRegUses(RegUses, MI, LIS, *MRI);
315  for (const RegisterMaskPair &U : RegUses) {
316    LaneBitmask &LiveMask = LiveRegs[U.RegUnit];
317    LaneBitmask PrevMask = LiveMask;
318    LiveMask |= U.LaneMask;
319    CurPressure.inc(U.RegUnit, PrevMask, LiveMask, *MRI);
320  }
321
322  // Update MaxPressure with uses plus early-clobber defs pressure.
323  MaxPressure = HasECDefs ? max(CurPressure + ECDefPressure, MaxPressure)
324                          : max(CurPressure, MaxPressure);
325
326  assert(CurPressure == getRegPressure(*MRI, LiveRegs));
327}
328
329////////////////////////////////////////////////////////////////////////////////
330// GCNDownwardRPTracker
331
332bool GCNDownwardRPTracker::reset(const MachineInstr &MI,
333                                 const LiveRegSet *LiveRegsCopy) {
334  MRI = &MI.getParent()->getParent()->getRegInfo();
335  LastTrackedMI = nullptr;
336  MBBEnd = MI.getParent()->end();
337  NextMI = &MI;
338  NextMI = skipDebugInstructionsForward(NextMI, MBBEnd);
339  if (NextMI == MBBEnd)
340    return false;
341  GCNRPTracker::reset(*NextMI, LiveRegsCopy, false);
342  return true;
343}
344
345bool GCNDownwardRPTracker::advanceBeforeNext() {
346  assert(MRI && "call reset first");
347  if (!LastTrackedMI)
348    return NextMI == MBBEnd;
349
350  assert(NextMI == MBBEnd || !NextMI->isDebugInstr());
351
352  SlotIndex SI = NextMI == MBBEnd
353                     ? LIS.getInstructionIndex(*LastTrackedMI).getDeadSlot()
354                     : LIS.getInstructionIndex(*NextMI).getBaseIndex();
355  assert(SI.isValid());
356
357  // Remove dead registers or mask bits.
358  SmallSet<Register, 8> SeenRegs;
359  for (auto &MO : LastTrackedMI->operands()) {
360    if (!MO.isReg() || !MO.getReg().isVirtual())
361      continue;
362    if (MO.isUse() && !MO.readsReg())
363      continue;
364    if (!SeenRegs.insert(MO.getReg()).second)
365      continue;
366    const LiveInterval &LI = LIS.getInterval(MO.getReg());
367    if (LI.hasSubRanges()) {
368      auto It = LiveRegs.end();
369      for (const auto &S : LI.subranges()) {
370        if (!S.liveAt(SI)) {
371          if (It == LiveRegs.end()) {
372            It = LiveRegs.find(MO.getReg());
373            if (It == LiveRegs.end())
374              llvm_unreachable("register isn't live");
375          }
376          auto PrevMask = It->second;
377          It->second &= ~S.LaneMask;
378          CurPressure.inc(MO.getReg(), PrevMask, It->second, *MRI);
379        }
380      }
381      if (It != LiveRegs.end() && It->second.none())
382        LiveRegs.erase(It);
383    } else if (!LI.liveAt(SI)) {
384      auto It = LiveRegs.find(MO.getReg());
385      if (It == LiveRegs.end())
386        llvm_unreachable("register isn't live");
387      CurPressure.inc(MO.getReg(), It->second, LaneBitmask::getNone(), *MRI);
388      LiveRegs.erase(It);
389    }
390  }
391
392  MaxPressure = max(MaxPressure, CurPressure);
393
394  LastTrackedMI = nullptr;
395
396  return NextMI == MBBEnd;
397}
398
399void GCNDownwardRPTracker::advanceToNext() {
400  LastTrackedMI = &*NextMI++;
401  NextMI = skipDebugInstructionsForward(NextMI, MBBEnd);
402
403  // Add new registers or mask bits.
404  for (const auto &MO : LastTrackedMI->all_defs()) {
405    Register Reg = MO.getReg();
406    if (!Reg.isVirtual())
407      continue;
408    auto &LiveMask = LiveRegs[Reg];
409    auto PrevMask = LiveMask;
410    LiveMask |= getDefRegMask(MO, *MRI);
411    CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
412  }
413
414  MaxPressure = max(MaxPressure, CurPressure);
415}
416
417bool GCNDownwardRPTracker::advance() {
418  if (NextMI == MBBEnd)
419    return false;
420  advanceBeforeNext();
421  advanceToNext();
422  return true;
423}
424
425bool GCNDownwardRPTracker::advance(MachineBasicBlock::const_iterator End) {
426  while (NextMI != End)
427    if (!advance()) return false;
428  return true;
429}
430
431bool GCNDownwardRPTracker::advance(MachineBasicBlock::const_iterator Begin,
432                                   MachineBasicBlock::const_iterator End,
433                                   const LiveRegSet *LiveRegsCopy) {
434  reset(*Begin, LiveRegsCopy);
435  return advance(End);
436}
437
438Printable llvm::reportMismatch(const GCNRPTracker::LiveRegSet &LISLR,
439                               const GCNRPTracker::LiveRegSet &TrackedLR,
440                               const TargetRegisterInfo *TRI, StringRef Pfx) {
441  return Printable([&LISLR, &TrackedLR, TRI, Pfx](raw_ostream &OS) {
442    for (auto const &P : TrackedLR) {
443      auto I = LISLR.find(P.first);
444      if (I == LISLR.end()) {
445        OS << Pfx << printReg(P.first, TRI) << ":L" << PrintLaneMask(P.second)
446           << " isn't found in LIS reported set\n";
447      } else if (I->second != P.second) {
448        OS << Pfx << printReg(P.first, TRI)
449           << " masks doesn't match: LIS reported " << PrintLaneMask(I->second)
450           << ", tracked " << PrintLaneMask(P.second) << '\n';
451      }
452    }
453    for (auto const &P : LISLR) {
454      auto I = TrackedLR.find(P.first);
455      if (I == TrackedLR.end()) {
456        OS << Pfx << printReg(P.first, TRI) << ":L" << PrintLaneMask(P.second)
457           << " isn't found in tracked set\n";
458      }
459    }
460  });
461}
462
463bool GCNUpwardRPTracker::isValid() const {
464  const auto &SI = LIS.getInstructionIndex(*LastTrackedMI).getBaseIndex();
465  const auto LISLR = llvm::getLiveRegs(SI, LIS, *MRI);
466  const auto &TrackedLR = LiveRegs;
467
468  if (!isEqual(LISLR, TrackedLR)) {
469    dbgs() << "\nGCNUpwardRPTracker error: Tracked and"
470              " LIS reported livesets mismatch:\n"
471           << print(LISLR, *MRI);
472    reportMismatch(LISLR, TrackedLR, MRI->getTargetRegisterInfo());
473    return false;
474  }
475
476  auto LISPressure = getRegPressure(*MRI, LISLR);
477  if (LISPressure != CurPressure) {
478    dbgs() << "GCNUpwardRPTracker error: Pressure sets different\nTracked: "
479           << print(CurPressure) << "LIS rpt: " << print(LISPressure);
480    return false;
481  }
482  return true;
483}
484
485Printable llvm::print(const GCNRPTracker::LiveRegSet &LiveRegs,
486                      const MachineRegisterInfo &MRI) {
487  return Printable([&LiveRegs, &MRI](raw_ostream &OS) {
488    const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
489    for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
490      Register Reg = Register::index2VirtReg(I);
491      auto It = LiveRegs.find(Reg);
492      if (It != LiveRegs.end() && It->second.any())
493        OS << ' ' << printVRegOrUnit(Reg, TRI) << ':'
494           << PrintLaneMask(It->second);
495    }
496    OS << '\n';
497  });
498}
499
500void GCNRegPressure::dump() const { dbgs() << print(*this); }
501
502static cl::opt<bool> UseDownwardTracker(
503    "amdgpu-print-rp-downward",
504    cl::desc("Use GCNDownwardRPTracker for GCNRegPressurePrinter pass"),
505    cl::init(false), cl::Hidden);
506
507char llvm::GCNRegPressurePrinter::ID = 0;
508char &llvm::GCNRegPressurePrinterID = GCNRegPressurePrinter::ID;
509
510INITIALIZE_PASS(GCNRegPressurePrinter, "amdgpu-print-rp", "", true, true)
511
512// Return lanemask of Reg's subregs that are live-through at [Begin, End] and
513// are fully covered by Mask.
514static LaneBitmask
515getRegLiveThroughMask(const MachineRegisterInfo &MRI, const LiveIntervals &LIS,
516                      Register Reg, SlotIndex Begin, SlotIndex End,
517                      LaneBitmask Mask = LaneBitmask::getAll()) {
518
519  auto IsInOneSegment = [Begin, End](const LiveRange &LR) -> bool {
520    auto *Segment = LR.getSegmentContaining(Begin);
521    return Segment && Segment->contains(End);
522  };
523
524  LaneBitmask LiveThroughMask;
525  const LiveInterval &LI = LIS.getInterval(Reg);
526  if (LI.hasSubRanges()) {
527    for (auto &SR : LI.subranges()) {
528      if ((SR.LaneMask & Mask) == SR.LaneMask && IsInOneSegment(SR))
529        LiveThroughMask |= SR.LaneMask;
530    }
531  } else {
532    LaneBitmask RegMask = MRI.getMaxLaneMaskForVReg(Reg);
533    if ((RegMask & Mask) == RegMask && IsInOneSegment(LI))
534      LiveThroughMask = RegMask;
535  }
536
537  return LiveThroughMask;
538}
539
540bool GCNRegPressurePrinter::runOnMachineFunction(MachineFunction &MF) {
541  const MachineRegisterInfo &MRI = MF.getRegInfo();
542  const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
543  const LiveIntervals &LIS = getAnalysis<LiveIntervals>();
544
545  auto &OS = dbgs();
546
547// Leading spaces are important for YAML syntax.
548#define PFX "  "
549
550  OS << "---\nname: " << MF.getName() << "\nbody:             |\n";
551
552  auto printRP = [](const GCNRegPressure &RP) {
553    return Printable([&RP](raw_ostream &OS) {
554      OS << format(PFX "  %-5d", RP.getSGPRNum())
555         << format(" %-5d", RP.getVGPRNum(false));
556    });
557  };
558
559  auto ReportLISMismatchIfAny = [&](const GCNRPTracker::LiveRegSet &TrackedLR,
560                                    const GCNRPTracker::LiveRegSet &LISLR) {
561    if (LISLR != TrackedLR) {
562      OS << PFX "  mis LIS: " << llvm::print(LISLR, MRI)
563         << reportMismatch(LISLR, TrackedLR, TRI, PFX "    ");
564    }
565  };
566
567  // Register pressure before and at an instruction (in program order).
568  SmallVector<std::pair<GCNRegPressure, GCNRegPressure>, 16> RP;
569
570  for (auto &MBB : MF) {
571    RP.clear();
572    RP.reserve(MBB.size());
573
574    OS << PFX;
575    MBB.printName(OS);
576    OS << ":\n";
577
578    SlotIndex MBBStartSlot = LIS.getSlotIndexes()->getMBBStartIdx(&MBB);
579    SlotIndex MBBEndSlot = LIS.getSlotIndexes()->getMBBEndIdx(&MBB);
580
581    GCNRPTracker::LiveRegSet LiveIn, LiveOut;
582    GCNRegPressure RPAtMBBEnd;
583
584    if (UseDownwardTracker) {
585      if (MBB.empty()) {
586        LiveIn = LiveOut = getLiveRegs(MBBStartSlot, LIS, MRI);
587        RPAtMBBEnd = getRegPressure(MRI, LiveIn);
588      } else {
589        GCNDownwardRPTracker RPT(LIS);
590        RPT.reset(MBB.front());
591
592        LiveIn = RPT.getLiveRegs();
593
594        while (!RPT.advanceBeforeNext()) {
595          GCNRegPressure RPBeforeMI = RPT.getPressure();
596          RPT.advanceToNext();
597          RP.emplace_back(RPBeforeMI, RPT.getPressure());
598        }
599
600        LiveOut = RPT.getLiveRegs();
601        RPAtMBBEnd = RPT.getPressure();
602      }
603    } else {
604      GCNUpwardRPTracker RPT(LIS);
605      RPT.reset(MRI, MBBEndSlot);
606
607      LiveOut = RPT.getLiveRegs();
608      RPAtMBBEnd = RPT.getPressure();
609
610      for (auto &MI : reverse(MBB)) {
611        RPT.resetMaxPressure();
612        RPT.recede(MI);
613        if (!MI.isDebugInstr())
614          RP.emplace_back(RPT.getPressure(), RPT.getMaxPressure());
615      }
616
617      LiveIn = RPT.getLiveRegs();
618    }
619
620    OS << PFX "  Live-in: " << llvm::print(LiveIn, MRI);
621    if (!UseDownwardTracker)
622      ReportLISMismatchIfAny(LiveIn, getLiveRegs(MBBStartSlot, LIS, MRI));
623
624    OS << PFX "  SGPR  VGPR\n";
625    int I = 0;
626    for (auto &MI : MBB) {
627      if (!MI.isDebugInstr()) {
628        auto &[RPBeforeInstr, RPAtInstr] =
629            RP[UseDownwardTracker ? I : (RP.size() - 1 - I)];
630        ++I;
631        OS << printRP(RPBeforeInstr) << '\n' << printRP(RPAtInstr) << "  ";
632      } else
633        OS << PFX "               ";
634      MI.print(OS);
635    }
636    OS << printRP(RPAtMBBEnd) << '\n';
637
638    OS << PFX "  Live-out:" << llvm::print(LiveOut, MRI);
639    if (UseDownwardTracker)
640      ReportLISMismatchIfAny(LiveOut, getLiveRegs(MBBEndSlot, LIS, MRI));
641
642    GCNRPTracker::LiveRegSet LiveThrough;
643    for (auto [Reg, Mask] : LiveIn) {
644      LaneBitmask MaskIntersection = Mask & LiveOut.lookup(Reg);
645      if (MaskIntersection.any()) {
646        LaneBitmask LTMask = getRegLiveThroughMask(
647            MRI, LIS, Reg, MBBStartSlot, MBBEndSlot, MaskIntersection);
648        if (LTMask.any())
649          LiveThrough[Reg] = LTMask;
650      }
651    }
652    OS << PFX "  Live-thr:" << llvm::print(LiveThrough, MRI);
653    OS << printRP(getRegPressure(MRI, LiveThrough)) << '\n';
654  }
655  OS << "...\n";
656  return false;
657
658#undef PFX
659}