1212793Sdim//===---------- SplitKit.cpp - Toolkit for splitting live ranges ----------===// 2212793Sdim// 3212793Sdim// The LLVM Compiler Infrastructure 4212793Sdim// 5212793Sdim// This file is distributed under the University of Illinois Open Source 6212793Sdim// License. See LICENSE.TXT for details. 7212793Sdim// 8212793Sdim//===----------------------------------------------------------------------===// 9212793Sdim// 10212793Sdim// This file contains the SplitAnalysis class as well as mutator functions for 11212793Sdim// live range splitting. 12212793Sdim// 13212793Sdim//===----------------------------------------------------------------------===// 14212793Sdim 15212793Sdim#include "SplitKit.h" 16221345Sdim#include "llvm/ADT/Statistic.h" 17212793Sdim#include "llvm/CodeGen/LiveIntervalAnalysis.h" 18234353Sdim#include "llvm/CodeGen/LiveRangeEdit.h" 19218893Sdim#include "llvm/CodeGen/MachineDominators.h" 20212793Sdim#include "llvm/CodeGen/MachineInstrBuilder.h" 21226633Sdim#include "llvm/CodeGen/MachineLoopInfo.h" 22212793Sdim#include "llvm/CodeGen/MachineRegisterInfo.h" 23249423Sdim#include "llvm/CodeGen/VirtRegMap.h" 24212793Sdim#include "llvm/Support/Debug.h" 25212793Sdim#include "llvm/Support/raw_ostream.h" 26212793Sdim#include "llvm/Target/TargetInstrInfo.h" 27212793Sdim#include "llvm/Target/TargetMachine.h" 28212793Sdim 29212793Sdimusing namespace llvm; 30212793Sdim 31276479Sdim#define DEBUG_TYPE "regalloc" 32276479Sdim 33221345SdimSTATISTIC(NumFinished, "Number of splits finished"); 34221345SdimSTATISTIC(NumSimple, "Number of splits that were simple"); 35223017SdimSTATISTIC(NumCopies, "Number of copies inserted for splitting"); 36223017SdimSTATISTIC(NumRemats, "Number of rematerialized defs for splitting"); 37223017SdimSTATISTIC(NumRepairs, "Number of invalid live ranges repaired"); 38212793Sdim 39212793Sdim//===----------------------------------------------------------------------===// 40212793Sdim// Split Analysis 41212793Sdim//===----------------------------------------------------------------------===// 42212793Sdim 43280031SdimSplitAnalysis::SplitAnalysis(const VirtRegMap &vrm, const LiveIntervals &lis, 44212793Sdim const MachineLoopInfo &mli) 45280031Sdim : MF(vrm.getMachineFunction()), VRM(vrm), LIS(lis), Loops(mli), 46280031Sdim TII(*MF.getSubtarget().getInstrInfo()), CurLI(nullptr), 47280031Sdim LastSplitPoint(MF.getNumBlockIDs()) {} 48212793Sdim 49212793Sdimvoid SplitAnalysis::clear() { 50218893Sdim UseSlots.clear(); 51221345Sdim UseBlocks.clear(); 52221345Sdim ThroughBlocks.clear(); 53276479Sdim CurLI = nullptr; 54223017Sdim DidRepairRange = false; 55212793Sdim} 56212793Sdim 57221345SdimSlotIndex SplitAnalysis::computeLastSplitPoint(unsigned Num) { 58221345Sdim const MachineBasicBlock *MBB = MF.getBlockNumbered(Num); 59296417Sdim // FIXME: Handle multiple EH pad successors. 60221345Sdim const MachineBasicBlock *LPad = MBB->getLandingPadSuccessor(); 61221345Sdim std::pair<SlotIndex, SlotIndex> &LSP = LastSplitPoint[Num]; 62234353Sdim SlotIndex MBBEnd = LIS.getMBBEndIdx(MBB); 63221345Sdim 64221345Sdim // Compute split points on the first call. The pair is independent of the 65221345Sdim // current live interval. 66221345Sdim if (!LSP.first.isValid()) { 67221345Sdim MachineBasicBlock::const_iterator FirstTerm = MBB->getFirstTerminator(); 68221345Sdim if (FirstTerm == MBB->end()) 69234353Sdim LSP.first = MBBEnd; 70221345Sdim else 71221345Sdim LSP.first = LIS.getInstructionIndex(FirstTerm); 72221345Sdim 73221345Sdim // If there is a landing pad successor, also find the call instruction. 74221345Sdim if (!LPad) 75221345Sdim return LSP.first; 76221345Sdim // There may not be a call instruction (?) in which case we ignore LPad. 77221345Sdim LSP.second = LSP.first; 78224145Sdim for (MachineBasicBlock::const_iterator I = MBB->end(), E = MBB->begin(); 79224145Sdim I != E;) { 80224145Sdim --I; 81234353Sdim if (I->isCall()) { 82221345Sdim LSP.second = LIS.getInstructionIndex(I); 83221345Sdim break; 84221345Sdim } 85224145Sdim } 86221345Sdim } 87221345Sdim 88221345Sdim // If CurLI is live into a landing pad successor, move the last split point 89221345Sdim // back to the call that may throw. 90234353Sdim if (!LPad || !LSP.second || !LIS.isLiveInToMBB(*CurLI, LPad)) 91221345Sdim return LSP.first; 92234353Sdim 93234353Sdim // Find the value leaving MBB. 94234353Sdim const VNInfo *VNI = CurLI->getVNInfoBefore(MBBEnd); 95234353Sdim if (!VNI) 96234353Sdim return LSP.first; 97234353Sdim 98234353Sdim // If the value leaving MBB was defined after the call in MBB, it can't 99234353Sdim // really be live-in to the landing pad. This can happen if the landing pad 100234353Sdim // has a PHI, and this register is undef on the exceptional edge. 101234353Sdim // <rdar://problem/10664933> 102234353Sdim if (!SlotIndex::isEarlierInstr(VNI->def, LSP.second) && VNI->def < MBBEnd) 103234353Sdim return LSP.first; 104234353Sdim 105234353Sdim // Value is properly live-in to the landing pad. 106234353Sdim // Only allow splits before the call. 107234353Sdim return LSP.second; 108212793Sdim} 109212793Sdim 110234353SdimMachineBasicBlock::iterator 111234353SdimSplitAnalysis::getLastSplitPointIter(MachineBasicBlock *MBB) { 112234353Sdim SlotIndex LSP = getLastSplitPoint(MBB->getNumber()); 113234353Sdim if (LSP == LIS.getMBBEndIdx(MBB)) 114234353Sdim return MBB->end(); 115234353Sdim return LIS.getInstructionFromIndex(LSP); 116234353Sdim} 117234353Sdim 118218893Sdim/// analyzeUses - Count instructions, basic blocks, and loops using CurLI. 119212793Sdimvoid SplitAnalysis::analyzeUses() { 120221345Sdim assert(UseSlots.empty() && "Call clear first"); 121221345Sdim 122221345Sdim // First get all the defs from the interval values. This provides the correct 123221345Sdim // slots for early clobbers. 124280031Sdim for (const VNInfo *VNI : CurLI->valnos) 125280031Sdim if (!VNI->isPHIDef() && !VNI->isUnused()) 126280031Sdim UseSlots.push_back(VNI->def); 127221345Sdim 128221345Sdim // Get use slots form the use-def chain. 129218893Sdim const MachineRegisterInfo &MRI = MF.getRegInfo(); 130276479Sdim for (MachineOperand &MO : MRI.use_nodbg_operands(CurLI->reg)) 131276479Sdim if (!MO.isUndef()) 132276479Sdim UseSlots.push_back(LIS.getInstructionIndex(MO.getParent()).getRegSlot()); 133221345Sdim 134221345Sdim array_pod_sort(UseSlots.begin(), UseSlots.end()); 135221345Sdim 136221345Sdim // Remove duplicates, keeping the smaller slot for each instruction. 137221345Sdim // That is what we want for early clobbers. 138221345Sdim UseSlots.erase(std::unique(UseSlots.begin(), UseSlots.end(), 139221345Sdim SlotIndex::isSameInstr), 140221345Sdim UseSlots.end()); 141221345Sdim 142221345Sdim // Compute per-live block info. 143221345Sdim if (!calcLiveBlockInfo()) { 144221345Sdim // FIXME: calcLiveBlockInfo found inconsistencies in the live range. 145224145Sdim // I am looking at you, RegisterCoalescer! 146223017Sdim DidRepairRange = true; 147223017Sdim ++NumRepairs; 148221345Sdim DEBUG(dbgs() << "*** Fixing inconsistent live interval! ***\n"); 149221345Sdim const_cast<LiveIntervals&>(LIS) 150221345Sdim .shrinkToUses(const_cast<LiveInterval*>(CurLI)); 151221345Sdim UseBlocks.clear(); 152221345Sdim ThroughBlocks.clear(); 153221345Sdim bool fixed = calcLiveBlockInfo(); 154221345Sdim (void)fixed; 155221345Sdim assert(fixed && "Couldn't fix broken live interval"); 156212793Sdim } 157221345Sdim 158221345Sdim DEBUG(dbgs() << "Analyze counted " 159221345Sdim << UseSlots.size() << " instrs in " 160221345Sdim << UseBlocks.size() << " blocks, through " 161221345Sdim << NumThroughBlocks << " blocks.\n"); 162212793Sdim} 163212793Sdim 164218893Sdim/// calcLiveBlockInfo - Fill the LiveBlocks array with information about blocks 165218893Sdim/// where CurLI is live. 166221345Sdimbool SplitAnalysis::calcLiveBlockInfo() { 167221345Sdim ThroughBlocks.resize(MF.getNumBlockIDs()); 168223017Sdim NumThroughBlocks = NumGapBlocks = 0; 169218893Sdim if (CurLI->empty()) 170221345Sdim return true; 171212793Sdim 172218893Sdim LiveInterval::const_iterator LVI = CurLI->begin(); 173218893Sdim LiveInterval::const_iterator LVE = CurLI->end(); 174212793Sdim 175218893Sdim SmallVectorImpl<SlotIndex>::const_iterator UseI, UseE; 176218893Sdim UseI = UseSlots.begin(); 177218893Sdim UseE = UseSlots.end(); 178212793Sdim 179218893Sdim // Loop over basic blocks where CurLI is live. 180296417Sdim MachineFunction::iterator MFI = 181296417Sdim LIS.getMBBFromIndex(LVI->start)->getIterator(); 182218893Sdim for (;;) { 183218893Sdim BlockInfo BI; 184296417Sdim BI.MBB = &*MFI; 185218893Sdim SlotIndex Start, Stop; 186276479Sdim std::tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(BI.MBB); 187212793Sdim 188223017Sdim // If the block contains no uses, the range must be live through. At one 189224145Sdim // point, RegisterCoalescer could create dangling ranges that ended 190223017Sdim // mid-block. 191223017Sdim if (UseI == UseE || *UseI >= Stop) { 192223017Sdim ++NumThroughBlocks; 193223017Sdim ThroughBlocks.set(BI.MBB->getNumber()); 194223017Sdim // The range shouldn't end mid-block if there are no uses. This shouldn't 195223017Sdim // happen. 196223017Sdim if (LVI->end < Stop) 197223017Sdim return false; 198223017Sdim } else { 199223017Sdim // This block has uses. Find the first and last uses in the block. 200226633Sdim BI.FirstInstr = *UseI; 201226633Sdim assert(BI.FirstInstr >= Start); 202218893Sdim do ++UseI; 203218893Sdim while (UseI != UseE && *UseI < Stop); 204226633Sdim BI.LastInstr = UseI[-1]; 205226633Sdim assert(BI.LastInstr < Stop); 206212793Sdim 207223017Sdim // LVI is the first live segment overlapping MBB. 208223017Sdim BI.LiveIn = LVI->start <= Start; 209223017Sdim 210226633Sdim // When not live in, the first use should be a def. 211226633Sdim if (!BI.LiveIn) { 212261991Sdim assert(LVI->start == LVI->valno->def && "Dangling Segment start"); 213226633Sdim assert(LVI->start == BI.FirstInstr && "First instr should be a def"); 214226633Sdim BI.FirstDef = BI.FirstInstr; 215226633Sdim } 216226633Sdim 217223017Sdim // Look for gaps in the live range. 218223017Sdim BI.LiveOut = true; 219223017Sdim while (LVI->end < Stop) { 220223017Sdim SlotIndex LastStop = LVI->end; 221223017Sdim if (++LVI == LVE || LVI->start >= Stop) { 222223017Sdim BI.LiveOut = false; 223226633Sdim BI.LastInstr = LastStop; 224223017Sdim break; 225223017Sdim } 226226633Sdim 227223017Sdim if (LastStop < LVI->start) { 228223017Sdim // There is a gap in the live range. Create duplicate entries for the 229223017Sdim // live-in snippet and the live-out snippet. 230223017Sdim ++NumGapBlocks; 231223017Sdim 232223017Sdim // Push the Live-in part. 233223017Sdim BI.LiveOut = false; 234223017Sdim UseBlocks.push_back(BI); 235226633Sdim UseBlocks.back().LastInstr = LastStop; 236223017Sdim 237223017Sdim // Set up BI for the live-out part. 238223017Sdim BI.LiveIn = false; 239223017Sdim BI.LiveOut = true; 240226633Sdim BI.FirstInstr = BI.FirstDef = LVI->start; 241223017Sdim } 242226633Sdim 243261991Sdim // A Segment that starts in the middle of the block must be a def. 244261991Sdim assert(LVI->start == LVI->valno->def && "Dangling Segment start"); 245226633Sdim if (!BI.FirstDef) 246226633Sdim BI.FirstDef = LVI->start; 247218893Sdim } 248212793Sdim 249221345Sdim UseBlocks.push_back(BI); 250223017Sdim 251223017Sdim // LVI is now at LVE or LVI->end >= Stop. 252223017Sdim if (LVI == LVE) 253223017Sdim break; 254221345Sdim } 255212793Sdim 256218893Sdim // Live segment ends exactly at Stop. Move to the next segment. 257218893Sdim if (LVI->end == Stop && ++LVI == LVE) 258218893Sdim break; 259218893Sdim 260218893Sdim // Pick the next basic block. 261218893Sdim if (LVI->start < Stop) 262218893Sdim ++MFI; 263218893Sdim else 264296417Sdim MFI = LIS.getMBBFromIndex(LVI->start)->getIterator(); 265212793Sdim } 266223017Sdim 267223017Sdim assert(getNumLiveBlocks() == countLiveBlocks(CurLI) && "Bad block count"); 268221345Sdim return true; 269212793Sdim} 270212793Sdim 271221345Sdimunsigned SplitAnalysis::countLiveBlocks(const LiveInterval *cli) const { 272221345Sdim if (cli->empty()) 273221345Sdim return 0; 274221345Sdim LiveInterval *li = const_cast<LiveInterval*>(cli); 275221345Sdim LiveInterval::iterator LVI = li->begin(); 276221345Sdim LiveInterval::iterator LVE = li->end(); 277221345Sdim unsigned Count = 0; 278221345Sdim 279221345Sdim // Loop over basic blocks where li is live. 280296417Sdim MachineFunction::const_iterator MFI = 281296417Sdim LIS.getMBBFromIndex(LVI->start)->getIterator(); 282296417Sdim SlotIndex Stop = LIS.getMBBEndIdx(&*MFI); 283221345Sdim for (;;) { 284221345Sdim ++Count; 285221345Sdim LVI = li->advanceTo(LVI, Stop); 286221345Sdim if (LVI == LVE) 287221345Sdim return Count; 288221345Sdim do { 289221345Sdim ++MFI; 290296417Sdim Stop = LIS.getMBBEndIdx(&*MFI); 291221345Sdim } while (Stop <= LVI->start); 292221345Sdim } 293221345Sdim} 294221345Sdim 295219077Sdimbool SplitAnalysis::isOriginalEndpoint(SlotIndex Idx) const { 296219077Sdim unsigned OrigReg = VRM.getOriginal(CurLI->reg); 297219077Sdim const LiveInterval &Orig = LIS.getInterval(OrigReg); 298219077Sdim assert(!Orig.empty() && "Splitting empty interval?"); 299219077Sdim LiveInterval::const_iterator I = Orig.find(Idx); 300219077Sdim 301219077Sdim // Range containing Idx should begin at Idx. 302219077Sdim if (I != Orig.end() && I->start <= Idx) 303219077Sdim return I->start == Idx; 304219077Sdim 305219077Sdim // Range does not contain Idx, previous must end at Idx. 306219077Sdim return I != Orig.begin() && (--I)->end == Idx; 307219077Sdim} 308219077Sdim 309212793Sdimvoid SplitAnalysis::analyze(const LiveInterval *li) { 310212793Sdim clear(); 311218893Sdim CurLI = li; 312212793Sdim analyzeUses(); 313212793Sdim} 314212793Sdim 315212793Sdim 316218893Sdim//===----------------------------------------------------------------------===// 317221345Sdim// Split Editor 318218893Sdim//===----------------------------------------------------------------------===// 319212793Sdim 320221345Sdim/// Create a new SplitEditor for editing the LiveInterval analyzed by SA. 321280031SdimSplitEditor::SplitEditor(SplitAnalysis &sa, LiveIntervals &lis, VirtRegMap &vrm, 322261991Sdim MachineDominatorTree &mdt, 323261991Sdim MachineBlockFrequencyInfo &mbfi) 324280031Sdim : SA(sa), LIS(lis), VRM(vrm), MRI(vrm.getMachineFunction().getRegInfo()), 325280031Sdim MDT(mdt), TII(*vrm.getMachineFunction().getSubtarget().getInstrInfo()), 326280031Sdim TRI(*vrm.getMachineFunction().getSubtarget().getRegisterInfo()), 327280031Sdim MBFI(mbfi), Edit(nullptr), OpenIdx(0), SpillMode(SM_Partition), 328280031Sdim RegAssign(Allocator) {} 329212793Sdim 330226633Sdimvoid SplitEditor::reset(LiveRangeEdit &LRE, ComplementSpillMode SM) { 331226633Sdim Edit = &LRE; 332226633Sdim SpillMode = SM; 333221345Sdim OpenIdx = 0; 334221345Sdim RegAssign.clear(); 335218893Sdim Values.clear(); 336221345Sdim 337226633Sdim // Reset the LiveRangeCalc instances needed for this spill mode. 338239462Sdim LRCalc[0].reset(&VRM.getMachineFunction(), LIS.getSlotIndexes(), &MDT, 339239462Sdim &LIS.getVNInfoAllocator()); 340226633Sdim if (SpillMode) 341239462Sdim LRCalc[1].reset(&VRM.getMachineFunction(), LIS.getSlotIndexes(), &MDT, 342239462Sdim &LIS.getVNInfoAllocator()); 343221345Sdim 344221345Sdim // We don't need an AliasAnalysis since we will only be performing 345221345Sdim // cheap-as-a-copy remats anyway. 346276479Sdim Edit->anyRematerializable(nullptr); 347212793Sdim} 348212793Sdim 349243830Sdim#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 350221345Sdimvoid SplitEditor::dump() const { 351221345Sdim if (RegAssign.empty()) { 352221345Sdim dbgs() << " empty\n"; 353221345Sdim return; 354221345Sdim } 355221345Sdim 356221345Sdim for (RegAssignMap::const_iterator I = RegAssign.begin(); I.valid(); ++I) 357221345Sdim dbgs() << " [" << I.start() << ';' << I.stop() << "):" << I.value(); 358221345Sdim dbgs() << '\n'; 359212793Sdim} 360243830Sdim#endif 361212793Sdim 362221345SdimVNInfo *SplitEditor::defValue(unsigned RegIdx, 363221345Sdim const VNInfo *ParentVNI, 364221345Sdim SlotIndex Idx) { 365212793Sdim assert(ParentVNI && "Mapping NULL value"); 366212793Sdim assert(Idx.isValid() && "Invalid SlotIndex"); 367221345Sdim assert(Edit->getParent().getVNInfoAt(Idx) == ParentVNI && "Bad Parent VNI"); 368261991Sdim LiveInterval *LI = &LIS.getInterval(Edit->get(RegIdx)); 369212793Sdim 370218893Sdim // Create a new value. 371234353Sdim VNInfo *VNI = LI->getNextValue(Idx, LIS.getVNInfoAllocator()); 372212793Sdim 373218893Sdim // Use insert for lookup, so we can add missing values with a second lookup. 374221345Sdim std::pair<ValueMap::iterator, bool> InsP = 375226633Sdim Values.insert(std::make_pair(std::make_pair(RegIdx, ParentVNI->id), 376226633Sdim ValueForcePair(VNI, false))); 377218893Sdim 378221345Sdim // This was the first time (RegIdx, ParentVNI) was mapped. 379221345Sdim // Keep it as a simple def without any liveness. 380221345Sdim if (InsP.second) 381221345Sdim return VNI; 382221345Sdim 383221345Sdim // If the previous value was a simple mapping, add liveness for it now. 384226633Sdim if (VNInfo *OldVNI = InsP.first->second.getPointer()) { 385221345Sdim SlotIndex Def = OldVNI->def; 386261991Sdim LI->addSegment(LiveInterval::Segment(Def, Def.getDeadSlot(), OldVNI)); 387226633Sdim // No longer a simple mapping. Switch to a complex, non-forced mapping. 388226633Sdim InsP.first->second = ValueForcePair(); 389221345Sdim } 390218893Sdim 391221345Sdim // This is a complex mapping, add liveness for VNI 392221345Sdim SlotIndex Def = VNI->def; 393261991Sdim LI->addSegment(LiveInterval::Segment(Def, Def.getDeadSlot(), VNI)); 394221345Sdim 395212793Sdim return VNI; 396212793Sdim} 397212793Sdim 398226633Sdimvoid SplitEditor::forceRecompute(unsigned RegIdx, const VNInfo *ParentVNI) { 399212793Sdim assert(ParentVNI && "Mapping NULL value"); 400226633Sdim ValueForcePair &VFP = Values[std::make_pair(RegIdx, ParentVNI->id)]; 401226633Sdim VNInfo *VNI = VFP.getPointer(); 402212793Sdim 403226633Sdim // ParentVNI was either unmapped or already complex mapped. Either way, just 404226633Sdim // set the force bit. 405226633Sdim if (!VNI) { 406226633Sdim VFP.setInt(true); 407221345Sdim return; 408226633Sdim } 409212793Sdim 410221345Sdim // This was previously a single mapping. Make sure the old def is represented 411221345Sdim // by a trivial live range. 412221345Sdim SlotIndex Def = VNI->def; 413261991Sdim LiveInterval *LI = &LIS.getInterval(Edit->get(RegIdx)); 414261991Sdim LI->addSegment(LiveInterval::Segment(Def, Def.getDeadSlot(), VNI)); 415226633Sdim // Mark as complex mapped, forced. 416276479Sdim VFP = ValueForcePair(nullptr, true); 417221345Sdim} 418218893Sdim 419218893SdimVNInfo *SplitEditor::defFromParent(unsigned RegIdx, 420218893Sdim VNInfo *ParentVNI, 421218893Sdim SlotIndex UseIdx, 422218893Sdim MachineBasicBlock &MBB, 423218893Sdim MachineBasicBlock::iterator I) { 424276479Sdim MachineInstr *CopyMI = nullptr; 425218893Sdim SlotIndex Def; 426261991Sdim LiveInterval *LI = &LIS.getInterval(Edit->get(RegIdx)); 427212793Sdim 428221345Sdim // We may be trying to avoid interference that ends at a deleted instruction, 429221345Sdim // so always begin RegIdx 0 early and all others late. 430221345Sdim bool Late = RegIdx != 0; 431221345Sdim 432218893Sdim // Attempt cheap-as-a-copy rematerialization. 433218893Sdim LiveRangeEdit::Remat RM(ParentVNI); 434234353Sdim if (Edit->canRematerializeAt(RM, UseIdx, true)) { 435234353Sdim Def = Edit->rematerializeAt(MBB, I, LI->reg, RM, TRI, Late); 436223017Sdim ++NumRemats; 437218893Sdim } else { 438218893Sdim // Can't remat, just insert a copy from parent. 439218893Sdim CopyMI = BuildMI(MBB, I, DebugLoc(), TII.get(TargetOpcode::COPY), LI->reg) 440221345Sdim .addReg(Edit->getReg()); 441221345Sdim Def = LIS.getSlotIndexes()->insertMachineInstrInMaps(CopyMI, Late) 442234353Sdim .getRegSlot(); 443223017Sdim ++NumCopies; 444212793Sdim } 445212793Sdim 446218893Sdim // Define the value in Reg. 447234353Sdim return defValue(RegIdx, ParentVNI, Def); 448212793Sdim} 449212793Sdim 450212793Sdim/// Create a new virtual register and live interval. 451221345Sdimunsigned SplitEditor::openIntv() { 452218893Sdim // Create the complement as index 0. 453221345Sdim if (Edit->empty()) 454261991Sdim Edit->createEmptyInterval(); 455212793Sdim 456218893Sdim // Create the open interval. 457221345Sdim OpenIdx = Edit->size(); 458261991Sdim Edit->createEmptyInterval(); 459221345Sdim return OpenIdx; 460212793Sdim} 461212793Sdim 462221345Sdimvoid SplitEditor::selectIntv(unsigned Idx) { 463221345Sdim assert(Idx != 0 && "Cannot select the complement interval"); 464221345Sdim assert(Idx < Edit->size() && "Can only select previously opened interval"); 465224145Sdim DEBUG(dbgs() << " selectIntv " << OpenIdx << " -> " << Idx << '\n'); 466221345Sdim OpenIdx = Idx; 467221345Sdim} 468221345Sdim 469218893SdimSlotIndex SplitEditor::enterIntvBefore(SlotIndex Idx) { 470218893Sdim assert(OpenIdx && "openIntv not called before enterIntvBefore"); 471218893Sdim DEBUG(dbgs() << " enterIntvBefore " << Idx); 472218893Sdim Idx = Idx.getBaseIndex(); 473221345Sdim VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx); 474218893Sdim if (!ParentVNI) { 475218893Sdim DEBUG(dbgs() << ": not live\n"); 476218893Sdim return Idx; 477212793Sdim } 478218893Sdim DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n'); 479218893Sdim MachineInstr *MI = LIS.getInstructionFromIndex(Idx); 480218893Sdim assert(MI && "enterIntvBefore called with invalid index"); 481212793Sdim 482218893Sdim VNInfo *VNI = defFromParent(OpenIdx, ParentVNI, Idx, *MI->getParent(), MI); 483218893Sdim return VNI->def; 484218893Sdim} 485212793Sdim 486224145SdimSlotIndex SplitEditor::enterIntvAfter(SlotIndex Idx) { 487224145Sdim assert(OpenIdx && "openIntv not called before enterIntvAfter"); 488224145Sdim DEBUG(dbgs() << " enterIntvAfter " << Idx); 489224145Sdim Idx = Idx.getBoundaryIndex(); 490224145Sdim VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx); 491224145Sdim if (!ParentVNI) { 492224145Sdim DEBUG(dbgs() << ": not live\n"); 493224145Sdim return Idx; 494224145Sdim } 495224145Sdim DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n'); 496224145Sdim MachineInstr *MI = LIS.getInstructionFromIndex(Idx); 497224145Sdim assert(MI && "enterIntvAfter called with invalid index"); 498224145Sdim 499224145Sdim VNInfo *VNI = defFromParent(OpenIdx, ParentVNI, Idx, *MI->getParent(), 500276479Sdim std::next(MachineBasicBlock::iterator(MI))); 501224145Sdim return VNI->def; 502224145Sdim} 503224145Sdim 504218893SdimSlotIndex SplitEditor::enterIntvAtEnd(MachineBasicBlock &MBB) { 505218893Sdim assert(OpenIdx && "openIntv not called before enterIntvAtEnd"); 506218893Sdim SlotIndex End = LIS.getMBBEndIdx(&MBB); 507218893Sdim SlotIndex Last = End.getPrevSlot(); 508218893Sdim DEBUG(dbgs() << " enterIntvAtEnd BB#" << MBB.getNumber() << ", " << Last); 509221345Sdim VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Last); 510218893Sdim if (!ParentVNI) { 511218893Sdim DEBUG(dbgs() << ": not live\n"); 512218893Sdim return End; 513212793Sdim } 514218893Sdim DEBUG(dbgs() << ": valno " << ParentVNI->id); 515218893Sdim VNInfo *VNI = defFromParent(OpenIdx, ParentVNI, Last, MBB, 516234353Sdim SA.getLastSplitPointIter(&MBB)); 517218893Sdim RegAssign.insert(VNI->def, End, OpenIdx); 518218893Sdim DEBUG(dump()); 519218893Sdim return VNI->def; 520212793Sdim} 521212793Sdim 522218893Sdim/// useIntv - indicate that all instructions in MBB should use OpenLI. 523212793Sdimvoid SplitEditor::useIntv(const MachineBasicBlock &MBB) { 524218893Sdim useIntv(LIS.getMBBStartIdx(&MBB), LIS.getMBBEndIdx(&MBB)); 525212793Sdim} 526212793Sdim 527212793Sdimvoid SplitEditor::useIntv(SlotIndex Start, SlotIndex End) { 528218893Sdim assert(OpenIdx && "openIntv not called before useIntv"); 529218893Sdim DEBUG(dbgs() << " useIntv [" << Start << ';' << End << "):"); 530218893Sdim RegAssign.insert(Start, End, OpenIdx); 531218893Sdim DEBUG(dump()); 532218893Sdim} 533212793Sdim 534218893SdimSlotIndex SplitEditor::leaveIntvAfter(SlotIndex Idx) { 535218893Sdim assert(OpenIdx && "openIntv not called before leaveIntvAfter"); 536218893Sdim DEBUG(dbgs() << " leaveIntvAfter " << Idx); 537212793Sdim 538218893Sdim // The interval must be live beyond the instruction at Idx. 539226633Sdim SlotIndex Boundary = Idx.getBoundaryIndex(); 540226633Sdim VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Boundary); 541218893Sdim if (!ParentVNI) { 542218893Sdim DEBUG(dbgs() << ": not live\n"); 543226633Sdim return Boundary.getNextSlot(); 544212793Sdim } 545218893Sdim DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n'); 546226633Sdim MachineInstr *MI = LIS.getInstructionFromIndex(Boundary); 547226633Sdim assert(MI && "No instruction at index"); 548212793Sdim 549226633Sdim // In spill mode, make live ranges as short as possible by inserting the copy 550226633Sdim // before MI. This is only possible if that instruction doesn't redefine the 551226633Sdim // value. The inserted COPY is not a kill, and we don't need to recompute 552226633Sdim // the source live range. The spiller also won't try to hoist this copy. 553226633Sdim if (SpillMode && !SlotIndex::isSameInstr(ParentVNI->def, Idx) && 554226633Sdim MI->readsVirtualRegister(Edit->getReg())) { 555226633Sdim forceRecompute(0, ParentVNI); 556226633Sdim defFromParent(0, ParentVNI, Idx, *MI->getParent(), MI); 557226633Sdim return Idx; 558226633Sdim } 559226633Sdim 560226633Sdim VNInfo *VNI = defFromParent(0, ParentVNI, Boundary, *MI->getParent(), 561276479Sdim std::next(MachineBasicBlock::iterator(MI))); 562218893Sdim return VNI->def; 563212793Sdim} 564212793Sdim 565218893SdimSlotIndex SplitEditor::leaveIntvBefore(SlotIndex Idx) { 566218893Sdim assert(OpenIdx && "openIntv not called before leaveIntvBefore"); 567218893Sdim DEBUG(dbgs() << " leaveIntvBefore " << Idx); 568212793Sdim 569218893Sdim // The interval must be live into the instruction at Idx. 570226633Sdim Idx = Idx.getBaseIndex(); 571221345Sdim VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx); 572218893Sdim if (!ParentVNI) { 573218893Sdim DEBUG(dbgs() << ": not live\n"); 574218893Sdim return Idx.getNextSlot(); 575212793Sdim } 576218893Sdim DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n'); 577212793Sdim 578218893Sdim MachineInstr *MI = LIS.getInstructionFromIndex(Idx); 579218893Sdim assert(MI && "No instruction at index"); 580218893Sdim VNInfo *VNI = defFromParent(0, ParentVNI, Idx, *MI->getParent(), MI); 581218893Sdim return VNI->def; 582212793Sdim} 583212793Sdim 584218893SdimSlotIndex SplitEditor::leaveIntvAtTop(MachineBasicBlock &MBB) { 585218893Sdim assert(OpenIdx && "openIntv not called before leaveIntvAtTop"); 586218893Sdim SlotIndex Start = LIS.getMBBStartIdx(&MBB); 587218893Sdim DEBUG(dbgs() << " leaveIntvAtTop BB#" << MBB.getNumber() << ", " << Start); 588212793Sdim 589221345Sdim VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Start); 590218893Sdim if (!ParentVNI) { 591218893Sdim DEBUG(dbgs() << ": not live\n"); 592218893Sdim return Start; 593212793Sdim } 594212793Sdim 595218893Sdim VNInfo *VNI = defFromParent(0, ParentVNI, Start, MBB, 596218893Sdim MBB.SkipPHIsAndLabels(MBB.begin())); 597218893Sdim RegAssign.insert(Start, VNI->def, OpenIdx); 598218893Sdim DEBUG(dump()); 599218893Sdim return VNI->def; 600218893Sdim} 601212793Sdim 602218893Sdimvoid SplitEditor::overlapIntv(SlotIndex Start, SlotIndex End) { 603218893Sdim assert(OpenIdx && "openIntv not called before overlapIntv"); 604221345Sdim const VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Start); 605234353Sdim assert(ParentVNI == Edit->getParent().getVNInfoBefore(End) && 606218893Sdim "Parent changes value in extended range"); 607218893Sdim assert(LIS.getMBBFromIndex(Start) == LIS.getMBBFromIndex(End) && 608218893Sdim "Range cannot span basic blocks"); 609212793Sdim 610226633Sdim // The complement interval will be extended as needed by LRCalc.extend(). 611221345Sdim if (ParentVNI) 612226633Sdim forceRecompute(0, ParentVNI); 613218893Sdim DEBUG(dbgs() << " overlapIntv [" << Start << ';' << End << "):"); 614218893Sdim RegAssign.insert(Start, End, OpenIdx); 615218893Sdim DEBUG(dump()); 616212793Sdim} 617212793Sdim 618226633Sdim//===----------------------------------------------------------------------===// 619226633Sdim// Spill modes 620226633Sdim//===----------------------------------------------------------------------===// 621226633Sdim 622226633Sdimvoid SplitEditor::removeBackCopies(SmallVectorImpl<VNInfo*> &Copies) { 623261991Sdim LiveInterval *LI = &LIS.getInterval(Edit->get(0)); 624226633Sdim DEBUG(dbgs() << "Removing " << Copies.size() << " back-copies.\n"); 625226633Sdim RegAssignMap::iterator AssignI; 626226633Sdim AssignI.setMap(RegAssign); 627226633Sdim 628226633Sdim for (unsigned i = 0, e = Copies.size(); i != e; ++i) { 629288943Sdim SlotIndex Def = Copies[i]->def; 630226633Sdim MachineInstr *MI = LIS.getInstructionFromIndex(Def); 631226633Sdim assert(MI && "No instruction for back-copy"); 632226633Sdim 633226633Sdim MachineBasicBlock *MBB = MI->getParent(); 634226633Sdim MachineBasicBlock::iterator MBBI(MI); 635226633Sdim bool AtBegin; 636226633Sdim do AtBegin = MBBI == MBB->begin(); 637226633Sdim while (!AtBegin && (--MBBI)->isDebugValue()); 638226633Sdim 639226633Sdim DEBUG(dbgs() << "Removing " << Def << '\t' << *MI); 640288943Sdim LIS.removeVRegDefAt(*LI, Def); 641226633Sdim LIS.RemoveMachineInstrFromMaps(MI); 642226633Sdim MI->eraseFromParent(); 643226633Sdim 644288943Sdim // Adjust RegAssign if a register assignment is killed at Def. We want to 645288943Sdim // avoid calculating the live range of the source register if possible. 646239462Sdim AssignI.find(Def.getPrevSlot()); 647226633Sdim if (!AssignI.valid() || AssignI.start() >= Def) 648226633Sdim continue; 649226633Sdim // If MI doesn't kill the assigned register, just leave it. 650226633Sdim if (AssignI.stop() != Def) 651226633Sdim continue; 652226633Sdim unsigned RegIdx = AssignI.value(); 653226633Sdim if (AtBegin || !MBBI->readsVirtualRegister(Edit->getReg())) { 654226633Sdim DEBUG(dbgs() << " cannot find simple kill of RegIdx " << RegIdx << '\n'); 655226633Sdim forceRecompute(RegIdx, Edit->getParent().getVNInfoAt(Def)); 656226633Sdim } else { 657234353Sdim SlotIndex Kill = LIS.getInstructionIndex(MBBI).getRegSlot(); 658226633Sdim DEBUG(dbgs() << " move kill to " << Kill << '\t' << *MBBI); 659226633Sdim AssignI.setStop(Kill); 660226633Sdim } 661226633Sdim } 662226633Sdim} 663226633Sdim 664226633SdimMachineBasicBlock* 665226633SdimSplitEditor::findShallowDominator(MachineBasicBlock *MBB, 666226633Sdim MachineBasicBlock *DefMBB) { 667226633Sdim if (MBB == DefMBB) 668226633Sdim return MBB; 669226633Sdim assert(MDT.dominates(DefMBB, MBB) && "MBB must be dominated by the def."); 670226633Sdim 671226633Sdim const MachineLoopInfo &Loops = SA.Loops; 672226633Sdim const MachineLoop *DefLoop = Loops.getLoopFor(DefMBB); 673226633Sdim MachineDomTreeNode *DefDomNode = MDT[DefMBB]; 674226633Sdim 675226633Sdim // Best candidate so far. 676226633Sdim MachineBasicBlock *BestMBB = MBB; 677226633Sdim unsigned BestDepth = UINT_MAX; 678226633Sdim 679226633Sdim for (;;) { 680226633Sdim const MachineLoop *Loop = Loops.getLoopFor(MBB); 681226633Sdim 682226633Sdim // MBB isn't in a loop, it doesn't get any better. All dominators have a 683226633Sdim // higher frequency by definition. 684226633Sdim if (!Loop) { 685226633Sdim DEBUG(dbgs() << "Def in BB#" << DefMBB->getNumber() << " dominates BB#" 686226633Sdim << MBB->getNumber() << " at depth 0\n"); 687226633Sdim return MBB; 688226633Sdim } 689226633Sdim 690226633Sdim // We'll never be able to exit the DefLoop. 691226633Sdim if (Loop == DefLoop) { 692226633Sdim DEBUG(dbgs() << "Def in BB#" << DefMBB->getNumber() << " dominates BB#" 693226633Sdim << MBB->getNumber() << " in the same loop\n"); 694226633Sdim return MBB; 695226633Sdim } 696226633Sdim 697226633Sdim // Least busy dominator seen so far. 698226633Sdim unsigned Depth = Loop->getLoopDepth(); 699226633Sdim if (Depth < BestDepth) { 700226633Sdim BestMBB = MBB; 701226633Sdim BestDepth = Depth; 702226633Sdim DEBUG(dbgs() << "Def in BB#" << DefMBB->getNumber() << " dominates BB#" 703226633Sdim << MBB->getNumber() << " at depth " << Depth << '\n'); 704226633Sdim } 705226633Sdim 706226633Sdim // Leave loop by going to the immediate dominator of the loop header. 707226633Sdim // This is a bigger stride than simply walking up the dominator tree. 708226633Sdim MachineDomTreeNode *IDom = MDT[Loop->getHeader()]->getIDom(); 709226633Sdim 710226633Sdim // Too far up the dominator tree? 711226633Sdim if (!IDom || !MDT.dominates(DefDomNode, IDom)) 712226633Sdim return BestMBB; 713226633Sdim 714226633Sdim MBB = IDom->getBlock(); 715226633Sdim } 716226633Sdim} 717226633Sdim 718226633Sdimvoid SplitEditor::hoistCopiesForSize() { 719226633Sdim // Get the complement interval, always RegIdx 0. 720261991Sdim LiveInterval *LI = &LIS.getInterval(Edit->get(0)); 721226633Sdim LiveInterval *Parent = &Edit->getParent(); 722226633Sdim 723226633Sdim // Track the nearest common dominator for all back-copies for each ParentVNI, 724226633Sdim // indexed by ParentVNI->id. 725226633Sdim typedef std::pair<MachineBasicBlock*, SlotIndex> DomPair; 726226633Sdim SmallVector<DomPair, 8> NearestDom(Parent->getNumValNums()); 727226633Sdim 728226633Sdim // Find the nearest common dominator for parent values with multiple 729226633Sdim // back-copies. If a single back-copy dominates, put it in DomPair.second. 730280031Sdim for (VNInfo *VNI : LI->valnos) { 731239462Sdim if (VNI->isUnused()) 732239462Sdim continue; 733226633Sdim VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(VNI->def); 734226633Sdim assert(ParentVNI && "Parent not live at complement def"); 735226633Sdim 736226633Sdim // Don't hoist remats. The complement is probably going to disappear 737226633Sdim // completely anyway. 738226633Sdim if (Edit->didRematerialize(ParentVNI)) 739226633Sdim continue; 740226633Sdim 741226633Sdim MachineBasicBlock *ValMBB = LIS.getMBBFromIndex(VNI->def); 742226633Sdim DomPair &Dom = NearestDom[ParentVNI->id]; 743226633Sdim 744226633Sdim // Keep directly defined parent values. This is either a PHI or an 745226633Sdim // instruction in the complement range. All other copies of ParentVNI 746226633Sdim // should be eliminated. 747226633Sdim if (VNI->def == ParentVNI->def) { 748226633Sdim DEBUG(dbgs() << "Direct complement def at " << VNI->def << '\n'); 749226633Sdim Dom = DomPair(ValMBB, VNI->def); 750226633Sdim continue; 751226633Sdim } 752226633Sdim // Skip the singly mapped values. There is nothing to gain from hoisting a 753226633Sdim // single back-copy. 754226633Sdim if (Values.lookup(std::make_pair(0, ParentVNI->id)).getPointer()) { 755226633Sdim DEBUG(dbgs() << "Single complement def at " << VNI->def << '\n'); 756226633Sdim continue; 757226633Sdim } 758226633Sdim 759226633Sdim if (!Dom.first) { 760226633Sdim // First time we see ParentVNI. VNI dominates itself. 761226633Sdim Dom = DomPair(ValMBB, VNI->def); 762226633Sdim } else if (Dom.first == ValMBB) { 763226633Sdim // Two defs in the same block. Pick the earlier def. 764226633Sdim if (!Dom.second.isValid() || VNI->def < Dom.second) 765226633Sdim Dom.second = VNI->def; 766226633Sdim } else { 767226633Sdim // Different basic blocks. Check if one dominates. 768226633Sdim MachineBasicBlock *Near = 769226633Sdim MDT.findNearestCommonDominator(Dom.first, ValMBB); 770226633Sdim if (Near == ValMBB) 771226633Sdim // Def ValMBB dominates. 772226633Sdim Dom = DomPair(ValMBB, VNI->def); 773226633Sdim else if (Near != Dom.first) 774226633Sdim // None dominate. Hoist to common dominator, need new def. 775226633Sdim Dom = DomPair(Near, SlotIndex()); 776226633Sdim } 777226633Sdim 778226633Sdim DEBUG(dbgs() << "Multi-mapped complement " << VNI->id << '@' << VNI->def 779226633Sdim << " for parent " << ParentVNI->id << '@' << ParentVNI->def 780226633Sdim << " hoist to BB#" << Dom.first->getNumber() << ' ' 781226633Sdim << Dom.second << '\n'); 782226633Sdim } 783226633Sdim 784226633Sdim // Insert the hoisted copies. 785226633Sdim for (unsigned i = 0, e = Parent->getNumValNums(); i != e; ++i) { 786226633Sdim DomPair &Dom = NearestDom[i]; 787226633Sdim if (!Dom.first || Dom.second.isValid()) 788226633Sdim continue; 789226633Sdim // This value needs a hoisted copy inserted at the end of Dom.first. 790226633Sdim VNInfo *ParentVNI = Parent->getValNumInfo(i); 791226633Sdim MachineBasicBlock *DefMBB = LIS.getMBBFromIndex(ParentVNI->def); 792226633Sdim // Get a less loopy dominator than Dom.first. 793226633Sdim Dom.first = findShallowDominator(Dom.first, DefMBB); 794226633Sdim SlotIndex Last = LIS.getMBBEndIdx(Dom.first).getPrevSlot(); 795226633Sdim Dom.second = 796226633Sdim defFromParent(0, ParentVNI, Last, *Dom.first, 797234353Sdim SA.getLastSplitPointIter(Dom.first))->def; 798226633Sdim } 799226633Sdim 800226633Sdim // Remove redundant back-copies that are now known to be dominated by another 801226633Sdim // def with the same value. 802226633Sdim SmallVector<VNInfo*, 8> BackCopies; 803280031Sdim for (VNInfo *VNI : LI->valnos) { 804239462Sdim if (VNI->isUnused()) 805239462Sdim continue; 806226633Sdim VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(VNI->def); 807226633Sdim const DomPair &Dom = NearestDom[ParentVNI->id]; 808226633Sdim if (!Dom.first || Dom.second == VNI->def) 809226633Sdim continue; 810226633Sdim BackCopies.push_back(VNI); 811226633Sdim forceRecompute(0, ParentVNI); 812226633Sdim } 813226633Sdim removeBackCopies(BackCopies); 814226633Sdim} 815226633Sdim 816226633Sdim 817221345Sdim/// transferValues - Transfer all possible values to the new live ranges. 818226633Sdim/// Values that were rematerialized are left alone, they need LRCalc.extend(). 819221345Sdimbool SplitEditor::transferValues() { 820221345Sdim bool Skipped = false; 821221345Sdim RegAssignMap::const_iterator AssignI = RegAssign.begin(); 822280031Sdim for (const LiveRange::Segment &S : Edit->getParent()) { 823280031Sdim DEBUG(dbgs() << " blit " << S << ':'); 824280031Sdim VNInfo *ParentVNI = S.valno; 825221345Sdim // RegAssign has holes where RegIdx 0 should be used. 826280031Sdim SlotIndex Start = S.start; 827221345Sdim AssignI.advanceTo(Start); 828221345Sdim do { 829221345Sdim unsigned RegIdx; 830280031Sdim SlotIndex End = S.end; 831221345Sdim if (!AssignI.valid()) { 832221345Sdim RegIdx = 0; 833221345Sdim } else if (AssignI.start() <= Start) { 834221345Sdim RegIdx = AssignI.value(); 835221345Sdim if (AssignI.stop() < End) { 836221345Sdim End = AssignI.stop(); 837221345Sdim ++AssignI; 838221345Sdim } 839221345Sdim } else { 840221345Sdim RegIdx = 0; 841221345Sdim End = std::min(End, AssignI.start()); 842221345Sdim } 843221345Sdim 844221345Sdim // The interval [Start;End) is continuously mapped to RegIdx, ParentVNI. 845221345Sdim DEBUG(dbgs() << " [" << Start << ';' << End << ")=" << RegIdx); 846261991Sdim LiveRange &LR = LIS.getInterval(Edit->get(RegIdx)); 847221345Sdim 848221345Sdim // Check for a simply defined value that can be blitted directly. 849226633Sdim ValueForcePair VFP = Values.lookup(std::make_pair(RegIdx, ParentVNI->id)); 850226633Sdim if (VNInfo *VNI = VFP.getPointer()) { 851221345Sdim DEBUG(dbgs() << ':' << VNI->id); 852261991Sdim LR.addSegment(LiveInterval::Segment(Start, End, VNI)); 853221345Sdim Start = End; 854221345Sdim continue; 855221345Sdim } 856221345Sdim 857226633Sdim // Skip values with forced recomputation. 858226633Sdim if (VFP.getInt()) { 859226633Sdim DEBUG(dbgs() << "(recalc)"); 860221345Sdim Skipped = true; 861221345Sdim Start = End; 862221345Sdim continue; 863221345Sdim } 864221345Sdim 865226633Sdim LiveRangeCalc &LRC = getLRCalc(RegIdx); 866221345Sdim 867221345Sdim // This value has multiple defs in RegIdx, but it wasn't rematerialized, 868221345Sdim // so the live range is accurate. Add live-in blocks in [Start;End) to the 869221345Sdim // LiveInBlocks. 870296417Sdim MachineFunction::iterator MBB = LIS.getMBBFromIndex(Start)->getIterator(); 871221345Sdim SlotIndex BlockStart, BlockEnd; 872296417Sdim std::tie(BlockStart, BlockEnd) = LIS.getSlotIndexes()->getMBBRange(&*MBB); 873221345Sdim 874221345Sdim // The first block may be live-in, or it may have its own def. 875221345Sdim if (Start != BlockStart) { 876261991Sdim VNInfo *VNI = LR.extendInBlock(BlockStart, std::min(BlockEnd, End)); 877221345Sdim assert(VNI && "Missing def for complex mapped value"); 878221345Sdim DEBUG(dbgs() << ':' << VNI->id << "*BB#" << MBB->getNumber()); 879221345Sdim // MBB has its own def. Is it also live-out? 880226633Sdim if (BlockEnd <= End) 881296417Sdim LRC.setLiveOutValue(&*MBB, VNI); 882226633Sdim 883221345Sdim // Skip to the next block for live-in. 884221345Sdim ++MBB; 885221345Sdim BlockStart = BlockEnd; 886221345Sdim } 887221345Sdim 888221345Sdim // Handle the live-in blocks covered by [Start;End). 889221345Sdim assert(Start <= BlockStart && "Expected live-in block"); 890221345Sdim while (BlockStart < End) { 891221345Sdim DEBUG(dbgs() << ">BB#" << MBB->getNumber()); 892296417Sdim BlockEnd = LIS.getMBBEndIdx(&*MBB); 893221345Sdim if (BlockStart == ParentVNI->def) { 894221345Sdim // This block has the def of a parent PHI, so it isn't live-in. 895221345Sdim assert(ParentVNI->isPHIDef() && "Non-phi defined at block start?"); 896261991Sdim VNInfo *VNI = LR.extendInBlock(BlockStart, std::min(BlockEnd, End)); 897221345Sdim assert(VNI && "Missing def for complex mapped parent PHI"); 898226633Sdim if (End >= BlockEnd) 899296417Sdim LRC.setLiveOutValue(&*MBB, VNI); // Live-out as well. 900221345Sdim } else { 901226633Sdim // This block needs a live-in value. The last block covered may not 902226633Sdim // be live-out. 903221345Sdim if (End < BlockEnd) 904296417Sdim LRC.addLiveInBlock(LR, MDT[&*MBB], End); 905221345Sdim else { 906226633Sdim // Live-through, and we don't know the value. 907296417Sdim LRC.addLiveInBlock(LR, MDT[&*MBB]); 908296417Sdim LRC.setLiveOutValue(&*MBB, nullptr); 909221345Sdim } 910221345Sdim } 911221345Sdim BlockStart = BlockEnd; 912221345Sdim ++MBB; 913221345Sdim } 914221345Sdim Start = End; 915280031Sdim } while (Start != S.end); 916221345Sdim DEBUG(dbgs() << '\n'); 917221345Sdim } 918221345Sdim 919239462Sdim LRCalc[0].calculateValues(); 920226633Sdim if (SpillMode) 921239462Sdim LRCalc[1].calculateValues(); 922221345Sdim 923221345Sdim return Skipped; 924212793Sdim} 925212793Sdim 926221345Sdimvoid SplitEditor::extendPHIKillRanges() { 927221345Sdim // Extend live ranges to be live-out for successor PHI values. 928280031Sdim for (const VNInfo *PHIVNI : Edit->getParent().valnos) { 929221345Sdim if (PHIVNI->isUnused() || !PHIVNI->isPHIDef()) 930221345Sdim continue; 931221345Sdim unsigned RegIdx = RegAssign.lookup(PHIVNI->def); 932261991Sdim LiveRange &LR = LIS.getInterval(Edit->get(RegIdx)); 933226633Sdim LiveRangeCalc &LRC = getLRCalc(RegIdx); 934221345Sdim MachineBasicBlock *MBB = LIS.getMBBFromIndex(PHIVNI->def); 935221345Sdim for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), 936221345Sdim PE = MBB->pred_end(); PI != PE; ++PI) { 937226633Sdim SlotIndex End = LIS.getMBBEndIdx(*PI); 938226633Sdim SlotIndex LastUse = End.getPrevSlot(); 939221345Sdim // The predecessor may not have a live-out value. That is OK, like an 940221345Sdim // undef PHI operand. 941226633Sdim if (Edit->getParent().liveAt(LastUse)) { 942226633Sdim assert(RegAssign.lookup(LastUse) == RegIdx && 943221345Sdim "Different register assignment in phi predecessor"); 944261991Sdim LRC.extend(LR, End); 945221345Sdim } 946221345Sdim } 947221345Sdim } 948221345Sdim} 949221345Sdim 950221345Sdim/// rewriteAssigned - Rewrite all uses of Edit->getReg(). 951221345Sdimvoid SplitEditor::rewriteAssigned(bool ExtendRanges) { 952221345Sdim for (MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(Edit->getReg()), 953218893Sdim RE = MRI.reg_end(); RI != RE;) { 954276479Sdim MachineOperand &MO = *RI; 955212793Sdim MachineInstr *MI = MO.getParent(); 956212793Sdim ++RI; 957218893Sdim // LiveDebugVariables should have handled all DBG_VALUE instructions. 958212793Sdim if (MI->isDebugValue()) { 959212793Sdim DEBUG(dbgs() << "Zapping " << *MI); 960212793Sdim MO.setReg(0); 961212793Sdim continue; 962212793Sdim } 963218893Sdim 964226633Sdim // <undef> operands don't really read the register, so it doesn't matter 965226633Sdim // which register we choose. When the use operand is tied to a def, we must 966226633Sdim // use the same register as the def, so just do that always. 967218893Sdim SlotIndex Idx = LIS.getInstructionIndex(MI); 968226633Sdim if (MO.isDef() || MO.isUndef()) 969234353Sdim Idx = Idx.getRegSlot(MO.isEarlyClobber()); 970218893Sdim 971218893Sdim // Rewrite to the mapped register at Idx. 972218893Sdim unsigned RegIdx = RegAssign.lookup(Idx); 973261991Sdim LiveInterval *LI = &LIS.getInterval(Edit->get(RegIdx)); 974226633Sdim MO.setReg(LI->reg); 975218893Sdim DEBUG(dbgs() << " rewr BB#" << MI->getParent()->getNumber() << '\t' 976218893Sdim << Idx << ':' << RegIdx << '\t' << *MI); 977218893Sdim 978221345Sdim // Extend liveness to Idx if the instruction reads reg. 979226633Sdim if (!ExtendRanges || MO.isUndef()) 980221345Sdim continue; 981221345Sdim 982221345Sdim // Skip instructions that don't read Reg. 983221345Sdim if (MO.isDef()) { 984221345Sdim if (!MO.getSubReg() && !MO.isEarlyClobber()) 985221345Sdim continue; 986221345Sdim // We may wan't to extend a live range for a partial redef, or for a use 987221345Sdim // tied to an early clobber. 988221345Sdim Idx = Idx.getPrevSlot(); 989221345Sdim if (!Edit->getParent().liveAt(Idx)) 990221345Sdim continue; 991221345Sdim } else 992234353Sdim Idx = Idx.getRegSlot(true); 993221345Sdim 994261991Sdim getLRCalc(RegIdx).extend(*LI, Idx.getNextSlot()); 995212793Sdim } 996218893Sdim} 997212793Sdim 998221345Sdimvoid SplitEditor::deleteRematVictims() { 999221345Sdim SmallVector<MachineInstr*, 8> Dead; 1000221345Sdim for (LiveRangeEdit::iterator I = Edit->begin(), E = Edit->end(); I != E; ++I){ 1001261991Sdim LiveInterval *LI = &LIS.getInterval(*I); 1002280031Sdim for (const LiveRange::Segment &S : LI->segments) { 1003234353Sdim // Dead defs end at the dead slot. 1004280031Sdim if (S.end != S.valno->def.getDeadSlot()) 1005221345Sdim continue; 1006280031Sdim MachineInstr *MI = LIS.getInstructionFromIndex(S.valno->def); 1007221345Sdim assert(MI && "Missing instruction for dead def"); 1008221345Sdim MI->addRegisterDead(LI->reg, &TRI); 1009221345Sdim 1010221345Sdim if (!MI->allDefsAreDead()) 1011221345Sdim continue; 1012221345Sdim 1013221345Sdim DEBUG(dbgs() << "All defs dead: " << *MI); 1014221345Sdim Dead.push_back(MI); 1015221345Sdim } 1016212793Sdim } 1017221345Sdim 1018221345Sdim if (Dead.empty()) 1019221345Sdim return; 1020221345Sdim 1021234353Sdim Edit->eliminateDeadDefs(Dead); 1022212793Sdim} 1023212793Sdim 1024221345Sdimvoid SplitEditor::finish(SmallVectorImpl<unsigned> *LRMap) { 1025221345Sdim ++NumFinished; 1026212793Sdim 1027218893Sdim // At this point, the live intervals in Edit contain VNInfos corresponding to 1028218893Sdim // the inserted copies. 1029212793Sdim 1030218893Sdim // Add the original defs from the parent interval. 1031280031Sdim for (const VNInfo *ParentVNI : Edit->getParent().valnos) { 1032218893Sdim if (ParentVNI->isUnused()) 1033218893Sdim continue; 1034221345Sdim unsigned RegIdx = RegAssign.lookup(ParentVNI->def); 1035239462Sdim defValue(RegIdx, ParentVNI, ParentVNI->def); 1036221345Sdim 1037226633Sdim // Force rematted values to be recomputed everywhere. 1038221345Sdim // The new live ranges may be truncated. 1039221345Sdim if (Edit->didRematerialize(ParentVNI)) 1040221345Sdim for (unsigned i = 0, e = Edit->size(); i != e; ++i) 1041226633Sdim forceRecompute(i, ParentVNI); 1042218893Sdim } 1043212793Sdim 1044226633Sdim // Hoist back-copies to the complement interval when in spill mode. 1045226633Sdim switch (SpillMode) { 1046226633Sdim case SM_Partition: 1047226633Sdim // Leave all back-copies as is. 1048226633Sdim break; 1049226633Sdim case SM_Size: 1050226633Sdim hoistCopiesForSize(); 1051226633Sdim break; 1052226633Sdim case SM_Speed: 1053226633Sdim llvm_unreachable("Spill mode 'speed' not implemented yet"); 1054226633Sdim } 1055226633Sdim 1056221345Sdim // Transfer the simply mapped values, check if any are skipped. 1057221345Sdim bool Skipped = transferValues(); 1058221345Sdim if (Skipped) 1059221345Sdim extendPHIKillRanges(); 1060221345Sdim else 1061221345Sdim ++NumSimple; 1062212793Sdim 1063221345Sdim // Rewrite virtual registers, possibly extending ranges. 1064221345Sdim rewriteAssigned(Skipped); 1065212793Sdim 1066221345Sdim // Delete defs that were rematted everywhere. 1067221345Sdim if (Skipped) 1068221345Sdim deleteRematVictims(); 1069212793Sdim 1070218893Sdim // Get rid of unused values and set phi-kill flags. 1071261991Sdim for (LiveRangeEdit::iterator I = Edit->begin(), E = Edit->end(); I != E; ++I) { 1072261991Sdim LiveInterval &LI = LIS.getInterval(*I); 1073261991Sdim LI.RenumberValues(); 1074261991Sdim } 1075218893Sdim 1076221345Sdim // Provide a reverse mapping from original indices to Edit ranges. 1077221345Sdim if (LRMap) { 1078221345Sdim LRMap->clear(); 1079221345Sdim for (unsigned i = 0, e = Edit->size(); i != e; ++i) 1080221345Sdim LRMap->push_back(i); 1081221345Sdim } 1082221345Sdim 1083218893Sdim // Now check if any registers were separated into multiple components. 1084218893Sdim ConnectedVNInfoEqClasses ConEQ(LIS); 1085221345Sdim for (unsigned i = 0, e = Edit->size(); i != e; ++i) { 1086218893Sdim // Don't use iterators, they are invalidated by create() below. 1087296417Sdim unsigned VReg = Edit->get(i); 1088296417Sdim LiveInterval &LI = LIS.getInterval(VReg); 1089296417Sdim SmallVector<LiveInterval*, 8> SplitLIs; 1090296417Sdim LIS.splitSeparateComponents(LI, SplitLIs); 1091296417Sdim unsigned Original = VRM.getOriginal(VReg); 1092296417Sdim for (LiveInterval *SplitLI : SplitLIs) 1093296417Sdim VRM.setIsSplitFromReg(SplitLI->reg, Original); 1094296417Sdim 1095221345Sdim // The new intervals all map back to i. 1096221345Sdim if (LRMap) 1097221345Sdim LRMap->resize(Edit->size(), i); 1098212793Sdim } 1099212793Sdim 1100218893Sdim // Calculate spill weight and allocation hints for new intervals. 1101261991Sdim Edit->calculateRegClassAndHint(VRM.getMachineFunction(), SA.Loops, MBFI); 1102221345Sdim 1103221345Sdim assert(!LRMap || LRMap->size() == Edit->size()); 1104212793Sdim} 1105212793Sdim 1106212793Sdim 1107212793Sdim//===----------------------------------------------------------------------===// 1108212793Sdim// Single Block Splitting 1109212793Sdim//===----------------------------------------------------------------------===// 1110212793Sdim 1111226633Sdimbool SplitAnalysis::shouldSplitSingleBlock(const BlockInfo &BI, 1112226633Sdim bool SingleInstrs) const { 1113226633Sdim // Always split for multiple instructions. 1114226633Sdim if (!BI.isOneInstr()) 1115226633Sdim return true; 1116226633Sdim // Don't split for single instructions unless explicitly requested. 1117226633Sdim if (!SingleInstrs) 1118218893Sdim return false; 1119226633Sdim // Splitting a live-through range always makes progress. 1120226633Sdim if (BI.LiveIn && BI.LiveOut) 1121226633Sdim return true; 1122226633Sdim // No point in isolating a copy. It has no register class constraints. 1123226633Sdim if (LIS.getInstructionFromIndex(BI.FirstInstr)->isCopyLike()) 1124226633Sdim return false; 1125226633Sdim // Finally, don't isolate an end point that was created by earlier splits. 1126226633Sdim return isOriginalEndpoint(BI.FirstInstr); 1127218893Sdim} 1128212793Sdim 1129221345Sdimvoid SplitEditor::splitSingleBlock(const SplitAnalysis::BlockInfo &BI) { 1130221345Sdim openIntv(); 1131221345Sdim SlotIndex LastSplitPoint = SA.getLastSplitPoint(BI.MBB->getNumber()); 1132226633Sdim SlotIndex SegStart = enterIntvBefore(std::min(BI.FirstInstr, 1133221345Sdim LastSplitPoint)); 1134226633Sdim if (!BI.LiveOut || BI.LastInstr < LastSplitPoint) { 1135226633Sdim useIntv(SegStart, leaveIntvAfter(BI.LastInstr)); 1136221345Sdim } else { 1137221345Sdim // The last use is after the last valid split point. 1138221345Sdim SlotIndex SegStop = leaveIntvBefore(LastSplitPoint); 1139221345Sdim useIntv(SegStart, SegStop); 1140226633Sdim overlapIntv(SegStop, BI.LastInstr); 1141221345Sdim } 1142221345Sdim} 1143221345Sdim 1144224145Sdim 1145224145Sdim//===----------------------------------------------------------------------===// 1146224145Sdim// Global Live Range Splitting Support 1147224145Sdim//===----------------------------------------------------------------------===// 1148224145Sdim 1149224145Sdim// These methods support a method of global live range splitting that uses a 1150224145Sdim// global algorithm to decide intervals for CFG edges. They will insert split 1151224145Sdim// points and color intervals in basic blocks while avoiding interference. 1152224145Sdim// 1153224145Sdim// Note that splitSingleBlock is also useful for blocks where both CFG edges 1154224145Sdim// are on the stack. 1155224145Sdim 1156224145Sdimvoid SplitEditor::splitLiveThroughBlock(unsigned MBBNum, 1157224145Sdim unsigned IntvIn, SlotIndex LeaveBefore, 1158224145Sdim unsigned IntvOut, SlotIndex EnterAfter){ 1159224145Sdim SlotIndex Start, Stop; 1160276479Sdim std::tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(MBBNum); 1161224145Sdim 1162224145Sdim DEBUG(dbgs() << "BB#" << MBBNum << " [" << Start << ';' << Stop 1163224145Sdim << ") intf " << LeaveBefore << '-' << EnterAfter 1164224145Sdim << ", live-through " << IntvIn << " -> " << IntvOut); 1165224145Sdim 1166224145Sdim assert((IntvIn || IntvOut) && "Use splitSingleBlock for isolated blocks"); 1167224145Sdim 1168226633Sdim assert((!LeaveBefore || LeaveBefore < Stop) && "Interference after block"); 1169226633Sdim assert((!IntvIn || !LeaveBefore || LeaveBefore > Start) && "Impossible intf"); 1170226633Sdim assert((!EnterAfter || EnterAfter >= Start) && "Interference before block"); 1171226633Sdim 1172226633Sdim MachineBasicBlock *MBB = VRM.getMachineFunction().getBlockNumbered(MBBNum); 1173226633Sdim 1174224145Sdim if (!IntvOut) { 1175224145Sdim DEBUG(dbgs() << ", spill on entry.\n"); 1176224145Sdim // 1177224145Sdim // <<<<<<<<< Possible LeaveBefore interference. 1178224145Sdim // |-----------| Live through. 1179224145Sdim // -____________ Spill on entry. 1180224145Sdim // 1181224145Sdim selectIntv(IntvIn); 1182224145Sdim SlotIndex Idx = leaveIntvAtTop(*MBB); 1183224145Sdim assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference"); 1184224145Sdim (void)Idx; 1185224145Sdim return; 1186224145Sdim } 1187224145Sdim 1188224145Sdim if (!IntvIn) { 1189224145Sdim DEBUG(dbgs() << ", reload on exit.\n"); 1190224145Sdim // 1191224145Sdim // >>>>>>> Possible EnterAfter interference. 1192224145Sdim // |-----------| Live through. 1193224145Sdim // ___________-- Reload on exit. 1194224145Sdim // 1195224145Sdim selectIntv(IntvOut); 1196224145Sdim SlotIndex Idx = enterIntvAtEnd(*MBB); 1197224145Sdim assert((!EnterAfter || Idx >= EnterAfter) && "Interference"); 1198224145Sdim (void)Idx; 1199224145Sdim return; 1200224145Sdim } 1201224145Sdim 1202224145Sdim if (IntvIn == IntvOut && !LeaveBefore && !EnterAfter) { 1203224145Sdim DEBUG(dbgs() << ", straight through.\n"); 1204224145Sdim // 1205224145Sdim // |-----------| Live through. 1206224145Sdim // ------------- Straight through, same intv, no interference. 1207224145Sdim // 1208224145Sdim selectIntv(IntvOut); 1209224145Sdim useIntv(Start, Stop); 1210224145Sdim return; 1211224145Sdim } 1212224145Sdim 1213224145Sdim // We cannot legally insert splits after LSP. 1214224145Sdim SlotIndex LSP = SA.getLastSplitPoint(MBBNum); 1215226633Sdim assert((!IntvOut || !EnterAfter || EnterAfter < LSP) && "Impossible intf"); 1216224145Sdim 1217224145Sdim if (IntvIn != IntvOut && (!LeaveBefore || !EnterAfter || 1218224145Sdim LeaveBefore.getBaseIndex() > EnterAfter.getBoundaryIndex())) { 1219224145Sdim DEBUG(dbgs() << ", switch avoiding interference.\n"); 1220224145Sdim // 1221224145Sdim // >>>> <<<< Non-overlapping EnterAfter/LeaveBefore interference. 1222224145Sdim // |-----------| Live through. 1223224145Sdim // ------======= Switch intervals between interference. 1224224145Sdim // 1225224145Sdim selectIntv(IntvOut); 1226226633Sdim SlotIndex Idx; 1227226633Sdim if (LeaveBefore && LeaveBefore < LSP) { 1228226633Sdim Idx = enterIntvBefore(LeaveBefore); 1229226633Sdim useIntv(Idx, Stop); 1230226633Sdim } else { 1231226633Sdim Idx = enterIntvAtEnd(*MBB); 1232226633Sdim } 1233224145Sdim selectIntv(IntvIn); 1234224145Sdim useIntv(Start, Idx); 1235224145Sdim assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference"); 1236224145Sdim assert((!EnterAfter || Idx >= EnterAfter) && "Interference"); 1237224145Sdim return; 1238224145Sdim } 1239224145Sdim 1240224145Sdim DEBUG(dbgs() << ", create local intv for interference.\n"); 1241224145Sdim // 1242224145Sdim // >>><><><><<<< Overlapping EnterAfter/LeaveBefore interference. 1243224145Sdim // |-----------| Live through. 1244224145Sdim // ==---------== Switch intervals before/after interference. 1245224145Sdim // 1246224145Sdim assert(LeaveBefore <= EnterAfter && "Missed case"); 1247224145Sdim 1248224145Sdim selectIntv(IntvOut); 1249224145Sdim SlotIndex Idx = enterIntvAfter(EnterAfter); 1250224145Sdim useIntv(Idx, Stop); 1251224145Sdim assert((!EnterAfter || Idx >= EnterAfter) && "Interference"); 1252224145Sdim 1253224145Sdim selectIntv(IntvIn); 1254224145Sdim Idx = leaveIntvBefore(LeaveBefore); 1255224145Sdim useIntv(Start, Idx); 1256224145Sdim assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference"); 1257224145Sdim} 1258224145Sdim 1259224145Sdim 1260224145Sdimvoid SplitEditor::splitRegInBlock(const SplitAnalysis::BlockInfo &BI, 1261224145Sdim unsigned IntvIn, SlotIndex LeaveBefore) { 1262224145Sdim SlotIndex Start, Stop; 1263276479Sdim std::tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(BI.MBB); 1264224145Sdim 1265224145Sdim DEBUG(dbgs() << "BB#" << BI.MBB->getNumber() << " [" << Start << ';' << Stop 1266226633Sdim << "), uses " << BI.FirstInstr << '-' << BI.LastInstr 1267224145Sdim << ", reg-in " << IntvIn << ", leave before " << LeaveBefore 1268224145Sdim << (BI.LiveOut ? ", stack-out" : ", killed in block")); 1269224145Sdim 1270224145Sdim assert(IntvIn && "Must have register in"); 1271224145Sdim assert(BI.LiveIn && "Must be live-in"); 1272224145Sdim assert((!LeaveBefore || LeaveBefore > Start) && "Bad interference"); 1273224145Sdim 1274226633Sdim if (!BI.LiveOut && (!LeaveBefore || LeaveBefore >= BI.LastInstr)) { 1275224145Sdim DEBUG(dbgs() << " before interference.\n"); 1276224145Sdim // 1277224145Sdim // <<< Interference after kill. 1278224145Sdim // |---o---x | Killed in block. 1279224145Sdim // ========= Use IntvIn everywhere. 1280224145Sdim // 1281224145Sdim selectIntv(IntvIn); 1282226633Sdim useIntv(Start, BI.LastInstr); 1283224145Sdim return; 1284224145Sdim } 1285224145Sdim 1286224145Sdim SlotIndex LSP = SA.getLastSplitPoint(BI.MBB->getNumber()); 1287224145Sdim 1288226633Sdim if (!LeaveBefore || LeaveBefore > BI.LastInstr.getBoundaryIndex()) { 1289224145Sdim // 1290224145Sdim // <<< Possible interference after last use. 1291224145Sdim // |---o---o---| Live-out on stack. 1292224145Sdim // =========____ Leave IntvIn after last use. 1293224145Sdim // 1294224145Sdim // < Interference after last use. 1295224145Sdim // |---o---o--o| Live-out on stack, late last use. 1296224145Sdim // ============ Copy to stack after LSP, overlap IntvIn. 1297224145Sdim // \_____ Stack interval is live-out. 1298224145Sdim // 1299226633Sdim if (BI.LastInstr < LSP) { 1300224145Sdim DEBUG(dbgs() << ", spill after last use before interference.\n"); 1301224145Sdim selectIntv(IntvIn); 1302226633Sdim SlotIndex Idx = leaveIntvAfter(BI.LastInstr); 1303224145Sdim useIntv(Start, Idx); 1304224145Sdim assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference"); 1305224145Sdim } else { 1306224145Sdim DEBUG(dbgs() << ", spill before last split point.\n"); 1307224145Sdim selectIntv(IntvIn); 1308224145Sdim SlotIndex Idx = leaveIntvBefore(LSP); 1309226633Sdim overlapIntv(Idx, BI.LastInstr); 1310224145Sdim useIntv(Start, Idx); 1311224145Sdim assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference"); 1312224145Sdim } 1313224145Sdim return; 1314224145Sdim } 1315224145Sdim 1316224145Sdim // The interference is overlapping somewhere we wanted to use IntvIn. That 1317224145Sdim // means we need to create a local interval that can be allocated a 1318224145Sdim // different register. 1319224145Sdim unsigned LocalIntv = openIntv(); 1320224145Sdim (void)LocalIntv; 1321224145Sdim DEBUG(dbgs() << ", creating local interval " << LocalIntv << ".\n"); 1322224145Sdim 1323226633Sdim if (!BI.LiveOut || BI.LastInstr < LSP) { 1324224145Sdim // 1325224145Sdim // <<<<<<< Interference overlapping uses. 1326224145Sdim // |---o---o---| Live-out on stack. 1327224145Sdim // =====----____ Leave IntvIn before interference, then spill. 1328224145Sdim // 1329226633Sdim SlotIndex To = leaveIntvAfter(BI.LastInstr); 1330224145Sdim SlotIndex From = enterIntvBefore(LeaveBefore); 1331224145Sdim useIntv(From, To); 1332224145Sdim selectIntv(IntvIn); 1333224145Sdim useIntv(Start, From); 1334224145Sdim assert((!LeaveBefore || From <= LeaveBefore) && "Interference"); 1335224145Sdim return; 1336224145Sdim } 1337224145Sdim 1338224145Sdim // <<<<<<< Interference overlapping uses. 1339224145Sdim // |---o---o--o| Live-out on stack, late last use. 1340224145Sdim // =====------- Copy to stack before LSP, overlap LocalIntv. 1341224145Sdim // \_____ Stack interval is live-out. 1342224145Sdim // 1343224145Sdim SlotIndex To = leaveIntvBefore(LSP); 1344226633Sdim overlapIntv(To, BI.LastInstr); 1345224145Sdim SlotIndex From = enterIntvBefore(std::min(To, LeaveBefore)); 1346224145Sdim useIntv(From, To); 1347224145Sdim selectIntv(IntvIn); 1348224145Sdim useIntv(Start, From); 1349224145Sdim assert((!LeaveBefore || From <= LeaveBefore) && "Interference"); 1350224145Sdim} 1351224145Sdim 1352224145Sdimvoid SplitEditor::splitRegOutBlock(const SplitAnalysis::BlockInfo &BI, 1353224145Sdim unsigned IntvOut, SlotIndex EnterAfter) { 1354224145Sdim SlotIndex Start, Stop; 1355276479Sdim std::tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(BI.MBB); 1356224145Sdim 1357224145Sdim DEBUG(dbgs() << "BB#" << BI.MBB->getNumber() << " [" << Start << ';' << Stop 1358226633Sdim << "), uses " << BI.FirstInstr << '-' << BI.LastInstr 1359224145Sdim << ", reg-out " << IntvOut << ", enter after " << EnterAfter 1360224145Sdim << (BI.LiveIn ? ", stack-in" : ", defined in block")); 1361224145Sdim 1362224145Sdim SlotIndex LSP = SA.getLastSplitPoint(BI.MBB->getNumber()); 1363224145Sdim 1364224145Sdim assert(IntvOut && "Must have register out"); 1365224145Sdim assert(BI.LiveOut && "Must be live-out"); 1366224145Sdim assert((!EnterAfter || EnterAfter < LSP) && "Bad interference"); 1367224145Sdim 1368226633Sdim if (!BI.LiveIn && (!EnterAfter || EnterAfter <= BI.FirstInstr)) { 1369224145Sdim DEBUG(dbgs() << " after interference.\n"); 1370224145Sdim // 1371224145Sdim // >>>> Interference before def. 1372224145Sdim // | o---o---| Defined in block. 1373224145Sdim // ========= Use IntvOut everywhere. 1374224145Sdim // 1375224145Sdim selectIntv(IntvOut); 1376226633Sdim useIntv(BI.FirstInstr, Stop); 1377224145Sdim return; 1378224145Sdim } 1379224145Sdim 1380226633Sdim if (!EnterAfter || EnterAfter < BI.FirstInstr.getBaseIndex()) { 1381224145Sdim DEBUG(dbgs() << ", reload after interference.\n"); 1382224145Sdim // 1383224145Sdim // >>>> Interference before def. 1384224145Sdim // |---o---o---| Live-through, stack-in. 1385224145Sdim // ____========= Enter IntvOut before first use. 1386224145Sdim // 1387224145Sdim selectIntv(IntvOut); 1388226633Sdim SlotIndex Idx = enterIntvBefore(std::min(LSP, BI.FirstInstr)); 1389224145Sdim useIntv(Idx, Stop); 1390224145Sdim assert((!EnterAfter || Idx >= EnterAfter) && "Interference"); 1391224145Sdim return; 1392224145Sdim } 1393224145Sdim 1394224145Sdim // The interference is overlapping somewhere we wanted to use IntvOut. That 1395224145Sdim // means we need to create a local interval that can be allocated a 1396224145Sdim // different register. 1397224145Sdim DEBUG(dbgs() << ", interference overlaps uses.\n"); 1398224145Sdim // 1399224145Sdim // >>>>>>> Interference overlapping uses. 1400224145Sdim // |---o---o---| Live-through, stack-in. 1401224145Sdim // ____---====== Create local interval for interference range. 1402224145Sdim // 1403224145Sdim selectIntv(IntvOut); 1404224145Sdim SlotIndex Idx = enterIntvAfter(EnterAfter); 1405224145Sdim useIntv(Idx, Stop); 1406224145Sdim assert((!EnterAfter || Idx >= EnterAfter) && "Interference"); 1407224145Sdim 1408224145Sdim openIntv(); 1409226633Sdim SlotIndex From = enterIntvBefore(std::min(Idx, BI.FirstInstr)); 1410224145Sdim useIntv(From, Idx); 1411224145Sdim} 1412