GCNHazardRecognizer.cpp revision 303231
1212795Sdim//===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===// 2212795Sdim// 3212795Sdim// The LLVM Compiler Infrastructure 4212795Sdim// 5212795Sdim// This file is distributed under the University of Illinois Open Source 6212795Sdim// License. See LICENSE.TXT for details. 7212795Sdim// 8212795Sdim//===----------------------------------------------------------------------===// 9212795Sdim// 10212795Sdim// This file implements hazard recognizers for scheduling on GCN processors. 11212795Sdim// 12212795Sdim//===----------------------------------------------------------------------===// 13212795Sdim 14212795Sdim#include "GCNHazardRecognizer.h" 15212795Sdim#include "AMDGPUSubtarget.h" 16212795Sdim#include "SIInstrInfo.h" 17212795Sdim#include "llvm/CodeGen/ScheduleDAG.h" 18212795Sdim#include "llvm/Support/Debug.h" 19212795Sdim 20212795Sdimusing namespace llvm; 21212795Sdim 22212795Sdim//===----------------------------------------------------------------------===// 23212795Sdim// Hazard Recoginizer Implementation 24212795Sdim//===----------------------------------------------------------------------===// 25212795Sdim 26212795SdimGCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) : 27212795Sdim CurrCycleInstr(nullptr), 28212795Sdim MF(MF), 29212795Sdim ST(MF.getSubtarget<SISubtarget>()) { 30212795Sdim MaxLookAhead = 5; 31212795Sdim} 32212795Sdim 33212795Sdimvoid GCNHazardRecognizer::EmitInstruction(SUnit *SU) { 34212795Sdim EmitInstruction(SU->getInstr()); 35212795Sdim} 36212795Sdim 37212795Sdimvoid GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) { 38212795Sdim CurrCycleInstr = MI; 39212795Sdim} 40212795Sdim 41212795SdimScheduleHazardRecognizer::HazardType 42212795SdimGCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { 43212795Sdim MachineInstr *MI = SU->getInstr(); 44212795Sdim 45212795Sdim if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0) 46212795Sdim return NoopHazard; 47212795Sdim 48212795Sdim if (SIInstrInfo::isVMEM(*MI) && checkVMEMHazards(MI) > 0) 49212795Sdim return NoopHazard; 50212795Sdim 51212795Sdim if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0) 52212795Sdim return NoopHazard; 53212795Sdim 54212795Sdim return NoHazard; 55212795Sdim} 56212795Sdim 57212795Sdimunsigned GCNHazardRecognizer::PreEmitNoops(SUnit *SU) { 58212795Sdim return PreEmitNoops(SU->getInstr()); 59212795Sdim} 60212795Sdim 61212795Sdimunsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) { 62212795Sdim if (SIInstrInfo::isSMRD(*MI)) 63212795Sdim return std::max(0, checkSMRDHazards(MI)); 64212795Sdim 65212795Sdim if (SIInstrInfo::isVMEM(*MI)) 66212795Sdim return std::max(0, checkVMEMHazards(MI)); 67212795Sdim 68212795Sdim if (SIInstrInfo::isDPP(*MI)) 69212795Sdim return std::max(0, checkDPPHazards(MI)); 70212795Sdim 71212795Sdim return 0; 72212795Sdim} 73212795Sdim 74212795Sdimvoid GCNHazardRecognizer::EmitNoop() { 75212795Sdim EmittedInstrs.push_front(nullptr); 76212795Sdim} 77212795Sdim 78212795Sdimvoid GCNHazardRecognizer::AdvanceCycle() { 79212795Sdim 80212795Sdim // When the scheduler detects a stall, it will call AdvanceCycle() without 81212795Sdim // emitting any instructions. 82212795Sdim if (!CurrCycleInstr) 83212795Sdim return; 84212795Sdim 85212795Sdim const SIInstrInfo *TII = ST.getInstrInfo(); 86212795Sdim unsigned NumWaitStates = TII->getNumWaitStates(*CurrCycleInstr); 87212795Sdim 88212795Sdim // Keep track of emitted instructions 89212795Sdim EmittedInstrs.push_front(CurrCycleInstr); 90212795Sdim 91212795Sdim // Add a nullptr for each additional wait state after the first. Make sure 92212795Sdim // not to add more than getMaxLookAhead() items to the list, since we 93212795Sdim // truncate the list to that size right after this loop. 94212795Sdim for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead()); 95212795Sdim i < e; ++i) { 96212795Sdim EmittedInstrs.push_front(nullptr); 97212795Sdim } 98212795Sdim 99212795Sdim // getMaxLookahead() is the largest number of wait states we will ever need 100212795Sdim // to insert, so there is no point in keeping track of more than that many 101212795Sdim // wait states. 102212795Sdim EmittedInstrs.resize(getMaxLookAhead()); 103212795Sdim 104212795Sdim CurrCycleInstr = nullptr; 105212795Sdim} 106212795Sdim 107212795Sdimvoid GCNHazardRecognizer::RecedeCycle() { 108212795Sdim llvm_unreachable("hazard recognizer does not support bottom-up scheduling."); 109212795Sdim} 110212795Sdim 111212795Sdim//===----------------------------------------------------------------------===// 112212795Sdim// Helper Functions 113212795Sdim//===----------------------------------------------------------------------===// 114212795Sdim 115212795Sdimint GCNHazardRecognizer::getWaitStatesSinceDef( 116212795Sdim unsigned Reg, function_ref<bool(MachineInstr *)> IsHazardDef) { 117212795Sdim const SIRegisterInfo *TRI = ST.getRegisterInfo(); 118212795Sdim 119212795Sdim int WaitStates = -1; 120212795Sdim for (MachineInstr *MI : EmittedInstrs) { 121212795Sdim ++WaitStates; 122212795Sdim if (!MI || !IsHazardDef(MI)) 123212795Sdim continue; 124212795Sdim if (MI->modifiesRegister(Reg, TRI)) 125212795Sdim return WaitStates; 126212795Sdim } 127212795Sdim return std::numeric_limits<int>::max(); 128212795Sdim} 129212795Sdim 130212795Sdim//===----------------------------------------------------------------------===// 131212795Sdim// No-op Hazard Detection 132212795Sdim//===----------------------------------------------------------------------===// 133212795Sdim 134212795Sdimstatic void addRegsToSet(iterator_range<MachineInstr::const_mop_iterator> Ops, 135212795Sdim std::set<unsigned> &Set) { 136212795Sdim for (const MachineOperand &Op : Ops) { 137212795Sdim if (Op.isReg()) 138212795Sdim Set.insert(Op.getReg()); 139212795Sdim } 140212795Sdim} 141212795Sdim 142212795Sdimint GCNHazardRecognizer::checkSMEMSoftClauseHazards(MachineInstr *SMEM) { 143212795Sdim // SMEM soft clause are only present on VI+ 144212795Sdim if (ST.getGeneration() < SISubtarget::VOLCANIC_ISLANDS) 145212795Sdim return 0; 146212795Sdim 147212795Sdim // A soft-clause is any group of consecutive SMEM instructions. The 148212795Sdim // instructions in this group may return out of order and/or may be 149212795Sdim // replayed (i.e. the same instruction issued more than once). 150212795Sdim // 151212795Sdim // In order to handle these situations correctly we need to make sure 152212795Sdim // that when a clause has more than one instruction, no instruction in the 153212795Sdim // clause writes to a register that is read another instruction in the clause 154212795Sdim // (including itself). If we encounter this situaion, we need to break the 155212795Sdim // clause by inserting a non SMEM instruction. 156212795Sdim 157212795Sdim std::set<unsigned> ClauseDefs; 158212795Sdim std::set<unsigned> ClauseUses; 159212795Sdim 160212795Sdim for (MachineInstr *MI : EmittedInstrs) { 161212795Sdim 162212795Sdim // When we hit a non-SMEM instruction then we have passed the start of the 163212795Sdim // clause and we can stop. 164212795Sdim if (!MI || !SIInstrInfo::isSMRD(*MI)) 165212795Sdim break; 166212795Sdim 167212795Sdim addRegsToSet(MI->defs(), ClauseDefs); 168212795Sdim addRegsToSet(MI->uses(), ClauseUses); 169212795Sdim } 170212795Sdim 171212795Sdim if (ClauseDefs.empty()) 172212795Sdim return 0; 173212795Sdim 174212795Sdim // FIXME: When we support stores, we need to make sure not to put loads and 175212795Sdim // stores in the same clause if they use the same address. For now, just 176212795Sdim // start a new clause whenever we see a store. 177212795Sdim if (SMEM->mayStore()) 178212795Sdim return 1; 179212795Sdim 180212795Sdim addRegsToSet(SMEM->defs(), ClauseDefs); 181212795Sdim addRegsToSet(SMEM->uses(), ClauseUses); 182212795Sdim 183212795Sdim std::vector<unsigned> Result(std::max(ClauseDefs.size(), ClauseUses.size())); 184212795Sdim std::vector<unsigned>::iterator End; 185212795Sdim 186212795Sdim End = std::set_intersection(ClauseDefs.begin(), ClauseDefs.end(), 187212795Sdim ClauseUses.begin(), ClauseUses.end(), Result.begin()); 188226890Sdim 189212795Sdim // If the set of defs and uses intersect then we cannot add this instruction 190212795Sdim // to the clause, so we have a hazard. 191212795Sdim if (End != Result.begin()) 192212795Sdim return 1; 193212795Sdim 194212795Sdim return 0; 195212795Sdim} 196212795Sdim 197212795Sdimint GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) { 198212795Sdim const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); 199212795Sdim const SIInstrInfo *TII = ST.getInstrInfo(); 200212795Sdim int WaitStatesNeeded = 0; 201212795Sdim 202212795Sdim WaitStatesNeeded = checkSMEMSoftClauseHazards(SMRD); 203212795Sdim 204212795Sdim // This SMRD hazard only affects SI. 205212795Sdim if (ST.getGeneration() != SISubtarget::SOUTHERN_ISLANDS) 206212795Sdim return WaitStatesNeeded; 207212795Sdim 208212795Sdim // A read of an SGPR by SMRD instruction requires 4 wait states when the 209212795Sdim // SGPR was written by a VALU instruction. 210212795Sdim int SmrdSgprWaitStates = 4; 211212795Sdim auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); }; 212212795Sdim 213212795Sdim for (const MachineOperand &Use : SMRD->uses()) { 214212795Sdim if (!Use.isReg()) 215212795Sdim continue; 216212795Sdim int WaitStatesNeededForUse = 217212795Sdim SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn); 218212795Sdim WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 219 } 220 return WaitStatesNeeded; 221} 222 223int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) { 224 const SIInstrInfo *TII = ST.getInstrInfo(); 225 226 if (ST.getGeneration() < SISubtarget::VOLCANIC_ISLANDS) 227 return 0; 228 229 const SIRegisterInfo &TRI = TII->getRegisterInfo(); 230 231 // A read of an SGPR by a VMEM instruction requires 5 wait states when the 232 // SGPR was written by a VALU Instruction. 233 int VmemSgprWaitStates = 5; 234 int WaitStatesNeeded = 0; 235 auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); }; 236 237 for (const MachineOperand &Use : VMEM->uses()) { 238 if (!Use.isReg() || TRI.isVGPR(MF.getRegInfo(), Use.getReg())) 239 continue; 240 241 int WaitStatesNeededForUse = 242 VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn); 243 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 244 } 245 return WaitStatesNeeded; 246} 247 248int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) { 249 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 250 251 // Check for DPP VGPR read after VALU VGPR write. 252 int DppVgprWaitStates = 2; 253 int WaitStatesNeeded = 0; 254 255 for (const MachineOperand &Use : DPP->uses()) { 256 if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg())) 257 continue; 258 int WaitStatesNeededForUse = 259 DppVgprWaitStates - getWaitStatesSinceDef(Use.getReg()); 260 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 261 } 262 263 return WaitStatesNeeded; 264} 265