1//===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "SIMachineFunctionInfo.h"
10#include "AMDGPUArgumentUsageInfo.h"
11#include "AMDGPUSubtarget.h"
12#include "SIRegisterInfo.h"
13#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
14#include "Utils/AMDGPUBaseInfo.h"
15#include "llvm/ADT/Optional.h"
16#include "llvm/CodeGen/MachineBasicBlock.h"
17#include "llvm/CodeGen/MachineFrameInfo.h"
18#include "llvm/CodeGen/MachineFunction.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/IR/CallingConv.h"
21#include "llvm/IR/Function.h"
22#include <cassert>
23#include <vector>
24
25#define MAX_LANES 64
26
27using namespace llvm;
28
29SIMachineFunctionInfo::SIMachineFunctionInfo(const MachineFunction &MF)
30  : AMDGPUMachineFunction(MF),
31    PrivateSegmentBuffer(false),
32    DispatchPtr(false),
33    QueuePtr(false),
34    KernargSegmentPtr(false),
35    DispatchID(false),
36    FlatScratchInit(false),
37    WorkGroupIDX(false),
38    WorkGroupIDY(false),
39    WorkGroupIDZ(false),
40    WorkGroupInfo(false),
41    PrivateSegmentWaveByteOffset(false),
42    WorkItemIDX(false),
43    WorkItemIDY(false),
44    WorkItemIDZ(false),
45    ImplicitBufferPtr(false),
46    ImplicitArgPtr(false),
47    GITPtrHigh(0xffffffff),
48    HighBitsOf32BitAddress(0),
49    GDSSize(0) {
50  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
51  const Function &F = MF.getFunction();
52  FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F);
53  WavesPerEU = ST.getWavesPerEU(F);
54
55  Occupancy = ST.computeOccupancy(MF, getLDSSize());
56  CallingConv::ID CC = F.getCallingConv();
57
58  if (CC == CallingConv::AMDGPU_KERNEL || CC == CallingConv::SPIR_KERNEL) {
59    if (!F.arg_empty())
60      KernargSegmentPtr = true;
61    WorkGroupIDX = true;
62    WorkItemIDX = true;
63  } else if (CC == CallingConv::AMDGPU_PS) {
64    PSInputAddr = AMDGPU::getInitialPSInputAddr(F);
65  }
66
67  if (!isEntryFunction()) {
68    // Non-entry functions have no special inputs for now, other registers
69    // required for scratch access.
70    ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
71    ScratchWaveOffsetReg = AMDGPU::SGPR33;
72
73    // TODO: Pick a high register, and shift down, similar to a kernel.
74    FrameOffsetReg = AMDGPU::SGPR34;
75    StackPtrOffsetReg = AMDGPU::SGPR32;
76
77    ArgInfo.PrivateSegmentBuffer =
78      ArgDescriptor::createRegister(ScratchRSrcReg);
79    ArgInfo.PrivateSegmentWaveByteOffset =
80      ArgDescriptor::createRegister(ScratchWaveOffsetReg);
81
82    if (F.hasFnAttribute("amdgpu-implicitarg-ptr"))
83      ImplicitArgPtr = true;
84  } else {
85    if (F.hasFnAttribute("amdgpu-implicitarg-ptr")) {
86      KernargSegmentPtr = true;
87      MaxKernArgAlign = std::max(ST.getAlignmentForImplicitArgPtr(),
88                                 MaxKernArgAlign);
89    }
90  }
91
92  if (F.hasFnAttribute("amdgpu-work-group-id-x"))
93    WorkGroupIDX = true;
94
95  if (F.hasFnAttribute("amdgpu-work-group-id-y"))
96    WorkGroupIDY = true;
97
98  if (F.hasFnAttribute("amdgpu-work-group-id-z"))
99    WorkGroupIDZ = true;
100
101  if (F.hasFnAttribute("amdgpu-work-item-id-x"))
102    WorkItemIDX = true;
103
104  if (F.hasFnAttribute("amdgpu-work-item-id-y"))
105    WorkItemIDY = true;
106
107  if (F.hasFnAttribute("amdgpu-work-item-id-z"))
108    WorkItemIDZ = true;
109
110  const MachineFrameInfo &FrameInfo = MF.getFrameInfo();
111  bool HasStackObjects = FrameInfo.hasStackObjects();
112
113  if (isEntryFunction()) {
114    // X, XY, and XYZ are the only supported combinations, so make sure Y is
115    // enabled if Z is.
116    if (WorkItemIDZ)
117      WorkItemIDY = true;
118
119    PrivateSegmentWaveByteOffset = true;
120
121    // HS and GS always have the scratch wave offset in SGPR5 on GFX9.
122    if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 &&
123        (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS))
124      ArgInfo.PrivateSegmentWaveByteOffset =
125          ArgDescriptor::createRegister(AMDGPU::SGPR5);
126  }
127
128  bool isAmdHsaOrMesa = ST.isAmdHsaOrMesa(F);
129  if (isAmdHsaOrMesa) {
130    PrivateSegmentBuffer = true;
131
132    if (F.hasFnAttribute("amdgpu-dispatch-ptr"))
133      DispatchPtr = true;
134
135    if (F.hasFnAttribute("amdgpu-queue-ptr"))
136      QueuePtr = true;
137
138    if (F.hasFnAttribute("amdgpu-dispatch-id"))
139      DispatchID = true;
140  } else if (ST.isMesaGfxShader(F)) {
141    ImplicitBufferPtr = true;
142  }
143
144  if (F.hasFnAttribute("amdgpu-kernarg-segment-ptr"))
145    KernargSegmentPtr = true;
146
147  if (ST.hasFlatAddressSpace() && isEntryFunction() && isAmdHsaOrMesa) {
148    auto hasNonSpillStackObjects = [&]() {
149      // Avoid expensive checking if there's no stack objects.
150      if (!HasStackObjects)
151        return false;
152      for (auto OI = FrameInfo.getObjectIndexBegin(),
153                OE = FrameInfo.getObjectIndexEnd(); OI != OE; ++OI)
154        if (!FrameInfo.isSpillSlotObjectIndex(OI))
155          return true;
156      // All stack objects are spill slots.
157      return false;
158    };
159    // TODO: This could be refined a lot. The attribute is a poor way of
160    // detecting calls that may require it before argument lowering.
161    if (hasNonSpillStackObjects() || F.hasFnAttribute("amdgpu-flat-scratch"))
162      FlatScratchInit = true;
163  }
164
165  Attribute A = F.getFnAttribute("amdgpu-git-ptr-high");
166  StringRef S = A.getValueAsString();
167  if (!S.empty())
168    S.consumeInteger(0, GITPtrHigh);
169
170  A = F.getFnAttribute("amdgpu-32bit-address-high-bits");
171  S = A.getValueAsString();
172  if (!S.empty())
173    S.consumeInteger(0, HighBitsOf32BitAddress);
174
175  S = F.getFnAttribute("amdgpu-gds-size").getValueAsString();
176  if (!S.empty())
177    S.consumeInteger(0, GDSSize);
178}
179
180void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) {
181  limitOccupancy(getMaxWavesPerEU());
182  const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>();
183  limitOccupancy(ST.getOccupancyWithLocalMemSize(getLDSSize(),
184                 MF.getFunction()));
185}
186
187unsigned SIMachineFunctionInfo::addPrivateSegmentBuffer(
188  const SIRegisterInfo &TRI) {
189  ArgInfo.PrivateSegmentBuffer =
190    ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
191    getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SGPR_128RegClass));
192  NumUserSGPRs += 4;
193  return ArgInfo.PrivateSegmentBuffer.getRegister();
194}
195
196unsigned SIMachineFunctionInfo::addDispatchPtr(const SIRegisterInfo &TRI) {
197  ArgInfo.DispatchPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
198    getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
199  NumUserSGPRs += 2;
200  return ArgInfo.DispatchPtr.getRegister();
201}
202
203unsigned SIMachineFunctionInfo::addQueuePtr(const SIRegisterInfo &TRI) {
204  ArgInfo.QueuePtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
205    getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
206  NumUserSGPRs += 2;
207  return ArgInfo.QueuePtr.getRegister();
208}
209
210unsigned SIMachineFunctionInfo::addKernargSegmentPtr(const SIRegisterInfo &TRI) {
211  ArgInfo.KernargSegmentPtr
212    = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
213    getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
214  NumUserSGPRs += 2;
215  return ArgInfo.KernargSegmentPtr.getRegister();
216}
217
218unsigned SIMachineFunctionInfo::addDispatchID(const SIRegisterInfo &TRI) {
219  ArgInfo.DispatchID = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
220    getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
221  NumUserSGPRs += 2;
222  return ArgInfo.DispatchID.getRegister();
223}
224
225unsigned SIMachineFunctionInfo::addFlatScratchInit(const SIRegisterInfo &TRI) {
226  ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
227    getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
228  NumUserSGPRs += 2;
229  return ArgInfo.FlatScratchInit.getRegister();
230}
231
232unsigned SIMachineFunctionInfo::addImplicitBufferPtr(const SIRegisterInfo &TRI) {
233  ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
234    getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
235  NumUserSGPRs += 2;
236  return ArgInfo.ImplicitBufferPtr.getRegister();
237}
238
239static bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) {
240  for (unsigned I = 0; CSRegs[I]; ++I) {
241    if (CSRegs[I] == Reg)
242      return true;
243  }
244
245  return false;
246}
247
248/// \p returns true if \p NumLanes slots are available in VGPRs already used for
249/// SGPR spilling.
250//
251// FIXME: This only works after processFunctionBeforeFrameFinalized
252bool SIMachineFunctionInfo::haveFreeLanesForSGPRSpill(const MachineFunction &MF,
253                                                      unsigned NumNeed) const {
254  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
255  unsigned WaveSize = ST.getWavefrontSize();
256  return NumVGPRSpillLanes + NumNeed <= WaveSize * SpillVGPRs.size();
257}
258
259/// Reserve a slice of a VGPR to support spilling for FrameIndex \p FI.
260bool SIMachineFunctionInfo::allocateSGPRSpillToVGPR(MachineFunction &MF,
261                                                    int FI) {
262  std::vector<SpilledReg> &SpillLanes = SGPRToVGPRSpills[FI];
263
264  // This has already been allocated.
265  if (!SpillLanes.empty())
266    return true;
267
268  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
269  const SIRegisterInfo *TRI = ST.getRegisterInfo();
270  MachineFrameInfo &FrameInfo = MF.getFrameInfo();
271  MachineRegisterInfo &MRI = MF.getRegInfo();
272  unsigned WaveSize = ST.getWavefrontSize();
273
274  unsigned Size = FrameInfo.getObjectSize(FI);
275  assert(Size >= 4 && Size <= 64 && "invalid sgpr spill size");
276  assert(TRI->spillSGPRToVGPR() && "not spilling SGPRs to VGPRs");
277
278  int NumLanes = Size / 4;
279
280  const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
281
282  // Make sure to handle the case where a wide SGPR spill may span between two
283  // VGPRs.
284  for (int I = 0; I < NumLanes; ++I, ++NumVGPRSpillLanes) {
285    unsigned LaneVGPR;
286    unsigned VGPRIndex = (NumVGPRSpillLanes % WaveSize);
287
288    if (VGPRIndex == 0) {
289      LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF);
290      if (LaneVGPR == AMDGPU::NoRegister) {
291        // We have no VGPRs left for spilling SGPRs. Reset because we will not
292        // partially spill the SGPR to VGPRs.
293        SGPRToVGPRSpills.erase(FI);
294        NumVGPRSpillLanes -= I;
295        return false;
296      }
297
298      Optional<int> CSRSpillFI;
299      if ((FrameInfo.hasCalls() || !isEntryFunction()) && CSRegs &&
300          isCalleeSavedReg(CSRegs, LaneVGPR)) {
301        CSRSpillFI = FrameInfo.CreateSpillStackObject(4, 4);
302      }
303
304      SpillVGPRs.push_back(SGPRSpillVGPRCSR(LaneVGPR, CSRSpillFI));
305
306      // Add this register as live-in to all blocks to avoid machine verifer
307      // complaining about use of an undefined physical register.
308      for (MachineBasicBlock &BB : MF)
309        BB.addLiveIn(LaneVGPR);
310    } else {
311      LaneVGPR = SpillVGPRs.back().VGPR;
312    }
313
314    SpillLanes.push_back(SpilledReg(LaneVGPR, VGPRIndex));
315  }
316
317  return true;
318}
319
320/// Reserve AGPRs or VGPRs to support spilling for FrameIndex \p FI.
321/// Either AGPR is spilled to VGPR to vice versa.
322/// Returns true if a \p FI can be eliminated completely.
323bool SIMachineFunctionInfo::allocateVGPRSpillToAGPR(MachineFunction &MF,
324                                                    int FI,
325                                                    bool isAGPRtoVGPR) {
326  MachineRegisterInfo &MRI = MF.getRegInfo();
327  MachineFrameInfo &FrameInfo = MF.getFrameInfo();
328  const GCNSubtarget &ST =  MF.getSubtarget<GCNSubtarget>();
329
330  assert(ST.hasMAIInsts() && FrameInfo.isSpillSlotObjectIndex(FI));
331
332  auto &Spill = VGPRToAGPRSpills[FI];
333
334  // This has already been allocated.
335  if (!Spill.Lanes.empty())
336    return Spill.FullyAllocated;
337
338  unsigned Size = FrameInfo.getObjectSize(FI);
339  unsigned NumLanes = Size / 4;
340  Spill.Lanes.resize(NumLanes, AMDGPU::NoRegister);
341
342  const TargetRegisterClass &RC =
343      isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass;
344  auto Regs = RC.getRegisters();
345
346  auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR;
347  const SIRegisterInfo *TRI = ST.getRegisterInfo();
348  Spill.FullyAllocated = true;
349
350  // FIXME: Move allocation logic out of MachineFunctionInfo and initialize
351  // once.
352  BitVector OtherUsedRegs;
353  OtherUsedRegs.resize(TRI->getNumRegs());
354
355  const uint32_t *CSRMask =
356      TRI->getCallPreservedMask(MF, MF.getFunction().getCallingConv());
357  if (CSRMask)
358    OtherUsedRegs.setBitsInMask(CSRMask);
359
360  // TODO: Should include register tuples, but doesn't matter with current
361  // usage.
362  for (MCPhysReg Reg : SpillAGPR)
363    OtherUsedRegs.set(Reg);
364  for (MCPhysReg Reg : SpillVGPR)
365    OtherUsedRegs.set(Reg);
366
367  SmallVectorImpl<MCPhysReg>::const_iterator NextSpillReg = Regs.begin();
368  for (unsigned I = 0; I < NumLanes; ++I) {
369    NextSpillReg = std::find_if(
370        NextSpillReg, Regs.end(), [&MRI, &OtherUsedRegs](MCPhysReg Reg) {
371          return MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg) &&
372                 !OtherUsedRegs[Reg];
373        });
374
375    if (NextSpillReg == Regs.end()) { // Registers exhausted
376      Spill.FullyAllocated = false;
377      break;
378    }
379
380    OtherUsedRegs.set(*NextSpillReg);
381    SpillRegs.push_back(*NextSpillReg);
382    Spill.Lanes[I] = *NextSpillReg++;
383  }
384
385  return Spill.FullyAllocated;
386}
387
388void SIMachineFunctionInfo::removeDeadFrameIndices(MachineFrameInfo &MFI) {
389  // The FP spill hasn't been inserted yet, so keep it around.
390  for (auto &R : SGPRToVGPRSpills) {
391    if (R.first != FramePointerSaveIndex)
392      MFI.RemoveStackObject(R.first);
393  }
394
395  // All other SPGRs must be allocated on the default stack, so reset the stack
396  // ID.
397  for (int i = MFI.getObjectIndexBegin(), e = MFI.getObjectIndexEnd(); i != e;
398       ++i)
399    if (i != FramePointerSaveIndex)
400      MFI.setStackID(i, TargetStackID::Default);
401
402  for (auto &R : VGPRToAGPRSpills) {
403    if (R.second.FullyAllocated)
404      MFI.RemoveStackObject(R.first);
405  }
406}
407
408MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const {
409  assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs");
410  return AMDGPU::SGPR0 + NumUserSGPRs;
411}
412
413MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const {
414  return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
415}
416
417static yaml::StringValue regToString(unsigned Reg,
418                                     const TargetRegisterInfo &TRI) {
419  yaml::StringValue Dest;
420  {
421    raw_string_ostream OS(Dest.Value);
422    OS << printReg(Reg, &TRI);
423  }
424  return Dest;
425}
426
427static Optional<yaml::SIArgumentInfo>
428convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo,
429                    const TargetRegisterInfo &TRI) {
430  yaml::SIArgumentInfo AI;
431
432  auto convertArg = [&](Optional<yaml::SIArgument> &A,
433                        const ArgDescriptor &Arg) {
434    if (!Arg)
435      return false;
436
437    // Create a register or stack argument.
438    yaml::SIArgument SA = yaml::SIArgument::createArgument(Arg.isRegister());
439    if (Arg.isRegister()) {
440      raw_string_ostream OS(SA.RegisterName.Value);
441      OS << printReg(Arg.getRegister(), &TRI);
442    } else
443      SA.StackOffset = Arg.getStackOffset();
444    // Check and update the optional mask.
445    if (Arg.isMasked())
446      SA.Mask = Arg.getMask();
447
448    A = SA;
449    return true;
450  };
451
452  bool Any = false;
453  Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer);
454  Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr);
455  Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr);
456  Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr);
457  Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID);
458  Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit);
459  Any |= convertArg(AI.PrivateSegmentSize, ArgInfo.PrivateSegmentSize);
460  Any |= convertArg(AI.WorkGroupIDX, ArgInfo.WorkGroupIDX);
461  Any |= convertArg(AI.WorkGroupIDY, ArgInfo.WorkGroupIDY);
462  Any |= convertArg(AI.WorkGroupIDZ, ArgInfo.WorkGroupIDZ);
463  Any |= convertArg(AI.WorkGroupInfo, ArgInfo.WorkGroupInfo);
464  Any |= convertArg(AI.PrivateSegmentWaveByteOffset,
465                    ArgInfo.PrivateSegmentWaveByteOffset);
466  Any |= convertArg(AI.ImplicitArgPtr, ArgInfo.ImplicitArgPtr);
467  Any |= convertArg(AI.ImplicitBufferPtr, ArgInfo.ImplicitBufferPtr);
468  Any |= convertArg(AI.WorkItemIDX, ArgInfo.WorkItemIDX);
469  Any |= convertArg(AI.WorkItemIDY, ArgInfo.WorkItemIDY);
470  Any |= convertArg(AI.WorkItemIDZ, ArgInfo.WorkItemIDZ);
471
472  if (Any)
473    return AI;
474
475  return None;
476}
477
478yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(
479  const llvm::SIMachineFunctionInfo& MFI,
480  const TargetRegisterInfo &TRI)
481  : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
482    MaxKernArgAlign(MFI.getMaxKernArgAlign()),
483    LDSSize(MFI.getLDSSize()),
484    IsEntryFunction(MFI.isEntryFunction()),
485    NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()),
486    MemoryBound(MFI.isMemoryBound()),
487    WaveLimiter(MFI.needsWaveLimiter()),
488    HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
489    ScratchRSrcReg(regToString(MFI.getScratchRSrcReg(), TRI)),
490    ScratchWaveOffsetReg(regToString(MFI.getScratchWaveOffsetReg(), TRI)),
491    FrameOffsetReg(regToString(MFI.getFrameOffsetReg(), TRI)),
492    StackPtrOffsetReg(regToString(MFI.getStackPtrOffsetReg(), TRI)),
493    ArgInfo(convertArgumentInfo(MFI.getArgInfo(), TRI)),
494    Mode(MFI.getMode()) {}
495
496void yaml::SIMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
497  MappingTraits<SIMachineFunctionInfo>::mapping(YamlIO, *this);
498}
499
500bool SIMachineFunctionInfo::initializeBaseYamlFields(
501  const yaml::SIMachineFunctionInfo &YamlMFI) {
502  ExplicitKernArgSize = YamlMFI.ExplicitKernArgSize;
503  MaxKernArgAlign = assumeAligned(YamlMFI.MaxKernArgAlign);
504  LDSSize = YamlMFI.LDSSize;
505  HighBitsOf32BitAddress = YamlMFI.HighBitsOf32BitAddress;
506  IsEntryFunction = YamlMFI.IsEntryFunction;
507  NoSignedZerosFPMath = YamlMFI.NoSignedZerosFPMath;
508  MemoryBound = YamlMFI.MemoryBound;
509  WaveLimiter = YamlMFI.WaveLimiter;
510  return false;
511}
512