StackMaps.cpp revision 280031 
1//===---------------------------- StackMaps.cpp ---------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9
10#include "llvm/CodeGen/StackMaps.h"
11#include "llvm/CodeGen/AsmPrinter.h"
12#include "llvm/CodeGen/MachineFrameInfo.h"
13#include "llvm/CodeGen/MachineFunction.h"
14#include "llvm/CodeGen/MachineInstr.h"
15#include "llvm/IR/DataLayout.h"
16#include "llvm/MC/MCContext.h"
17#include "llvm/MC/MCExpr.h"
18#include "llvm/MC/MCObjectFileInfo.h"
19#include "llvm/MC/MCSectionMachO.h"
20#include "llvm/MC/MCStreamer.h"
21#include "llvm/Support/CommandLine.h"
22#include "llvm/Support/Debug.h"
23#include "llvm/Support/raw_ostream.h"
24#include "llvm/Target/TargetMachine.h"
25#include "llvm/Target/TargetOpcodes.h"
26#include "llvm/Target/TargetRegisterInfo.h"
27#include "llvm/Target/TargetSubtargetInfo.h"
28#include <iterator>
29
30using namespace llvm;
31
32#define DEBUG_TYPE "stackmaps"
33
34static cl::opt<int> StackMapVersion("stackmap-version", cl::init(1),
35  cl::desc("Specify the stackmap encoding version (default = 1)"));
36
37const char *StackMaps::WSMP = "Stack Maps: ";
38
39PatchPointOpers::PatchPointOpers(const MachineInstr *MI)
40  : MI(MI),
41    HasDef(MI->getOperand(0).isReg() && MI->getOperand(0).isDef() &&
42           !MI->getOperand(0).isImplicit()),
43    IsAnyReg(MI->getOperand(getMetaIdx(CCPos)).getImm() == CallingConv::AnyReg)
44{
45#ifndef NDEBUG
46  unsigned CheckStartIdx = 0, e = MI->getNumOperands();
47  while (CheckStartIdx < e && MI->getOperand(CheckStartIdx).isReg() &&
48         MI->getOperand(CheckStartIdx).isDef() &&
49         !MI->getOperand(CheckStartIdx).isImplicit())
50    ++CheckStartIdx;
51
52  assert(getMetaIdx() == CheckStartIdx &&
53         "Unexpected additional definition in Patchpoint intrinsic.");
54#endif
55}
56
57unsigned PatchPointOpers::getNextScratchIdx(unsigned StartIdx) const {
58  if (!StartIdx)
59    StartIdx = getVarIdx();
60
61  // Find the next scratch register (implicit def and early clobber)
62  unsigned ScratchIdx = StartIdx, e = MI->getNumOperands();
63  while (ScratchIdx < e &&
64         !(MI->getOperand(ScratchIdx).isReg() &&
65           MI->getOperand(ScratchIdx).isDef() &&
66           MI->getOperand(ScratchIdx).isImplicit() &&
67           MI->getOperand(ScratchIdx).isEarlyClobber()))
68    ++ScratchIdx;
69
70  assert(ScratchIdx != e && "No scratch register available");
71  return ScratchIdx;
72}
73
74StackMaps::StackMaps(AsmPrinter &AP) : AP(AP) {
75  if (StackMapVersion != 1)
76    llvm_unreachable("Unsupported stackmap version!");
77}
78
79MachineInstr::const_mop_iterator
80StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI,
81                        MachineInstr::const_mop_iterator MOE,
82                        LocationVec &Locs, LiveOutVec &LiveOuts) const {
83  if (MOI->isImm()) {
84    switch (MOI->getImm()) {
85    default: llvm_unreachable("Unrecognized operand type.");
86    case StackMaps::DirectMemRefOp: {
87      unsigned Size =
88          AP.TM.getSubtargetImpl()->getDataLayout()->getPointerSizeInBits();
89      assert((Size % 8) == 0 && "Need pointer size in bytes.");
90      Size /= 8;
91      unsigned Reg = (++MOI)->getReg();
92      int64_t Imm = (++MOI)->getImm();
93      Locs.push_back(Location(StackMaps::Location::Direct, Size, Reg, Imm));
94      break;
95    }
96    case StackMaps::IndirectMemRefOp: {
97      int64_t Size = (++MOI)->getImm();
98      assert(Size > 0 && "Need a valid size for indirect memory locations.");
99      unsigned Reg = (++MOI)->getReg();
100      int64_t Imm = (++MOI)->getImm();
101      Locs.push_back(Location(StackMaps::Location::Indirect, Size, Reg, Imm));
102      break;
103    }
104    case StackMaps::ConstantOp: {
105      ++MOI;
106      assert(MOI->isImm() && "Expected constant operand.");
107      int64_t Imm = MOI->getImm();
108      Locs.push_back(Location(Location::Constant, sizeof(int64_t), 0, Imm));
109      break;
110    }
111    }
112    return ++MOI;
113  }
114
115  // The physical register number will ultimately be encoded as a DWARF regno.
116  // The stack map also records the size of a spill slot that can hold the
117  // register content. (The runtime can track the actual size of the data type
118  // if it needs to.)
119  if (MOI->isReg()) {
120    // Skip implicit registers (this includes our scratch registers)
121    if (MOI->isImplicit())
122      return ++MOI;
123
124    assert(TargetRegisterInfo::isPhysicalRegister(MOI->getReg()) &&
125           "Virtreg operands should have been rewritten before now.");
126    const TargetRegisterClass *RC =
127        AP.TM.getSubtargetImpl()->getRegisterInfo()->getMinimalPhysRegClass(
128            MOI->getReg());
129    assert(!MOI->getSubReg() && "Physical subreg still around.");
130    Locs.push_back(
131      Location(Location::Register, RC->getSize(), MOI->getReg(), 0));
132    return ++MOI;
133  }
134
135  if (MOI->isRegLiveOut())
136    LiveOuts = parseRegisterLiveOutMask(MOI->getRegLiveOut());
137
138  return ++MOI;
139}
140
141/// Go up the super-register chain until we hit a valid dwarf register number.
142static unsigned getDwarfRegNum(unsigned Reg, const TargetRegisterInfo *TRI) {
143  int RegNo = TRI->getDwarfRegNum(Reg, false);
144  for (MCSuperRegIterator SR(Reg, TRI); SR.isValid() && RegNo < 0; ++SR)
145    RegNo = TRI->getDwarfRegNum(*SR, false);
146
147  assert(RegNo >= 0 && "Invalid Dwarf register number.");
148  return (unsigned) RegNo;
149}
150
151/// Create a live-out register record for the given register Reg.
152StackMaps::LiveOutReg
153StackMaps::createLiveOutReg(unsigned Reg, const TargetRegisterInfo *TRI) const {
154  unsigned RegNo = getDwarfRegNum(Reg, TRI);
155  unsigned Size = TRI->getMinimalPhysRegClass(Reg)->getSize();
156  return LiveOutReg(Reg, RegNo, Size);
157}
158
159/// Parse the register live-out mask and return a vector of live-out registers
160/// that need to be recorded in the stackmap.
161StackMaps::LiveOutVec
162StackMaps::parseRegisterLiveOutMask(const uint32_t *Mask) const {
163  assert(Mask && "No register mask specified");
164  const TargetRegisterInfo *TRI = AP.TM.getSubtargetImpl()->getRegisterInfo();
165  LiveOutVec LiveOuts;
166
167  // Create a LiveOutReg for each bit that is set in the register mask.
168  for (unsigned Reg = 0, NumRegs = TRI->getNumRegs(); Reg != NumRegs; ++Reg)
169    if ((Mask[Reg / 32] >> Reg % 32) & 1)
170      LiveOuts.push_back(createLiveOutReg(Reg, TRI));
171
172  // We don't need to keep track of a register if its super-register is already
173  // in the list. Merge entries that refer to the same dwarf register and use
174  // the maximum size that needs to be spilled.
175  std::sort(LiveOuts.begin(), LiveOuts.end());
176  for (LiveOutVec::iterator I = LiveOuts.begin(), E = LiveOuts.end();
177       I != E; ++I) {
178    for (LiveOutVec::iterator II = std::next(I); II != E; ++II) {
179      if (I->RegNo != II->RegNo) {
180        // Skip all the now invalid entries.
181        I = --II;
182        break;
183      }
184      I->Size = std::max(I->Size, II->Size);
185      if (TRI->isSuperRegister(I->Reg, II->Reg))
186        I->Reg = II->Reg;
187      II->MarkInvalid();
188    }
189  }
190  LiveOuts.erase(std::remove_if(LiveOuts.begin(), LiveOuts.end(),
191                                LiveOutReg::IsInvalid), LiveOuts.end());
192  return LiveOuts;
193}
194
195void StackMaps::recordStackMapOpers(const MachineInstr &MI, uint64_t ID,
196                                    MachineInstr::const_mop_iterator MOI,
197                                    MachineInstr::const_mop_iterator MOE,
198                                    bool recordResult) {
199
200  MCContext &OutContext = AP.OutStreamer.getContext();
201  MCSymbol *MILabel = OutContext.CreateTempSymbol();
202  AP.OutStreamer.EmitLabel(MILabel);
203
204  LocationVec Locations;
205  LiveOutVec LiveOuts;
206
207  if (recordResult) {
208    assert(PatchPointOpers(&MI).hasDef() && "Stackmap has no return value.");
209    parseOperand(MI.operands_begin(), std::next(MI.operands_begin()),
210                 Locations, LiveOuts);
211  }
212
213  // Parse operands.
214  while (MOI != MOE) {
215    MOI = parseOperand(MOI, MOE, Locations, LiveOuts);
216  }
217
218  // Move large constants into the constant pool.
219  for (LocationVec::iterator I = Locations.begin(), E = Locations.end();
220       I != E; ++I) {
221    // Constants are encoded as sign-extended integers.
222    // -1 is directly encoded as .long 0xFFFFFFFF with no constant pool.
223    if (I->LocType == Location::Constant && !isInt<32>(I->Offset)) {
224      I->LocType = Location::ConstantIndex;
225      // ConstPool is intentionally a MapVector of 'uint64_t's (as
226      // opposed to 'int64_t's).  We should never be in a situation
227      // where we have to insert either the tombstone or the empty
228      // keys into a map, and for a DenseMap<uint64_t, T> these are
229      // (uint64_t)0 and (uint64_t)-1.  They can be and are
230      // represented using 32 bit integers.
231
232      assert((uint64_t)I->Offset != DenseMapInfo<uint64_t>::getEmptyKey() &&
233             (uint64_t)I->Offset != DenseMapInfo<uint64_t>::getTombstoneKey() &&
234             "empty and tombstone keys should fit in 32 bits!");
235      auto Result = ConstPool.insert(std::make_pair(I->Offset, I->Offset));
236      I->Offset = Result.first - ConstPool.begin();
237    }
238  }
239
240  // Create an expression to calculate the offset of the callsite from function
241  // entry.
242  const MCExpr *CSOffsetExpr = MCBinaryExpr::CreateSub(
243    MCSymbolRefExpr::Create(MILabel, OutContext),
244    MCSymbolRefExpr::Create(AP.CurrentFnSymForSize, OutContext),
245    OutContext);
246
247  CSInfos.emplace_back(CSOffsetExpr, ID, std::move(Locations),
248                       std::move(LiveOuts));
249
250  // Record the stack size of the current function.
251  const MachineFrameInfo *MFI = AP.MF->getFrameInfo();
252  const TargetRegisterInfo *RegInfo = AP.MF->getSubtarget().getRegisterInfo();
253  const bool DynamicFrameSize = MFI->hasVarSizedObjects() ||
254    RegInfo->needsStackRealignment(*(AP.MF));
255  FnStackSize[AP.CurrentFnSym] =
256    DynamicFrameSize ? UINT64_MAX : MFI->getStackSize();
257}
258
259void StackMaps::recordStackMap(const MachineInstr &MI) {
260  assert(MI.getOpcode() == TargetOpcode::STACKMAP && "expected stackmap");
261
262  int64_t ID = MI.getOperand(0).getImm();
263  recordStackMapOpers(MI, ID, std::next(MI.operands_begin(), 2),
264                      MI.operands_end());
265}
266
267void StackMaps::recordPatchPoint(const MachineInstr &MI) {
268  assert(MI.getOpcode() == TargetOpcode::PATCHPOINT && "expected patchpoint");
269
270  PatchPointOpers opers(&MI);
271  int64_t ID = opers.getMetaOper(PatchPointOpers::IDPos).getImm();
272
273  MachineInstr::const_mop_iterator MOI =
274    std::next(MI.operands_begin(), opers.getStackMapStartIdx());
275  recordStackMapOpers(MI, ID, MOI, MI.operands_end(),
276                      opers.isAnyReg() && opers.hasDef());
277
278#ifndef NDEBUG
279  // verify anyregcc
280  LocationVec &Locations = CSInfos.back().Locations;
281  if (opers.isAnyReg()) {
282    unsigned NArgs = opers.getMetaOper(PatchPointOpers::NArgPos).getImm();
283    for (unsigned i = 0, e = (opers.hasDef() ? NArgs+1 : NArgs); i != e; ++i)
284      assert(Locations[i].LocType == Location::Register &&
285             "anyreg arg must be in reg.");
286  }
287#endif
288}
289void StackMaps::recordStatepoint(const MachineInstr &MI) {
290  assert(MI.getOpcode() == TargetOpcode::STATEPOINT &&
291         "expected statepoint");
292
293  StatepointOpers opers(&MI);
294  // Record all the deopt and gc operands (they're contiguous and run from the
295  // initial index to the end of the operand list)
296  const unsigned StartIdx = opers.getVarIdx();
297  recordStackMapOpers(MI, 0xABCDEF00,
298                      MI.operands_begin() + StartIdx, MI.operands_end(),
299                      false);
300}
301
302/// Emit the stackmap header.
303///
304/// Header {
305///   uint8  : Stack Map Version (currently 1)
306///   uint8  : Reserved (expected to be 0)
307///   uint16 : Reserved (expected to be 0)
308/// }
309/// uint32 : NumFunctions
310/// uint32 : NumConstants
311/// uint32 : NumRecords
312void StackMaps::emitStackmapHeader(MCStreamer &OS) {
313  // Header.
314  OS.EmitIntValue(StackMapVersion, 1); // Version.
315  OS.EmitIntValue(0, 1); // Reserved.
316  OS.EmitIntValue(0, 2); // Reserved.
317
318  // Num functions.
319  DEBUG(dbgs() << WSMP << "#functions = " << FnStackSize.size() << '\n');
320  OS.EmitIntValue(FnStackSize.size(), 4);
321  // Num constants.
322  DEBUG(dbgs() << WSMP << "#constants = " << ConstPool.size() << '\n');
323  OS.EmitIntValue(ConstPool.size(), 4);
324  // Num callsites.
325  DEBUG(dbgs() << WSMP << "#callsites = " << CSInfos.size() << '\n');
326  OS.EmitIntValue(CSInfos.size(), 4);
327}
328
329/// Emit the function frame record for each function.
330///
331/// StkSizeRecord[NumFunctions] {
332///   uint64 : Function Address
333///   uint64 : Stack Size
334/// }
335void StackMaps::emitFunctionFrameRecords(MCStreamer &OS) {
336  // Function Frame records.
337  DEBUG(dbgs() << WSMP << "functions:\n");
338  for (auto const &FR : FnStackSize) {
339    DEBUG(dbgs() << WSMP << "function addr: " << FR.first
340                         << " frame size: " << FR.second);
341    OS.EmitSymbolValue(FR.first, 8);
342    OS.EmitIntValue(FR.second, 8);
343  }
344}
345
346/// Emit the constant pool.
347///
348/// int64  : Constants[NumConstants]
349void StackMaps::emitConstantPoolEntries(MCStreamer &OS) {
350  // Constant pool entries.
351  DEBUG(dbgs() << WSMP << "constants:\n");
352  for (auto ConstEntry : ConstPool) {
353    DEBUG(dbgs() << WSMP << ConstEntry.second << '\n');
354    OS.EmitIntValue(ConstEntry.second, 8);
355  }
356}
357
358/// Emit the callsite info for each callsite.
359///
360/// StkMapRecord[NumRecords] {
361///   uint64 : PatchPoint ID
362///   uint32 : Instruction Offset
363///   uint16 : Reserved (record flags)
364///   uint16 : NumLocations
365///   Location[NumLocations] {
366///     uint8  : Register | Direct | Indirect | Constant | ConstantIndex
367///     uint8  : Size in Bytes
368///     uint16 : Dwarf RegNum
369///     int32  : Offset
370///   }
371///   uint16 : Padding
372///   uint16 : NumLiveOuts
373///   LiveOuts[NumLiveOuts] {
374///     uint16 : Dwarf RegNum
375///     uint8  : Reserved
376///     uint8  : Size in Bytes
377///   }
378///   uint32 : Padding (only if required to align to 8 byte)
379/// }
380///
381/// Location Encoding, Type, Value:
382///   0x1, Register, Reg                 (value in register)
383///   0x2, Direct, Reg + Offset          (frame index)
384///   0x3, Indirect, [Reg + Offset]      (spilled value)
385///   0x4, Constant, Offset              (small constant)
386///   0x5, ConstIndex, Constants[Offset] (large constant)
387void StackMaps::emitCallsiteEntries(MCStreamer &OS,
388                                    const TargetRegisterInfo *TRI) {
389  // Callsite entries.
390  DEBUG(dbgs() << WSMP << "callsites:\n");
391  for (const auto &CSI : CSInfos) {
392    const LocationVec &CSLocs = CSI.Locations;
393    const LiveOutVec &LiveOuts = CSI.LiveOuts;
394
395    DEBUG(dbgs() << WSMP << "callsite " << CSI.ID << "\n");
396
397    // Verify stack map entry. It's better to communicate a problem to the
398    // runtime than crash in case of in-process compilation. Currently, we do
399    // simple overflow checks, but we may eventually communicate other
400    // compilation errors this way.
401    if (CSLocs.size() > UINT16_MAX || LiveOuts.size() > UINT16_MAX) {
402      OS.EmitIntValue(UINT64_MAX, 8); // Invalid ID.
403      OS.EmitValue(CSI.CSOffsetExpr, 4);
404      OS.EmitIntValue(0, 2); // Reserved.
405      OS.EmitIntValue(0, 2); // 0 locations.
406      OS.EmitIntValue(0, 2); // padding.
407      OS.EmitIntValue(0, 2); // 0 live-out registers.
408      OS.EmitIntValue(0, 4); // padding.
409      continue;
410    }
411
412    OS.EmitIntValue(CSI.ID, 8);
413    OS.EmitValue(CSI.CSOffsetExpr, 4);
414
415    // Reserved for flags.
416    OS.EmitIntValue(0, 2);
417
418    DEBUG(dbgs() << WSMP << "  has " << CSLocs.size() << " locations\n");
419
420    OS.EmitIntValue(CSLocs.size(), 2);
421
422    unsigned OperIdx = 0;
423    for (const auto &Loc : CSLocs) {
424      unsigned RegNo = 0;
425      int Offset = Loc.Offset;
426      if(Loc.Reg) {
427        RegNo = getDwarfRegNum(Loc.Reg, TRI);
428
429        // If this is a register location, put the subregister byte offset in
430        // the location offset.
431        if (Loc.LocType == Location::Register) {
432          assert(!Loc.Offset && "Register location should have zero offset");
433          unsigned LLVMRegNo = TRI->getLLVMRegNum(RegNo, false);
434          unsigned SubRegIdx = TRI->getSubRegIndex(LLVMRegNo, Loc.Reg);
435          if (SubRegIdx)
436            Offset = TRI->getSubRegIdxOffset(SubRegIdx);
437        }
438      }
439      else {
440        assert(Loc.LocType != Location::Register &&
441               "Missing location register");
442      }
443
444      DEBUG(dbgs() << WSMP << "  Loc " << OperIdx << ": ";
445            switch (Loc.LocType) {
446            case Location::Unprocessed:
447              dbgs() << "<Unprocessed operand>";
448              break;
449            case Location::Register:
450              dbgs() << "Register " << TRI->getName(Loc.Reg);
451              break;
452            case Location::Direct:
453              dbgs() << "Direct " << TRI->getName(Loc.Reg);
454              if (Loc.Offset)
455              dbgs() << " + " << Loc.Offset;
456              break;
457            case Location::Indirect:
458              dbgs() << "Indirect " << TRI->getName(Loc.Reg)
459              << " + " << Loc.Offset;
460              break;
461            case Location::Constant:
462              dbgs() << "Constant " << Loc.Offset;
463              break;
464            case Location::ConstantIndex:
465              dbgs() << "Constant Index " << Loc.Offset;
466              break;
467              }
468            dbgs() << "     [encoding: .byte " << Loc.LocType
469            << ", .byte " << Loc.Size
470            << ", .short " << RegNo
471            << ", .int " << Offset << "]\n";
472            );
473
474      OS.EmitIntValue(Loc.LocType, 1);
475      OS.EmitIntValue(Loc.Size, 1);
476      OS.EmitIntValue(RegNo, 2);
477      OS.EmitIntValue(Offset, 4);
478      OperIdx++;
479    }
480
481    DEBUG(dbgs() << WSMP << "  has " << LiveOuts.size()
482                         << " live-out registers\n");
483
484    // Num live-out registers and padding to align to 4 byte.
485    OS.EmitIntValue(0, 2);
486    OS.EmitIntValue(LiveOuts.size(), 2);
487
488    OperIdx = 0;
489    for (const auto &LO : LiveOuts) {
490      DEBUG(dbgs() << WSMP << "  LO " << OperIdx << ": "
491                           << TRI->getName(LO.Reg)
492                           << "     [encoding: .short " << LO.RegNo
493                           << ", .byte 0, .byte " << LO.Size << "]\n");
494      OS.EmitIntValue(LO.RegNo, 2);
495      OS.EmitIntValue(0, 1);
496      OS.EmitIntValue(LO.Size, 1);
497    }
498    // Emit alignment to 8 byte.
499    OS.EmitValueToAlignment(8);
500  }
501}
502
503/// Serialize the stackmap data.
504void StackMaps::serializeToStackMapSection() {
505  (void) WSMP;
506  // Bail out if there's no stack map data.
507  assert((!CSInfos.empty() || (CSInfos.empty() && ConstPool.empty())) &&
508         "Expected empty constant pool too!");
509  assert((!CSInfos.empty() || (CSInfos.empty() && FnStackSize.empty())) &&
510         "Expected empty function record too!");
511  if (CSInfos.empty())
512    return;
513
514  MCContext &OutContext = AP.OutStreamer.getContext();
515  MCStreamer &OS = AP.OutStreamer;
516  const TargetRegisterInfo *TRI = AP.TM.getSubtargetImpl()->getRegisterInfo();
517
518  // Create the section.
519  const MCSection *StackMapSection =
520    OutContext.getObjectFileInfo()->getStackMapSection();
521  OS.SwitchSection(StackMapSection);
522
523  // Emit a dummy symbol to force section inclusion.
524  OS.EmitLabel(OutContext.GetOrCreateSymbol(Twine("__LLVM_StackMaps")));
525
526  // Serialize data.
527  DEBUG(dbgs() << "********** Stack Map Output **********\n");
528  emitStackmapHeader(OS);
529  emitFunctionFrameRecords(OS);
530  emitConstantPoolEntries(OS);
531  emitCallsiteEntries(OS, TRI);
532  OS.AddBlankLine();
533
534  // Clean up.
535  CSInfos.clear();
536  ConstPool.clear();
537}
538