1//===-- xray_fdr_controller.h ---------------------------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file is a part of XRay, a function call tracing system.
11//
12//===----------------------------------------------------------------------===//
13#ifndef COMPILER_RT_LIB_XRAY_XRAY_FDR_CONTROLLER_H_
14#define COMPILER_RT_LIB_XRAY_XRAY_FDR_CONTROLLER_H_
15
16#include <limits>
17#include <time.h>
18
19#include "xray/xray_interface.h"
20#include "xray/xray_records.h"
21#include "xray_buffer_queue.h"
22#include "xray_fdr_log_writer.h"
23
24namespace __xray {
25
26template <size_t Version = 5> class FDRController {
27  BufferQueue *BQ;
28  BufferQueue::Buffer &B;
29  FDRLogWriter &W;
30  int (*WallClockReader)(clockid_t, struct timespec *) = 0;
31  uint64_t CycleThreshold = 0;
32
33  uint64_t LastFunctionEntryTSC = 0;
34  uint64_t LatestTSC = 0;
35  uint16_t LatestCPU = 0;
36  tid_t TId = 0;
37  pid_t PId = 0;
38  bool First = true;
39
40  uint32_t UndoableFunctionEnters = 0;
41  uint32_t UndoableTailExits = 0;
42
43  bool finalized() const XRAY_NEVER_INSTRUMENT {
44    return BQ == nullptr || BQ->finalizing();
45  }
46
47  bool hasSpace(size_t S) XRAY_NEVER_INSTRUMENT {
48    return B.Data != nullptr && B.Generation == BQ->generation() &&
49           W.getNextRecord() + S <= reinterpret_cast<char *>(B.Data) + B.Size;
50  }
51
52  constexpr int32_t mask(int32_t FuncId) const XRAY_NEVER_INSTRUMENT {
53    return FuncId & ((1 << 29) - 1);
54  }
55
56  bool getNewBuffer() XRAY_NEVER_INSTRUMENT {
57    if (BQ->getBuffer(B) != BufferQueue::ErrorCode::Ok)
58      return false;
59
60    W.resetRecord();
61    DCHECK_EQ(W.getNextRecord(), B.Data);
62    LatestTSC = 0;
63    LatestCPU = 0;
64    First = true;
65    UndoableFunctionEnters = 0;
66    UndoableTailExits = 0;
67    atomic_store(B.Extents, 0, memory_order_release);
68    return true;
69  }
70
71  bool setupNewBuffer() XRAY_NEVER_INSTRUMENT {
72    if (finalized())
73      return false;
74
75    DCHECK(hasSpace(sizeof(MetadataRecord) * 3));
76    TId = GetTid();
77    PId = internal_getpid();
78    struct timespec TS {
79      0, 0
80    };
81    WallClockReader(CLOCK_MONOTONIC, &TS);
82
83    MetadataRecord Metadata[] = {
84        // Write out a MetadataRecord to signify that this is the start of a new
85        // buffer, associated with a particular thread, with a new CPU. For the
86        // data, we have 15 bytes to squeeze as much information as we can. At
87        // this point we only write down the following bytes:
88        //   - Thread ID (tid_t, cast to 4 bytes type due to Darwin being 8
89        //   bytes)
90        createMetadataRecord<MetadataRecord::RecordKinds::NewBuffer>(
91            static_cast<int32_t>(TId)),
92
93        // Also write the WalltimeMarker record. We only really need microsecond
94        // precision here, and enforce across platforms that we need 64-bit
95        // seconds and 32-bit microseconds encoded in the Metadata record.
96        createMetadataRecord<MetadataRecord::RecordKinds::WalltimeMarker>(
97            static_cast<int64_t>(TS.tv_sec),
98            static_cast<int32_t>(TS.tv_nsec / 1000)),
99
100        // Also write the Pid record.
101        createMetadataRecord<MetadataRecord::RecordKinds::Pid>(
102            static_cast<int32_t>(PId)),
103    };
104
105    if (finalized())
106      return false;
107    return W.writeMetadataRecords(Metadata);
108  }
109
110  bool prepareBuffer(size_t S) XRAY_NEVER_INSTRUMENT {
111    if (finalized())
112      return returnBuffer();
113
114    if (UNLIKELY(!hasSpace(S))) {
115      if (!returnBuffer())
116        return false;
117      if (!getNewBuffer())
118        return false;
119      if (!setupNewBuffer())
120        return false;
121    }
122
123    if (First) {
124      First = false;
125      W.resetRecord();
126      atomic_store(B.Extents, 0, memory_order_release);
127      return setupNewBuffer();
128    }
129
130    return true;
131  }
132
133  bool returnBuffer() XRAY_NEVER_INSTRUMENT {
134    if (BQ == nullptr)
135      return false;
136
137    First = true;
138    if (finalized()) {
139      BQ->releaseBuffer(B); // ignore result.
140      return false;
141    }
142
143    return BQ->releaseBuffer(B) == BufferQueue::ErrorCode::Ok;
144  }
145
146  enum class PreambleResult { NoChange, WroteMetadata, InvalidBuffer };
147  PreambleResult recordPreamble(uint64_t TSC,
148                                uint16_t CPU) XRAY_NEVER_INSTRUMENT {
149    if (UNLIKELY(LatestCPU != CPU || LatestTSC == 0)) {
150      // We update our internal tracking state for the Latest TSC and CPU we've
151      // seen, then write out the appropriate metadata and function records.
152      LatestTSC = TSC;
153      LatestCPU = CPU;
154
155      if (B.Generation != BQ->generation())
156        return PreambleResult::InvalidBuffer;
157
158      W.writeMetadata<MetadataRecord::RecordKinds::NewCPUId>(CPU, TSC);
159      return PreambleResult::WroteMetadata;
160    }
161
162    DCHECK_EQ(LatestCPU, CPU);
163
164    if (UNLIKELY(LatestTSC > TSC ||
165                 TSC - LatestTSC >
166                     uint64_t{std::numeric_limits<int32_t>::max()})) {
167      // Either the TSC has wrapped around from the last TSC we've seen or the
168      // delta is too large to fit in a 32-bit signed integer, so we write a
169      // wrap-around record.
170      LatestTSC = TSC;
171
172      if (B.Generation != BQ->generation())
173        return PreambleResult::InvalidBuffer;
174
175      W.writeMetadata<MetadataRecord::RecordKinds::TSCWrap>(TSC);
176      return PreambleResult::WroteMetadata;
177    }
178
179    return PreambleResult::NoChange;
180  }
181
182  bool rewindRecords(int32_t FuncId, uint64_t TSC,
183                     uint16_t CPU) XRAY_NEVER_INSTRUMENT {
184    // Undo one enter record, because at this point we are either at the state
185    // of:
186    // - We are exiting a function that we recently entered.
187    // - We are exiting a function that was the result of a sequence of tail
188    //   exits, and we can check whether the tail exits can be re-wound.
189    //
190    FunctionRecord F;
191    W.undoWrites(sizeof(FunctionRecord));
192    if (B.Generation != BQ->generation())
193      return false;
194    internal_memcpy(&F, W.getNextRecord(), sizeof(FunctionRecord));
195
196    DCHECK(F.RecordKind ==
197               uint8_t(FunctionRecord::RecordKinds::FunctionEnter) &&
198           "Expected to find function entry recording when rewinding.");
199    DCHECK_EQ(F.FuncId, FuncId & ~(0x0F << 28));
200
201    LatestTSC -= F.TSCDelta;
202    if (--UndoableFunctionEnters != 0) {
203      LastFunctionEntryTSC -= F.TSCDelta;
204      return true;
205    }
206
207    LastFunctionEntryTSC = 0;
208    auto RewindingTSC = LatestTSC;
209    auto RewindingRecordPtr = W.getNextRecord() - sizeof(FunctionRecord);
210    while (UndoableTailExits) {
211      if (B.Generation != BQ->generation())
212        return false;
213      internal_memcpy(&F, RewindingRecordPtr, sizeof(FunctionRecord));
214      DCHECK_EQ(F.RecordKind,
215                uint8_t(FunctionRecord::RecordKinds::FunctionTailExit));
216      RewindingTSC -= F.TSCDelta;
217      RewindingRecordPtr -= sizeof(FunctionRecord);
218      if (B.Generation != BQ->generation())
219        return false;
220      internal_memcpy(&F, RewindingRecordPtr, sizeof(FunctionRecord));
221
222      // This tail call exceeded the threshold duration. It will not be erased.
223      if ((TSC - RewindingTSC) >= CycleThreshold) {
224        UndoableTailExits = 0;
225        return true;
226      }
227
228      --UndoableTailExits;
229      W.undoWrites(sizeof(FunctionRecord) * 2);
230      LatestTSC = RewindingTSC;
231    }
232    return true;
233  }
234
235public:
236  template <class WallClockFunc>
237  FDRController(BufferQueue *BQ, BufferQueue::Buffer &B, FDRLogWriter &W,
238                WallClockFunc R, uint64_t C) XRAY_NEVER_INSTRUMENT
239      : BQ(BQ),
240        B(B),
241        W(W),
242        WallClockReader(R),
243        CycleThreshold(C) {}
244
245  bool functionEnter(int32_t FuncId, uint64_t TSC,
246                     uint16_t CPU) XRAY_NEVER_INSTRUMENT {
247    if (finalized() ||
248        !prepareBuffer(sizeof(MetadataRecord) + sizeof(FunctionRecord)))
249      return returnBuffer();
250
251    auto PreambleStatus = recordPreamble(TSC, CPU);
252    if (PreambleStatus == PreambleResult::InvalidBuffer)
253      return returnBuffer();
254
255    if (PreambleStatus == PreambleResult::WroteMetadata) {
256      UndoableFunctionEnters = 1;
257      UndoableTailExits = 0;
258    } else {
259      ++UndoableFunctionEnters;
260    }
261
262    auto Delta = TSC - LatestTSC;
263    LastFunctionEntryTSC = TSC;
264    LatestTSC = TSC;
265    return W.writeFunction(FDRLogWriter::FunctionRecordKind::Enter,
266                           mask(FuncId), Delta);
267  }
268
269  bool functionTailExit(int32_t FuncId, uint64_t TSC,
270                        uint16_t CPU) XRAY_NEVER_INSTRUMENT {
271    if (finalized())
272      return returnBuffer();
273
274    if (!prepareBuffer(sizeof(MetadataRecord) + sizeof(FunctionRecord)))
275      return returnBuffer();
276
277    auto PreambleStatus = recordPreamble(TSC, CPU);
278    if (PreambleStatus == PreambleResult::InvalidBuffer)
279      return returnBuffer();
280
281    if (PreambleStatus == PreambleResult::NoChange &&
282        UndoableFunctionEnters != 0 &&
283        TSC - LastFunctionEntryTSC < CycleThreshold)
284      return rewindRecords(FuncId, TSC, CPU);
285
286    UndoableTailExits = UndoableFunctionEnters ? UndoableTailExits + 1 : 0;
287    UndoableFunctionEnters = 0;
288    auto Delta = TSC - LatestTSC;
289    LatestTSC = TSC;
290    return W.writeFunction(FDRLogWriter::FunctionRecordKind::TailExit,
291                           mask(FuncId), Delta);
292  }
293
294  bool functionEnterArg(int32_t FuncId, uint64_t TSC, uint16_t CPU,
295                        uint64_t Arg) XRAY_NEVER_INSTRUMENT {
296    if (finalized() ||
297        !prepareBuffer((2 * sizeof(MetadataRecord)) + sizeof(FunctionRecord)) ||
298        recordPreamble(TSC, CPU) == PreambleResult::InvalidBuffer)
299      return returnBuffer();
300
301    auto Delta = TSC - LatestTSC;
302    LatestTSC = TSC;
303    LastFunctionEntryTSC = 0;
304    UndoableFunctionEnters = 0;
305    UndoableTailExits = 0;
306
307    return W.writeFunctionWithArg(FDRLogWriter::FunctionRecordKind::EnterArg,
308                                  mask(FuncId), Delta, Arg);
309  }
310
311  bool functionExit(int32_t FuncId, uint64_t TSC,
312                    uint16_t CPU) XRAY_NEVER_INSTRUMENT {
313    if (finalized() ||
314        !prepareBuffer(sizeof(MetadataRecord) + sizeof(FunctionRecord)))
315      return returnBuffer();
316
317    auto PreambleStatus = recordPreamble(TSC, CPU);
318    if (PreambleStatus == PreambleResult::InvalidBuffer)
319      return returnBuffer();
320
321    if (PreambleStatus == PreambleResult::NoChange &&
322        UndoableFunctionEnters != 0 &&
323        TSC - LastFunctionEntryTSC < CycleThreshold)
324      return rewindRecords(FuncId, TSC, CPU);
325
326    auto Delta = TSC - LatestTSC;
327    LatestTSC = TSC;
328    UndoableFunctionEnters = 0;
329    UndoableTailExits = 0;
330    return W.writeFunction(FDRLogWriter::FunctionRecordKind::Exit, mask(FuncId),
331                           Delta);
332  }
333
334  bool customEvent(uint64_t TSC, uint16_t CPU, const void *Event,
335                   int32_t EventSize) XRAY_NEVER_INSTRUMENT {
336    if (finalized() ||
337        !prepareBuffer((2 * sizeof(MetadataRecord)) + EventSize) ||
338        recordPreamble(TSC, CPU) == PreambleResult::InvalidBuffer)
339      return returnBuffer();
340
341    auto Delta = TSC - LatestTSC;
342    LatestTSC = TSC;
343    UndoableFunctionEnters = 0;
344    UndoableTailExits = 0;
345    return W.writeCustomEvent(Delta, Event, EventSize);
346  }
347
348  bool typedEvent(uint64_t TSC, uint16_t CPU, uint16_t EventType,
349                  const void *Event, int32_t EventSize) XRAY_NEVER_INSTRUMENT {
350    if (finalized() ||
351        !prepareBuffer((2 * sizeof(MetadataRecord)) + EventSize) ||
352        recordPreamble(TSC, CPU) == PreambleResult::InvalidBuffer)
353      return returnBuffer();
354
355    auto Delta = TSC - LatestTSC;
356    LatestTSC = TSC;
357    UndoableFunctionEnters = 0;
358    UndoableTailExits = 0;
359    return W.writeTypedEvent(Delta, EventType, Event, EventSize);
360  }
361
362  bool flush() XRAY_NEVER_INSTRUMENT {
363    if (finalized()) {
364      returnBuffer(); // ignore result.
365      return true;
366    }
367    return returnBuffer();
368  }
369};
370
371} // namespace __xray
372
373#endif // COMPILER-RT_LIB_XRAY_XRAY_FDR_CONTROLLER_H_
374