1/*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7\*===----------------------------------------------------------------------===*/
8
9#ifdef _MSC_VER
10/* For _alloca */
11#include <malloc.h>
12#endif
13#include <string.h>
14
15#include "InstrProfiling.h"
16#include "InstrProfilingInternal.h"
17#include "InstrProfilingPort.h"
18
19#define INSTR_PROF_VALUE_PROF_DATA
20#include "profile/InstrProfData.inc"
21
22COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL;
23static ProfBufferIO TheBufferIO;
24#define VP_BUFFER_SIZE 8 * 1024
25static uint8_t BufferIOBuffer[VP_BUFFER_SIZE];
26static InstrProfValueData VPDataArray[16];
27static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray);
28
29COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0;
30COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0;
31
32/* The buffer writer is reponsponsible in keeping writer state
33 * across the call.
34 */
35COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This,
36                                                  ProfDataIOVec *IOVecs,
37                                                  uint32_t NumIOVecs) {
38  uint32_t I;
39  char **Buffer = (char **)&This->WriterCtx;
40  for (I = 0; I < NumIOVecs; I++) {
41    size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm;
42    if (IOVecs[I].Data)
43      memcpy(*Buffer, IOVecs[I].Data, Length);
44    else if (IOVecs[I].UseZeroPadding) {
45      /* Allocating the buffer should zero fill. */
46    }
47    *Buffer += Length;
48  }
49  return 0;
50}
51
52static void llvmInitBufferIO(ProfBufferIO *BufferIO, ProfDataWriter *FileWriter,
53                             uint8_t *Buffer, uint32_t BufferSz) {
54  BufferIO->FileWriter = FileWriter;
55  BufferIO->OwnFileWriter = 0;
56  BufferIO->BufferStart = Buffer;
57  BufferIO->BufferSz = BufferSz;
58  BufferIO->CurOffset = 0;
59}
60
61COMPILER_RT_VISIBILITY ProfBufferIO *
62lprofCreateBufferIO(ProfDataWriter *FileWriter) {
63  uint8_t *Buffer = DynamicBufferIOBuffer;
64  uint32_t BufferSize = VPBufferSize;
65  if (!Buffer) {
66    Buffer = &BufferIOBuffer[0];
67    BufferSize = sizeof(BufferIOBuffer);
68  }
69  llvmInitBufferIO(&TheBufferIO, FileWriter, Buffer, BufferSize);
70  return &TheBufferIO;
71}
72
73COMPILER_RT_VISIBILITY void lprofDeleteBufferIO(ProfBufferIO *BufferIO) {
74  if (BufferIO->OwnFileWriter)
75    FreeHook(BufferIO->FileWriter);
76  if (DynamicBufferIOBuffer) {
77    FreeHook(DynamicBufferIOBuffer);
78    DynamicBufferIOBuffer = 0;
79    VPBufferSize = 0;
80  }
81}
82
83COMPILER_RT_VISIBILITY int
84lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) {
85  /* Buffer is not large enough, it is time to flush.  */
86  if (Size + BufferIO->CurOffset > BufferIO->BufferSz) {
87    if (lprofBufferIOFlush(BufferIO) != 0)
88      return -1;
89  }
90  /* Special case, bypass the buffer completely. */
91  ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size, 0}};
92  if (Size > BufferIO->BufferSz) {
93    if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))
94      return -1;
95  } else {
96    /* Write the data to buffer */
97    uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset;
98    ProfDataWriter BufferWriter;
99    initBufferWriter(&BufferWriter, (char *)Buffer);
100    lprofBufferWriter(&BufferWriter, IO, 1);
101    BufferIO->CurOffset =
102        (uint8_t *)BufferWriter.WriterCtx - BufferIO->BufferStart;
103  }
104  return 0;
105}
106
107COMPILER_RT_VISIBILITY int lprofBufferIOFlush(ProfBufferIO *BufferIO) {
108  if (BufferIO->CurOffset) {
109    ProfDataIOVec IO[] = {
110        {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset, 0}};
111    if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))
112      return -1;
113    BufferIO->CurOffset = 0;
114  }
115  return 0;
116}
117
118/* Write out value profile data for function specified with \c Data.
119 * The implementation does not use the method \c serializeValueProfData
120 * which depends on dynamic memory allocation. In this implementation,
121 * value profile data is written out to \c BufferIO piecemeal.
122 */
123static int writeOneValueProfData(ProfBufferIO *BufferIO,
124                                 VPDataReaderType *VPDataReader,
125                                 const __llvm_profile_data *Data) {
126  unsigned I, NumValueKinds = 0;
127  ValueProfData VPHeader;
128  uint8_t *SiteCountArray[IPVK_Last + 1];
129
130  for (I = 0; I <= IPVK_Last; I++) {
131    if (!Data->NumValueSites[I])
132      SiteCountArray[I] = 0;
133    else {
134      uint32_t Sz =
135          VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
136          offsetof(ValueProfRecord, SiteCountArray);
137      /* Only use alloca for this small byte array to avoid excessive
138       * stack growth.  */
139      SiteCountArray[I] = (uint8_t *)COMPILER_RT_ALLOCA(Sz);
140      memset(SiteCountArray[I], 0, Sz);
141    }
142  }
143
144  /* If NumValueKinds returned is 0, there is nothing to write, report
145     success and return. This should match the raw profile reader's behavior. */
146  if (!(NumValueKinds = VPDataReader->InitRTRecord(Data, SiteCountArray)))
147    return 0;
148
149  /* First write the header structure. */
150  VPHeader.TotalSize = VPDataReader->GetValueProfDataSize();
151  VPHeader.NumValueKinds = NumValueKinds;
152  if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPHeader,
153                         sizeof(ValueProfData)))
154    return -1;
155
156  /* Make sure nothing else needs to be written before value profile
157   * records. */
158  if ((void *)VPDataReader->GetFirstValueProfRecord(&VPHeader) !=
159      (void *)(&VPHeader + 1))
160    return -1;
161
162  /* Write out the value profile record for each value kind
163   * one by one. */
164  for (I = 0; I <= IPVK_Last; I++) {
165    uint32_t J;
166    ValueProfRecord RecordHeader;
167    /* The size of the value prof record header without counting the
168     * site count array .*/
169    uint32_t RecordHeaderSize = offsetof(ValueProfRecord, SiteCountArray);
170    uint32_t SiteCountArraySize;
171
172    if (!Data->NumValueSites[I])
173      continue;
174
175    /* Write out the record header.  */
176    RecordHeader.Kind = I;
177    RecordHeader.NumValueSites = Data->NumValueSites[I];
178    if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&RecordHeader,
179                           RecordHeaderSize))
180      return -1;
181
182    /* Write out the site value count array including padding space. */
183    SiteCountArraySize =
184        VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
185        RecordHeaderSize;
186    if (lprofBufferIOWrite(BufferIO, SiteCountArray[I], SiteCountArraySize))
187      return -1;
188
189    /* Write out the value profile data for each value site.  */
190    for (J = 0; J < Data->NumValueSites[I]; J++) {
191      uint32_t NRead, NRemain;
192      ValueProfNode *NextStartNode = 0;
193      NRemain = VPDataReader->GetNumValueDataForSite(I, J);
194      if (!NRemain)
195        continue;
196      /* Read and write out value data in small chunks till it is done. */
197      do {
198        NRead = (NRemain > VPDataArraySize ? VPDataArraySize : NRemain);
199        NextStartNode =
200            VPDataReader->GetValueData(I, /* ValueKind */
201                                       J, /* Site */
202                                       &VPDataArray[0], NextStartNode, NRead);
203        if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPDataArray[0],
204                               NRead * sizeof(InstrProfValueData)))
205          return -1;
206        NRemain -= NRead;
207      } while (NRemain != 0);
208    }
209  }
210  /* All done report success.  */
211  return 0;
212}
213
214static int writeValueProfData(ProfDataWriter *Writer,
215                              VPDataReaderType *VPDataReader,
216                              const __llvm_profile_data *DataBegin,
217                              const __llvm_profile_data *DataEnd) {
218  ProfBufferIO *BufferIO;
219  const __llvm_profile_data *DI = 0;
220
221  if (!VPDataReader)
222    return 0;
223
224  BufferIO = lprofCreateBufferIO(Writer);
225
226  for (DI = DataBegin; DI < DataEnd; DI++) {
227    if (writeOneValueProfData(BufferIO, VPDataReader, DI))
228      return -1;
229  }
230
231  if (lprofBufferIOFlush(BufferIO) != 0)
232    return -1;
233  lprofDeleteBufferIO(BufferIO);
234
235  return 0;
236}
237
238COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer,
239                                          VPDataReaderType *VPDataReader,
240                                          int SkipNameDataWrite) {
241  /* Match logic in __llvm_profile_write_buffer(). */
242  const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
243  const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
244  const uint64_t *CountersBegin = __llvm_profile_begin_counters();
245  const uint64_t *CountersEnd = __llvm_profile_end_counters();
246  const char *NamesBegin = __llvm_profile_begin_names();
247  const char *NamesEnd = __llvm_profile_end_names();
248  return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin,
249                            CountersEnd, VPDataReader, NamesBegin, NamesEnd,
250                            SkipNameDataWrite);
251}
252
253COMPILER_RT_VISIBILITY int
254lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
255                   const __llvm_profile_data *DataEnd,
256                   const uint64_t *CountersBegin, const uint64_t *CountersEnd,
257                   VPDataReaderType *VPDataReader, const char *NamesBegin,
258                   const char *NamesEnd, int SkipNameDataWrite) {
259
260  /* Calculate size of sections. */
261  const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
262  const uint64_t CountersSize = CountersEnd - CountersBegin;
263  const uint64_t NamesSize = NamesEnd - NamesBegin;
264
265  /* Create the header. */
266  __llvm_profile_header Header;
267
268  if (!DataSize)
269    return 0;
270
271  /* Determine how much padding is needed before/after the counters and after
272   * the names. */
273  uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
274      PaddingBytesAfterNames;
275  __llvm_profile_get_padding_sizes_for_counters(
276      DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters,
277      &PaddingBytesAfterCounters, &PaddingBytesAfterNames);
278
279/* Initialize header structure.  */
280#define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init;
281#include "profile/InstrProfData.inc"
282
283  /* Write the data. */
284  ProfDataIOVec IOVec[] = {
285      {&Header, sizeof(__llvm_profile_header), 1, 0},
286      {DataBegin, sizeof(__llvm_profile_data), DataSize, 0},
287      {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1},
288      {CountersBegin, sizeof(uint64_t), CountersSize, 0},
289      {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1},
290      {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0},
291      {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}};
292  if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec)))
293    return -1;
294
295  /* Value profiling is not yet supported in continuous mode. */
296  if (__llvm_profile_is_continuous_mode_enabled())
297    return 0;
298
299  return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd);
300}
301