1/*===- InstrProfilingMerge.c - Profile in-process Merging  ---------------===*\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7|*===----------------------------------------------------------------------===*
8|* This file defines the API needed for in-process merging of profile data
9|* stored in memory buffer.
10\*===---------------------------------------------------------------------===*/
11
12#include "InstrProfiling.h"
13#include "InstrProfilingInternal.h"
14#include "InstrProfilingUtil.h"
15
16#define INSTR_PROF_VALUE_PROF_DATA
17#include "profile/InstrProfData.inc"
18
19COMPILER_RT_VISIBILITY
20void (*VPMergeHook)(ValueProfData *, __llvm_profile_data *);
21
22COMPILER_RT_VISIBILITY
23uint64_t lprofGetLoadModuleSignature(void) {
24  /* A very fast way to compute a module signature.  */
25  uint64_t Version = __llvm_profile_get_version();
26  uint64_t NumCounters = __llvm_profile_get_num_counters(
27      __llvm_profile_begin_counters(), __llvm_profile_end_counters());
28  uint64_t NumData = __llvm_profile_get_num_data(__llvm_profile_begin_data(),
29                                                 __llvm_profile_end_data());
30  uint64_t NamesSize =
31      (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names());
32  uint64_t NumVnodes =
33      (uint64_t)(__llvm_profile_end_vnodes() - __llvm_profile_begin_vnodes());
34  const __llvm_profile_data *FirstD = __llvm_profile_begin_data();
35
36  return (NamesSize << 40) + (NumCounters << 30) + (NumData << 20) +
37         (NumVnodes << 10) + (NumData > 0 ? FirstD->NameRef : 0) + Version +
38         __llvm_profile_get_magic();
39}
40
41#ifdef __GNUC__
42#pragma GCC diagnostic push
43#pragma GCC diagnostic ignored "-Wcast-qual"
44#elif defined(__clang__)
45#pragma clang diagnostic push
46#pragma clang diagnostic ignored "-Wcast-qual"
47#endif
48
49/* Returns 1 if profile is not structurally compatible.  */
50COMPILER_RT_VISIBILITY
51int __llvm_profile_check_compatibility(const char *ProfileData,
52                                       uint64_t ProfileSize) {
53  __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
54  __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
55  SrcDataStart =
56      (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
57                              Header->BinaryIdsSize);
58  SrcDataEnd = SrcDataStart + Header->NumData;
59
60  if (ProfileSize < sizeof(__llvm_profile_header))
61    return 1;
62
63  /* Check the header first.  */
64  if (Header->Magic != __llvm_profile_get_magic() ||
65      Header->Version != __llvm_profile_get_version() ||
66      Header->NumData !=
67          __llvm_profile_get_num_data(__llvm_profile_begin_data(),
68                                      __llvm_profile_end_data()) ||
69      Header->NumCounters !=
70          __llvm_profile_get_num_counters(__llvm_profile_begin_counters(),
71                                          __llvm_profile_end_counters()) ||
72      Header->NumBitmapBytes !=
73          __llvm_profile_get_num_bitmap_bytes(__llvm_profile_begin_bitmap(),
74                                              __llvm_profile_end_bitmap()) ||
75      Header->NamesSize !=
76          __llvm_profile_get_name_size(__llvm_profile_begin_names(),
77                                       __llvm_profile_end_names()) ||
78      Header->ValueKindLast != IPVK_Last)
79    return 1;
80
81  if (ProfileSize <
82      sizeof(__llvm_profile_header) + Header->BinaryIdsSize +
83          Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize +
84          Header->NumCounters * __llvm_profile_counter_entry_size() +
85          Header->NumBitmapBytes)
86    return 1;
87
88  for (SrcData = SrcDataStart,
89       DstData = (__llvm_profile_data *)__llvm_profile_begin_data();
90       SrcData < SrcDataEnd; ++SrcData, ++DstData) {
91    if (SrcData->NameRef != DstData->NameRef ||
92        SrcData->FuncHash != DstData->FuncHash ||
93        SrcData->NumCounters != DstData->NumCounters ||
94        SrcData->NumBitmapBytes != DstData->NumBitmapBytes)
95      return 1;
96  }
97
98  /* Matched! */
99  return 0;
100}
101
102static uintptr_t signextIfWin64(void *V) {
103#ifdef _WIN64
104  return (uintptr_t)(int32_t)(uintptr_t)V;
105#else
106  return (uintptr_t)V;
107#endif
108}
109
110COMPILER_RT_VISIBILITY
111int __llvm_profile_merge_from_buffer(const char *ProfileData,
112                                     uint64_t ProfileSize) {
113  if (__llvm_profile_get_version() & VARIANT_MASK_TEMPORAL_PROF) {
114    PROF_ERR("%s\n",
115             "Temporal profiles do not support profile merging at runtime. "
116             "Instead, merge raw profiles using the llvm-profdata tool.");
117    return 1;
118  }
119
120  __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
121  __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
122  char *SrcCountersStart, *DstCounter;
123  const char *SrcCountersEnd, *SrcCounter;
124  const char *SrcBitmapStart;
125  const char *SrcNameStart;
126  const char *SrcValueProfDataStart, *SrcValueProfData;
127  uintptr_t CountersDelta = Header->CountersDelta;
128  uintptr_t BitmapDelta = Header->BitmapDelta;
129
130  SrcDataStart =
131      (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
132                              Header->BinaryIdsSize);
133  SrcDataEnd = SrcDataStart + Header->NumData;
134  SrcCountersStart = (char *)SrcDataEnd;
135  SrcCountersEnd = SrcCountersStart +
136                   Header->NumCounters * __llvm_profile_counter_entry_size();
137  SrcBitmapStart = SrcCountersEnd;
138  SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes;
139  SrcValueProfDataStart =
140      SrcNameStart + Header->NamesSize +
141      __llvm_profile_get_num_padding_bytes(Header->NamesSize);
142  if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart)
143    return 1;
144
145  // Merge counters by iterating the entire counter section when data section is
146  // empty due to correlation.
147  if (Header->NumData == 0) {
148    for (SrcCounter = SrcCountersStart,
149        DstCounter = __llvm_profile_begin_counters();
150         SrcCounter < SrcCountersEnd;) {
151      if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
152        *DstCounter &= *SrcCounter;
153      } else {
154        *(uint64_t *)DstCounter += *(uint64_t *)SrcCounter;
155      }
156      SrcCounter += __llvm_profile_counter_entry_size();
157      DstCounter += __llvm_profile_counter_entry_size();
158    }
159    return 0;
160  }
161
162  for (SrcData = SrcDataStart,
163      DstData = (__llvm_profile_data *)__llvm_profile_begin_data(),
164      SrcValueProfData = SrcValueProfDataStart;
165       SrcData < SrcDataEnd; ++SrcData, ++DstData) {
166    // For the in-memory destination, CounterPtr is the distance from the start
167    // address of the data to the start address of the counter. On WIN64,
168    // CounterPtr is a truncated 32-bit value due to COFF limitation. Sign
169    // extend CounterPtr to get the original value.
170    char *DstCounters =
171        (char *)((uintptr_t)DstData + signextIfWin64(DstData->CounterPtr));
172    char *DstBitmap =
173        (char *)((uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr));
174    unsigned NVK = 0;
175
176    // SrcData is a serialized representation of the memory image. We need to
177    // compute the in-buffer counter offset from the in-memory address distance.
178    // The initial CountersDelta is the in-memory address difference
179    // start(__llvm_prf_cnts)-start(__llvm_prf_data), so SrcData->CounterPtr -
180    // CountersDelta computes the offset into the in-buffer counter section.
181    //
182    // On WIN64, CountersDelta is truncated as well, so no need for signext.
183    char *SrcCounters =
184        SrcCountersStart + ((uintptr_t)SrcData->CounterPtr - CountersDelta);
185    // CountersDelta needs to be decreased as we advance to the next data
186    // record.
187    CountersDelta -= sizeof(*SrcData);
188    unsigned NC = SrcData->NumCounters;
189    if (NC == 0)
190      return 1;
191    if (SrcCounters < SrcCountersStart || SrcCounters >= SrcNameStart ||
192        (SrcCounters + __llvm_profile_counter_entry_size() * NC) > SrcNameStart)
193      return 1;
194    for (unsigned I = 0; I < NC; I++) {
195      if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
196        // A value of zero signifies the function is covered.
197        DstCounters[I] &= SrcCounters[I];
198      } else {
199        ((uint64_t *)DstCounters)[I] += ((uint64_t *)SrcCounters)[I];
200      }
201    }
202
203    const char *SrcBitmap =
204        SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta);
205    // BitmapDelta also needs to be decreased as we advance to the next data
206    // record.
207    BitmapDelta -= sizeof(*SrcData);
208    unsigned NB = SrcData->NumBitmapBytes;
209    // NumBitmapBytes may legitimately be 0. Just keep going.
210    if (NB != 0) {
211      if (SrcBitmap < SrcBitmapStart || (SrcBitmap + NB) > SrcNameStart)
212        return 1;
213      // Merge Src and Dst Bitmap bytes by simply ORing them together.
214      for (unsigned I = 0; I < NB; I++)
215        DstBitmap[I] |= SrcBitmap[I];
216    }
217
218    /* Now merge value profile data. */
219    if (!VPMergeHook)
220      continue;
221
222    for (unsigned I = 0; I <= IPVK_Last; I++)
223      NVK += (SrcData->NumValueSites[I] != 0);
224
225    if (!NVK)
226      continue;
227
228    if (SrcValueProfData >= ProfileData + ProfileSize)
229      return 1;
230    VPMergeHook((ValueProfData *)SrcValueProfData, DstData);
231    SrcValueProfData =
232        SrcValueProfData + ((ValueProfData *)SrcValueProfData)->TotalSize;
233  }
234
235  return 0;
236}
237
238#ifdef __GNUC__
239#pragma GCC diagnostic pop
240#elif defined(__clang__)
241#pragma clang diagnostic pop
242#endif
243