1//===-- xray_interface.cpp --------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of XRay, a dynamic runtime instrumentation system.
10//
11// Implementation of the API functions.
12//
13//===----------------------------------------------------------------------===//
14
15#include "xray_interface_internal.h"
16
17#include <cstdint>
18#include <cstdio>
19#include <errno.h>
20#include <limits>
21#include <string.h>
22#include <sys/mman.h>
23
24#if SANITIZER_FUCHSIA
25#include <zircon/process.h>
26#include <zircon/sanitizer.h>
27#include <zircon/status.h>
28#include <zircon/syscalls.h>
29#endif
30
31#include "sanitizer_common/sanitizer_addrhashmap.h"
32#include "sanitizer_common/sanitizer_common.h"
33
34#include "xray_defs.h"
35#include "xray_flags.h"
36
37extern __sanitizer::SpinMutex XRayInstrMapMutex;
38extern __sanitizer::atomic_uint8_t XRayInitialized;
39extern __xray::XRaySledMap XRayInstrMap;
40
41namespace __xray {
42
43#if defined(__x86_64__)
44static const int16_t cSledLength = 12;
45#elif defined(__aarch64__)
46static const int16_t cSledLength = 32;
47#elif defined(__arm__)
48static const int16_t cSledLength = 28;
49#elif SANITIZER_MIPS32
50static const int16_t cSledLength = 48;
51#elif SANITIZER_MIPS64
52static const int16_t cSledLength = 64;
53#elif defined(__powerpc64__)
54static const int16_t cSledLength = 8;
55#else
56#error "Unsupported CPU Architecture"
57#endif /* CPU architecture */
58
59// This is the function to call when we encounter the entry or exit sleds.
60atomic_uintptr_t XRayPatchedFunction{0};
61
62// This is the function to call from the arg1-enabled sleds/trampolines.
63atomic_uintptr_t XRayArgLogger{0};
64
65// This is the function to call when we encounter a custom event log call.
66atomic_uintptr_t XRayPatchedCustomEvent{0};
67
68// This is the function to call when we encounter a typed event log call.
69atomic_uintptr_t XRayPatchedTypedEvent{0};
70
71// This is the global status to determine whether we are currently
72// patching/unpatching.
73atomic_uint8_t XRayPatching{0};
74
75struct TypeDescription {
76  uint32_t type_id;
77  std::size_t description_string_length;
78};
79
80using TypeDescriptorMapType = AddrHashMap<TypeDescription, 11>;
81// An address map from immutable descriptors to type ids.
82TypeDescriptorMapType TypeDescriptorAddressMap{};
83
84atomic_uint32_t TypeEventDescriptorCounter{0};
85
86// MProtectHelper is an RAII wrapper for calls to mprotect(...) that will
87// undo any successful mprotect(...) changes. This is used to make a page
88// writeable and executable, and upon destruction if it was successful in
89// doing so returns the page into a read-only and executable page.
90//
91// This is only used specifically for runtime-patching of the XRay
92// instrumentation points. This assumes that the executable pages are
93// originally read-and-execute only.
94class MProtectHelper {
95  void *PageAlignedAddr;
96  std::size_t MProtectLen;
97  bool MustCleanup;
98
99public:
100  explicit MProtectHelper(void *PageAlignedAddr,
101                          std::size_t MProtectLen,
102                          std::size_t PageSize) XRAY_NEVER_INSTRUMENT
103      : PageAlignedAddr(PageAlignedAddr),
104        MProtectLen(MProtectLen),
105        MustCleanup(false) {
106#if SANITIZER_FUCHSIA
107    MProtectLen = RoundUpTo(MProtectLen, PageSize);
108#endif
109  }
110
111  int MakeWriteable() XRAY_NEVER_INSTRUMENT {
112#if SANITIZER_FUCHSIA
113    auto R = __sanitizer_change_code_protection(
114        reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, true);
115    if (R != ZX_OK) {
116      Report("XRay: cannot change code protection: %s\n",
117             _zx_status_get_string(R));
118      return -1;
119    }
120    MustCleanup = true;
121    return 0;
122#else
123    auto R = mprotect(PageAlignedAddr, MProtectLen,
124                      PROT_READ | PROT_WRITE | PROT_EXEC);
125    if (R != -1)
126      MustCleanup = true;
127    return R;
128#endif
129  }
130
131  ~MProtectHelper() XRAY_NEVER_INSTRUMENT {
132    if (MustCleanup) {
133#if SANITIZER_FUCHSIA
134      auto R = __sanitizer_change_code_protection(
135          reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, false);
136      if (R != ZX_OK) {
137        Report("XRay: cannot change code protection: %s\n",
138               _zx_status_get_string(R));
139      }
140#else
141      mprotect(PageAlignedAddr, MProtectLen, PROT_READ | PROT_EXEC);
142#endif
143    }
144  }
145};
146
147namespace {
148
149bool patchSled(const XRaySledEntry &Sled, bool Enable,
150               int32_t FuncId) XRAY_NEVER_INSTRUMENT {
151  bool Success = false;
152  switch (Sled.Kind) {
153  case XRayEntryType::ENTRY:
154    Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_FunctionEntry);
155    break;
156  case XRayEntryType::EXIT:
157    Success = patchFunctionExit(Enable, FuncId, Sled);
158    break;
159  case XRayEntryType::TAIL:
160    Success = patchFunctionTailExit(Enable, FuncId, Sled);
161    break;
162  case XRayEntryType::LOG_ARGS_ENTRY:
163    Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_ArgLoggerEntry);
164    break;
165  case XRayEntryType::CUSTOM_EVENT:
166    Success = patchCustomEvent(Enable, FuncId, Sled);
167    break;
168  case XRayEntryType::TYPED_EVENT:
169    Success = patchTypedEvent(Enable, FuncId, Sled);
170    break;
171  default:
172    Report("Unsupported sled kind '%d' @%04x\n", Sled.Address, int(Sled.Kind));
173    return false;
174  }
175  return Success;
176}
177
178const XRayFunctionSledIndex
179findFunctionSleds(int32_t FuncId,
180                  const XRaySledMap &InstrMap) XRAY_NEVER_INSTRUMENT {
181  int32_t CurFn = 0;
182  uint64_t LastFnAddr = 0;
183  XRayFunctionSledIndex Index = {nullptr, nullptr};
184
185  for (std::size_t I = 0; I < InstrMap.Entries && CurFn <= FuncId; I++) {
186    const auto &Sled = InstrMap.Sleds[I];
187    const auto Function = Sled.function();
188    if (Function != LastFnAddr) {
189      CurFn++;
190      LastFnAddr = Function;
191    }
192
193    if (CurFn == FuncId) {
194      if (Index.Begin == nullptr)
195        Index.Begin = &Sled;
196      Index.End = &Sled;
197    }
198  }
199
200  Index.End += 1;
201
202  return Index;
203}
204
205XRayPatchingStatus patchFunction(int32_t FuncId,
206                                 bool Enable) XRAY_NEVER_INSTRUMENT {
207  if (!atomic_load(&XRayInitialized,
208                                memory_order_acquire))
209    return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
210
211  uint8_t NotPatching = false;
212  if (!atomic_compare_exchange_strong(
213          &XRayPatching, &NotPatching, true, memory_order_acq_rel))
214    return XRayPatchingStatus::ONGOING; // Already patching.
215
216  // Next, we look for the function index.
217  XRaySledMap InstrMap;
218  {
219    SpinMutexLock Guard(&XRayInstrMapMutex);
220    InstrMap = XRayInstrMap;
221  }
222
223  // If we don't have an index, we can't patch individual functions.
224  if (InstrMap.Functions == 0)
225    return XRayPatchingStatus::NOT_INITIALIZED;
226
227  // FuncId must be a positive number, less than the number of functions
228  // instrumented.
229  if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
230    Report("Invalid function id provided: %d\n", FuncId);
231    return XRayPatchingStatus::FAILED;
232  }
233
234  // Now we patch ths sleds for this specific function.
235  auto SledRange = InstrMap.SledsIndex ? InstrMap.SledsIndex[FuncId - 1]
236                                       : findFunctionSleds(FuncId, InstrMap);
237  auto *f = SledRange.Begin;
238  auto *e = SledRange.End;
239  bool SucceedOnce = false;
240  while (f != e)
241    SucceedOnce |= patchSled(*f++, Enable, FuncId);
242
243  atomic_store(&XRayPatching, false,
244                            memory_order_release);
245
246  if (!SucceedOnce) {
247    Report("Failed patching any sled for function '%d'.", FuncId);
248    return XRayPatchingStatus::FAILED;
249  }
250
251  return XRayPatchingStatus::SUCCESS;
252}
253
254// controlPatching implements the common internals of the patching/unpatching
255// implementation. |Enable| defines whether we're enabling or disabling the
256// runtime XRay instrumentation.
257XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
258  if (!atomic_load(&XRayInitialized,
259                                memory_order_acquire))
260    return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
261
262  uint8_t NotPatching = false;
263  if (!atomic_compare_exchange_strong(
264          &XRayPatching, &NotPatching, true, memory_order_acq_rel))
265    return XRayPatchingStatus::ONGOING; // Already patching.
266
267  uint8_t PatchingSuccess = false;
268  auto XRayPatchingStatusResetter =
269      at_scope_exit([&PatchingSuccess] {
270        if (!PatchingSuccess)
271          atomic_store(&XRayPatching, false,
272                                    memory_order_release);
273      });
274
275  XRaySledMap InstrMap;
276  {
277    SpinMutexLock Guard(&XRayInstrMapMutex);
278    InstrMap = XRayInstrMap;
279  }
280  if (InstrMap.Entries == 0)
281    return XRayPatchingStatus::NOT_INITIALIZED;
282
283  uint32_t FuncId = 1;
284  uint64_t CurFun = 0;
285
286  // First we want to find the bounds for which we have instrumentation points,
287  // and try to get as few calls to mprotect(...) as possible. We're assuming
288  // that all the sleds for the instrumentation map are contiguous as a single
289  // set of pages. When we do support dynamic shared object instrumentation,
290  // we'll need to do this for each set of page load offsets per DSO loaded. For
291  // now we're assuming we can mprotect the whole section of text between the
292  // minimum sled address and the maximum sled address (+ the largest sled
293  // size).
294  auto *MinSled = &InstrMap.Sleds[0];
295  auto *MaxSled = &InstrMap.Sleds[InstrMap.Entries - 1];
296  for (std::size_t I = 0; I < InstrMap.Entries; I++) {
297    const auto &Sled = InstrMap.Sleds[I];
298    if (Sled.address() < MinSled->address())
299      MinSled = &Sled;
300    if (Sled.address() > MaxSled->address())
301      MaxSled = &Sled;
302  }
303
304  const size_t PageSize = flags()->xray_page_size_override > 0
305                              ? flags()->xray_page_size_override
306                              : GetPageSizeCached();
307  if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
308    Report("System page size is not a power of two: %lld\n", PageSize);
309    return XRayPatchingStatus::FAILED;
310  }
311
312  void *PageAlignedAddr =
313      reinterpret_cast<void *>(MinSled->address() & ~(PageSize - 1));
314  size_t MProtectLen =
315      (MaxSled->address() - reinterpret_cast<uptr>(PageAlignedAddr)) +
316      cSledLength;
317  MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
318  if (Protector.MakeWriteable() == -1) {
319    Report("Failed mprotect: %d\n", errno);
320    return XRayPatchingStatus::FAILED;
321  }
322
323  for (std::size_t I = 0; I < InstrMap.Entries; ++I) {
324    auto &Sled = InstrMap.Sleds[I];
325    auto F = Sled.function();
326    if (CurFun == 0)
327      CurFun = F;
328    if (F != CurFun) {
329      ++FuncId;
330      CurFun = F;
331    }
332    patchSled(Sled, Enable, FuncId);
333  }
334  atomic_store(&XRayPatching, false,
335                            memory_order_release);
336  PatchingSuccess = true;
337  return XRayPatchingStatus::SUCCESS;
338}
339
340XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId,
341                                            bool Enable) XRAY_NEVER_INSTRUMENT {
342  XRaySledMap InstrMap;
343  {
344    SpinMutexLock Guard(&XRayInstrMapMutex);
345    InstrMap = XRayInstrMap;
346  }
347
348  // FuncId must be a positive number, less than the number of functions
349  // instrumented.
350  if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
351    Report("Invalid function id provided: %d\n", FuncId);
352    return XRayPatchingStatus::FAILED;
353  }
354
355  const size_t PageSize = flags()->xray_page_size_override > 0
356                              ? flags()->xray_page_size_override
357                              : GetPageSizeCached();
358  if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
359    Report("Provided page size is not a power of two: %lld\n", PageSize);
360    return XRayPatchingStatus::FAILED;
361  }
362
363  // Here we compute the minumum sled and maximum sled associated with a
364  // particular function ID.
365  auto SledRange = InstrMap.SledsIndex ? InstrMap.SledsIndex[FuncId - 1]
366                                       : findFunctionSleds(FuncId, InstrMap);
367  auto *f = SledRange.Begin;
368  auto *e = SledRange.End;
369  auto *MinSled = f;
370  auto *MaxSled = (SledRange.End - 1);
371  while (f != e) {
372    if (f->address() < MinSled->address())
373      MinSled = f;
374    if (f->address() > MaxSled->address())
375      MaxSled = f;
376    ++f;
377  }
378
379  void *PageAlignedAddr =
380      reinterpret_cast<void *>(MinSled->address() & ~(PageSize - 1));
381  size_t MProtectLen =
382      (MaxSled->address() - reinterpret_cast<uptr>(PageAlignedAddr)) +
383      cSledLength;
384  MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
385  if (Protector.MakeWriteable() == -1) {
386    Report("Failed mprotect: %d\n", errno);
387    return XRayPatchingStatus::FAILED;
388  }
389  return patchFunction(FuncId, Enable);
390}
391
392} // namespace
393
394} // namespace __xray
395
396using namespace __xray;
397
398// The following functions are declared `extern "C" {...}` in the header, hence
399// they're defined in the global namespace.
400
401int __xray_set_handler(void (*entry)(int32_t,
402                                     XRayEntryType)) XRAY_NEVER_INSTRUMENT {
403  if (atomic_load(&XRayInitialized,
404                               memory_order_acquire)) {
405
406    atomic_store(&__xray::XRayPatchedFunction,
407                              reinterpret_cast<uintptr_t>(entry),
408                              memory_order_release);
409    return 1;
410  }
411  return 0;
412}
413
414int __xray_set_customevent_handler(void (*entry)(void *, size_t))
415    XRAY_NEVER_INSTRUMENT {
416  if (atomic_load(&XRayInitialized,
417                               memory_order_acquire)) {
418    atomic_store(&__xray::XRayPatchedCustomEvent,
419                              reinterpret_cast<uintptr_t>(entry),
420                              memory_order_release);
421    return 1;
422  }
423  return 0;
424}
425
426int __xray_set_typedevent_handler(void (*entry)(
427    uint16_t, const void *, size_t)) XRAY_NEVER_INSTRUMENT {
428  if (atomic_load(&XRayInitialized,
429                               memory_order_acquire)) {
430    atomic_store(&__xray::XRayPatchedTypedEvent,
431                              reinterpret_cast<uintptr_t>(entry),
432                              memory_order_release);
433    return 1;
434  }
435  return 0;
436}
437
438int __xray_remove_handler() XRAY_NEVER_INSTRUMENT {
439  return __xray_set_handler(nullptr);
440}
441
442int __xray_remove_customevent_handler() XRAY_NEVER_INSTRUMENT {
443  return __xray_set_customevent_handler(nullptr);
444}
445
446int __xray_remove_typedevent_handler() XRAY_NEVER_INSTRUMENT {
447  return __xray_set_typedevent_handler(nullptr);
448}
449
450uint16_t __xray_register_event_type(
451    const char *const event_type) XRAY_NEVER_INSTRUMENT {
452  TypeDescriptorMapType::Handle h(&TypeDescriptorAddressMap, (uptr)event_type);
453  if (h.created()) {
454    h->type_id = atomic_fetch_add(
455        &TypeEventDescriptorCounter, 1, memory_order_acq_rel);
456    h->description_string_length = strnlen(event_type, 1024);
457  }
458  return h->type_id;
459}
460
461XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT {
462  return controlPatching(true);
463}
464
465XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT {
466  return controlPatching(false);
467}
468
469XRayPatchingStatus __xray_patch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
470  return mprotectAndPatchFunction(FuncId, true);
471}
472
473XRayPatchingStatus
474__xray_unpatch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
475  return mprotectAndPatchFunction(FuncId, false);
476}
477
478int __xray_set_handler_arg1(void (*entry)(int32_t, XRayEntryType, uint64_t)) {
479  if (!atomic_load(&XRayInitialized,
480                                memory_order_acquire))
481    return 0;
482
483  // A relaxed write might not be visible even if the current thread gets
484  // scheduled on a different CPU/NUMA node.  We need to wait for everyone to
485  // have this handler installed for consistency of collected data across CPUs.
486  atomic_store(&XRayArgLogger, reinterpret_cast<uint64_t>(entry),
487                            memory_order_release);
488  return 1;
489}
490
491int __xray_remove_handler_arg1() { return __xray_set_handler_arg1(nullptr); }
492
493uintptr_t __xray_function_address(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
494  XRaySledMap InstrMap;
495  {
496    SpinMutexLock Guard(&XRayInstrMapMutex);
497    InstrMap = XRayInstrMap;
498  }
499
500  if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions)
501    return 0;
502  const XRaySledEntry *Sled = InstrMap.SledsIndex
503                                  ? InstrMap.SledsIndex[FuncId - 1].Begin
504                                  : findFunctionSleds(FuncId, InstrMap).Begin;
505  return Sled->function()
506// On PPC, function entries are always aligned to 16 bytes. The beginning of a
507// sled might be a local entry, which is always +8 based on the global entry.
508// Always return the global entry.
509#ifdef __PPC__
510         & ~0xf
511#endif
512      ;
513}
514
515size_t __xray_max_function_id() XRAY_NEVER_INSTRUMENT {
516  SpinMutexLock Guard(&XRayInstrMapMutex);
517  return XRayInstrMap.Functions;
518}
519