1//===-- xray_interface.cpp --------------------------------------*- C++ -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file is a part of XRay, a dynamic runtime instrumentation system. 10// 11// Implementation of the API functions. 12// 13//===----------------------------------------------------------------------===// 14 15#include "xray_interface_internal.h" 16 17#include <cstdint> 18#include <cstdio> 19#include <errno.h> 20#include <limits> 21#include <string.h> 22#include <sys/mman.h> 23 24#if SANITIZER_FUCHSIA 25#include <zircon/process.h> 26#include <zircon/sanitizer.h> 27#include <zircon/status.h> 28#include <zircon/syscalls.h> 29#endif 30 31#include "sanitizer_common/sanitizer_addrhashmap.h" 32#include "sanitizer_common/sanitizer_common.h" 33 34#include "xray_defs.h" 35#include "xray_flags.h" 36 37extern __sanitizer::SpinMutex XRayInstrMapMutex; 38extern __sanitizer::atomic_uint8_t XRayInitialized; 39extern __xray::XRaySledMap XRayInstrMap; 40 41namespace __xray { 42 43#if defined(__x86_64__) 44static const int16_t cSledLength = 12; 45#elif defined(__aarch64__) 46static const int16_t cSledLength = 32; 47#elif defined(__arm__) 48static const int16_t cSledLength = 28; 49#elif SANITIZER_MIPS32 50static const int16_t cSledLength = 48; 51#elif SANITIZER_MIPS64 52static const int16_t cSledLength = 64; 53#elif defined(__powerpc64__) 54static const int16_t cSledLength = 8; 55#else 56#error "Unsupported CPU Architecture" 57#endif /* CPU architecture */ 58 59// This is the function to call when we encounter the entry or exit sleds. 60atomic_uintptr_t XRayPatchedFunction{0}; 61 62// This is the function to call from the arg1-enabled sleds/trampolines. 63atomic_uintptr_t XRayArgLogger{0}; 64 65// This is the function to call when we encounter a custom event log call. 66atomic_uintptr_t XRayPatchedCustomEvent{0}; 67 68// This is the function to call when we encounter a typed event log call. 69atomic_uintptr_t XRayPatchedTypedEvent{0}; 70 71// This is the global status to determine whether we are currently 72// patching/unpatching. 73atomic_uint8_t XRayPatching{0}; 74 75struct TypeDescription { 76 uint32_t type_id; 77 std::size_t description_string_length; 78}; 79 80using TypeDescriptorMapType = AddrHashMap<TypeDescription, 11>; 81// An address map from immutable descriptors to type ids. 82TypeDescriptorMapType TypeDescriptorAddressMap{}; 83 84atomic_uint32_t TypeEventDescriptorCounter{0}; 85 86// MProtectHelper is an RAII wrapper for calls to mprotect(...) that will 87// undo any successful mprotect(...) changes. This is used to make a page 88// writeable and executable, and upon destruction if it was successful in 89// doing so returns the page into a read-only and executable page. 90// 91// This is only used specifically for runtime-patching of the XRay 92// instrumentation points. This assumes that the executable pages are 93// originally read-and-execute only. 94class MProtectHelper { 95 void *PageAlignedAddr; 96 std::size_t MProtectLen; 97 bool MustCleanup; 98 99public: 100 explicit MProtectHelper(void *PageAlignedAddr, 101 std::size_t MProtectLen, 102 std::size_t PageSize) XRAY_NEVER_INSTRUMENT 103 : PageAlignedAddr(PageAlignedAddr), 104 MProtectLen(MProtectLen), 105 MustCleanup(false) { 106#if SANITIZER_FUCHSIA 107 MProtectLen = RoundUpTo(MProtectLen, PageSize); 108#endif 109 } 110 111 int MakeWriteable() XRAY_NEVER_INSTRUMENT { 112#if SANITIZER_FUCHSIA 113 auto R = __sanitizer_change_code_protection( 114 reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, true); 115 if (R != ZX_OK) { 116 Report("XRay: cannot change code protection: %s\n", 117 _zx_status_get_string(R)); 118 return -1; 119 } 120 MustCleanup = true; 121 return 0; 122#else 123 auto R = mprotect(PageAlignedAddr, MProtectLen, 124 PROT_READ | PROT_WRITE | PROT_EXEC); 125 if (R != -1) 126 MustCleanup = true; 127 return R; 128#endif 129 } 130 131 ~MProtectHelper() XRAY_NEVER_INSTRUMENT { 132 if (MustCleanup) { 133#if SANITIZER_FUCHSIA 134 auto R = __sanitizer_change_code_protection( 135 reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, false); 136 if (R != ZX_OK) { 137 Report("XRay: cannot change code protection: %s\n", 138 _zx_status_get_string(R)); 139 } 140#else 141 mprotect(PageAlignedAddr, MProtectLen, PROT_READ | PROT_EXEC); 142#endif 143 } 144 } 145}; 146 147namespace { 148 149bool patchSled(const XRaySledEntry &Sled, bool Enable, 150 int32_t FuncId) XRAY_NEVER_INSTRUMENT { 151 bool Success = false; 152 switch (Sled.Kind) { 153 case XRayEntryType::ENTRY: 154 Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_FunctionEntry); 155 break; 156 case XRayEntryType::EXIT: 157 Success = patchFunctionExit(Enable, FuncId, Sled); 158 break; 159 case XRayEntryType::TAIL: 160 Success = patchFunctionTailExit(Enable, FuncId, Sled); 161 break; 162 case XRayEntryType::LOG_ARGS_ENTRY: 163 Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_ArgLoggerEntry); 164 break; 165 case XRayEntryType::CUSTOM_EVENT: 166 Success = patchCustomEvent(Enable, FuncId, Sled); 167 break; 168 case XRayEntryType::TYPED_EVENT: 169 Success = patchTypedEvent(Enable, FuncId, Sled); 170 break; 171 default: 172 Report("Unsupported sled kind '%d' @%04x\n", Sled.Address, int(Sled.Kind)); 173 return false; 174 } 175 return Success; 176} 177 178const XRayFunctionSledIndex 179findFunctionSleds(int32_t FuncId, 180 const XRaySledMap &InstrMap) XRAY_NEVER_INSTRUMENT { 181 int32_t CurFn = 0; 182 uint64_t LastFnAddr = 0; 183 XRayFunctionSledIndex Index = {nullptr, nullptr}; 184 185 for (std::size_t I = 0; I < InstrMap.Entries && CurFn <= FuncId; I++) { 186 const auto &Sled = InstrMap.Sleds[I]; 187 const auto Function = Sled.function(); 188 if (Function != LastFnAddr) { 189 CurFn++; 190 LastFnAddr = Function; 191 } 192 193 if (CurFn == FuncId) { 194 if (Index.Begin == nullptr) 195 Index.Begin = &Sled; 196 Index.End = &Sled; 197 } 198 } 199 200 Index.End += 1; 201 202 return Index; 203} 204 205XRayPatchingStatus patchFunction(int32_t FuncId, 206 bool Enable) XRAY_NEVER_INSTRUMENT { 207 if (!atomic_load(&XRayInitialized, 208 memory_order_acquire)) 209 return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized. 210 211 uint8_t NotPatching = false; 212 if (!atomic_compare_exchange_strong( 213 &XRayPatching, &NotPatching, true, memory_order_acq_rel)) 214 return XRayPatchingStatus::ONGOING; // Already patching. 215 216 // Next, we look for the function index. 217 XRaySledMap InstrMap; 218 { 219 SpinMutexLock Guard(&XRayInstrMapMutex); 220 InstrMap = XRayInstrMap; 221 } 222 223 // If we don't have an index, we can't patch individual functions. 224 if (InstrMap.Functions == 0) 225 return XRayPatchingStatus::NOT_INITIALIZED; 226 227 // FuncId must be a positive number, less than the number of functions 228 // instrumented. 229 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) { 230 Report("Invalid function id provided: %d\n", FuncId); 231 return XRayPatchingStatus::FAILED; 232 } 233 234 // Now we patch ths sleds for this specific function. 235 auto SledRange = InstrMap.SledsIndex ? InstrMap.SledsIndex[FuncId - 1] 236 : findFunctionSleds(FuncId, InstrMap); 237 auto *f = SledRange.Begin; 238 auto *e = SledRange.End; 239 bool SucceedOnce = false; 240 while (f != e) 241 SucceedOnce |= patchSled(*f++, Enable, FuncId); 242 243 atomic_store(&XRayPatching, false, 244 memory_order_release); 245 246 if (!SucceedOnce) { 247 Report("Failed patching any sled for function '%d'.", FuncId); 248 return XRayPatchingStatus::FAILED; 249 } 250 251 return XRayPatchingStatus::SUCCESS; 252} 253 254// controlPatching implements the common internals of the patching/unpatching 255// implementation. |Enable| defines whether we're enabling or disabling the 256// runtime XRay instrumentation. 257XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT { 258 if (!atomic_load(&XRayInitialized, 259 memory_order_acquire)) 260 return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized. 261 262 uint8_t NotPatching = false; 263 if (!atomic_compare_exchange_strong( 264 &XRayPatching, &NotPatching, true, memory_order_acq_rel)) 265 return XRayPatchingStatus::ONGOING; // Already patching. 266 267 uint8_t PatchingSuccess = false; 268 auto XRayPatchingStatusResetter = 269 at_scope_exit([&PatchingSuccess] { 270 if (!PatchingSuccess) 271 atomic_store(&XRayPatching, false, 272 memory_order_release); 273 }); 274 275 XRaySledMap InstrMap; 276 { 277 SpinMutexLock Guard(&XRayInstrMapMutex); 278 InstrMap = XRayInstrMap; 279 } 280 if (InstrMap.Entries == 0) 281 return XRayPatchingStatus::NOT_INITIALIZED; 282 283 uint32_t FuncId = 1; 284 uint64_t CurFun = 0; 285 286 // First we want to find the bounds for which we have instrumentation points, 287 // and try to get as few calls to mprotect(...) as possible. We're assuming 288 // that all the sleds for the instrumentation map are contiguous as a single 289 // set of pages. When we do support dynamic shared object instrumentation, 290 // we'll need to do this for each set of page load offsets per DSO loaded. For 291 // now we're assuming we can mprotect the whole section of text between the 292 // minimum sled address and the maximum sled address (+ the largest sled 293 // size). 294 auto *MinSled = &InstrMap.Sleds[0]; 295 auto *MaxSled = &InstrMap.Sleds[InstrMap.Entries - 1]; 296 for (std::size_t I = 0; I < InstrMap.Entries; I++) { 297 const auto &Sled = InstrMap.Sleds[I]; 298 if (Sled.address() < MinSled->address()) 299 MinSled = &Sled; 300 if (Sled.address() > MaxSled->address()) 301 MaxSled = &Sled; 302 } 303 304 const size_t PageSize = flags()->xray_page_size_override > 0 305 ? flags()->xray_page_size_override 306 : GetPageSizeCached(); 307 if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) { 308 Report("System page size is not a power of two: %lld\n", PageSize); 309 return XRayPatchingStatus::FAILED; 310 } 311 312 void *PageAlignedAddr = 313 reinterpret_cast<void *>(MinSled->address() & ~(PageSize - 1)); 314 size_t MProtectLen = 315 (MaxSled->address() - reinterpret_cast<uptr>(PageAlignedAddr)) + 316 cSledLength; 317 MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize); 318 if (Protector.MakeWriteable() == -1) { 319 Report("Failed mprotect: %d\n", errno); 320 return XRayPatchingStatus::FAILED; 321 } 322 323 for (std::size_t I = 0; I < InstrMap.Entries; ++I) { 324 auto &Sled = InstrMap.Sleds[I]; 325 auto F = Sled.function(); 326 if (CurFun == 0) 327 CurFun = F; 328 if (F != CurFun) { 329 ++FuncId; 330 CurFun = F; 331 } 332 patchSled(Sled, Enable, FuncId); 333 } 334 atomic_store(&XRayPatching, false, 335 memory_order_release); 336 PatchingSuccess = true; 337 return XRayPatchingStatus::SUCCESS; 338} 339 340XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId, 341 bool Enable) XRAY_NEVER_INSTRUMENT { 342 XRaySledMap InstrMap; 343 { 344 SpinMutexLock Guard(&XRayInstrMapMutex); 345 InstrMap = XRayInstrMap; 346 } 347 348 // FuncId must be a positive number, less than the number of functions 349 // instrumented. 350 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) { 351 Report("Invalid function id provided: %d\n", FuncId); 352 return XRayPatchingStatus::FAILED; 353 } 354 355 const size_t PageSize = flags()->xray_page_size_override > 0 356 ? flags()->xray_page_size_override 357 : GetPageSizeCached(); 358 if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) { 359 Report("Provided page size is not a power of two: %lld\n", PageSize); 360 return XRayPatchingStatus::FAILED; 361 } 362 363 // Here we compute the minumum sled and maximum sled associated with a 364 // particular function ID. 365 auto SledRange = InstrMap.SledsIndex ? InstrMap.SledsIndex[FuncId - 1] 366 : findFunctionSleds(FuncId, InstrMap); 367 auto *f = SledRange.Begin; 368 auto *e = SledRange.End; 369 auto *MinSled = f; 370 auto *MaxSled = (SledRange.End - 1); 371 while (f != e) { 372 if (f->address() < MinSled->address()) 373 MinSled = f; 374 if (f->address() > MaxSled->address()) 375 MaxSled = f; 376 ++f; 377 } 378 379 void *PageAlignedAddr = 380 reinterpret_cast<void *>(MinSled->address() & ~(PageSize - 1)); 381 size_t MProtectLen = 382 (MaxSled->address() - reinterpret_cast<uptr>(PageAlignedAddr)) + 383 cSledLength; 384 MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize); 385 if (Protector.MakeWriteable() == -1) { 386 Report("Failed mprotect: %d\n", errno); 387 return XRayPatchingStatus::FAILED; 388 } 389 return patchFunction(FuncId, Enable); 390} 391 392} // namespace 393 394} // namespace __xray 395 396using namespace __xray; 397 398// The following functions are declared `extern "C" {...}` in the header, hence 399// they're defined in the global namespace. 400 401int __xray_set_handler(void (*entry)(int32_t, 402 XRayEntryType)) XRAY_NEVER_INSTRUMENT { 403 if (atomic_load(&XRayInitialized, 404 memory_order_acquire)) { 405 406 atomic_store(&__xray::XRayPatchedFunction, 407 reinterpret_cast<uintptr_t>(entry), 408 memory_order_release); 409 return 1; 410 } 411 return 0; 412} 413 414int __xray_set_customevent_handler(void (*entry)(void *, size_t)) 415 XRAY_NEVER_INSTRUMENT { 416 if (atomic_load(&XRayInitialized, 417 memory_order_acquire)) { 418 atomic_store(&__xray::XRayPatchedCustomEvent, 419 reinterpret_cast<uintptr_t>(entry), 420 memory_order_release); 421 return 1; 422 } 423 return 0; 424} 425 426int __xray_set_typedevent_handler(void (*entry)( 427 uint16_t, const void *, size_t)) XRAY_NEVER_INSTRUMENT { 428 if (atomic_load(&XRayInitialized, 429 memory_order_acquire)) { 430 atomic_store(&__xray::XRayPatchedTypedEvent, 431 reinterpret_cast<uintptr_t>(entry), 432 memory_order_release); 433 return 1; 434 } 435 return 0; 436} 437 438int __xray_remove_handler() XRAY_NEVER_INSTRUMENT { 439 return __xray_set_handler(nullptr); 440} 441 442int __xray_remove_customevent_handler() XRAY_NEVER_INSTRUMENT { 443 return __xray_set_customevent_handler(nullptr); 444} 445 446int __xray_remove_typedevent_handler() XRAY_NEVER_INSTRUMENT { 447 return __xray_set_typedevent_handler(nullptr); 448} 449 450uint16_t __xray_register_event_type( 451 const char *const event_type) XRAY_NEVER_INSTRUMENT { 452 TypeDescriptorMapType::Handle h(&TypeDescriptorAddressMap, (uptr)event_type); 453 if (h.created()) { 454 h->type_id = atomic_fetch_add( 455 &TypeEventDescriptorCounter, 1, memory_order_acq_rel); 456 h->description_string_length = strnlen(event_type, 1024); 457 } 458 return h->type_id; 459} 460 461XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT { 462 return controlPatching(true); 463} 464 465XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT { 466 return controlPatching(false); 467} 468 469XRayPatchingStatus __xray_patch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT { 470 return mprotectAndPatchFunction(FuncId, true); 471} 472 473XRayPatchingStatus 474__xray_unpatch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT { 475 return mprotectAndPatchFunction(FuncId, false); 476} 477 478int __xray_set_handler_arg1(void (*entry)(int32_t, XRayEntryType, uint64_t)) { 479 if (!atomic_load(&XRayInitialized, 480 memory_order_acquire)) 481 return 0; 482 483 // A relaxed write might not be visible even if the current thread gets 484 // scheduled on a different CPU/NUMA node. We need to wait for everyone to 485 // have this handler installed for consistency of collected data across CPUs. 486 atomic_store(&XRayArgLogger, reinterpret_cast<uint64_t>(entry), 487 memory_order_release); 488 return 1; 489} 490 491int __xray_remove_handler_arg1() { return __xray_set_handler_arg1(nullptr); } 492 493uintptr_t __xray_function_address(int32_t FuncId) XRAY_NEVER_INSTRUMENT { 494 XRaySledMap InstrMap; 495 { 496 SpinMutexLock Guard(&XRayInstrMapMutex); 497 InstrMap = XRayInstrMap; 498 } 499 500 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) 501 return 0; 502 const XRaySledEntry *Sled = InstrMap.SledsIndex 503 ? InstrMap.SledsIndex[FuncId - 1].Begin 504 : findFunctionSleds(FuncId, InstrMap).Begin; 505 return Sled->function() 506// On PPC, function entries are always aligned to 16 bytes. The beginning of a 507// sled might be a local entry, which is always +8 based on the global entry. 508// Always return the global entry. 509#ifdef __PPC__ 510 & ~0xf 511#endif 512 ; 513} 514 515size_t __xray_max_function_id() XRAY_NEVER_INSTRUMENT { 516 SpinMutexLock Guard(&XRayInstrMapMutex); 517 return XRayInstrMap.Functions; 518} 519