PerfJITEventListener.cpp revision 336916
1//===-- PerfJITEventListener.cpp - Tell Linux's perf about JITted code ----===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines a JITEventListener object that tells perf about JITted 11// functions, including source line information. 12// 13// Documentation for perf jit integration is available at: 14// https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jitdump-specification.txt 15// https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jit-interface.txt 16// 17//===----------------------------------------------------------------------===// 18 19#include "llvm/ADT/Twine.h" 20#include "llvm/Config/config.h" 21#include "llvm/DebugInfo/DWARF/DWARFContext.h" 22#include "llvm/ExecutionEngine/JITEventListener.h" 23#include "llvm/Object/ObjectFile.h" 24#include "llvm/Object/SymbolSize.h" 25#include "llvm/Support/Debug.h" 26#include "llvm/Support/Errno.h" 27#include "llvm/Support/FileSystem.h" 28#include "llvm/Support/MemoryBuffer.h" 29#include "llvm/Support/Mutex.h" 30#include "llvm/Support/MutexGuard.h" 31#include "llvm/Support/Path.h" 32#include "llvm/Support/Process.h" 33#include "llvm/Support/Threading.h" 34#include "llvm/Support/raw_ostream.h" 35 36#include <sys/mman.h> // mmap() 37#include <sys/types.h> // getpid() 38#include <time.h> // clock_gettime(), time(), localtime_r() */ 39#include <unistd.h> // for getpid(), read(), close() 40 41using namespace llvm; 42using namespace llvm::object; 43typedef DILineInfoSpecifier::FileLineInfoKind FileLineInfoKind; 44 45namespace { 46 47// language identifier (XXX: should we generate something better from debug 48// info?) 49#define JIT_LANG "llvm-IR" 50#define LLVM_PERF_JIT_MAGIC \ 51 ((uint32_t)'J' << 24 | (uint32_t)'i' << 16 | (uint32_t)'T' << 8 | \ 52 (uint32_t)'D') 53#define LLVM_PERF_JIT_VERSION 1 54 55// bit 0: set if the jitdump file is using an architecture-specific timestamp 56// clock source 57#define JITDUMP_FLAGS_ARCH_TIMESTAMP (1ULL << 0) 58 59struct LLVMPerfJitHeader; 60 61class PerfJITEventListener : public JITEventListener { 62public: 63 PerfJITEventListener(); 64 ~PerfJITEventListener() { 65 if (MarkerAddr) 66 CloseMarker(); 67 } 68 69 void NotifyObjectEmitted(const ObjectFile &Obj, 70 const RuntimeDyld::LoadedObjectInfo &L) override; 71 void NotifyFreeingObject(const ObjectFile &Obj) override; 72 73private: 74 bool InitDebuggingDir(); 75 bool OpenMarker(); 76 void CloseMarker(); 77 static bool FillMachine(LLVMPerfJitHeader &hdr); 78 79 void NotifyCode(Expected<llvm::StringRef> &Symbol, uint64_t CodeAddr, 80 uint64_t CodeSize); 81 void NotifyDebug(uint64_t CodeAddr, DILineInfoTable Lines); 82 83 // cache lookups 84 pid_t Pid; 85 86 // base directory for output data 87 std::string JitPath; 88 89 // output data stream, closed via Dumpstream 90 int DumpFd = -1; 91 92 // output data stream 93 std::unique_ptr<raw_fd_ostream> Dumpstream; 94 95 // prevent concurrent dumps from messing up the output file 96 sys::Mutex Mutex; 97 98 // perf mmap marker 99 void *MarkerAddr = NULL; 100 101 // perf support ready 102 bool SuccessfullyInitialized = false; 103 104 // identifier for functions, primarily to identify when moving them around 105 uint64_t CodeGeneration = 1; 106}; 107 108// The following are POD struct definitions from the perf jit specification 109 110enum LLVMPerfJitRecordType { 111 JIT_CODE_LOAD = 0, 112 JIT_CODE_MOVE = 1, // not emitted, code isn't moved 113 JIT_CODE_DEBUG_INFO = 2, 114 JIT_CODE_CLOSE = 3, // not emitted, unnecessary 115 JIT_CODE_UNWINDING_INFO = 4, // not emitted 116 117 JIT_CODE_MAX 118}; 119 120struct LLVMPerfJitHeader { 121 uint32_t Magic; // characters "JiTD" 122 uint32_t Version; // header version 123 uint32_t TotalSize; // total size of header 124 uint32_t ElfMach; // elf mach target 125 uint32_t Pad1; // reserved 126 uint32_t Pid; 127 uint64_t Timestamp; // timestamp 128 uint64_t Flags; // flags 129}; 130 131// record prefix (mandatory in each record) 132struct LLVMPerfJitRecordPrefix { 133 uint32_t Id; // record type identifier 134 uint32_t TotalSize; 135 uint64_t Timestamp; 136}; 137 138struct LLVMPerfJitRecordCodeLoad { 139 LLVMPerfJitRecordPrefix Prefix; 140 141 uint32_t Pid; 142 uint32_t Tid; 143 uint64_t Vma; 144 uint64_t CodeAddr; 145 uint64_t CodeSize; 146 uint64_t CodeIndex; 147}; 148 149struct LLVMPerfJitDebugEntry { 150 uint64_t Addr; 151 int Lineno; // source line number starting at 1 152 int Discrim; // column discriminator, 0 is default 153 // followed by null terminated filename, \xff\0 if same as previous entry 154}; 155 156struct LLVMPerfJitRecordDebugInfo { 157 LLVMPerfJitRecordPrefix Prefix; 158 159 uint64_t CodeAddr; 160 uint64_t NrEntry; 161 // followed by NrEntry LLVMPerfJitDebugEntry records 162}; 163 164static inline uint64_t timespec_to_ns(const struct timespec *ts) { 165 const uint64_t NanoSecPerSec = 1000000000; 166 return ((uint64_t)ts->tv_sec * NanoSecPerSec) + ts->tv_nsec; 167} 168 169static inline uint64_t perf_get_timestamp(void) { 170 struct timespec ts; 171 int ret; 172 173 ret = clock_gettime(CLOCK_MONOTONIC, &ts); 174 if (ret) 175 return 0; 176 177 return timespec_to_ns(&ts); 178} 179 180PerfJITEventListener::PerfJITEventListener() : Pid(::getpid()) { 181 // check if clock-source is supported 182 if (!perf_get_timestamp()) { 183 errs() << "kernel does not support CLOCK_MONOTONIC\n"; 184 return; 185 } 186 187 if (!InitDebuggingDir()) { 188 errs() << "could not initialize debugging directory\n"; 189 return; 190 } 191 192 std::string Filename; 193 raw_string_ostream FilenameBuf(Filename); 194 FilenameBuf << JitPath << "/jit-" << Pid << ".dump"; 195 196 // Need to open ourselves, because we need to hand the FD to OpenMarker() and 197 // raw_fd_ostream doesn't expose the FD. 198 using sys::fs::openFileForWrite; 199 if (auto EC = 200 openFileForReadWrite(FilenameBuf.str(), DumpFd, 201 sys::fs::CD_CreateNew, sys::fs::OF_None)) { 202 errs() << "could not open JIT dump file " << FilenameBuf.str() << ": " 203 << EC.message() << "\n"; 204 return; 205 } 206 207 Dumpstream = make_unique<raw_fd_ostream>(DumpFd, true); 208 209 LLVMPerfJitHeader Header = {0}; 210 if (!FillMachine(Header)) 211 return; 212 213 // signal this process emits JIT information 214 if (!OpenMarker()) 215 return; 216 217 // emit dumpstream header 218 Header.Magic = LLVM_PERF_JIT_MAGIC; 219 Header.Version = LLVM_PERF_JIT_VERSION; 220 Header.TotalSize = sizeof(Header); 221 Header.Pid = Pid; 222 Header.Timestamp = perf_get_timestamp(); 223 Dumpstream->write(reinterpret_cast<const char *>(&Header), sizeof(Header)); 224 225 // Everything initialized, can do profiling now. 226 if (!Dumpstream->has_error()) 227 SuccessfullyInitialized = true; 228} 229 230void PerfJITEventListener::NotifyObjectEmitted( 231 const ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &L) { 232 233 if (!SuccessfullyInitialized) 234 return; 235 236 OwningBinary<ObjectFile> DebugObjOwner = L.getObjectForDebug(Obj); 237 const ObjectFile &DebugObj = *DebugObjOwner.getBinary(); 238 239 // Get the address of the object image for use as a unique identifier 240 std::unique_ptr<DIContext> Context = DWARFContext::create(DebugObj); 241 242 // Use symbol info to iterate over functions in the object. 243 for (const std::pair<SymbolRef, uint64_t> &P : computeSymbolSizes(DebugObj)) { 244 SymbolRef Sym = P.first; 245 std::string SourceFileName; 246 247 Expected<SymbolRef::Type> SymTypeOrErr = Sym.getType(); 248 if (!SymTypeOrErr) { 249 // There's not much we can with errors here 250 consumeError(SymTypeOrErr.takeError()); 251 continue; 252 } 253 SymbolRef::Type SymType = *SymTypeOrErr; 254 if (SymType != SymbolRef::ST_Function) 255 continue; 256 257 Expected<StringRef> Name = Sym.getName(); 258 if (!Name) { 259 consumeError(Name.takeError()); 260 continue; 261 } 262 263 Expected<uint64_t> AddrOrErr = Sym.getAddress(); 264 if (!AddrOrErr) { 265 consumeError(AddrOrErr.takeError()); 266 continue; 267 } 268 uint64_t Addr = *AddrOrErr; 269 uint64_t Size = P.second; 270 271 // According to spec debugging info has to come before loading the 272 // corresonding code load. 273 DILineInfoTable Lines = Context->getLineInfoForAddressRange( 274 Addr, Size, FileLineInfoKind::AbsoluteFilePath); 275 276 NotifyDebug(Addr, Lines); 277 NotifyCode(Name, Addr, Size); 278 } 279 280 Dumpstream->flush(); 281} 282 283void PerfJITEventListener::NotifyFreeingObject(const ObjectFile &Obj) { 284 // perf currently doesn't have an interface for unloading. But munmap()ing the 285 // code section does, so that's ok. 286} 287 288bool PerfJITEventListener::InitDebuggingDir() { 289 time_t Time; 290 struct tm LocalTime; 291 char TimeBuffer[sizeof("YYYYMMDD")]; 292 SmallString<64> Path; 293 294 // search for location to dump data to 295 if (const char *BaseDir = getenv("JITDUMPDIR")) 296 Path.append(BaseDir); 297 else if (!sys::path::home_directory(Path)) 298 Path = "."; 299 300 // create debug directory 301 Path += "/.debug/jit/"; 302 if (auto EC = sys::fs::create_directories(Path)) { 303 errs() << "could not create jit cache directory " << Path << ": " 304 << EC.message() << "\n"; 305 return false; 306 } 307 308 // create unique directory for dump data related to this process 309 time(&Time); 310 localtime_r(&Time, &LocalTime); 311 strftime(TimeBuffer, sizeof(TimeBuffer), "%Y%m%d", &LocalTime); 312 Path += JIT_LANG "-jit-"; 313 Path += TimeBuffer; 314 315 SmallString<128> UniqueDebugDir; 316 317 using sys::fs::createUniqueDirectory; 318 if (auto EC = createUniqueDirectory(Path, UniqueDebugDir)) { 319 errs() << "could not create unique jit cache directory " << UniqueDebugDir 320 << ": " << EC.message() << "\n"; 321 return false; 322 } 323 324 JitPath = UniqueDebugDir.str(); 325 326 return true; 327} 328 329bool PerfJITEventListener::OpenMarker() { 330 // We mmap the jitdump to create an MMAP RECORD in perf.data file. The mmap 331 // is captured either live (perf record running when we mmap) or in deferred 332 // mode, via /proc/PID/maps. The MMAP record is used as a marker of a jitdump 333 // file for more meta data info about the jitted code. Perf report/annotate 334 // detect this special filename and process the jitdump file. 335 // 336 // Mapping must be PROT_EXEC to ensure it is captured by perf record 337 // even when not using -d option. 338 MarkerAddr = ::mmap(NULL, sys::Process::getPageSize(), PROT_READ | PROT_EXEC, 339 MAP_PRIVATE, DumpFd, 0); 340 341 if (MarkerAddr == MAP_FAILED) { 342 errs() << "could not mmap JIT marker\n"; 343 return false; 344 } 345 return true; 346} 347 348void PerfJITEventListener::CloseMarker() { 349 if (!MarkerAddr) 350 return; 351 352 munmap(MarkerAddr, sys::Process::getPageSize()); 353 MarkerAddr = nullptr; 354} 355 356bool PerfJITEventListener::FillMachine(LLVMPerfJitHeader &hdr) { 357 char id[16]; 358 struct { 359 uint16_t e_type; 360 uint16_t e_machine; 361 } info; 362 363 size_t RequiredMemory = sizeof(id) + sizeof(info); 364 365 ErrorOr<std::unique_ptr<MemoryBuffer>> MB = 366 MemoryBuffer::getFileSlice("/proc/self/exe", 367 RequiredMemory, 368 0); 369 370 // This'll not guarantee that enough data was actually read from the 371 // underlying file. Instead the trailing part of the buffer would be 372 // zeroed. Given the ELF signature check below that seems ok though, 373 // it's unlikely that the file ends just after that, and the 374 // consequence would just be that perf wouldn't recognize the 375 // signature. 376 if (auto EC = MB.getError()) { 377 errs() << "could not open /proc/self/exe: " << EC.message() << "\n"; 378 return false; 379 } 380 381 memcpy(&id, (*MB)->getBufferStart(), sizeof(id)); 382 memcpy(&info, (*MB)->getBufferStart() + sizeof(id), sizeof(info)); 383 384 // check ELF signature 385 if (id[0] != 0x7f || id[1] != 'E' || id[2] != 'L' || id[3] != 'F') { 386 errs() << "invalid elf signature\n"; 387 return false; 388 } 389 390 hdr.ElfMach = info.e_machine; 391 392 return true; 393} 394 395void PerfJITEventListener::NotifyCode(Expected<llvm::StringRef> &Symbol, 396 uint64_t CodeAddr, uint64_t CodeSize) { 397 assert(SuccessfullyInitialized); 398 399 // 0 length functions can't have samples. 400 if (CodeSize == 0) 401 return; 402 403 LLVMPerfJitRecordCodeLoad rec; 404 rec.Prefix.Id = JIT_CODE_LOAD; 405 rec.Prefix.TotalSize = sizeof(rec) + // debug record itself 406 Symbol->size() + 1 + // symbol name 407 CodeSize; // and code 408 rec.Prefix.Timestamp = perf_get_timestamp(); 409 410 rec.CodeSize = CodeSize; 411 rec.Vma = 0; 412 rec.CodeAddr = CodeAddr; 413 rec.Pid = Pid; 414 rec.Tid = get_threadid(); 415 416 // avoid interspersing output 417 MutexGuard Guard(Mutex); 418 419 rec.CodeIndex = CodeGeneration++; // under lock! 420 421 Dumpstream->write(reinterpret_cast<const char *>(&rec), sizeof(rec)); 422 Dumpstream->write(Symbol->data(), Symbol->size() + 1); 423 Dumpstream->write(reinterpret_cast<const char *>(CodeAddr), CodeSize); 424} 425 426void PerfJITEventListener::NotifyDebug(uint64_t CodeAddr, 427 DILineInfoTable Lines) { 428 assert(SuccessfullyInitialized); 429 430 // Didn't get useful debug info. 431 if (Lines.empty()) 432 return; 433 434 LLVMPerfJitRecordDebugInfo rec; 435 rec.Prefix.Id = JIT_CODE_DEBUG_INFO; 436 rec.Prefix.TotalSize = sizeof(rec); // will be increased further 437 rec.Prefix.Timestamp = perf_get_timestamp(); 438 rec.CodeAddr = CodeAddr; 439 rec.NrEntry = Lines.size(); 440 441 // compute total size size of record (variable due to filenames) 442 DILineInfoTable::iterator Begin = Lines.begin(); 443 DILineInfoTable::iterator End = Lines.end(); 444 for (DILineInfoTable::iterator It = Begin; It != End; ++It) { 445 DILineInfo &line = It->second; 446 rec.Prefix.TotalSize += sizeof(LLVMPerfJitDebugEntry); 447 rec.Prefix.TotalSize += line.FileName.size() + 1; 448 } 449 450 // The debug_entry describes the source line information. It is defined as 451 // follows in order: 452 // * uint64_t code_addr: address of function for which the debug information 453 // is generated 454 // * uint32_t line : source file line number (starting at 1) 455 // * uint32_t discrim : column discriminator, 0 is default 456 // * char name[n] : source file name in ASCII, including null termination 457 458 // avoid interspersing output 459 MutexGuard Guard(Mutex); 460 461 Dumpstream->write(reinterpret_cast<const char *>(&rec), sizeof(rec)); 462 463 for (DILineInfoTable::iterator It = Begin; It != End; ++It) { 464 LLVMPerfJitDebugEntry LineInfo; 465 DILineInfo &Line = It->second; 466 467 LineInfo.Addr = It->first; 468 // The function re-created by perf is preceded by a elf 469 // header. Need to adjust for that, otherwise the results are 470 // wrong. 471 LineInfo.Addr += 0x40; 472 LineInfo.Lineno = Line.Line; 473 LineInfo.Discrim = Line.Discriminator; 474 475 Dumpstream->write(reinterpret_cast<const char *>(&LineInfo), 476 sizeof(LineInfo)); 477 Dumpstream->write(Line.FileName.c_str(), Line.FileName.size() + 1); 478 } 479} 480 481// There should be only a single event listener per process, otherwise perf gets 482// confused. 483llvm::ManagedStatic<PerfJITEventListener> PerfListener; 484 485} // end anonymous namespace 486 487namespace llvm { 488JITEventListener *JITEventListener::createPerfJITEventListener() { 489 return &*PerfListener; 490} 491 492} // namespace llvm 493 494LLVMJITEventListenerRef LLVMCreatePerfJITEventListener(void) 495{ 496 return wrap(JITEventListener::createPerfJITEventListener()); 497} 498