1277323Sdim//===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===// 2277323Sdim// 3353358Sdim// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4353358Sdim// See https://llvm.org/LICENSE.txt for license information. 5353358Sdim// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6277323Sdim// 7277323Sdim//===----------------------------------------------------------------------===// 8277323Sdim// 9277323Sdim// This file implements the class that reads LLVM sample profiles. It 10296417Sdim// supports three file formats: text, binary and gcov. 11277323Sdim// 12296417Sdim// The textual representation is useful for debugging and testing purposes. The 13296417Sdim// binary representation is more compact, resulting in smaller file sizes. 14277323Sdim// 15296417Sdim// The gcov encoding is the one generated by GCC's AutoFDO profile creation 16296417Sdim// tool (https://github.com/google/autofdo) 17277323Sdim// 18296417Sdim// All three encodings can be used interchangeably as an input sample profile. 19277323Sdim// 20277323Sdim//===----------------------------------------------------------------------===// 21277323Sdim 22277323Sdim#include "llvm/ProfileData/SampleProfReader.h" 23296417Sdim#include "llvm/ADT/DenseMap.h" 24309124Sdim#include "llvm/ADT/STLExtras.h" 25321369Sdim#include "llvm/ADT/StringRef.h" 26321369Sdim#include "llvm/IR/ProfileSummary.h" 27321369Sdim#include "llvm/ProfileData/ProfileCommon.h" 28321369Sdim#include "llvm/ProfileData/SampleProf.h" 29360784Sdim#include "llvm/Support/Compression.h" 30277323Sdim#include "llvm/Support/ErrorOr.h" 31277323Sdim#include "llvm/Support/LEB128.h" 32277323Sdim#include "llvm/Support/LineIterator.h" 33344779Sdim#include "llvm/Support/MD5.h" 34277323Sdim#include "llvm/Support/MemoryBuffer.h" 35321369Sdim#include "llvm/Support/raw_ostream.h" 36321369Sdim#include <algorithm> 37321369Sdim#include <cstddef> 38321369Sdim#include <cstdint> 39321369Sdim#include <limits> 40321369Sdim#include <memory> 41321369Sdim#include <system_error> 42321369Sdim#include <vector> 43277323Sdim 44277323Sdimusing namespace llvm; 45321369Sdimusing namespace sampleprof; 46277323Sdim 47341825Sdim/// Dump the function profile for \p FName. 48277323Sdim/// 49277323Sdim/// \param FName Name of the function to print. 50277323Sdim/// \param OS Stream to emit the output to. 51277323Sdimvoid SampleProfileReader::dumpFunctionProfile(StringRef FName, 52277323Sdim raw_ostream &OS) { 53296417Sdim OS << "Function: " << FName << ": " << Profiles[FName]; 54277323Sdim} 55277323Sdim 56341825Sdim/// Dump all the function profiles found on stream \p OS. 57277323Sdimvoid SampleProfileReader::dump(raw_ostream &OS) { 58277323Sdim for (const auto &I : Profiles) 59277323Sdim dumpFunctionProfile(I.getKey(), OS); 60277323Sdim} 61277323Sdim 62341825Sdim/// Parse \p Input as function head. 63296417Sdim/// 64296417Sdim/// Parse one line of \p Input, and update function name in \p FName, 65296417Sdim/// function's total sample count in \p NumSamples, function's entry 66296417Sdim/// count in \p NumHeadSamples. 67296417Sdim/// 68296417Sdim/// \returns true if parsing is successful. 69296417Sdimstatic bool ParseHead(const StringRef &Input, StringRef &FName, 70296417Sdim uint64_t &NumSamples, uint64_t &NumHeadSamples) { 71296417Sdim if (Input[0] == ' ') 72296417Sdim return false; 73296417Sdim size_t n2 = Input.rfind(':'); 74296417Sdim size_t n1 = Input.rfind(':', n2 - 1); 75296417Sdim FName = Input.substr(0, n1); 76296417Sdim if (Input.substr(n1 + 1, n2 - n1 - 1).getAsInteger(10, NumSamples)) 77296417Sdim return false; 78296417Sdim if (Input.substr(n2 + 1).getAsInteger(10, NumHeadSamples)) 79296417Sdim return false; 80296417Sdim return true; 81296417Sdim} 82296417Sdim 83341825Sdim/// Returns true if line offset \p L is legal (only has 16 bits). 84309124Sdimstatic bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; } 85296417Sdim 86341825Sdim/// Parse \p Input as line sample. 87296417Sdim/// 88296417Sdim/// \param Input input line. 89296417Sdim/// \param IsCallsite true if the line represents an inlined callsite. 90296417Sdim/// \param Depth the depth of the inline stack. 91296417Sdim/// \param NumSamples total samples of the line/inlined callsite. 92296417Sdim/// \param LineOffset line offset to the start of the function. 93296417Sdim/// \param Discriminator discriminator of the line. 94296417Sdim/// \param TargetCountMap map from indirect call target to count. 95296417Sdim/// 96296417Sdim/// returns true if parsing is successful. 97296417Sdimstatic bool ParseLine(const StringRef &Input, bool &IsCallsite, uint32_t &Depth, 98296417Sdim uint64_t &NumSamples, uint32_t &LineOffset, 99296417Sdim uint32_t &Discriminator, StringRef &CalleeName, 100296417Sdim DenseMap<StringRef, uint64_t> &TargetCountMap) { 101296417Sdim for (Depth = 0; Input[Depth] == ' '; Depth++) 102296417Sdim ; 103296417Sdim if (Depth == 0) 104296417Sdim return false; 105296417Sdim 106296417Sdim size_t n1 = Input.find(':'); 107296417Sdim StringRef Loc = Input.substr(Depth, n1 - Depth); 108296417Sdim size_t n2 = Loc.find('.'); 109296417Sdim if (n2 == StringRef::npos) { 110296417Sdim if (Loc.getAsInteger(10, LineOffset) || !isOffsetLegal(LineOffset)) 111296417Sdim return false; 112296417Sdim Discriminator = 0; 113296417Sdim } else { 114296417Sdim if (Loc.substr(0, n2).getAsInteger(10, LineOffset)) 115296417Sdim return false; 116296417Sdim if (Loc.substr(n2 + 1).getAsInteger(10, Discriminator)) 117296417Sdim return false; 118296417Sdim } 119296417Sdim 120296417Sdim StringRef Rest = Input.substr(n1 + 2); 121296417Sdim if (Rest[0] >= '0' && Rest[0] <= '9') { 122296417Sdim IsCallsite = false; 123296417Sdim size_t n3 = Rest.find(' '); 124296417Sdim if (n3 == StringRef::npos) { 125296417Sdim if (Rest.getAsInteger(10, NumSamples)) 126296417Sdim return false; 127296417Sdim } else { 128296417Sdim if (Rest.substr(0, n3).getAsInteger(10, NumSamples)) 129296417Sdim return false; 130296417Sdim } 131341825Sdim // Find call targets and their sample counts. 132341825Sdim // Note: In some cases, there are symbols in the profile which are not 133341825Sdim // mangled. To accommodate such cases, use colon + integer pairs as the 134341825Sdim // anchor points. 135341825Sdim // An example: 136341825Sdim // _M_construct<char *>:1000 string_view<std::allocator<char> >:437 137341825Sdim // ":1000" and ":437" are used as anchor points so the string above will 138341825Sdim // be interpreted as 139341825Sdim // target: _M_construct<char *> 140341825Sdim // count: 1000 141341825Sdim // target: string_view<std::allocator<char> > 142341825Sdim // count: 437 143296417Sdim while (n3 != StringRef::npos) { 144296417Sdim n3 += Rest.substr(n3).find_first_not_of(' '); 145296417Sdim Rest = Rest.substr(n3); 146341825Sdim n3 = Rest.find_first_of(':'); 147341825Sdim if (n3 == StringRef::npos || n3 == 0) 148341825Sdim return false; 149341825Sdim 150341825Sdim StringRef Target; 151341825Sdim uint64_t count, n4; 152341825Sdim while (true) { 153341825Sdim // Get the segment after the current colon. 154341825Sdim StringRef AfterColon = Rest.substr(n3 + 1); 155341825Sdim // Get the target symbol before the current colon. 156341825Sdim Target = Rest.substr(0, n3); 157341825Sdim // Check if the word after the current colon is an integer. 158341825Sdim n4 = AfterColon.find_first_of(' '); 159341825Sdim n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size(); 160341825Sdim StringRef WordAfterColon = Rest.substr(n3 + 1, n4 - n3 - 1); 161341825Sdim if (!WordAfterColon.getAsInteger(10, count)) 162341825Sdim break; 163341825Sdim 164341825Sdim // Try to find the next colon. 165341825Sdim uint64_t n5 = AfterColon.find_first_of(':'); 166341825Sdim if (n5 == StringRef::npos) 167341825Sdim return false; 168341825Sdim n3 += n5 + 1; 169296417Sdim } 170341825Sdim 171341825Sdim // An anchor point is found. Save the {target, count} pair 172341825Sdim TargetCountMap[Target] = count; 173341825Sdim if (n4 == Rest.size()) 174341825Sdim break; 175341825Sdim // Change n3 to the next blank space after colon + integer pair. 176341825Sdim n3 = n4; 177296417Sdim } 178296417Sdim } else { 179296417Sdim IsCallsite = true; 180296417Sdim size_t n3 = Rest.find_last_of(':'); 181296417Sdim CalleeName = Rest.substr(0, n3); 182296417Sdim if (Rest.substr(n3 + 1).getAsInteger(10, NumSamples)) 183296417Sdim return false; 184296417Sdim } 185296417Sdim return true; 186296417Sdim} 187296417Sdim 188341825Sdim/// Load samples from a text file. 189277323Sdim/// 190277323Sdim/// See the documentation at the top of the file for an explanation of 191277323Sdim/// the expected format. 192277323Sdim/// 193277323Sdim/// \returns true if the file was loaded successfully, false otherwise. 194360784Sdimstd::error_code SampleProfileReaderText::readImpl() { 195277323Sdim line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#'); 196296417Sdim sampleprof_error Result = sampleprof_error::success; 197277323Sdim 198296417Sdim InlineCallStack InlineStack; 199296417Sdim 200296417Sdim for (; !LineIt.is_at_eof(); ++LineIt) { 201296417Sdim if ((*LineIt)[(*LineIt).find_first_not_of(' ')] == '#') 202296417Sdim continue; 203277323Sdim // Read the header of each function. 204277323Sdim // 205277323Sdim // Note that for function identifiers we are actually expecting 206277323Sdim // mangled names, but we may not always get them. This happens when 207277323Sdim // the compiler decides not to emit the function (e.g., it was inlined 208277323Sdim // and removed). In this case, the binary will not have the linkage 209277323Sdim // name for the function, so the profiler will emit the function's 210277323Sdim // unmangled name, which may contain characters like ':' and '>' in its 211277323Sdim // name (member functions, templates, etc). 212277323Sdim // 213277323Sdim // The only requirement we place on the identifier, then, is that it 214277323Sdim // should not begin with a number. 215296417Sdim if ((*LineIt)[0] != ' ') { 216296417Sdim uint64_t NumSamples, NumHeadSamples; 217296417Sdim StringRef FName; 218296417Sdim if (!ParseHead(*LineIt, FName, NumSamples, NumHeadSamples)) { 219296417Sdim reportError(LineIt.line_number(), 220296417Sdim "Expected 'mangled_name:NUM:NUM', found " + *LineIt); 221277323Sdim return sampleprof_error::malformed; 222277323Sdim } 223296417Sdim Profiles[FName] = FunctionSamples(); 224296417Sdim FunctionSamples &FProfile = Profiles[FName]; 225309124Sdim FProfile.setName(FName); 226296417Sdim MergeResult(Result, FProfile.addTotalSamples(NumSamples)); 227296417Sdim MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples)); 228296417Sdim InlineStack.clear(); 229296417Sdim InlineStack.push_back(&FProfile); 230296417Sdim } else { 231296417Sdim uint64_t NumSamples; 232296417Sdim StringRef FName; 233296417Sdim DenseMap<StringRef, uint64_t> TargetCountMap; 234296417Sdim bool IsCallsite; 235296417Sdim uint32_t Depth, LineOffset, Discriminator; 236296417Sdim if (!ParseLine(*LineIt, IsCallsite, Depth, NumSamples, LineOffset, 237296417Sdim Discriminator, FName, TargetCountMap)) { 238296417Sdim reportError(LineIt.line_number(), 239296417Sdim "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " + 240296417Sdim *LineIt); 241296417Sdim return sampleprof_error::malformed; 242296417Sdim } 243296417Sdim if (IsCallsite) { 244296417Sdim while (InlineStack.size() > Depth) { 245296417Sdim InlineStack.pop_back(); 246277323Sdim } 247296417Sdim FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt( 248321369Sdim LineLocation(LineOffset, Discriminator))[FName]; 249309124Sdim FSamples.setName(FName); 250296417Sdim MergeResult(Result, FSamples.addTotalSamples(NumSamples)); 251296417Sdim InlineStack.push_back(&FSamples); 252296417Sdim } else { 253296417Sdim while (InlineStack.size() > Depth) { 254296417Sdim InlineStack.pop_back(); 255296417Sdim } 256296417Sdim FunctionSamples &FProfile = *InlineStack.back(); 257296417Sdim for (const auto &name_count : TargetCountMap) { 258296417Sdim MergeResult(Result, FProfile.addCalledTargetSamples( 259296417Sdim LineOffset, Discriminator, name_count.first, 260296417Sdim name_count.second)); 261296417Sdim } 262296417Sdim MergeResult(Result, FProfile.addBodySamples(LineOffset, Discriminator, 263296417Sdim NumSamples)); 264277323Sdim } 265296417Sdim } 266296417Sdim } 267309124Sdim if (Result == sampleprof_error::success) 268309124Sdim computeSummary(); 269277323Sdim 270296417Sdim return Result; 271296417Sdim} 272296417Sdim 273296417Sdimbool SampleProfileReaderText::hasFormat(const MemoryBuffer &Buffer) { 274296417Sdim bool result = false; 275296417Sdim 276296417Sdim // Check that the first non-comment line is a valid function header. 277296417Sdim line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#'); 278296417Sdim if (!LineIt.is_at_eof()) { 279296417Sdim if ((*LineIt)[0] != ' ') { 280296417Sdim uint64_t NumSamples, NumHeadSamples; 281296417Sdim StringRef FName; 282296417Sdim result = ParseHead(*LineIt, FName, NumSamples, NumHeadSamples); 283277323Sdim } 284277323Sdim } 285277323Sdim 286296417Sdim return result; 287277323Sdim} 288277323Sdim 289277323Sdimtemplate <typename T> ErrorOr<T> SampleProfileReaderBinary::readNumber() { 290277323Sdim unsigned NumBytesRead = 0; 291277323Sdim std::error_code EC; 292277323Sdim uint64_t Val = decodeULEB128(Data, &NumBytesRead); 293277323Sdim 294277323Sdim if (Val > std::numeric_limits<T>::max()) 295277323Sdim EC = sampleprof_error::malformed; 296277323Sdim else if (Data + NumBytesRead > End) 297277323Sdim EC = sampleprof_error::truncated; 298277323Sdim else 299277323Sdim EC = sampleprof_error::success; 300277323Sdim 301277323Sdim if (EC) { 302296417Sdim reportError(0, EC.message()); 303277323Sdim return EC; 304277323Sdim } 305277323Sdim 306277323Sdim Data += NumBytesRead; 307277323Sdim return static_cast<T>(Val); 308277323Sdim} 309277323Sdim 310277323SdimErrorOr<StringRef> SampleProfileReaderBinary::readString() { 311277323Sdim std::error_code EC; 312277323Sdim StringRef Str(reinterpret_cast<const char *>(Data)); 313277323Sdim if (Data + Str.size() + 1 > End) { 314277323Sdim EC = sampleprof_error::truncated; 315296417Sdim reportError(0, EC.message()); 316277323Sdim return EC; 317277323Sdim } 318277323Sdim 319277323Sdim Data += Str.size() + 1; 320277323Sdim return Str; 321277323Sdim} 322277323Sdim 323341825Sdimtemplate <typename T> 324344779SdimErrorOr<T> SampleProfileReaderBinary::readUnencodedNumber() { 325344779Sdim std::error_code EC; 326344779Sdim 327344779Sdim if (Data + sizeof(T) > End) { 328344779Sdim EC = sampleprof_error::truncated; 329344779Sdim reportError(0, EC.message()); 330344779Sdim return EC; 331344779Sdim } 332344779Sdim 333344779Sdim using namespace support; 334344779Sdim T Val = endian::readNext<T, little, unaligned>(Data); 335344779Sdim return Val; 336344779Sdim} 337344779Sdim 338344779Sdimtemplate <typename T> 339341825Sdiminline ErrorOr<uint32_t> SampleProfileReaderBinary::readStringIndex(T &Table) { 340296417Sdim std::error_code EC; 341296417Sdim auto Idx = readNumber<uint32_t>(); 342296417Sdim if (std::error_code EC = Idx.getError()) 343296417Sdim return EC; 344341825Sdim if (*Idx >= Table.size()) 345296417Sdim return sampleprof_error::truncated_name_table; 346341825Sdim return *Idx; 347341825Sdim} 348341825Sdim 349360784SdimErrorOr<StringRef> SampleProfileReaderBinary::readStringFromTable() { 350341825Sdim auto Idx = readStringIndex(NameTable); 351341825Sdim if (std::error_code EC = Idx.getError()) 352341825Sdim return EC; 353341825Sdim 354296417Sdim return NameTable[*Idx]; 355296417Sdim} 356296417Sdim 357341825SdimErrorOr<StringRef> SampleProfileReaderCompactBinary::readStringFromTable() { 358341825Sdim auto Idx = readStringIndex(NameTable); 359341825Sdim if (std::error_code EC = Idx.getError()) 360341825Sdim return EC; 361341825Sdim 362341825Sdim return StringRef(NameTable[*Idx]); 363341825Sdim} 364341825Sdim 365296417Sdimstd::error_code 366296417SdimSampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) { 367296417Sdim auto NumSamples = readNumber<uint64_t>(); 368296417Sdim if (std::error_code EC = NumSamples.getError()) 369296417Sdim return EC; 370296417Sdim FProfile.addTotalSamples(*NumSamples); 371296417Sdim 372296417Sdim // Read the samples in the body. 373296417Sdim auto NumRecords = readNumber<uint32_t>(); 374296417Sdim if (std::error_code EC = NumRecords.getError()) 375296417Sdim return EC; 376296417Sdim 377296417Sdim for (uint32_t I = 0; I < *NumRecords; ++I) { 378296417Sdim auto LineOffset = readNumber<uint64_t>(); 379296417Sdim if (std::error_code EC = LineOffset.getError()) 380277323Sdim return EC; 381277323Sdim 382296417Sdim if (!isOffsetLegal(*LineOffset)) { 383296417Sdim return std::error_code(); 384296417Sdim } 385277323Sdim 386296417Sdim auto Discriminator = readNumber<uint64_t>(); 387296417Sdim if (std::error_code EC = Discriminator.getError()) 388277323Sdim return EC; 389277323Sdim 390296417Sdim auto NumSamples = readNumber<uint64_t>(); 391296417Sdim if (std::error_code EC = NumSamples.getError()) 392277323Sdim return EC; 393277323Sdim 394296417Sdim auto NumCalls = readNumber<uint32_t>(); 395296417Sdim if (std::error_code EC = NumCalls.getError()) 396277323Sdim return EC; 397277323Sdim 398296417Sdim for (uint32_t J = 0; J < *NumCalls; ++J) { 399296417Sdim auto CalledFunction(readStringFromTable()); 400296417Sdim if (std::error_code EC = CalledFunction.getError()) 401277323Sdim return EC; 402277323Sdim 403296417Sdim auto CalledFunctionSamples = readNumber<uint64_t>(); 404296417Sdim if (std::error_code EC = CalledFunctionSamples.getError()) 405277323Sdim return EC; 406277323Sdim 407296417Sdim FProfile.addCalledTargetSamples(*LineOffset, *Discriminator, 408296417Sdim *CalledFunction, *CalledFunctionSamples); 409296417Sdim } 410277323Sdim 411296417Sdim FProfile.addBodySamples(*LineOffset, *Discriminator, *NumSamples); 412296417Sdim } 413277323Sdim 414296417Sdim // Read all the samples for inlined function calls. 415296417Sdim auto NumCallsites = readNumber<uint32_t>(); 416296417Sdim if (std::error_code EC = NumCallsites.getError()) 417296417Sdim return EC; 418277323Sdim 419296417Sdim for (uint32_t J = 0; J < *NumCallsites; ++J) { 420296417Sdim auto LineOffset = readNumber<uint64_t>(); 421296417Sdim if (std::error_code EC = LineOffset.getError()) 422296417Sdim return EC; 423277323Sdim 424296417Sdim auto Discriminator = readNumber<uint64_t>(); 425296417Sdim if (std::error_code EC = Discriminator.getError()) 426296417Sdim return EC; 427296417Sdim 428296417Sdim auto FName(readStringFromTable()); 429296417Sdim if (std::error_code EC = FName.getError()) 430296417Sdim return EC; 431296417Sdim 432321369Sdim FunctionSamples &CalleeProfile = FProfile.functionSamplesAt( 433321369Sdim LineLocation(*LineOffset, *Discriminator))[*FName]; 434309124Sdim CalleeProfile.setName(*FName); 435296417Sdim if (std::error_code EC = readProfile(CalleeProfile)) 436296417Sdim return EC; 437277323Sdim } 438277323Sdim 439277323Sdim return sampleprof_error::success; 440277323Sdim} 441277323Sdim 442360784Sdimstd::error_code 443360784SdimSampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) { 444360784Sdim Data = Start; 445344779Sdim auto NumHeadSamples = readNumber<uint64_t>(); 446344779Sdim if (std::error_code EC = NumHeadSamples.getError()) 447344779Sdim return EC; 448296417Sdim 449344779Sdim auto FName(readStringFromTable()); 450344779Sdim if (std::error_code EC = FName.getError()) 451344779Sdim return EC; 452296417Sdim 453344779Sdim Profiles[*FName] = FunctionSamples(); 454344779Sdim FunctionSamples &FProfile = Profiles[*FName]; 455344779Sdim FProfile.setName(*FName); 456296417Sdim 457344779Sdim FProfile.addHeadSamples(*NumHeadSamples); 458296417Sdim 459344779Sdim if (std::error_code EC = readProfile(FProfile)) 460344779Sdim return EC; 461344779Sdim return sampleprof_error::success; 462344779Sdim} 463344779Sdim 464360784Sdimstd::error_code SampleProfileReaderBinary::readImpl() { 465344779Sdim while (!at_eof()) { 466360784Sdim if (std::error_code EC = readFuncProfile(Data)) 467296417Sdim return EC; 468296417Sdim } 469296417Sdim 470296417Sdim return sampleprof_error::success; 471296417Sdim} 472296417Sdim 473360784Sdimstd::error_code 474360784SdimSampleProfileReaderExtBinary::readOneSection(const uint8_t *Start, 475360784Sdim uint64_t Size, SecType Type) { 476360784Sdim Data = Start; 477360784Sdim End = Start + Size; 478360784Sdim switch (Type) { 479360784Sdim case SecProfSummary: 480360784Sdim if (std::error_code EC = readSummary()) 481360784Sdim return EC; 482360784Sdim break; 483360784Sdim case SecNameTable: 484360784Sdim if (std::error_code EC = readNameTable()) 485360784Sdim return EC; 486360784Sdim break; 487360784Sdim case SecLBRProfile: 488360784Sdim if (std::error_code EC = readFuncProfiles()) 489360784Sdim return EC; 490360784Sdim break; 491360784Sdim case SecProfileSymbolList: 492360784Sdim if (std::error_code EC = readProfileSymbolList()) 493360784Sdim return EC; 494360784Sdim break; 495360784Sdim case SecFuncOffsetTable: 496360784Sdim if (std::error_code EC = readFuncOffsetTable()) 497360784Sdim return EC; 498360784Sdim break; 499360784Sdim default: 500360784Sdim break; 501360784Sdim } 502360784Sdim return sampleprof_error::success; 503360784Sdim} 504360784Sdim 505360784Sdimvoid SampleProfileReaderExtBinary::collectFuncsFrom(const Module &M) { 506360784Sdim UseAllFuncs = false; 507360784Sdim FuncsToUse.clear(); 508360784Sdim for (auto &F : M) 509360784Sdim FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F)); 510360784Sdim} 511360784Sdim 512360784Sdimstd::error_code SampleProfileReaderExtBinary::readFuncOffsetTable() { 513360784Sdim auto Size = readNumber<uint64_t>(); 514360784Sdim if (std::error_code EC = Size.getError()) 515360784Sdim return EC; 516360784Sdim 517360784Sdim FuncOffsetTable.reserve(*Size); 518360784Sdim for (uint32_t I = 0; I < *Size; ++I) { 519360784Sdim auto FName(readStringFromTable()); 520360784Sdim if (std::error_code EC = FName.getError()) 521360784Sdim return EC; 522360784Sdim 523360784Sdim auto Offset = readNumber<uint64_t>(); 524360784Sdim if (std::error_code EC = Offset.getError()) 525360784Sdim return EC; 526360784Sdim 527360784Sdim FuncOffsetTable[*FName] = *Offset; 528360784Sdim } 529360784Sdim return sampleprof_error::success; 530360784Sdim} 531360784Sdim 532360784Sdimstd::error_code SampleProfileReaderExtBinary::readFuncProfiles() { 533360784Sdim const uint8_t *Start = Data; 534360784Sdim if (UseAllFuncs) { 535360784Sdim while (Data < End) { 536360784Sdim if (std::error_code EC = readFuncProfile(Data)) 537360784Sdim return EC; 538360784Sdim } 539360784Sdim assert(Data == End && "More data is read than expected"); 540360784Sdim return sampleprof_error::success; 541360784Sdim } 542360784Sdim 543360784Sdim if (Remapper) { 544360784Sdim for (auto Name : FuncsToUse) { 545360784Sdim Remapper->insert(Name); 546360784Sdim } 547360784Sdim } 548360784Sdim 549360784Sdim for (auto NameOffset : FuncOffsetTable) { 550360784Sdim auto FuncName = NameOffset.first; 551360784Sdim if (!FuncsToUse.count(FuncName) && 552360784Sdim (!Remapper || !Remapper->exist(FuncName))) 553344779Sdim continue; 554360784Sdim const uint8_t *FuncProfileAddr = Start + NameOffset.second; 555360784Sdim assert(FuncProfileAddr < End && "out of LBRProfile section"); 556360784Sdim if (std::error_code EC = readFuncProfile(FuncProfileAddr)) 557360784Sdim return EC; 558360784Sdim } 559360784Sdim 560360784Sdim Data = End; 561360784Sdim return sampleprof_error::success; 562360784Sdim} 563360784Sdim 564360784Sdimstd::error_code SampleProfileReaderExtBinary::readProfileSymbolList() { 565360784Sdim if (!ProfSymList) 566360784Sdim ProfSymList = std::make_unique<ProfileSymbolList>(); 567360784Sdim 568360784Sdim if (std::error_code EC = ProfSymList->read(Data, End - Data)) 569360784Sdim return EC; 570360784Sdim 571360784Sdim Data = End; 572360784Sdim return sampleprof_error::success; 573360784Sdim} 574360784Sdim 575360784Sdimstd::error_code SampleProfileReaderExtBinaryBase::decompressSection( 576360784Sdim const uint8_t *SecStart, const uint64_t SecSize, 577360784Sdim const uint8_t *&DecompressBuf, uint64_t &DecompressBufSize) { 578360784Sdim Data = SecStart; 579360784Sdim End = SecStart + SecSize; 580360784Sdim auto DecompressSize = readNumber<uint64_t>(); 581360784Sdim if (std::error_code EC = DecompressSize.getError()) 582360784Sdim return EC; 583360784Sdim DecompressBufSize = *DecompressSize; 584360784Sdim 585360784Sdim auto CompressSize = readNumber<uint64_t>(); 586360784Sdim if (std::error_code EC = CompressSize.getError()) 587360784Sdim return EC; 588360784Sdim 589360784Sdim if (!llvm::zlib::isAvailable()) 590360784Sdim return sampleprof_error::zlib_unavailable; 591360784Sdim 592360784Sdim StringRef CompressedStrings(reinterpret_cast<const char *>(Data), 593360784Sdim *CompressSize); 594360784Sdim char *Buffer = Allocator.Allocate<char>(DecompressBufSize); 595360784Sdim size_t UCSize = DecompressBufSize; 596360784Sdim llvm::Error E = 597360784Sdim zlib::uncompress(CompressedStrings, Buffer, UCSize); 598360784Sdim if (E) 599360784Sdim return sampleprof_error::uncompress_failed; 600360784Sdim DecompressBuf = reinterpret_cast<const uint8_t *>(Buffer); 601360784Sdim return sampleprof_error::success; 602360784Sdim} 603360784Sdim 604360784Sdimstd::error_code SampleProfileReaderExtBinaryBase::readImpl() { 605360784Sdim const uint8_t *BufStart = 606360784Sdim reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()); 607360784Sdim 608360784Sdim for (auto &Entry : SecHdrTable) { 609360784Sdim // Skip empty section. 610360784Sdim if (!Entry.Size) 611360784Sdim continue; 612360784Sdim 613360784Sdim const uint8_t *SecStart = BufStart + Entry.Offset; 614360784Sdim uint64_t SecSize = Entry.Size; 615360784Sdim 616360784Sdim // If the section is compressed, decompress it into a buffer 617360784Sdim // DecompressBuf before reading the actual data. The pointee of 618360784Sdim // 'Data' will be changed to buffer hold by DecompressBuf 619360784Sdim // temporarily when reading the actual data. 620360784Sdim bool isCompressed = hasSecFlag(Entry, SecFlagCompress); 621360784Sdim if (isCompressed) { 622360784Sdim const uint8_t *DecompressBuf; 623360784Sdim uint64_t DecompressBufSize; 624360784Sdim if (std::error_code EC = decompressSection( 625360784Sdim SecStart, SecSize, DecompressBuf, DecompressBufSize)) 626360784Sdim return EC; 627360784Sdim SecStart = DecompressBuf; 628360784Sdim SecSize = DecompressBufSize; 629360784Sdim } 630360784Sdim 631360784Sdim if (std::error_code EC = readOneSection(SecStart, SecSize, Entry.Type)) 632360784Sdim return EC; 633360784Sdim if (Data != SecStart + SecSize) 634360784Sdim return sampleprof_error::malformed; 635360784Sdim 636360784Sdim // Change the pointee of 'Data' from DecompressBuf to original Buffer. 637360784Sdim if (isCompressed) { 638360784Sdim Data = BufStart + Entry.Offset; 639360784Sdim End = BufStart + Buffer->getBufferSize(); 640360784Sdim } 641360784Sdim } 642360784Sdim 643360784Sdim return sampleprof_error::success; 644360784Sdim} 645360784Sdim 646360784Sdimstd::error_code SampleProfileReaderCompactBinary::readImpl() { 647360784Sdim std::vector<uint64_t> OffsetsToUse; 648360784Sdim if (UseAllFuncs) { 649360784Sdim for (auto FuncEntry : FuncOffsetTable) { 650360784Sdim OffsetsToUse.push_back(FuncEntry.second); 651360784Sdim } 652360784Sdim } 653360784Sdim else { 654360784Sdim for (auto Name : FuncsToUse) { 655360784Sdim auto GUID = std::to_string(MD5Hash(Name)); 656360784Sdim auto iter = FuncOffsetTable.find(StringRef(GUID)); 657360784Sdim if (iter == FuncOffsetTable.end()) 658360784Sdim continue; 659360784Sdim OffsetsToUse.push_back(iter->second); 660360784Sdim } 661360784Sdim } 662360784Sdim 663360784Sdim for (auto Offset : OffsetsToUse) { 664344779Sdim const uint8_t *SavedData = Data; 665360784Sdim if (std::error_code EC = readFuncProfile( 666360784Sdim reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) + 667360784Sdim Offset)) 668344779Sdim return EC; 669344779Sdim Data = SavedData; 670344779Sdim } 671344779Sdim return sampleprof_error::success; 672344779Sdim} 673344779Sdim 674341825Sdimstd::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) { 675341825Sdim if (Magic == SPMagic()) 676341825Sdim return sampleprof_error::success; 677341825Sdim return sampleprof_error::bad_magic; 678341825Sdim} 679341825Sdim 680360784Sdimstd::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) { 681360784Sdim if (Magic == SPMagic(SPF_Ext_Binary)) 682360784Sdim return sampleprof_error::success; 683360784Sdim return sampleprof_error::bad_magic; 684360784Sdim} 685360784Sdim 686341825Sdimstd::error_code 687341825SdimSampleProfileReaderCompactBinary::verifySPMagic(uint64_t Magic) { 688341825Sdim if (Magic == SPMagic(SPF_Compact_Binary)) 689341825Sdim return sampleprof_error::success; 690341825Sdim return sampleprof_error::bad_magic; 691341825Sdim} 692341825Sdim 693360784Sdimstd::error_code SampleProfileReaderBinary::readNameTable() { 694341825Sdim auto Size = readNumber<uint32_t>(); 695341825Sdim if (std::error_code EC = Size.getError()) 696341825Sdim return EC; 697341825Sdim NameTable.reserve(*Size); 698341825Sdim for (uint32_t I = 0; I < *Size; ++I) { 699341825Sdim auto Name(readString()); 700341825Sdim if (std::error_code EC = Name.getError()) 701341825Sdim return EC; 702341825Sdim NameTable.push_back(*Name); 703341825Sdim } 704341825Sdim 705341825Sdim return sampleprof_error::success; 706341825Sdim} 707341825Sdim 708341825Sdimstd::error_code SampleProfileReaderCompactBinary::readNameTable() { 709341825Sdim auto Size = readNumber<uint64_t>(); 710341825Sdim if (std::error_code EC = Size.getError()) 711341825Sdim return EC; 712341825Sdim NameTable.reserve(*Size); 713341825Sdim for (uint32_t I = 0; I < *Size; ++I) { 714341825Sdim auto FID = readNumber<uint64_t>(); 715341825Sdim if (std::error_code EC = FID.getError()) 716341825Sdim return EC; 717341825Sdim NameTable.push_back(std::to_string(*FID)); 718341825Sdim } 719341825Sdim return sampleprof_error::success; 720341825Sdim} 721341825Sdim 722360784Sdimstd::error_code SampleProfileReaderExtBinaryBase::readSecHdrTableEntry() { 723360784Sdim SecHdrTableEntry Entry; 724360784Sdim auto Type = readUnencodedNumber<uint64_t>(); 725360784Sdim if (std::error_code EC = Type.getError()) 726360784Sdim return EC; 727360784Sdim Entry.Type = static_cast<SecType>(*Type); 728277323Sdim 729360784Sdim auto Flags = readUnencodedNumber<uint64_t>(); 730360784Sdim if (std::error_code EC = Flags.getError()) 731360784Sdim return EC; 732360784Sdim Entry.Flags = *Flags; 733360784Sdim 734360784Sdim auto Offset = readUnencodedNumber<uint64_t>(); 735360784Sdim if (std::error_code EC = Offset.getError()) 736360784Sdim return EC; 737360784Sdim Entry.Offset = *Offset; 738360784Sdim 739360784Sdim auto Size = readUnencodedNumber<uint64_t>(); 740360784Sdim if (std::error_code EC = Size.getError()) 741360784Sdim return EC; 742360784Sdim Entry.Size = *Size; 743360784Sdim 744360784Sdim SecHdrTable.push_back(std::move(Entry)); 745360784Sdim return sampleprof_error::success; 746360784Sdim} 747360784Sdim 748360784Sdimstd::error_code SampleProfileReaderExtBinaryBase::readSecHdrTable() { 749360784Sdim auto EntryNum = readUnencodedNumber<uint64_t>(); 750360784Sdim if (std::error_code EC = EntryNum.getError()) 751360784Sdim return EC; 752360784Sdim 753360784Sdim for (uint32_t i = 0; i < (*EntryNum); i++) 754360784Sdim if (std::error_code EC = readSecHdrTableEntry()) 755360784Sdim return EC; 756360784Sdim 757360784Sdim return sampleprof_error::success; 758360784Sdim} 759360784Sdim 760360784Sdimstd::error_code SampleProfileReaderExtBinaryBase::readHeader() { 761360784Sdim const uint8_t *BufStart = 762360784Sdim reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()); 763360784Sdim Data = BufStart; 764360784Sdim End = BufStart + Buffer->getBufferSize(); 765360784Sdim 766360784Sdim if (std::error_code EC = readMagicIdent()) 767360784Sdim return EC; 768360784Sdim 769360784Sdim if (std::error_code EC = readSecHdrTable()) 770360784Sdim return EC; 771360784Sdim 772360784Sdim return sampleprof_error::success; 773360784Sdim} 774360784Sdim 775360784Sdimuint64_t SampleProfileReaderExtBinaryBase::getSectionSize(SecType Type) { 776360784Sdim for (auto &Entry : SecHdrTable) { 777360784Sdim if (Entry.Type == Type) 778360784Sdim return Entry.Size; 779360784Sdim } 780360784Sdim return 0; 781360784Sdim} 782360784Sdim 783360784Sdimuint64_t SampleProfileReaderExtBinaryBase::getFileSize() { 784360784Sdim // Sections in SecHdrTable is not necessarily in the same order as 785360784Sdim // sections in the profile because section like FuncOffsetTable needs 786360784Sdim // to be written after section LBRProfile but needs to be read before 787360784Sdim // section LBRProfile, so we cannot simply use the last entry in 788360784Sdim // SecHdrTable to calculate the file size. 789360784Sdim uint64_t FileSize = 0; 790360784Sdim for (auto &Entry : SecHdrTable) { 791360784Sdim FileSize = std::max(Entry.Offset + Entry.Size, FileSize); 792360784Sdim } 793360784Sdim return FileSize; 794360784Sdim} 795360784Sdim 796360784Sdimbool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) { 797360784Sdim uint64_t TotalSecsSize = 0; 798360784Sdim for (auto &Entry : SecHdrTable) { 799360784Sdim OS << getSecName(Entry.Type) << " - Offset: " << Entry.Offset 800360784Sdim << ", Size: " << Entry.Size << "\n"; 801360784Sdim TotalSecsSize += getSectionSize(Entry.Type); 802360784Sdim } 803360784Sdim uint64_t HeaderSize = SecHdrTable.front().Offset; 804360784Sdim assert(HeaderSize + TotalSecsSize == getFileSize() && 805360784Sdim "Size of 'header + sections' doesn't match the total size of profile"); 806360784Sdim 807360784Sdim OS << "Header Size: " << HeaderSize << "\n"; 808360784Sdim OS << "Total Sections Size: " << TotalSecsSize << "\n"; 809360784Sdim OS << "File Size: " << getFileSize() << "\n"; 810360784Sdim return true; 811360784Sdim} 812360784Sdim 813360784Sdimstd::error_code SampleProfileReaderBinary::readMagicIdent() { 814277323Sdim // Read and check the magic identifier. 815277323Sdim auto Magic = readNumber<uint64_t>(); 816277323Sdim if (std::error_code EC = Magic.getError()) 817277323Sdim return EC; 818341825Sdim else if (std::error_code EC = verifySPMagic(*Magic)) 819341825Sdim return EC; 820277323Sdim 821277323Sdim // Read the version number. 822277323Sdim auto Version = readNumber<uint64_t>(); 823277323Sdim if (std::error_code EC = Version.getError()) 824277323Sdim return EC; 825277323Sdim else if (*Version != SPVersion()) 826277323Sdim return sampleprof_error::unsupported_version; 827277323Sdim 828360784Sdim return sampleprof_error::success; 829360784Sdim} 830360784Sdim 831360784Sdimstd::error_code SampleProfileReaderBinary::readHeader() { 832360784Sdim Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()); 833360784Sdim End = Data + Buffer->getBufferSize(); 834360784Sdim 835360784Sdim if (std::error_code EC = readMagicIdent()) 836360784Sdim return EC; 837360784Sdim 838309124Sdim if (std::error_code EC = readSummary()) 839309124Sdim return EC; 840309124Sdim 841341825Sdim if (std::error_code EC = readNameTable()) 842296417Sdim return EC; 843277323Sdim return sampleprof_error::success; 844277323Sdim} 845277323Sdim 846344779Sdimstd::error_code SampleProfileReaderCompactBinary::readHeader() { 847344779Sdim SampleProfileReaderBinary::readHeader(); 848344779Sdim if (std::error_code EC = readFuncOffsetTable()) 849344779Sdim return EC; 850344779Sdim return sampleprof_error::success; 851344779Sdim} 852344779Sdim 853344779Sdimstd::error_code SampleProfileReaderCompactBinary::readFuncOffsetTable() { 854344779Sdim auto TableOffset = readUnencodedNumber<uint64_t>(); 855344779Sdim if (std::error_code EC = TableOffset.getError()) 856344779Sdim return EC; 857344779Sdim 858344779Sdim const uint8_t *SavedData = Data; 859344779Sdim const uint8_t *TableStart = 860344779Sdim reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) + 861344779Sdim *TableOffset; 862344779Sdim Data = TableStart; 863344779Sdim 864344779Sdim auto Size = readNumber<uint64_t>(); 865344779Sdim if (std::error_code EC = Size.getError()) 866344779Sdim return EC; 867344779Sdim 868344779Sdim FuncOffsetTable.reserve(*Size); 869344779Sdim for (uint32_t I = 0; I < *Size; ++I) { 870344779Sdim auto FName(readStringFromTable()); 871344779Sdim if (std::error_code EC = FName.getError()) 872344779Sdim return EC; 873344779Sdim 874344779Sdim auto Offset = readNumber<uint64_t>(); 875344779Sdim if (std::error_code EC = Offset.getError()) 876344779Sdim return EC; 877344779Sdim 878344779Sdim FuncOffsetTable[*FName] = *Offset; 879344779Sdim } 880344779Sdim End = TableStart; 881344779Sdim Data = SavedData; 882344779Sdim return sampleprof_error::success; 883344779Sdim} 884344779Sdim 885360784Sdimvoid SampleProfileReaderCompactBinary::collectFuncsFrom(const Module &M) { 886360784Sdim UseAllFuncs = false; 887344779Sdim FuncsToUse.clear(); 888360784Sdim for (auto &F : M) 889360784Sdim FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F)); 890344779Sdim} 891344779Sdim 892309124Sdimstd::error_code SampleProfileReaderBinary::readSummaryEntry( 893309124Sdim std::vector<ProfileSummaryEntry> &Entries) { 894309124Sdim auto Cutoff = readNumber<uint64_t>(); 895309124Sdim if (std::error_code EC = Cutoff.getError()) 896309124Sdim return EC; 897309124Sdim 898309124Sdim auto MinBlockCount = readNumber<uint64_t>(); 899309124Sdim if (std::error_code EC = MinBlockCount.getError()) 900309124Sdim return EC; 901309124Sdim 902309124Sdim auto NumBlocks = readNumber<uint64_t>(); 903309124Sdim if (std::error_code EC = NumBlocks.getError()) 904309124Sdim return EC; 905309124Sdim 906309124Sdim Entries.emplace_back(*Cutoff, *MinBlockCount, *NumBlocks); 907309124Sdim return sampleprof_error::success; 908309124Sdim} 909309124Sdim 910309124Sdimstd::error_code SampleProfileReaderBinary::readSummary() { 911309124Sdim auto TotalCount = readNumber<uint64_t>(); 912309124Sdim if (std::error_code EC = TotalCount.getError()) 913309124Sdim return EC; 914309124Sdim 915309124Sdim auto MaxBlockCount = readNumber<uint64_t>(); 916309124Sdim if (std::error_code EC = MaxBlockCount.getError()) 917309124Sdim return EC; 918309124Sdim 919309124Sdim auto MaxFunctionCount = readNumber<uint64_t>(); 920309124Sdim if (std::error_code EC = MaxFunctionCount.getError()) 921309124Sdim return EC; 922309124Sdim 923309124Sdim auto NumBlocks = readNumber<uint64_t>(); 924309124Sdim if (std::error_code EC = NumBlocks.getError()) 925309124Sdim return EC; 926309124Sdim 927309124Sdim auto NumFunctions = readNumber<uint64_t>(); 928309124Sdim if (std::error_code EC = NumFunctions.getError()) 929309124Sdim return EC; 930309124Sdim 931309124Sdim auto NumSummaryEntries = readNumber<uint64_t>(); 932309124Sdim if (std::error_code EC = NumSummaryEntries.getError()) 933309124Sdim return EC; 934309124Sdim 935309124Sdim std::vector<ProfileSummaryEntry> Entries; 936309124Sdim for (unsigned i = 0; i < *NumSummaryEntries; i++) { 937309124Sdim std::error_code EC = readSummaryEntry(Entries); 938309124Sdim if (EC != sampleprof_error::success) 939309124Sdim return EC; 940309124Sdim } 941360784Sdim Summary = std::make_unique<ProfileSummary>( 942309124Sdim ProfileSummary::PSK_Sample, Entries, *TotalCount, *MaxBlockCount, 0, 943309124Sdim *MaxFunctionCount, *NumBlocks, *NumFunctions); 944309124Sdim 945309124Sdim return sampleprof_error::success; 946309124Sdim} 947309124Sdim 948341825Sdimbool SampleProfileReaderRawBinary::hasFormat(const MemoryBuffer &Buffer) { 949277323Sdim const uint8_t *Data = 950277323Sdim reinterpret_cast<const uint8_t *>(Buffer.getBufferStart()); 951277323Sdim uint64_t Magic = decodeULEB128(Data); 952277323Sdim return Magic == SPMagic(); 953277323Sdim} 954277323Sdim 955360784Sdimbool SampleProfileReaderExtBinary::hasFormat(const MemoryBuffer &Buffer) { 956360784Sdim const uint8_t *Data = 957360784Sdim reinterpret_cast<const uint8_t *>(Buffer.getBufferStart()); 958360784Sdim uint64_t Magic = decodeULEB128(Data); 959360784Sdim return Magic == SPMagic(SPF_Ext_Binary); 960360784Sdim} 961360784Sdim 962341825Sdimbool SampleProfileReaderCompactBinary::hasFormat(const MemoryBuffer &Buffer) { 963341825Sdim const uint8_t *Data = 964341825Sdim reinterpret_cast<const uint8_t *>(Buffer.getBufferStart()); 965341825Sdim uint64_t Magic = decodeULEB128(Data); 966341825Sdim return Magic == SPMagic(SPF_Compact_Binary); 967341825Sdim} 968341825Sdim 969296417Sdimstd::error_code SampleProfileReaderGCC::skipNextWord() { 970296417Sdim uint32_t dummy; 971296417Sdim if (!GcovBuffer.readInt(dummy)) 972296417Sdim return sampleprof_error::truncated; 973296417Sdim return sampleprof_error::success; 974296417Sdim} 975296417Sdim 976296417Sdimtemplate <typename T> ErrorOr<T> SampleProfileReaderGCC::readNumber() { 977296417Sdim if (sizeof(T) <= sizeof(uint32_t)) { 978296417Sdim uint32_t Val; 979296417Sdim if (GcovBuffer.readInt(Val) && Val <= std::numeric_limits<T>::max()) 980296417Sdim return static_cast<T>(Val); 981296417Sdim } else if (sizeof(T) <= sizeof(uint64_t)) { 982296417Sdim uint64_t Val; 983296417Sdim if (GcovBuffer.readInt64(Val) && Val <= std::numeric_limits<T>::max()) 984296417Sdim return static_cast<T>(Val); 985296417Sdim } 986296417Sdim 987296417Sdim std::error_code EC = sampleprof_error::malformed; 988296417Sdim reportError(0, EC.message()); 989296417Sdim return EC; 990296417Sdim} 991296417Sdim 992296417SdimErrorOr<StringRef> SampleProfileReaderGCC::readString() { 993296417Sdim StringRef Str; 994296417Sdim if (!GcovBuffer.readString(Str)) 995296417Sdim return sampleprof_error::truncated; 996296417Sdim return Str; 997296417Sdim} 998296417Sdim 999296417Sdimstd::error_code SampleProfileReaderGCC::readHeader() { 1000296417Sdim // Read the magic identifier. 1001296417Sdim if (!GcovBuffer.readGCDAFormat()) 1002296417Sdim return sampleprof_error::unrecognized_format; 1003296417Sdim 1004296417Sdim // Read the version number. Note - the GCC reader does not validate this 1005296417Sdim // version, but the profile creator generates v704. 1006296417Sdim GCOV::GCOVVersion version; 1007296417Sdim if (!GcovBuffer.readGCOVVersion(version)) 1008296417Sdim return sampleprof_error::unrecognized_format; 1009296417Sdim 1010296417Sdim if (version != GCOV::V704) 1011296417Sdim return sampleprof_error::unsupported_version; 1012296417Sdim 1013296417Sdim // Skip the empty integer. 1014296417Sdim if (std::error_code EC = skipNextWord()) 1015296417Sdim return EC; 1016296417Sdim 1017296417Sdim return sampleprof_error::success; 1018296417Sdim} 1019296417Sdim 1020296417Sdimstd::error_code SampleProfileReaderGCC::readSectionTag(uint32_t Expected) { 1021296417Sdim uint32_t Tag; 1022296417Sdim if (!GcovBuffer.readInt(Tag)) 1023296417Sdim return sampleprof_error::truncated; 1024296417Sdim 1025296417Sdim if (Tag != Expected) 1026296417Sdim return sampleprof_error::malformed; 1027296417Sdim 1028296417Sdim if (std::error_code EC = skipNextWord()) 1029296417Sdim return EC; 1030296417Sdim 1031296417Sdim return sampleprof_error::success; 1032296417Sdim} 1033296417Sdim 1034296417Sdimstd::error_code SampleProfileReaderGCC::readNameTable() { 1035296417Sdim if (std::error_code EC = readSectionTag(GCOVTagAFDOFileNames)) 1036296417Sdim return EC; 1037296417Sdim 1038296417Sdim uint32_t Size; 1039296417Sdim if (!GcovBuffer.readInt(Size)) 1040296417Sdim return sampleprof_error::truncated; 1041296417Sdim 1042296417Sdim for (uint32_t I = 0; I < Size; ++I) { 1043296417Sdim StringRef Str; 1044296417Sdim if (!GcovBuffer.readString(Str)) 1045296417Sdim return sampleprof_error::truncated; 1046296417Sdim Names.push_back(Str); 1047296417Sdim } 1048296417Sdim 1049296417Sdim return sampleprof_error::success; 1050296417Sdim} 1051296417Sdim 1052296417Sdimstd::error_code SampleProfileReaderGCC::readFunctionProfiles() { 1053296417Sdim if (std::error_code EC = readSectionTag(GCOVTagAFDOFunction)) 1054296417Sdim return EC; 1055296417Sdim 1056296417Sdim uint32_t NumFunctions; 1057296417Sdim if (!GcovBuffer.readInt(NumFunctions)) 1058296417Sdim return sampleprof_error::truncated; 1059296417Sdim 1060296417Sdim InlineCallStack Stack; 1061296417Sdim for (uint32_t I = 0; I < NumFunctions; ++I) 1062296417Sdim if (std::error_code EC = readOneFunctionProfile(Stack, true, 0)) 1063296417Sdim return EC; 1064296417Sdim 1065309124Sdim computeSummary(); 1066296417Sdim return sampleprof_error::success; 1067296417Sdim} 1068296417Sdim 1069296417Sdimstd::error_code SampleProfileReaderGCC::readOneFunctionProfile( 1070296417Sdim const InlineCallStack &InlineStack, bool Update, uint32_t Offset) { 1071296417Sdim uint64_t HeadCount = 0; 1072296417Sdim if (InlineStack.size() == 0) 1073296417Sdim if (!GcovBuffer.readInt64(HeadCount)) 1074296417Sdim return sampleprof_error::truncated; 1075296417Sdim 1076296417Sdim uint32_t NameIdx; 1077296417Sdim if (!GcovBuffer.readInt(NameIdx)) 1078296417Sdim return sampleprof_error::truncated; 1079296417Sdim 1080296417Sdim StringRef Name(Names[NameIdx]); 1081296417Sdim 1082296417Sdim uint32_t NumPosCounts; 1083296417Sdim if (!GcovBuffer.readInt(NumPosCounts)) 1084296417Sdim return sampleprof_error::truncated; 1085296417Sdim 1086296417Sdim uint32_t NumCallsites; 1087296417Sdim if (!GcovBuffer.readInt(NumCallsites)) 1088296417Sdim return sampleprof_error::truncated; 1089296417Sdim 1090296417Sdim FunctionSamples *FProfile = nullptr; 1091296417Sdim if (InlineStack.size() == 0) { 1092296417Sdim // If this is a top function that we have already processed, do not 1093296417Sdim // update its profile again. This happens in the presence of 1094296417Sdim // function aliases. Since these aliases share the same function 1095296417Sdim // body, there will be identical replicated profiles for the 1096296417Sdim // original function. In this case, we simply not bother updating 1097296417Sdim // the profile of the original function. 1098296417Sdim FProfile = &Profiles[Name]; 1099296417Sdim FProfile->addHeadSamples(HeadCount); 1100296417Sdim if (FProfile->getTotalSamples() > 0) 1101296417Sdim Update = false; 1102296417Sdim } else { 1103296417Sdim // Otherwise, we are reading an inlined instance. The top of the 1104296417Sdim // inline stack contains the profile of the caller. Insert this 1105296417Sdim // callee in the caller's CallsiteMap. 1106296417Sdim FunctionSamples *CallerProfile = InlineStack.front(); 1107296417Sdim uint32_t LineOffset = Offset >> 16; 1108296417Sdim uint32_t Discriminator = Offset & 0xffff; 1109296417Sdim FProfile = &CallerProfile->functionSamplesAt( 1110321369Sdim LineLocation(LineOffset, Discriminator))[Name]; 1111296417Sdim } 1112309124Sdim FProfile->setName(Name); 1113296417Sdim 1114296417Sdim for (uint32_t I = 0; I < NumPosCounts; ++I) { 1115296417Sdim uint32_t Offset; 1116296417Sdim if (!GcovBuffer.readInt(Offset)) 1117296417Sdim return sampleprof_error::truncated; 1118296417Sdim 1119296417Sdim uint32_t NumTargets; 1120296417Sdim if (!GcovBuffer.readInt(NumTargets)) 1121296417Sdim return sampleprof_error::truncated; 1122296417Sdim 1123296417Sdim uint64_t Count; 1124296417Sdim if (!GcovBuffer.readInt64(Count)) 1125296417Sdim return sampleprof_error::truncated; 1126296417Sdim 1127296417Sdim // The line location is encoded in the offset as: 1128296417Sdim // high 16 bits: line offset to the start of the function. 1129296417Sdim // low 16 bits: discriminator. 1130296417Sdim uint32_t LineOffset = Offset >> 16; 1131296417Sdim uint32_t Discriminator = Offset & 0xffff; 1132296417Sdim 1133296417Sdim InlineCallStack NewStack; 1134296417Sdim NewStack.push_back(FProfile); 1135296417Sdim NewStack.insert(NewStack.end(), InlineStack.begin(), InlineStack.end()); 1136296417Sdim if (Update) { 1137296417Sdim // Walk up the inline stack, adding the samples on this line to 1138296417Sdim // the total sample count of the callers in the chain. 1139296417Sdim for (auto CallerProfile : NewStack) 1140296417Sdim CallerProfile->addTotalSamples(Count); 1141296417Sdim 1142296417Sdim // Update the body samples for the current profile. 1143296417Sdim FProfile->addBodySamples(LineOffset, Discriminator, Count); 1144296417Sdim } 1145296417Sdim 1146296417Sdim // Process the list of functions called at an indirect call site. 1147296417Sdim // These are all the targets that a function pointer (or virtual 1148296417Sdim // function) resolved at runtime. 1149296417Sdim for (uint32_t J = 0; J < NumTargets; J++) { 1150296417Sdim uint32_t HistVal; 1151296417Sdim if (!GcovBuffer.readInt(HistVal)) 1152296417Sdim return sampleprof_error::truncated; 1153296417Sdim 1154296417Sdim if (HistVal != HIST_TYPE_INDIR_CALL_TOPN) 1155296417Sdim return sampleprof_error::malformed; 1156296417Sdim 1157296417Sdim uint64_t TargetIdx; 1158296417Sdim if (!GcovBuffer.readInt64(TargetIdx)) 1159296417Sdim return sampleprof_error::truncated; 1160296417Sdim StringRef TargetName(Names[TargetIdx]); 1161296417Sdim 1162296417Sdim uint64_t TargetCount; 1163296417Sdim if (!GcovBuffer.readInt64(TargetCount)) 1164296417Sdim return sampleprof_error::truncated; 1165296417Sdim 1166321369Sdim if (Update) 1167321369Sdim FProfile->addCalledTargetSamples(LineOffset, Discriminator, 1168321369Sdim TargetName, TargetCount); 1169296417Sdim } 1170296417Sdim } 1171296417Sdim 1172296417Sdim // Process all the inlined callers into the current function. These 1173296417Sdim // are all the callsites that were inlined into this function. 1174296417Sdim for (uint32_t I = 0; I < NumCallsites; I++) { 1175296417Sdim // The offset is encoded as: 1176296417Sdim // high 16 bits: line offset to the start of the function. 1177296417Sdim // low 16 bits: discriminator. 1178296417Sdim uint32_t Offset; 1179296417Sdim if (!GcovBuffer.readInt(Offset)) 1180296417Sdim return sampleprof_error::truncated; 1181296417Sdim InlineCallStack NewStack; 1182296417Sdim NewStack.push_back(FProfile); 1183296417Sdim NewStack.insert(NewStack.end(), InlineStack.begin(), InlineStack.end()); 1184296417Sdim if (std::error_code EC = readOneFunctionProfile(NewStack, Update, Offset)) 1185296417Sdim return EC; 1186296417Sdim } 1187296417Sdim 1188296417Sdim return sampleprof_error::success; 1189296417Sdim} 1190296417Sdim 1191341825Sdim/// Read a GCC AutoFDO profile. 1192296417Sdim/// 1193296417Sdim/// This format is generated by the Linux Perf conversion tool at 1194296417Sdim/// https://github.com/google/autofdo. 1195360784Sdimstd::error_code SampleProfileReaderGCC::readImpl() { 1196296417Sdim // Read the string table. 1197296417Sdim if (std::error_code EC = readNameTable()) 1198296417Sdim return EC; 1199296417Sdim 1200296417Sdim // Read the source profile. 1201296417Sdim if (std::error_code EC = readFunctionProfiles()) 1202296417Sdim return EC; 1203296417Sdim 1204296417Sdim return sampleprof_error::success; 1205296417Sdim} 1206296417Sdim 1207296417Sdimbool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) { 1208296417Sdim StringRef Magic(reinterpret_cast<const char *>(Buffer.getBufferStart())); 1209296417Sdim return Magic == "adcg*704"; 1210296417Sdim} 1211296417Sdim 1212360784Sdimvoid SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) { 1213360784Sdim // If the reader is in compact format, we can't remap it because 1214344779Sdim // we don't know what the original function names were. 1215360784Sdim if (Reader.getFormat() == SPF_Compact_Binary) { 1216344779Sdim Ctx.diagnose(DiagnosticInfoSampleProfile( 1217360784Sdim Reader.getBuffer()->getBufferIdentifier(), 1218344779Sdim "Profile data remapping cannot be applied to profile data " 1219344779Sdim "in compact format (original mangled names are not available).", 1220344779Sdim DS_Warning)); 1221360784Sdim return; 1222344779Sdim } 1223344779Sdim 1224360784Sdim assert(Remappings && "should be initialized while creating remapper"); 1225360784Sdim for (auto &Sample : Reader.getProfiles()) 1226360784Sdim if (auto Key = Remappings->insert(Sample.first())) 1227344779Sdim SampleMap.insert({Key, &Sample.second}); 1228344779Sdim 1229360784Sdim RemappingApplied = true; 1230344779Sdim} 1231344779Sdim 1232344779SdimFunctionSamples * 1233344779SdimSampleProfileReaderItaniumRemapper::getSamplesFor(StringRef Fname) { 1234360784Sdim if (auto Key = Remappings->lookup(Fname)) 1235344779Sdim return SampleMap.lookup(Key); 1236360784Sdim return nullptr; 1237344779Sdim} 1238344779Sdim 1239341825Sdim/// Prepare a memory buffer for the contents of \p Filename. 1240277323Sdim/// 1241277323Sdim/// \returns an error code indicating the status of the buffer. 1242277323Sdimstatic ErrorOr<std::unique_ptr<MemoryBuffer>> 1243309124SdimsetupMemoryBuffer(const Twine &Filename) { 1244277323Sdim auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename); 1245277323Sdim if (std::error_code EC = BufferOrErr.getError()) 1246277323Sdim return EC; 1247277323Sdim auto Buffer = std::move(BufferOrErr.get()); 1248277323Sdim 1249277323Sdim // Sanity check the file. 1250327952Sdim if (uint64_t(Buffer->getBufferSize()) > std::numeric_limits<uint32_t>::max()) 1251277323Sdim return sampleprof_error::too_large; 1252277323Sdim 1253277323Sdim return std::move(Buffer); 1254277323Sdim} 1255277323Sdim 1256341825Sdim/// Create a sample profile reader based on the format of the input file. 1257277323Sdim/// 1258277323Sdim/// \param Filename The file to open. 1259277323Sdim/// 1260277323Sdim/// \param C The LLVM context to use to emit diagnostics. 1261277323Sdim/// 1262360784Sdim/// \param RemapFilename The file used for profile remapping. 1263360784Sdim/// 1264277323Sdim/// \returns an error code indicating the status of the created reader. 1265277323SdimErrorOr<std::unique_ptr<SampleProfileReader>> 1266360784SdimSampleProfileReader::create(const std::string Filename, LLVMContext &C, 1267360784Sdim const std::string RemapFilename) { 1268277323Sdim auto BufferOrError = setupMemoryBuffer(Filename); 1269277323Sdim if (std::error_code EC = BufferOrError.getError()) 1270277323Sdim return EC; 1271360784Sdim return create(BufferOrError.get(), C, RemapFilename); 1272296417Sdim} 1273277323Sdim 1274344779Sdim/// Create a sample profile remapper from the given input, to remap the 1275344779Sdim/// function names in the given profile data. 1276344779Sdim/// 1277344779Sdim/// \param Filename The file to open. 1278344779Sdim/// 1279360784Sdim/// \param Reader The profile reader the remapper is going to be applied to. 1280360784Sdim/// 1281344779Sdim/// \param C The LLVM context to use to emit diagnostics. 1282344779Sdim/// 1283344779Sdim/// \returns an error code indicating the status of the created reader. 1284360784SdimErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>> 1285360784SdimSampleProfileReaderItaniumRemapper::create(const std::string Filename, 1286360784Sdim SampleProfileReader &Reader, 1287360784Sdim LLVMContext &C) { 1288344779Sdim auto BufferOrError = setupMemoryBuffer(Filename); 1289344779Sdim if (std::error_code EC = BufferOrError.getError()) 1290344779Sdim return EC; 1291360784Sdim return create(BufferOrError.get(), Reader, C); 1292344779Sdim} 1293344779Sdim 1294360784Sdim/// Create a sample profile remapper from the given input, to remap the 1295360784Sdim/// function names in the given profile data. 1296360784Sdim/// 1297360784Sdim/// \param B The memory buffer to create the reader from (assumes ownership). 1298360784Sdim/// 1299360784Sdim/// \param C The LLVM context to use to emit diagnostics. 1300360784Sdim/// 1301360784Sdim/// \param Reader The profile reader the remapper is going to be applied to. 1302360784Sdim/// 1303360784Sdim/// \returns an error code indicating the status of the created reader. 1304360784SdimErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>> 1305360784SdimSampleProfileReaderItaniumRemapper::create(std::unique_ptr<MemoryBuffer> &B, 1306360784Sdim SampleProfileReader &Reader, 1307360784Sdim LLVMContext &C) { 1308360784Sdim auto Remappings = std::make_unique<SymbolRemappingReader>(); 1309360784Sdim if (Error E = Remappings->read(*B.get())) { 1310360784Sdim handleAllErrors( 1311360784Sdim std::move(E), [&](const SymbolRemappingParseError &ParseError) { 1312360784Sdim C.diagnose(DiagnosticInfoSampleProfile(B->getBufferIdentifier(), 1313360784Sdim ParseError.getLineNum(), 1314360784Sdim ParseError.getMessage())); 1315360784Sdim }); 1316360784Sdim return sampleprof_error::malformed; 1317360784Sdim } 1318360784Sdim 1319360784Sdim return std::make_unique<SampleProfileReaderItaniumRemapper>( 1320360784Sdim std::move(B), std::move(Remappings), Reader); 1321360784Sdim} 1322360784Sdim 1323341825Sdim/// Create a sample profile reader based on the format of the input data. 1324296417Sdim/// 1325296417Sdim/// \param B The memory buffer to create the reader from (assumes ownership). 1326296417Sdim/// 1327296417Sdim/// \param C The LLVM context to use to emit diagnostics. 1328296417Sdim/// 1329360784Sdim/// \param RemapFilename The file used for profile remapping. 1330360784Sdim/// 1331296417Sdim/// \returns an error code indicating the status of the created reader. 1332296417SdimErrorOr<std::unique_ptr<SampleProfileReader>> 1333360784SdimSampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C, 1334360784Sdim const std::string RemapFilename) { 1335277323Sdim std::unique_ptr<SampleProfileReader> Reader; 1336341825Sdim if (SampleProfileReaderRawBinary::hasFormat(*B)) 1337341825Sdim Reader.reset(new SampleProfileReaderRawBinary(std::move(B), C)); 1338360784Sdim else if (SampleProfileReaderExtBinary::hasFormat(*B)) 1339360784Sdim Reader.reset(new SampleProfileReaderExtBinary(std::move(B), C)); 1340341825Sdim else if (SampleProfileReaderCompactBinary::hasFormat(*B)) 1341341825Sdim Reader.reset(new SampleProfileReaderCompactBinary(std::move(B), C)); 1342296417Sdim else if (SampleProfileReaderGCC::hasFormat(*B)) 1343296417Sdim Reader.reset(new SampleProfileReaderGCC(std::move(B), C)); 1344296417Sdim else if (SampleProfileReaderText::hasFormat(*B)) 1345296417Sdim Reader.reset(new SampleProfileReaderText(std::move(B), C)); 1346277323Sdim else 1347296417Sdim return sampleprof_error::unrecognized_format; 1348277323Sdim 1349360784Sdim if (!RemapFilename.empty()) { 1350360784Sdim auto ReaderOrErr = 1351360784Sdim SampleProfileReaderItaniumRemapper::create(RemapFilename, *Reader, C); 1352360784Sdim if (std::error_code EC = ReaderOrErr.getError()) { 1353360784Sdim std::string Msg = "Could not create remapper: " + EC.message(); 1354360784Sdim C.diagnose(DiagnosticInfoSampleProfile(RemapFilename, Msg)); 1355360784Sdim return EC; 1356360784Sdim } 1357360784Sdim Reader->Remapper = std::move(ReaderOrErr.get()); 1358360784Sdim } 1359360784Sdim 1360344779Sdim FunctionSamples::Format = Reader->getFormat(); 1361360784Sdim if (std::error_code EC = Reader->readHeader()) { 1362277323Sdim return EC; 1363360784Sdim } 1364277323Sdim 1365277323Sdim return std::move(Reader); 1366277323Sdim} 1367309124Sdim 1368309124Sdim// For text and GCC file formats, we compute the summary after reading the 1369309124Sdim// profile. Binary format has the profile summary in its header. 1370309124Sdimvoid SampleProfileReader::computeSummary() { 1371309124Sdim SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); 1372309124Sdim for (const auto &I : Profiles) { 1373309124Sdim const FunctionSamples &Profile = I.second; 1374309124Sdim Builder.addRecord(Profile); 1375309124Sdim } 1376309124Sdim Summary = Builder.getSummary(); 1377309124Sdim} 1378