1//===- SampleProfWriter.h - Write LLVM sample profile data ------*- C++ -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file contains definitions needed for writing sample profiles. 10// 11//===----------------------------------------------------------------------===// 12#ifndef LLVM_PROFILEDATA_SAMPLEPROFWRITER_H 13#define LLVM_PROFILEDATA_SAMPLEPROFWRITER_H 14 15#include "llvm/ADT/MapVector.h" 16#include "llvm/ADT/StringRef.h" 17#include "llvm/IR/ProfileSummary.h" 18#include "llvm/ProfileData/SampleProf.h" 19#include "llvm/Support/ErrorOr.h" 20#include "llvm/Support/raw_ostream.h" 21#include <cstdint> 22#include <memory> 23#include <set> 24#include <system_error> 25 26namespace llvm { 27namespace sampleprof { 28 29enum SectionLayout { 30 DefaultLayout, 31 // The layout splits profile with context information from profile without 32 // context information. When Thinlto is enabled, ThinLTO postlink phase only 33 // has to load profile with context information and can skip the other part. 34 CtxSplitLayout, 35 NumOfLayout, 36}; 37 38/// When writing a profile with size limit, user may want to use a different 39/// strategy to reduce function count other than dropping functions with fewest 40/// samples first. In this case a class implementing the same interfaces should 41/// be provided to SampleProfileWriter::writeWithSizeLimit(). 42class FunctionPruningStrategy { 43protected: 44 SampleProfileMap &ProfileMap; 45 size_t OutputSizeLimit; 46 47public: 48 /// \p ProfileMap A reference to the original profile map. It will be modified 49 /// by Erase(). 50 /// \p OutputSizeLimit Size limit in bytes of the output profile. This is 51 /// necessary to estimate how many functions to remove. 52 FunctionPruningStrategy(SampleProfileMap &ProfileMap, size_t OutputSizeLimit) 53 : ProfileMap(ProfileMap), OutputSizeLimit(OutputSizeLimit) {} 54 55 virtual ~FunctionPruningStrategy() = default; 56 57 /// SampleProfileWriter::writeWithSizeLimit() calls this after every write 58 /// iteration if the output size still exceeds the limit. This function 59 /// should erase some functions from the profile map so that the writer tries 60 /// to write the profile again with fewer functions. At least 1 entry from the 61 /// profile map must be erased. 62 /// 63 /// \p CurrentOutputSize Number of bytes in the output if current profile map 64 /// is written. 65 virtual void Erase(size_t CurrentOutputSize) = 0; 66}; 67 68class DefaultFunctionPruningStrategy : public FunctionPruningStrategy { 69 std::vector<NameFunctionSamples> SortedFunctions; 70 71public: 72 DefaultFunctionPruningStrategy(SampleProfileMap &ProfileMap, 73 size_t OutputSizeLimit); 74 75 /// In this default implementation, functions with fewest samples are dropped 76 /// first. Since the exact size of the output cannot be easily calculated due 77 /// to compression, we use a heuristic to remove as many functions as 78 /// necessary but not too many, aiming to minimize the number of write 79 /// iterations. 80 /// Empirically, functions with larger total sample count contain linearly 81 /// more sample entries, meaning it takes linearly more space to write them. 82 /// The cumulative length is therefore quadratic if all functions are sorted 83 /// by total sample count. 84 /// TODO: Find better heuristic. 85 void Erase(size_t CurrentOutputSize) override; 86}; 87 88/// Sample-based profile writer. Base class. 89class SampleProfileWriter { 90public: 91 virtual ~SampleProfileWriter() = default; 92 93 /// Write sample profiles in \p S. 94 /// 95 /// \returns status code of the file update operation. 96 virtual std::error_code writeSample(const FunctionSamples &S) = 0; 97 98 /// Write all the sample profiles in the given map of samples. 99 /// 100 /// \returns status code of the file update operation. 101 virtual std::error_code write(const SampleProfileMap &ProfileMap); 102 103 /// Write sample profiles up to given size limit, using the pruning strategy 104 /// to drop some functions if necessary. 105 /// 106 /// \returns status code of the file update operation. 107 template <typename FunctionPruningStrategy = DefaultFunctionPruningStrategy> 108 std::error_code writeWithSizeLimit(SampleProfileMap &ProfileMap, 109 size_t OutputSizeLimit) { 110 FunctionPruningStrategy Strategy(ProfileMap, OutputSizeLimit); 111 return writeWithSizeLimitInternal(ProfileMap, OutputSizeLimit, &Strategy); 112 } 113 114 raw_ostream &getOutputStream() { return *OutputStream; } 115 116 /// Profile writer factory. 117 /// 118 /// Create a new file writer based on the value of \p Format. 119 static ErrorOr<std::unique_ptr<SampleProfileWriter>> 120 create(StringRef Filename, SampleProfileFormat Format); 121 122 /// Create a new stream writer based on the value of \p Format. 123 /// For testing. 124 static ErrorOr<std::unique_ptr<SampleProfileWriter>> 125 create(std::unique_ptr<raw_ostream> &OS, SampleProfileFormat Format); 126 127 virtual void setProfileSymbolList(ProfileSymbolList *PSL) {} 128 virtual void setToCompressAllSections() {} 129 virtual void setUseMD5() {} 130 virtual void setPartialProfile() {} 131 virtual void resetSecLayout(SectionLayout SL) {} 132 133protected: 134 SampleProfileWriter(std::unique_ptr<raw_ostream> &OS) 135 : OutputStream(std::move(OS)) {} 136 137 /// Write a file header for the profile file. 138 virtual std::error_code writeHeader(const SampleProfileMap &ProfileMap) = 0; 139 140 // Write function profiles to the profile file. 141 virtual std::error_code writeFuncProfiles(const SampleProfileMap &ProfileMap); 142 143 std::error_code writeWithSizeLimitInternal(SampleProfileMap &ProfileMap, 144 size_t OutputSizeLimit, 145 FunctionPruningStrategy *Strategy); 146 147 /// For writeWithSizeLimit in text mode, each newline takes 1 additional byte 148 /// on Windows when actually written to the file, but not written to a memory 149 /// buffer. This needs to be accounted for when rewriting the profile. 150 size_t LineCount; 151 152 /// Output stream where to emit the profile to. 153 std::unique_ptr<raw_ostream> OutputStream; 154 155 /// Profile summary. 156 std::unique_ptr<ProfileSummary> Summary; 157 158 /// Compute summary for this profile. 159 void computeSummary(const SampleProfileMap &ProfileMap); 160 161 /// Profile format. 162 SampleProfileFormat Format = SPF_None; 163}; 164 165/// Sample-based profile writer (text format). 166class SampleProfileWriterText : public SampleProfileWriter { 167public: 168 std::error_code writeSample(const FunctionSamples &S) override; 169 170protected: 171 SampleProfileWriterText(std::unique_ptr<raw_ostream> &OS) 172 : SampleProfileWriter(OS), Indent(0) {} 173 174 std::error_code writeHeader(const SampleProfileMap &ProfileMap) override { 175 LineCount = 0; 176 return sampleprof_error::success; 177 } 178 179private: 180 /// Indent level to use when writing. 181 /// 182 /// This is used when printing inlined callees. 183 unsigned Indent; 184 185 friend ErrorOr<std::unique_ptr<SampleProfileWriter>> 186 SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS, 187 SampleProfileFormat Format); 188}; 189 190/// Sample-based profile writer (binary format). 191class SampleProfileWriterBinary : public SampleProfileWriter { 192public: 193 SampleProfileWriterBinary(std::unique_ptr<raw_ostream> &OS) 194 : SampleProfileWriter(OS) {} 195 196 std::error_code writeSample(const FunctionSamples &S) override; 197 198protected: 199 virtual MapVector<FunctionId, uint32_t> &getNameTable() { return NameTable; } 200 virtual std::error_code writeMagicIdent(SampleProfileFormat Format); 201 virtual std::error_code writeNameTable(); 202 std::error_code writeHeader(const SampleProfileMap &ProfileMap) override; 203 std::error_code writeSummary(); 204 virtual std::error_code writeContextIdx(const SampleContext &Context); 205 std::error_code writeNameIdx(FunctionId FName); 206 std::error_code writeBody(const FunctionSamples &S); 207 inline void stablizeNameTable(MapVector<FunctionId, uint32_t> &NameTable, 208 std::set<FunctionId> &V); 209 210 MapVector<FunctionId, uint32_t> NameTable; 211 212 void addName(FunctionId FName); 213 virtual void addContext(const SampleContext &Context); 214 void addNames(const FunctionSamples &S); 215 216private: 217 friend ErrorOr<std::unique_ptr<SampleProfileWriter>> 218 SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS, 219 SampleProfileFormat Format); 220}; 221 222class SampleProfileWriterRawBinary : public SampleProfileWriterBinary { 223 using SampleProfileWriterBinary::SampleProfileWriterBinary; 224}; 225 226const std::array<SmallVector<SecHdrTableEntry, 8>, NumOfLayout> 227 ExtBinaryHdrLayoutTable = { 228 // Note that SecFuncOffsetTable section is written after SecLBRProfile 229 // in the profile, but is put before SecLBRProfile in SectionHdrLayout. 230 // This is because sample reader follows the order in SectionHdrLayout 231 // to read each section. To read function profiles on demand, sample 232 // reader need to get the offset of each function profile first. 233 // 234 // DefaultLayout 235 SmallVector<SecHdrTableEntry, 8>({{SecProfSummary, 0, 0, 0, 0}, 236 {SecNameTable, 0, 0, 0, 0}, 237 {SecCSNameTable, 0, 0, 0, 0}, 238 {SecFuncOffsetTable, 0, 0, 0, 0}, 239 {SecLBRProfile, 0, 0, 0, 0}, 240 {SecProfileSymbolList, 0, 0, 0, 0}, 241 {SecFuncMetadata, 0, 0, 0, 0}}), 242 // CtxSplitLayout 243 SmallVector<SecHdrTableEntry, 8>({{SecProfSummary, 0, 0, 0, 0}, 244 {SecNameTable, 0, 0, 0, 0}, 245 // profile with context 246 // for next two sections 247 {SecFuncOffsetTable, 0, 0, 0, 0}, 248 {SecLBRProfile, 0, 0, 0, 0}, 249 // profile without context 250 // for next two sections 251 {SecFuncOffsetTable, 0, 0, 0, 0}, 252 {SecLBRProfile, 0, 0, 0, 0}, 253 {SecProfileSymbolList, 0, 0, 0, 0}, 254 {SecFuncMetadata, 0, 0, 0, 0}}), 255}; 256 257class SampleProfileWriterExtBinaryBase : public SampleProfileWriterBinary { 258 using SampleProfileWriterBinary::SampleProfileWriterBinary; 259public: 260 std::error_code write(const SampleProfileMap &ProfileMap) override; 261 262 void setToCompressAllSections() override; 263 void setToCompressSection(SecType Type); 264 std::error_code writeSample(const FunctionSamples &S) override; 265 266 // Set to use MD5 to represent string in NameTable. 267 void setUseMD5() override { 268 UseMD5 = true; 269 addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagMD5Name); 270 // MD5 will be stored as plain uint64_t instead of variable-length 271 // quantity format in NameTable section. 272 addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagFixedLengthMD5); 273 } 274 275 // Set the profile to be partial. It means the profile is for 276 // common/shared code. The common profile is usually merged from 277 // profiles collected from running other targets. 278 void setPartialProfile() override { 279 addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagPartial); 280 } 281 282 void setProfileSymbolList(ProfileSymbolList *PSL) override { 283 ProfSymList = PSL; 284 }; 285 286 void resetSecLayout(SectionLayout SL) override { 287 verifySecLayout(SL); 288#ifndef NDEBUG 289 // Make sure resetSecLayout is called before any flag setting. 290 for (auto &Entry : SectionHdrLayout) { 291 assert(Entry.Flags == 0 && 292 "resetSecLayout has to be called before any flag setting"); 293 } 294#endif 295 SecLayout = SL; 296 SectionHdrLayout = ExtBinaryHdrLayoutTable[SL]; 297 } 298 299protected: 300 uint64_t markSectionStart(SecType Type, uint32_t LayoutIdx); 301 std::error_code addNewSection(SecType Sec, uint32_t LayoutIdx, 302 uint64_t SectionStart); 303 template <class SecFlagType> 304 void addSectionFlag(SecType Type, SecFlagType Flag) { 305 for (auto &Entry : SectionHdrLayout) { 306 if (Entry.Type == Type) 307 addSecFlag(Entry, Flag); 308 } 309 } 310 template <class SecFlagType> 311 void addSectionFlag(uint32_t SectionIdx, SecFlagType Flag) { 312 addSecFlag(SectionHdrLayout[SectionIdx], Flag); 313 } 314 315 void addContext(const SampleContext &Context) override; 316 317 // placeholder for subclasses to dispatch their own section writers. 318 virtual std::error_code writeCustomSection(SecType Type) = 0; 319 // Verify the SecLayout is supported by the format. 320 virtual void verifySecLayout(SectionLayout SL) = 0; 321 322 // specify the order to write sections. 323 virtual std::error_code writeSections(const SampleProfileMap &ProfileMap) = 0; 324 325 // Dispatch section writer for each section. \p LayoutIdx is the sequence 326 // number indicating where the section is located in SectionHdrLayout. 327 virtual std::error_code writeOneSection(SecType Type, uint32_t LayoutIdx, 328 const SampleProfileMap &ProfileMap); 329 330 // Helper function to write name table. 331 std::error_code writeNameTable() override; 332 std::error_code writeContextIdx(const SampleContext &Context) override; 333 std::error_code writeCSNameIdx(const SampleContext &Context); 334 std::error_code writeCSNameTableSection(); 335 336 std::error_code writeFuncMetadata(const SampleProfileMap &Profiles); 337 std::error_code writeFuncMetadata(const FunctionSamples &Profile); 338 339 // Functions to write various kinds of sections. 340 std::error_code writeNameTableSection(const SampleProfileMap &ProfileMap); 341 std::error_code writeFuncOffsetTable(); 342 std::error_code writeProfileSymbolListSection(); 343 344 SectionLayout SecLayout = DefaultLayout; 345 // Specifiy the order of sections in section header table. Note 346 // the order of sections in SecHdrTable may be different that the 347 // order in SectionHdrLayout. sample Reader will follow the order 348 // in SectionHdrLayout to read each section. 349 SmallVector<SecHdrTableEntry, 8> SectionHdrLayout = 350 ExtBinaryHdrLayoutTable[DefaultLayout]; 351 352 // Save the start of SecLBRProfile so we can compute the offset to the 353 // start of SecLBRProfile for each Function's Profile and will keep it 354 // in FuncOffsetTable. 355 uint64_t SecLBRProfileStart = 0; 356 357private: 358 void allocSecHdrTable(); 359 std::error_code writeSecHdrTable(); 360 std::error_code writeHeader(const SampleProfileMap &ProfileMap) override; 361 std::error_code compressAndOutput(); 362 363 // We will swap the raw_ostream held by LocalBufStream and that 364 // held by OutputStream if we try to add a section which needs 365 // compression. After the swap, all the data written to output 366 // will be temporarily buffered into the underlying raw_string_ostream 367 // originally held by LocalBufStream. After the data writing for the 368 // section is completed, compress the data in the local buffer, 369 // swap the raw_ostream back and write the compressed data to the 370 // real output. 371 std::unique_ptr<raw_ostream> LocalBufStream; 372 // The location where the output stream starts. 373 uint64_t FileStart; 374 // The location in the output stream where the SecHdrTable should be 375 // written to. 376 uint64_t SecHdrTableOffset; 377 // The table contains SecHdrTableEntry entries in order of how they are 378 // populated in the writer. It may be different from the order in 379 // SectionHdrLayout which specifies the sequence in which sections will 380 // be read. 381 std::vector<SecHdrTableEntry> SecHdrTable; 382 383 // FuncOffsetTable maps function context to its profile offset in 384 // SecLBRProfile section. It is used to load function profile on demand. 385 MapVector<SampleContext, uint64_t> FuncOffsetTable; 386 // Whether to use MD5 to represent string. 387 bool UseMD5 = false; 388 389 /// CSNameTable maps function context to its offset in SecCSNameTable section. 390 /// The offset will be used everywhere where the context is referenced. 391 MapVector<SampleContext, uint32_t> CSNameTable; 392 393 ProfileSymbolList *ProfSymList = nullptr; 394}; 395 396class SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase { 397public: 398 SampleProfileWriterExtBinary(std::unique_ptr<raw_ostream> &OS) 399 : SampleProfileWriterExtBinaryBase(OS) {} 400 401private: 402 std::error_code writeDefaultLayout(const SampleProfileMap &ProfileMap); 403 std::error_code writeCtxSplitLayout(const SampleProfileMap &ProfileMap); 404 405 std::error_code writeSections(const SampleProfileMap &ProfileMap) override; 406 407 std::error_code writeCustomSection(SecType Type) override { 408 return sampleprof_error::success; 409 }; 410 411 void verifySecLayout(SectionLayout SL) override { 412 assert((SL == DefaultLayout || SL == CtxSplitLayout) && 413 "Unsupported layout"); 414 } 415}; 416 417} // end namespace sampleprof 418} // end namespace llvm 419 420#endif // LLVM_PROFILEDATA_SAMPLEPROFWRITER_H 421