1356843Sdim/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ 2356843Sdim|* 3356843Sdim|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4356843Sdim|* See https://llvm.org/LICENSE.txt for license information. 5356843Sdim|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6356843Sdim|* 7356843Sdim\*===----------------------------------------------------------------------===*/ 8356843Sdim/* 9356843Sdim * This is the master file that defines all the data structure, signature, 10356843Sdim * constant literals that are shared across profiling runtime library, 11356843Sdim * compiler (instrumentation), and host tools (reader/writer). The entities 12356843Sdim * defined in this file affect the profile runtime ABI, the raw profile format, 13356843Sdim * or both. 14356843Sdim * 15356843Sdim * The file has two identical copies. The master copy lives in LLVM and 16356843Sdim * the other one sits in compiler-rt/lib/profile directory. To make changes 17356843Sdim * in this file, first modify the master copy and copy it over to compiler-rt. 18356843Sdim * Testing of any change in this file can start only after the two copies are 19356843Sdim * synced up. 20356843Sdim * 21356843Sdim * The first part of the file includes macros that defines types, names, and 22356843Sdim * initializers for the member fields of the core data structures. The field 23356843Sdim * declarations for one structure is enabled by defining the field activation 24356843Sdim * macro associated with that structure. Only one field activation record 25356843Sdim * can be defined at one time and the rest definitions will be filtered out by 26356843Sdim * the preprocessor. 27356843Sdim * 28356843Sdim * Examples of how the template is used to instantiate structure definition: 29356843Sdim * 1. To declare a structure: 30356843Sdim * 31356843Sdim * struct ProfData { 32356843Sdim * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 33356843Sdim * Type Name; 34356843Sdim * #include "llvm/ProfileData/InstrProfData.inc" 35356843Sdim * }; 36356843Sdim * 37356843Sdim * 2. To construct LLVM type arrays for the struct type: 38356843Sdim * 39356843Sdim * Type *DataTypes[] = { 40356843Sdim * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 41356843Sdim * LLVMType, 42356843Sdim * #include "llvm/ProfileData/InstrProfData.inc" 43356843Sdim * }; 44356843Sdim * 45356843Sdim * 4. To construct constant array for the initializers: 46356843Sdim * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 47356843Sdim * Initializer, 48356843Sdim * Constant *ConstantVals[] = { 49356843Sdim * #include "llvm/ProfileData/InstrProfData.inc" 50356843Sdim * }; 51356843Sdim * 52356843Sdim * 53356843Sdim * The second part of the file includes definitions all other entities that 54356843Sdim * are related to runtime ABI and format. When no field activation macro is 55356843Sdim * defined, this file can be included to introduce the definitions. 56356843Sdim * 57356843Sdim\*===----------------------------------------------------------------------===*/ 58356843Sdim 59356843Sdim/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in 60356843Sdim * the compiler runtime. */ 61356843Sdim#ifndef INSTR_PROF_VISIBILITY 62356843Sdim#define INSTR_PROF_VISIBILITY 63356843Sdim#endif 64356843Sdim 65356843Sdim/* INSTR_PROF_DATA start. */ 66356843Sdim/* Definition of member fields of the per-function control structure. */ 67356843Sdim#ifndef INSTR_PROF_DATA 68356843Sdim#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) 69356843Sdim#else 70356843Sdim#define INSTR_PROF_DATA_DEFINED 71356843Sdim#endif 72356843SdimINSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 73356843Sdim ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 74356843Sdim IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName())))) 75356843SdimINSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 76356843Sdim ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 77356843Sdim Inc->getHash()->getZExtValue())) 78356843SdimINSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ 79356843Sdim ConstantExpr::getBitCast(CounterPtr, \ 80356843Sdim llvm::Type::getInt64PtrTy(Ctx))) 81356843Sdim/* This is used to map function pointers for the indirect call targets to 82356843Sdim * function name hashes during the conversion from raw to merged profile 83356843Sdim * data. 84356843Sdim */ 85356843SdimINSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \ 86356843Sdim FunctionAddr) 87356843SdimINSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ 88356843Sdim ValuesPtrExpr) 89356843SdimINSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ 90356843Sdim ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) 91356843SdimINSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ 92356843Sdim ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) 93356843Sdim#undef INSTR_PROF_DATA 94356843Sdim/* INSTR_PROF_DATA end. */ 95356843Sdim 96356843Sdim 97356843Sdim/* This is an internal data structure used by value profiler. It 98356843Sdim * is defined here to allow serialization code sharing by LLVM 99356843Sdim * to be used in unit test. 100356843Sdim * 101356843Sdim * typedef struct ValueProfNode { 102356843Sdim * // InstrProfValueData VData; 103356843Sdim * uint64_t Value; 104356843Sdim * uint64_t Count; 105356843Sdim * struct ValueProfNode *Next; 106356843Sdim * } ValueProfNode; 107356843Sdim */ 108356843Sdim/* INSTR_PROF_VALUE_NODE start. */ 109356843Sdim#ifndef INSTR_PROF_VALUE_NODE 110356843Sdim#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) 111356843Sdim#else 112356843Sdim#define INSTR_PROF_DATA_DEFINED 113356843Sdim#endif 114356843SdimINSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \ 115356843Sdim ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 116356843SdimINSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \ 117356843Sdim ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 118356843SdimINSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ 119356843Sdim ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0)) 120356843Sdim#undef INSTR_PROF_VALUE_NODE 121356843Sdim/* INSTR_PROF_VALUE_NODE end. */ 122356843Sdim 123356843Sdim/* INSTR_PROF_RAW_HEADER start */ 124356843Sdim/* Definition of member fields of the raw profile header data structure. */ 125356843Sdim#ifndef INSTR_PROF_RAW_HEADER 126356843Sdim#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) 127356843Sdim#else 128356843Sdim#define INSTR_PROF_DATA_DEFINED 129356843Sdim#endif 130356843SdimINSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) 131356843SdimINSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) 132356843SdimINSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) 133356843SdimINSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) 134356843SdimINSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) 135356843SdimINSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) 136356843SdimINSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) 137356843SdimINSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) 138356843SdimINSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) 139356843SdimINSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) 140356843Sdim#undef INSTR_PROF_RAW_HEADER 141356843Sdim/* INSTR_PROF_RAW_HEADER end */ 142356843Sdim 143356843Sdim/* VALUE_PROF_FUNC_PARAM start */ 144356843Sdim/* Definition of parameter types of the runtime API used to do value profiling 145356843Sdim * for a given value site. 146356843Sdim */ 147356843Sdim#ifndef VALUE_PROF_FUNC_PARAM 148356843Sdim#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) 149356843Sdim#define INSTR_PROF_COMMA 150356843Sdim#else 151356843Sdim#define INSTR_PROF_DATA_DEFINED 152356843Sdim#define INSTR_PROF_COMMA , 153356843Sdim#endif 154356843SdimVALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ 155356843Sdim INSTR_PROF_COMMA 156356843SdimVALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA 157356843Sdim#ifndef VALUE_RANGE_PROF 158356843SdimVALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) 159356843Sdim#else /* VALUE_RANGE_PROF */ 160356843SdimVALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \ 161356843Sdim INSTR_PROF_COMMA 162356843SdimVALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \ 163356843Sdim INSTR_PROF_COMMA 164356843SdimVALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeLast, Type::getInt64Ty(Ctx)) \ 165356843Sdim INSTR_PROF_COMMA 166356843SdimVALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) 167356843Sdim#endif /*VALUE_RANGE_PROF */ 168356843Sdim#undef VALUE_PROF_FUNC_PARAM 169356843Sdim#undef INSTR_PROF_COMMA 170356843Sdim/* VALUE_PROF_FUNC_PARAM end */ 171356843Sdim 172356843Sdim/* VALUE_PROF_KIND start */ 173356843Sdim#ifndef VALUE_PROF_KIND 174356843Sdim#define VALUE_PROF_KIND(Enumerator, Value, Descr) 175356843Sdim#else 176356843Sdim#define INSTR_PROF_DATA_DEFINED 177356843Sdim#endif 178356843Sdim/* For indirect function call value profiling, the addresses of the target 179356843Sdim * functions are profiled by the instrumented code. The target addresses are 180356843Sdim * written in the raw profile data and converted to target function name's MD5 181356843Sdim * hash by the profile reader during deserialization. Typically, this happens 182356843Sdim * when the raw profile data is read during profile merging. 183356843Sdim * 184356843Sdim * For this remapping the ProfData is used. ProfData contains both the function 185356843Sdim * name hash and the function address. 186356843Sdim */ 187356843SdimVALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") 188356843Sdim/* For memory intrinsic functions size profiling. */ 189356843SdimVALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") 190356843Sdim/* These two kinds must be the last to be 191356843Sdim * declared. This is to make sure the string 192356843Sdim * array created with the template can be 193356843Sdim * indexed with the kind value. 194356843Sdim */ 195356843SdimVALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") 196356843SdimVALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") 197356843Sdim 198356843Sdim#undef VALUE_PROF_KIND 199356843Sdim/* VALUE_PROF_KIND end */ 200356843Sdim 201356843Sdim/* COVMAP_FUNC_RECORD start */ 202356843Sdim/* Definition of member fields of the function record structure in coverage 203356843Sdim * map. 204356843Sdim */ 205356843Sdim#ifndef COVMAP_FUNC_RECORD 206356843Sdim#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) 207356843Sdim#else 208356843Sdim#define INSTR_PROF_DATA_DEFINED 209356843Sdim#endif 210356843Sdim#ifdef COVMAP_V1 211356843SdimCOVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ 212356843Sdim NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ 213356843Sdim llvm::Type::getInt8PtrTy(Ctx))) 214356843SdimCOVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ 215356843Sdim llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ 216356843Sdim NameValue.size())) 217356843Sdim#else 218356843SdimCOVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 219356843Sdim llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 220356843Sdim llvm::IndexedInstrProf::ComputeHash(NameValue))) 221356843Sdim#endif 222356843SdimCOVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ 223356843Sdim llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\ 224356843Sdim CoverageMapping.size())) 225356843SdimCOVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 226356843Sdim llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), FuncHash)) 227356843Sdim#undef COVMAP_FUNC_RECORD 228356843Sdim/* COVMAP_FUNC_RECORD end. */ 229356843Sdim 230356843Sdim/* COVMAP_HEADER start */ 231356843Sdim/* Definition of member fields of coverage map header. 232356843Sdim */ 233356843Sdim#ifndef COVMAP_HEADER 234356843Sdim#define COVMAP_HEADER(Type, LLVMType, Name, Initializer) 235356843Sdim#else 236356843Sdim#define INSTR_PROF_DATA_DEFINED 237356843Sdim#endif 238356843SdimCOVMAP_HEADER(uint32_t, Int32Ty, NRecords, \ 239356843Sdim llvm::ConstantInt::get(Int32Ty, FunctionRecords.size())) 240356843SdimCOVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ 241356843Sdim llvm::ConstantInt::get(Int32Ty, FilenamesSize)) 242356843SdimCOVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ 243356843Sdim llvm::ConstantInt::get(Int32Ty, CoverageMappingSize)) 244356843SdimCOVMAP_HEADER(uint32_t, Int32Ty, Version, \ 245356843Sdim llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion)) 246356843Sdim#undef COVMAP_HEADER 247356843Sdim/* COVMAP_HEADER end. */ 248356843Sdim 249356843Sdim 250356843Sdim#ifdef INSTR_PROF_SECT_ENTRY 251356843Sdim#define INSTR_PROF_DATA_DEFINED 252356843SdimINSTR_PROF_SECT_ENTRY(IPSK_data, \ 253356843Sdim INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ 254356843Sdim INSTR_PROF_DATA_COFF, "__DATA,") 255356843SdimINSTR_PROF_SECT_ENTRY(IPSK_cnts, \ 256356843Sdim INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ 257356843Sdim INSTR_PROF_CNTS_COFF, "__DATA,") 258356843SdimINSTR_PROF_SECT_ENTRY(IPSK_name, \ 259356843Sdim INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ 260356843Sdim INSTR_PROF_NAME_COFF, "__DATA,") 261356843SdimINSTR_PROF_SECT_ENTRY(IPSK_vals, \ 262356843Sdim INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ 263356843Sdim INSTR_PROF_VALS_COFF, "__DATA,") 264356843SdimINSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ 265356843Sdim INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ 266356843Sdim INSTR_PROF_VNODES_COFF, "__DATA,") 267356843SdimINSTR_PROF_SECT_ENTRY(IPSK_covmap, \ 268356843Sdim INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ 269356843Sdim INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") 270356843SdimINSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ 271356843Sdim INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ 272356843Sdim INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") 273356843Sdim 274356843Sdim#undef INSTR_PROF_SECT_ENTRY 275356843Sdim#endif 276356843Sdim 277356843Sdim 278356843Sdim#ifdef INSTR_PROF_VALUE_PROF_DATA 279356843Sdim#define INSTR_PROF_DATA_DEFINED 280356843Sdim 281356843Sdim#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255 282356843Sdim/*! 283356843Sdim * This is the header of the data structure that defines the on-disk 284356843Sdim * layout of the value profile data of a particular kind for one function. 285356843Sdim */ 286356843Sdimtypedef struct ValueProfRecord { 287356843Sdim /* The kind of the value profile record. */ 288356843Sdim uint32_t Kind; 289356843Sdim /* 290356843Sdim * The number of value profile sites. It is guaranteed to be non-zero; 291356843Sdim * otherwise the record for this kind won't be emitted. 292356843Sdim */ 293356843Sdim uint32_t NumValueSites; 294356843Sdim /* 295356843Sdim * The first element of the array that stores the number of profiled 296356843Sdim * values for each value site. The size of the array is NumValueSites. 297356843Sdim * Since NumValueSites is greater than zero, there is at least one 298356843Sdim * element in the array. 299356843Sdim */ 300356843Sdim uint8_t SiteCountArray[1]; 301356843Sdim 302356843Sdim /* 303356843Sdim * The fake declaration is for documentation purpose only. 304356843Sdim * Align the start of next field to be on 8 byte boundaries. 305356843Sdim uint8_t Padding[X]; 306356843Sdim */ 307356843Sdim 308356843Sdim /* The array of value profile data. The size of the array is the sum 309356843Sdim * of all elements in SiteCountArray[]. 310356843Sdim InstrProfValueData ValueData[]; 311356843Sdim */ 312356843Sdim 313356843Sdim#ifdef __cplusplus 314356843Sdim /*! 315356843Sdim * Return the number of value sites. 316356843Sdim */ 317356843Sdim uint32_t getNumValueSites() const { return NumValueSites; } 318356843Sdim /*! 319356843Sdim * Read data from this record and save it to Record. 320356843Sdim */ 321356843Sdim void deserializeTo(InstrProfRecord &Record, 322356843Sdim InstrProfSymtab *SymTab); 323356843Sdim /* 324356843Sdim * In-place byte swap: 325356843Sdim * Do byte swap for this instance. \c Old is the original order before 326356843Sdim * the swap, and \c New is the New byte order. 327356843Sdim */ 328356843Sdim void swapBytes(support::endianness Old, support::endianness New); 329356843Sdim#endif 330356843Sdim} ValueProfRecord; 331356843Sdim 332356843Sdim/*! 333356843Sdim * Per-function header/control data structure for value profiling 334356843Sdim * data in indexed format. 335356843Sdim */ 336356843Sdimtypedef struct ValueProfData { 337356843Sdim /* 338356843Sdim * Total size in bytes including this field. It must be a multiple 339356843Sdim * of sizeof(uint64_t). 340356843Sdim */ 341356843Sdim uint32_t TotalSize; 342356843Sdim /* 343356843Sdim *The number of value profile kinds that has value profile data. 344356843Sdim * In this implementation, a value profile kind is considered to 345356843Sdim * have profile data if the number of value profile sites for the 346356843Sdim * kind is not zero. More aggressively, the implementation can 347356843Sdim * choose to check the actual data value: if none of the value sites 348356843Sdim * has any profiled values, the kind can be skipped. 349356843Sdim */ 350356843Sdim uint32_t NumValueKinds; 351356843Sdim 352356843Sdim /* 353356843Sdim * Following are a sequence of variable length records. The prefix/header 354356843Sdim * of each record is defined by ValueProfRecord type. The number of 355356843Sdim * records is NumValueKinds. 356356843Sdim * ValueProfRecord Record_1; 357356843Sdim * ValueProfRecord Record_N; 358356843Sdim */ 359356843Sdim 360356843Sdim#if __cplusplus 361356843Sdim /*! 362356843Sdim * Return the total size in bytes of the on-disk value profile data 363356843Sdim * given the data stored in Record. 364356843Sdim */ 365356843Sdim static uint32_t getSize(const InstrProfRecord &Record); 366356843Sdim /*! 367356843Sdim * Return a pointer to \c ValueProfData instance ready to be streamed. 368356843Sdim */ 369356843Sdim static std::unique_ptr<ValueProfData> 370356843Sdim serializeFrom(const InstrProfRecord &Record); 371356843Sdim /*! 372356843Sdim * Check the integrity of the record. 373356843Sdim */ 374356843Sdim Error checkIntegrity(); 375356843Sdim /*! 376356843Sdim * Return a pointer to \c ValueProfileData instance ready to be read. 377356843Sdim * All data in the instance are properly byte swapped. The input 378356843Sdim * data is assumed to be in little endian order. 379356843Sdim */ 380356843Sdim static Expected<std::unique_ptr<ValueProfData>> 381356843Sdim getValueProfData(const unsigned char *SrcBuffer, 382356843Sdim const unsigned char *const SrcBufferEnd, 383356843Sdim support::endianness SrcDataEndianness); 384356843Sdim /*! 385356843Sdim * Swap byte order from \c Endianness order to host byte order. 386356843Sdim */ 387356843Sdim void swapBytesToHost(support::endianness Endianness); 388356843Sdim /*! 389356843Sdim * Swap byte order from host byte order to \c Endianness order. 390356843Sdim */ 391356843Sdim void swapBytesFromHost(support::endianness Endianness); 392356843Sdim /*! 393356843Sdim * Return the total size of \c ValueProfileData. 394356843Sdim */ 395356843Sdim uint32_t getSize() const { return TotalSize; } 396356843Sdim /*! 397356843Sdim * Read data from this data and save it to \c Record. 398356843Sdim */ 399356843Sdim void deserializeTo(InstrProfRecord &Record, 400356843Sdim InstrProfSymtab *SymTab); 401356843Sdim void operator delete(void *ptr) { ::operator delete(ptr); } 402356843Sdim#endif 403356843Sdim} ValueProfData; 404356843Sdim 405356843Sdim/* 406356843Sdim * The closure is designed to abstact away two types of value profile data: 407356843Sdim * - InstrProfRecord which is the primary data structure used to 408356843Sdim * represent profile data in host tools (reader, writer, and profile-use) 409356843Sdim * - value profile runtime data structure suitable to be used by C 410356843Sdim * runtime library. 411356843Sdim * 412356843Sdim * Both sources of data need to serialize to disk/memory-buffer in common 413356843Sdim * format: ValueProfData. The abstraction allows compiler-rt's raw profiler 414356843Sdim * writer to share the same format and code with indexed profile writer. 415356843Sdim * 416356843Sdim * For documentation of the member methods below, refer to corresponding methods 417356843Sdim * in class InstrProfRecord. 418356843Sdim */ 419356843Sdimtypedef struct ValueProfRecordClosure { 420356843Sdim const void *Record; 421356843Sdim uint32_t (*GetNumValueKinds)(const void *Record); 422356843Sdim uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); 423356843Sdim uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); 424356843Sdim uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); 425356843Sdim 426356843Sdim /* 427356843Sdim * After extracting the value profile data from the value profile record, 428356843Sdim * this method is used to map the in-memory value to on-disk value. If 429356843Sdim * the method is null, value will be written out untranslated. 430356843Sdim */ 431356843Sdim uint64_t (*RemapValueData)(uint32_t, uint64_t Value); 432356843Sdim void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, 433356843Sdim uint32_t S); 434356843Sdim ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); 435356843Sdim} ValueProfRecordClosure; 436356843Sdim 437356843SdimINSTR_PROF_VISIBILITY ValueProfRecord * 438356843SdimgetFirstValueProfRecord(ValueProfData *VPD); 439356843SdimINSTR_PROF_VISIBILITY ValueProfRecord * 440356843SdimgetValueProfRecordNext(ValueProfRecord *VPR); 441356843SdimINSTR_PROF_VISIBILITY InstrProfValueData * 442356843SdimgetValueProfRecordValueData(ValueProfRecord *VPR); 443356843SdimINSTR_PROF_VISIBILITY uint32_t 444356843SdimgetValueProfRecordHeaderSize(uint32_t NumValueSites); 445356843Sdim 446356843Sdim#undef INSTR_PROF_VALUE_PROF_DATA 447356843Sdim#endif /* INSTR_PROF_VALUE_PROF_DATA */ 448356843Sdim 449356843Sdim 450356843Sdim#ifdef INSTR_PROF_COMMON_API_IMPL 451356843Sdim#define INSTR_PROF_DATA_DEFINED 452356843Sdim#ifdef __cplusplus 453356843Sdim#define INSTR_PROF_INLINE inline 454356843Sdim#define INSTR_PROF_NULLPTR nullptr 455356843Sdim#else 456356843Sdim#define INSTR_PROF_INLINE 457356843Sdim#define INSTR_PROF_NULLPTR NULL 458356843Sdim#endif 459356843Sdim 460356843Sdim#ifndef offsetof 461356843Sdim#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) 462356843Sdim#endif 463356843Sdim 464356843Sdim/*! 465356843Sdim * Return the \c ValueProfRecord header size including the 466356843Sdim * padding bytes. 467356843Sdim */ 468356843SdimINSTR_PROF_VISIBILITY INSTR_PROF_INLINE 469356843Sdimuint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { 470356843Sdim uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + 471356843Sdim sizeof(uint8_t) * NumValueSites; 472356843Sdim /* Round the size to multiple of 8 bytes. */ 473356843Sdim Size = (Size + 7) & ~7; 474356843Sdim return Size; 475356843Sdim} 476356843Sdim 477356843Sdim/*! 478356843Sdim * Return the total size of the value profile record including the 479356843Sdim * header and the value data. 480356843Sdim */ 481356843SdimINSTR_PROF_VISIBILITY INSTR_PROF_INLINE 482356843Sdimuint32_t getValueProfRecordSize(uint32_t NumValueSites, 483356843Sdim uint32_t NumValueData) { 484356843Sdim return getValueProfRecordHeaderSize(NumValueSites) + 485356843Sdim sizeof(InstrProfValueData) * NumValueData; 486356843Sdim} 487356843Sdim 488356843Sdim/*! 489356843Sdim * Return the pointer to the start of value data array. 490356843Sdim */ 491356843SdimINSTR_PROF_VISIBILITY INSTR_PROF_INLINE 492356843SdimInstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { 493356843Sdim return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( 494356843Sdim This->NumValueSites)); 495356843Sdim} 496356843Sdim 497356843Sdim/*! 498356843Sdim * Return the total number of value data for \c This record. 499356843Sdim */ 500356843SdimINSTR_PROF_VISIBILITY INSTR_PROF_INLINE 501356843Sdimuint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { 502356843Sdim uint32_t NumValueData = 0; 503356843Sdim uint32_t I; 504356843Sdim for (I = 0; I < This->NumValueSites; I++) 505356843Sdim NumValueData += This->SiteCountArray[I]; 506356843Sdim return NumValueData; 507356843Sdim} 508356843Sdim 509356843Sdim/*! 510356843Sdim * Use this method to advance to the next \c This \c ValueProfRecord. 511356843Sdim */ 512356843SdimINSTR_PROF_VISIBILITY INSTR_PROF_INLINE 513356843SdimValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { 514356843Sdim uint32_t NumValueData = getValueProfRecordNumValueData(This); 515356843Sdim return (ValueProfRecord *)((char *)This + 516356843Sdim getValueProfRecordSize(This->NumValueSites, 517356843Sdim NumValueData)); 518356843Sdim} 519356843Sdim 520356843Sdim/*! 521356843Sdim * Return the first \c ValueProfRecord instance. 522356843Sdim */ 523356843SdimINSTR_PROF_VISIBILITY INSTR_PROF_INLINE 524356843SdimValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { 525356843Sdim return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); 526356843Sdim} 527356843Sdim 528356843Sdim/* Closure based interfaces. */ 529356843Sdim 530356843Sdim/*! 531356843Sdim * Return the total size in bytes of the on-disk value profile data 532356843Sdim * given the data stored in Record. 533356843Sdim */ 534356843SdimINSTR_PROF_VISIBILITY uint32_t 535356843SdimgetValueProfDataSize(ValueProfRecordClosure *Closure) { 536356843Sdim uint32_t Kind; 537356843Sdim uint32_t TotalSize = sizeof(ValueProfData); 538356843Sdim const void *Record = Closure->Record; 539356843Sdim 540356843Sdim for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 541356843Sdim uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); 542356843Sdim if (!NumValueSites) 543356843Sdim continue; 544356843Sdim TotalSize += getValueProfRecordSize(NumValueSites, 545356843Sdim Closure->GetNumValueData(Record, Kind)); 546356843Sdim } 547356843Sdim return TotalSize; 548356843Sdim} 549356843Sdim 550356843Sdim/*! 551356843Sdim * Extract value profile data of a function for the profile kind \c ValueKind 552356843Sdim * from the \c Closure and serialize the data into \c This record instance. 553356843Sdim */ 554356843SdimINSTR_PROF_VISIBILITY void 555356843SdimserializeValueProfRecordFrom(ValueProfRecord *This, 556356843Sdim ValueProfRecordClosure *Closure, 557356843Sdim uint32_t ValueKind, uint32_t NumValueSites) { 558356843Sdim uint32_t S; 559356843Sdim const void *Record = Closure->Record; 560356843Sdim This->Kind = ValueKind; 561356843Sdim This->NumValueSites = NumValueSites; 562356843Sdim InstrProfValueData *DstVD = getValueProfRecordValueData(This); 563356843Sdim 564356843Sdim for (S = 0; S < NumValueSites; S++) { 565356843Sdim uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); 566356843Sdim This->SiteCountArray[S] = ND; 567356843Sdim Closure->GetValueForSite(Record, DstVD, ValueKind, S); 568356843Sdim DstVD += ND; 569356843Sdim } 570356843Sdim} 571356843Sdim 572356843Sdim/*! 573356843Sdim * Extract value profile data of a function from the \c Closure 574356843Sdim * and serialize the data into \c DstData if it is not NULL or heap 575356843Sdim * memory allocated by the \c Closure's allocator method. If \c 576356843Sdim * DstData is not null, the caller is expected to set the TotalSize 577356843Sdim * in DstData. 578356843Sdim */ 579356843SdimINSTR_PROF_VISIBILITY ValueProfData * 580356843SdimserializeValueProfDataFrom(ValueProfRecordClosure *Closure, 581356843Sdim ValueProfData *DstData) { 582356843Sdim uint32_t Kind; 583356843Sdim uint32_t TotalSize = 584356843Sdim DstData ? DstData->TotalSize : getValueProfDataSize(Closure); 585356843Sdim 586356843Sdim ValueProfData *VPD = 587356843Sdim DstData ? DstData : Closure->AllocValueProfData(TotalSize); 588356843Sdim 589356843Sdim VPD->TotalSize = TotalSize; 590356843Sdim VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); 591356843Sdim ValueProfRecord *VR = getFirstValueProfRecord(VPD); 592356843Sdim for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 593356843Sdim uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); 594356843Sdim if (!NumValueSites) 595356843Sdim continue; 596356843Sdim serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); 597356843Sdim VR = getValueProfRecordNext(VR); 598356843Sdim } 599356843Sdim return VPD; 600356843Sdim} 601356843Sdim 602356843Sdim#undef INSTR_PROF_COMMON_API_IMPL 603356843Sdim#endif /* INSTR_PROF_COMMON_API_IMPL */ 604356843Sdim 605356843Sdim/*============================================================================*/ 606356843Sdim 607356843Sdim#ifndef INSTR_PROF_DATA_DEFINED 608356843Sdim 609356843Sdim#ifndef INSTR_PROF_DATA_INC 610356843Sdim#define INSTR_PROF_DATA_INC 611356843Sdim 612356843Sdim/* Helper macros. */ 613356843Sdim#define INSTR_PROF_SIMPLE_QUOTE(x) #x 614356843Sdim#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) 615356843Sdim#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y 616356843Sdim#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) 617356843Sdim 618356843Sdim/* Magic number to detect file format and endianness. 619356843Sdim * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, 620356843Sdim * so that utilities, like strings, don't grab it as a string. 129 is also 621356843Sdim * invalid UTF-8, and high enough to be interesting. 622356843Sdim * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" 623356843Sdim * for 32-bit platforms. 624356843Sdim */ 625356843Sdim#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 626356843Sdim (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 627356843Sdim (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 628356843Sdim#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 629356843Sdim (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 630356843Sdim (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 631356843Sdim 632356843Sdim/* Raw profile format version (start from 1). */ 633356843Sdim#define INSTR_PROF_RAW_VERSION 5 634356843Sdim/* Indexed profile format version (start from 1). */ 635356843Sdim#define INSTR_PROF_INDEX_VERSION 5 636356843Sdim/* Coverage mapping format vresion (start from 0). */ 637356843Sdim#define INSTR_PROF_COVMAP_VERSION 2 638356843Sdim 639356843Sdim/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the 640356843Sdim * version for other variants of profile. We set the lowest bit of the upper 8 641356843Sdim * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton 642356843Sdim * generated profile, and 0 if this is a Clang FE generated profile. 643356843Sdim * 1 in bit 57 indicates there are context-sensitive records in the profile. 644356843Sdim */ 645356843Sdim#define VARIANT_MASKS_ALL 0xff00000000000000ULL 646356843Sdim#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) 647356843Sdim#define VARIANT_MASK_IR_PROF (0x1ULL << 56) 648356843Sdim#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) 649356843Sdim#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version 650356843Sdim#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime 651356843Sdim 652356843Sdim/* The variable that holds the name of the profile data 653356843Sdim * specified via command line. */ 654356843Sdim#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename 655356843Sdim 656356843Sdim/* section name strings common to all targets other 657356843Sdim than WIN32 */ 658356843Sdim#define INSTR_PROF_DATA_COMMON __llvm_prf_data 659356843Sdim#define INSTR_PROF_NAME_COMMON __llvm_prf_names 660356843Sdim#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts 661356843Sdim#define INSTR_PROF_VALS_COMMON __llvm_prf_vals 662356843Sdim#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds 663356843Sdim#define INSTR_PROF_COVMAP_COMMON __llvm_covmap 664356843Sdim#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile 665356843Sdim/* Windows section names. Because these section names contain dollar characters, 666356843Sdim * they must be quoted. 667356843Sdim */ 668356843Sdim#define INSTR_PROF_DATA_COFF ".lprfd$M" 669356843Sdim#define INSTR_PROF_NAME_COFF ".lprfn$M" 670356843Sdim#define INSTR_PROF_CNTS_COFF ".lprfc$M" 671356843Sdim#define INSTR_PROF_VALS_COFF ".lprfv$M" 672356843Sdim#define INSTR_PROF_VNODES_COFF ".lprfnd$M" 673356843Sdim#define INSTR_PROF_COVMAP_COFF ".lcovmap$M" 674356843Sdim#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" 675356843Sdim 676356843Sdim#ifdef _WIN32 677356843Sdim/* Runtime section names and name strings. */ 678356843Sdim#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF 679356843Sdim#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF 680356843Sdim#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF 681356843Sdim/* Array of pointers. Each pointer points to a list 682356843Sdim * of value nodes associated with one value site. 683356843Sdim */ 684356843Sdim#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF 685356843Sdim/* Value profile nodes section. */ 686356843Sdim#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF 687356843Sdim#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF 688356843Sdim#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF 689356843Sdim#else 690356843Sdim/* Runtime section names and name strings. */ 691356843Sdim#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) 692356843Sdim#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) 693356843Sdim#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) 694356843Sdim/* Array of pointers. Each pointer points to a list 695356843Sdim * of value nodes associated with one value site. 696356843Sdim */ 697356843Sdim#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) 698356843Sdim/* Value profile nodes section. */ 699356843Sdim#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) 700356843Sdim#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) 701356843Sdim/* Order file instrumentation. */ 702356843Sdim#define INSTR_PROF_ORDERFILE_SECT_NAME \ 703356843Sdim INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) 704356843Sdim#endif 705356843Sdim 706356843Sdim#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer 707356843Sdim#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ 708356843Sdim INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) 709356843Sdim#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx 710356843Sdim#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ 711356843Sdim INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) 712356843Sdim 713356843Sdim/* Macros to define start/stop section symbol for a given 714356843Sdim * section on Linux. For instance 715356843Sdim * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will 716356843Sdim * expand to __start___llvm_prof_data 717356843Sdim */ 718356843Sdim#define INSTR_PROF_SECT_START(Sect) \ 719356843Sdim INSTR_PROF_CONCAT(__start_,Sect) 720356843Sdim#define INSTR_PROF_SECT_STOP(Sect) \ 721356843Sdim INSTR_PROF_CONCAT(__stop_,Sect) 722356843Sdim 723356843Sdim/* Value Profiling API linkage name. */ 724356843Sdim#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target 725356843Sdim#define INSTR_PROF_VALUE_PROF_FUNC_STR \ 726356843Sdim INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) 727356843Sdim#define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range 728356843Sdim#define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \ 729356843Sdim INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC) 730356843Sdim 731356843Sdim/* InstrProfile per-function control data alignment. */ 732356843Sdim#define INSTR_PROF_DATA_ALIGNMENT 8 733356843Sdim 734356843Sdim/* The data structure that represents a tracked value by the 735356843Sdim * value profiler. 736356843Sdim */ 737356843Sdimtypedef struct InstrProfValueData { 738356843Sdim /* Profiled value. */ 739356843Sdim uint64_t Value; 740356843Sdim /* Number of times the value appears in the training run. */ 741356843Sdim uint64_t Count; 742356843Sdim} InstrProfValueData; 743356843Sdim 744356843Sdim#endif /* INSTR_PROF_DATA_INC */ 745356843Sdim 746356843Sdim#ifndef INSTR_ORDER_FILE_INC 747356843Sdim/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */ 748356843Sdim#define INSTR_ORDER_FILE_BUFFER_SIZE 131072 749356843Sdim#define INSTR_ORDER_FILE_BUFFER_BITS 17 750356843Sdim#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff 751356843Sdim#endif /* INSTR_ORDER_FILE_INC */ 752356843Sdim#else 753356843Sdim#undef INSTR_PROF_DATA_DEFINED 754356843Sdim#endif 755