1/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ 2|* 3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4|* See https://llvm.org/LICENSE.txt for license information. 5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6|* 7\*===----------------------------------------------------------------------===*/ 8/* 9 * This is the master file that defines all the data structure, signature, 10 * constant literals that are shared across profiling runtime library, 11 * compiler (instrumentation), and host tools (reader/writer). The entities 12 * defined in this file affect the profile runtime ABI, the raw profile format, 13 * or both. 14 * 15 * The file has two identical copies. The master copy lives in LLVM and 16 * the other one sits in compiler-rt/lib/profile directory. To make changes 17 * in this file, first modify the master copy and copy it over to compiler-rt. 18 * Testing of any change in this file can start only after the two copies are 19 * synced up. 20 * 21 * The first part of the file includes macros that defines types, names, and 22 * initializers for the member fields of the core data structures. The field 23 * declarations for one structure is enabled by defining the field activation 24 * macro associated with that structure. Only one field activation record 25 * can be defined at one time and the rest definitions will be filtered out by 26 * the preprocessor. 27 * 28 * Examples of how the template is used to instantiate structure definition: 29 * 1. To declare a structure: 30 * 31 * struct ProfData { 32 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 33 * Type Name; 34 * #include "llvm/ProfileData/InstrProfData.inc" 35 * }; 36 * 37 * 2. To construct LLVM type arrays for the struct type: 38 * 39 * Type *DataTypes[] = { 40 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 41 * LLVMType, 42 * #include "llvm/ProfileData/InstrProfData.inc" 43 * }; 44 * 45 * 4. To construct constant array for the initializers: 46 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 47 * Initializer, 48 * Constant *ConstantVals[] = { 49 * #include "llvm/ProfileData/InstrProfData.inc" 50 * }; 51 * 52 * 53 * The second part of the file includes definitions all other entities that 54 * are related to runtime ABI and format. When no field activation macro is 55 * defined, this file can be included to introduce the definitions. 56 * 57\*===----------------------------------------------------------------------===*/ 58 59/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in 60 * the compiler runtime. */ 61#ifndef INSTR_PROF_VISIBILITY 62#define INSTR_PROF_VISIBILITY 63#endif 64 65/* INSTR_PROF_DATA start. */ 66/* Definition of member fields of the per-function control structure. */ 67#ifndef INSTR_PROF_DATA 68#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) 69#else 70#define INSTR_PROF_DATA_DEFINED 71#endif 72INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 73 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 74 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName())))) 75INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 76 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 77 Inc->getHash()->getZExtValue())) 78INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ 79 ConstantExpr::getBitCast(CounterPtr, \ 80 llvm::Type::getInt64PtrTy(Ctx))) 81/* This is used to map function pointers for the indirect call targets to 82 * function name hashes during the conversion from raw to merged profile 83 * data. 84 */ 85INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \ 86 FunctionAddr) 87INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ 88 ValuesPtrExpr) 89INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ 90 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) 91INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ 92 ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) 93#undef INSTR_PROF_DATA 94/* INSTR_PROF_DATA end. */ 95 96 97/* This is an internal data structure used by value profiler. It 98 * is defined here to allow serialization code sharing by LLVM 99 * to be used in unit test. 100 * 101 * typedef struct ValueProfNode { 102 * // InstrProfValueData VData; 103 * uint64_t Value; 104 * uint64_t Count; 105 * struct ValueProfNode *Next; 106 * } ValueProfNode; 107 */ 108/* INSTR_PROF_VALUE_NODE start. */ 109#ifndef INSTR_PROF_VALUE_NODE 110#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) 111#else 112#define INSTR_PROF_DATA_DEFINED 113#endif 114INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \ 115 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 116INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \ 117 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 118INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ 119 ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0)) 120#undef INSTR_PROF_VALUE_NODE 121/* INSTR_PROF_VALUE_NODE end. */ 122 123/* INSTR_PROF_RAW_HEADER start */ 124/* Definition of member fields of the raw profile header data structure. */ 125#ifndef INSTR_PROF_RAW_HEADER 126#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) 127#else 128#define INSTR_PROF_DATA_DEFINED 129#endif 130INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) 131INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) 132INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) 133INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) 134INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) 135INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) 136INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) 137INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) 138INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) 139INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) 140#undef INSTR_PROF_RAW_HEADER 141/* INSTR_PROF_RAW_HEADER end */ 142 143/* VALUE_PROF_FUNC_PARAM start */ 144/* Definition of parameter types of the runtime API used to do value profiling 145 * for a given value site. 146 */ 147#ifndef VALUE_PROF_FUNC_PARAM 148#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) 149#define INSTR_PROF_COMMA 150#else 151#define INSTR_PROF_DATA_DEFINED 152#define INSTR_PROF_COMMA , 153#endif 154VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ 155 INSTR_PROF_COMMA 156VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA 157#ifndef VALUE_RANGE_PROF 158VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) 159#else /* VALUE_RANGE_PROF */ 160VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \ 161 INSTR_PROF_COMMA 162VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \ 163 INSTR_PROF_COMMA 164VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeLast, Type::getInt64Ty(Ctx)) \ 165 INSTR_PROF_COMMA 166VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) 167#endif /*VALUE_RANGE_PROF */ 168#undef VALUE_PROF_FUNC_PARAM 169#undef INSTR_PROF_COMMA 170/* VALUE_PROF_FUNC_PARAM end */ 171 172/* VALUE_PROF_KIND start */ 173#ifndef VALUE_PROF_KIND 174#define VALUE_PROF_KIND(Enumerator, Value, Descr) 175#else 176#define INSTR_PROF_DATA_DEFINED 177#endif 178/* For indirect function call value profiling, the addresses of the target 179 * functions are profiled by the instrumented code. The target addresses are 180 * written in the raw profile data and converted to target function name's MD5 181 * hash by the profile reader during deserialization. Typically, this happens 182 * when the raw profile data is read during profile merging. 183 * 184 * For this remapping the ProfData is used. ProfData contains both the function 185 * name hash and the function address. 186 */ 187VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") 188/* For memory intrinsic functions size profiling. */ 189VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") 190/* These two kinds must be the last to be 191 * declared. This is to make sure the string 192 * array created with the template can be 193 * indexed with the kind value. 194 */ 195VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") 196VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") 197 198#undef VALUE_PROF_KIND 199/* VALUE_PROF_KIND end */ 200 201#undef COVMAP_V2_OR_V3 202#ifdef COVMAP_V2 203#define COVMAP_V2_OR_V3 204#endif 205#ifdef COVMAP_V3 206#define COVMAP_V2_OR_V3 207#endif 208 209/* COVMAP_FUNC_RECORD start */ 210/* Definition of member fields of the function record structure in coverage 211 * map. 212 */ 213#ifndef COVMAP_FUNC_RECORD 214#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) 215#else 216#define INSTR_PROF_DATA_DEFINED 217#endif 218#ifdef COVMAP_V1 219COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ 220 NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ 221 llvm::Type::getInt8PtrTy(Ctx))) 222COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ 223 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ 224 NameValue.size())) 225#endif 226#ifdef COVMAP_V2_OR_V3 227COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 228 llvm::ConstantInt::get( \ 229 llvm::Type::getInt64Ty(Ctx), NameHash)) 230#endif 231COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ 232 llvm::ConstantInt::get( \ 233 llvm::Type::getInt32Ty(Ctx), CoverageMapping.size())) 234COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 235 llvm::ConstantInt::get( \ 236 llvm::Type::getInt64Ty(Ctx), FuncHash)) 237#ifdef COVMAP_V3 238COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \ 239 llvm::ConstantInt::get( \ 240 llvm::Type::getInt64Ty(Ctx), FilenamesRef)) 241COVMAP_FUNC_RECORD(const char, \ 242 llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \ 243 CoverageMapping.size()), \ 244 CoverageMapping, 245 llvm::ConstantDataArray::getRaw( \ 246 CoverageMapping, CoverageMapping.size(), \ 247 llvm::Type::getInt8Ty(Ctx))) 248#endif 249#undef COVMAP_FUNC_RECORD 250/* COVMAP_FUNC_RECORD end. */ 251 252/* COVMAP_HEADER start */ 253/* Definition of member fields of coverage map header. 254 */ 255#ifndef COVMAP_HEADER 256#define COVMAP_HEADER(Type, LLVMType, Name, Initializer) 257#else 258#define INSTR_PROF_DATA_DEFINED 259#endif 260COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \ 261 llvm::ConstantInt::get(Int32Ty, NRecords)) 262COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ 263 llvm::ConstantInt::get(Int32Ty, FilenamesSize)) 264COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ 265 llvm::ConstantInt::get(Int32Ty, CoverageMappingSize)) 266COVMAP_HEADER(uint32_t, Int32Ty, Version, \ 267 llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion)) 268#undef COVMAP_HEADER 269/* COVMAP_HEADER end. */ 270 271 272#ifdef INSTR_PROF_SECT_ENTRY 273#define INSTR_PROF_DATA_DEFINED 274INSTR_PROF_SECT_ENTRY(IPSK_data, \ 275 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ 276 INSTR_PROF_DATA_COFF, "__DATA,") 277INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ 278 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ 279 INSTR_PROF_CNTS_COFF, "__DATA,") 280INSTR_PROF_SECT_ENTRY(IPSK_name, \ 281 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ 282 INSTR_PROF_NAME_COFF, "__DATA,") 283INSTR_PROF_SECT_ENTRY(IPSK_vals, \ 284 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ 285 INSTR_PROF_VALS_COFF, "__DATA,") 286INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ 287 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ 288 INSTR_PROF_VNODES_COFF, "__DATA,") 289INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ 290 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ 291 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") 292INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ 293 INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \ 294 INSTR_PROF_COVFUN_COFF, "__LLVM_COV,") 295INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ 296 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ 297 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") 298 299#undef INSTR_PROF_SECT_ENTRY 300#endif 301 302 303#ifdef INSTR_PROF_VALUE_PROF_DATA 304#define INSTR_PROF_DATA_DEFINED 305 306#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255 307/*! 308 * This is the header of the data structure that defines the on-disk 309 * layout of the value profile data of a particular kind for one function. 310 */ 311typedef struct ValueProfRecord { 312 /* The kind of the value profile record. */ 313 uint32_t Kind; 314 /* 315 * The number of value profile sites. It is guaranteed to be non-zero; 316 * otherwise the record for this kind won't be emitted. 317 */ 318 uint32_t NumValueSites; 319 /* 320 * The first element of the array that stores the number of profiled 321 * values for each value site. The size of the array is NumValueSites. 322 * Since NumValueSites is greater than zero, there is at least one 323 * element in the array. 324 */ 325 uint8_t SiteCountArray[1]; 326 327 /* 328 * The fake declaration is for documentation purpose only. 329 * Align the start of next field to be on 8 byte boundaries. 330 uint8_t Padding[X]; 331 */ 332 333 /* The array of value profile data. The size of the array is the sum 334 * of all elements in SiteCountArray[]. 335 InstrProfValueData ValueData[]; 336 */ 337 338#ifdef __cplusplus 339 /*! 340 * Return the number of value sites. 341 */ 342 uint32_t getNumValueSites() const { return NumValueSites; } 343 /*! 344 * Read data from this record and save it to Record. 345 */ 346 void deserializeTo(InstrProfRecord &Record, 347 InstrProfSymtab *SymTab); 348 /* 349 * In-place byte swap: 350 * Do byte swap for this instance. \c Old is the original order before 351 * the swap, and \c New is the New byte order. 352 */ 353 void swapBytes(support::endianness Old, support::endianness New); 354#endif 355} ValueProfRecord; 356 357/*! 358 * Per-function header/control data structure for value profiling 359 * data in indexed format. 360 */ 361typedef struct ValueProfData { 362 /* 363 * Total size in bytes including this field. It must be a multiple 364 * of sizeof(uint64_t). 365 */ 366 uint32_t TotalSize; 367 /* 368 *The number of value profile kinds that has value profile data. 369 * In this implementation, a value profile kind is considered to 370 * have profile data if the number of value profile sites for the 371 * kind is not zero. More aggressively, the implementation can 372 * choose to check the actual data value: if none of the value sites 373 * has any profiled values, the kind can be skipped. 374 */ 375 uint32_t NumValueKinds; 376 377 /* 378 * Following are a sequence of variable length records. The prefix/header 379 * of each record is defined by ValueProfRecord type. The number of 380 * records is NumValueKinds. 381 * ValueProfRecord Record_1; 382 * ValueProfRecord Record_N; 383 */ 384 385#if __cplusplus 386 /*! 387 * Return the total size in bytes of the on-disk value profile data 388 * given the data stored in Record. 389 */ 390 static uint32_t getSize(const InstrProfRecord &Record); 391 /*! 392 * Return a pointer to \c ValueProfData instance ready to be streamed. 393 */ 394 static std::unique_ptr<ValueProfData> 395 serializeFrom(const InstrProfRecord &Record); 396 /*! 397 * Check the integrity of the record. 398 */ 399 Error checkIntegrity(); 400 /*! 401 * Return a pointer to \c ValueProfileData instance ready to be read. 402 * All data in the instance are properly byte swapped. The input 403 * data is assumed to be in little endian order. 404 */ 405 static Expected<std::unique_ptr<ValueProfData>> 406 getValueProfData(const unsigned char *SrcBuffer, 407 const unsigned char *const SrcBufferEnd, 408 support::endianness SrcDataEndianness); 409 /*! 410 * Swap byte order from \c Endianness order to host byte order. 411 */ 412 void swapBytesToHost(support::endianness Endianness); 413 /*! 414 * Swap byte order from host byte order to \c Endianness order. 415 */ 416 void swapBytesFromHost(support::endianness Endianness); 417 /*! 418 * Return the total size of \c ValueProfileData. 419 */ 420 uint32_t getSize() const { return TotalSize; } 421 /*! 422 * Read data from this data and save it to \c Record. 423 */ 424 void deserializeTo(InstrProfRecord &Record, 425 InstrProfSymtab *SymTab); 426 void operator delete(void *ptr) { ::operator delete(ptr); } 427#endif 428} ValueProfData; 429 430/* 431 * The closure is designed to abstact away two types of value profile data: 432 * - InstrProfRecord which is the primary data structure used to 433 * represent profile data in host tools (reader, writer, and profile-use) 434 * - value profile runtime data structure suitable to be used by C 435 * runtime library. 436 * 437 * Both sources of data need to serialize to disk/memory-buffer in common 438 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler 439 * writer to share the same format and code with indexed profile writer. 440 * 441 * For documentation of the member methods below, refer to corresponding methods 442 * in class InstrProfRecord. 443 */ 444typedef struct ValueProfRecordClosure { 445 const void *Record; 446 uint32_t (*GetNumValueKinds)(const void *Record); 447 uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); 448 uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); 449 uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); 450 451 /* 452 * After extracting the value profile data from the value profile record, 453 * this method is used to map the in-memory value to on-disk value. If 454 * the method is null, value will be written out untranslated. 455 */ 456 uint64_t (*RemapValueData)(uint32_t, uint64_t Value); 457 void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, 458 uint32_t S); 459 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); 460} ValueProfRecordClosure; 461 462INSTR_PROF_VISIBILITY ValueProfRecord * 463getFirstValueProfRecord(ValueProfData *VPD); 464INSTR_PROF_VISIBILITY ValueProfRecord * 465getValueProfRecordNext(ValueProfRecord *VPR); 466INSTR_PROF_VISIBILITY InstrProfValueData * 467getValueProfRecordValueData(ValueProfRecord *VPR); 468INSTR_PROF_VISIBILITY uint32_t 469getValueProfRecordHeaderSize(uint32_t NumValueSites); 470 471#undef INSTR_PROF_VALUE_PROF_DATA 472#endif /* INSTR_PROF_VALUE_PROF_DATA */ 473 474 475#ifdef INSTR_PROF_COMMON_API_IMPL 476#define INSTR_PROF_DATA_DEFINED 477#ifdef __cplusplus 478#define INSTR_PROF_INLINE inline 479#define INSTR_PROF_NULLPTR nullptr 480#else 481#define INSTR_PROF_INLINE 482#define INSTR_PROF_NULLPTR NULL 483#endif 484 485#ifndef offsetof 486#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) 487#endif 488 489/*! 490 * Return the \c ValueProfRecord header size including the 491 * padding bytes. 492 */ 493INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 494uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { 495 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + 496 sizeof(uint8_t) * NumValueSites; 497 /* Round the size to multiple of 8 bytes. */ 498 Size = (Size + 7) & ~7; 499 return Size; 500} 501 502/*! 503 * Return the total size of the value profile record including the 504 * header and the value data. 505 */ 506INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 507uint32_t getValueProfRecordSize(uint32_t NumValueSites, 508 uint32_t NumValueData) { 509 return getValueProfRecordHeaderSize(NumValueSites) + 510 sizeof(InstrProfValueData) * NumValueData; 511} 512 513/*! 514 * Return the pointer to the start of value data array. 515 */ 516INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 517InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { 518 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( 519 This->NumValueSites)); 520} 521 522/*! 523 * Return the total number of value data for \c This record. 524 */ 525INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 526uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { 527 uint32_t NumValueData = 0; 528 uint32_t I; 529 for (I = 0; I < This->NumValueSites; I++) 530 NumValueData += This->SiteCountArray[I]; 531 return NumValueData; 532} 533 534/*! 535 * Use this method to advance to the next \c This \c ValueProfRecord. 536 */ 537INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 538ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { 539 uint32_t NumValueData = getValueProfRecordNumValueData(This); 540 return (ValueProfRecord *)((char *)This + 541 getValueProfRecordSize(This->NumValueSites, 542 NumValueData)); 543} 544 545/*! 546 * Return the first \c ValueProfRecord instance. 547 */ 548INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 549ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { 550 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); 551} 552 553/* Closure based interfaces. */ 554 555/*! 556 * Return the total size in bytes of the on-disk value profile data 557 * given the data stored in Record. 558 */ 559INSTR_PROF_VISIBILITY uint32_t 560getValueProfDataSize(ValueProfRecordClosure *Closure) { 561 uint32_t Kind; 562 uint32_t TotalSize = sizeof(ValueProfData); 563 const void *Record = Closure->Record; 564 565 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 566 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); 567 if (!NumValueSites) 568 continue; 569 TotalSize += getValueProfRecordSize(NumValueSites, 570 Closure->GetNumValueData(Record, Kind)); 571 } 572 return TotalSize; 573} 574 575/*! 576 * Extract value profile data of a function for the profile kind \c ValueKind 577 * from the \c Closure and serialize the data into \c This record instance. 578 */ 579INSTR_PROF_VISIBILITY void 580serializeValueProfRecordFrom(ValueProfRecord *This, 581 ValueProfRecordClosure *Closure, 582 uint32_t ValueKind, uint32_t NumValueSites) { 583 uint32_t S; 584 const void *Record = Closure->Record; 585 This->Kind = ValueKind; 586 This->NumValueSites = NumValueSites; 587 InstrProfValueData *DstVD = getValueProfRecordValueData(This); 588 589 for (S = 0; S < NumValueSites; S++) { 590 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); 591 This->SiteCountArray[S] = ND; 592 Closure->GetValueForSite(Record, DstVD, ValueKind, S); 593 DstVD += ND; 594 } 595} 596 597/*! 598 * Extract value profile data of a function from the \c Closure 599 * and serialize the data into \c DstData if it is not NULL or heap 600 * memory allocated by the \c Closure's allocator method. If \c 601 * DstData is not null, the caller is expected to set the TotalSize 602 * in DstData. 603 */ 604INSTR_PROF_VISIBILITY ValueProfData * 605serializeValueProfDataFrom(ValueProfRecordClosure *Closure, 606 ValueProfData *DstData) { 607 uint32_t Kind; 608 uint32_t TotalSize = 609 DstData ? DstData->TotalSize : getValueProfDataSize(Closure); 610 611 ValueProfData *VPD = 612 DstData ? DstData : Closure->AllocValueProfData(TotalSize); 613 614 VPD->TotalSize = TotalSize; 615 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); 616 ValueProfRecord *VR = getFirstValueProfRecord(VPD); 617 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 618 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); 619 if (!NumValueSites) 620 continue; 621 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); 622 VR = getValueProfRecordNext(VR); 623 } 624 return VPD; 625} 626 627#undef INSTR_PROF_COMMON_API_IMPL 628#endif /* INSTR_PROF_COMMON_API_IMPL */ 629 630/*============================================================================*/ 631 632#ifndef INSTR_PROF_DATA_DEFINED 633 634#ifndef INSTR_PROF_DATA_INC 635#define INSTR_PROF_DATA_INC 636 637/* Helper macros. */ 638#define INSTR_PROF_SIMPLE_QUOTE(x) #x 639#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) 640#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y 641#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) 642 643/* Magic number to detect file format and endianness. 644 * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, 645 * so that utilities, like strings, don't grab it as a string. 129 is also 646 * invalid UTF-8, and high enough to be interesting. 647 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" 648 * for 32-bit platforms. 649 */ 650#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 651 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 652 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 653#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 654 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 655 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 656 657/* Raw profile format version (start from 1). */ 658#define INSTR_PROF_RAW_VERSION 5 659/* Indexed profile format version (start from 1). */ 660#define INSTR_PROF_INDEX_VERSION 6 661/* Coverage mapping format version (start from 0). */ 662#define INSTR_PROF_COVMAP_VERSION 3 663 664/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the 665 * version for other variants of profile. We set the lowest bit of the upper 8 666 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton 667 * generated profile, and 0 if this is a Clang FE generated profile. 668 * 1 in bit 57 indicates there are context-sensitive records in the profile. 669 */ 670#define VARIANT_MASKS_ALL 0xff00000000000000ULL 671#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) 672#define VARIANT_MASK_IR_PROF (0x1ULL << 56) 673#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) 674#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version 675#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime 676 677/* The variable that holds the name of the profile data 678 * specified via command line. */ 679#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename 680 681/* section name strings common to all targets other 682 than WIN32 */ 683#define INSTR_PROF_DATA_COMMON __llvm_prf_data 684#define INSTR_PROF_NAME_COMMON __llvm_prf_names 685#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts 686#define INSTR_PROF_VALS_COMMON __llvm_prf_vals 687#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds 688#define INSTR_PROF_COVMAP_COMMON __llvm_covmap 689#define INSTR_PROF_COVFUN_COMMON __llvm_covfun 690#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile 691/* Windows section names. Because these section names contain dollar characters, 692 * they must be quoted. 693 */ 694#define INSTR_PROF_DATA_COFF ".lprfd$M" 695#define INSTR_PROF_NAME_COFF ".lprfn$M" 696#define INSTR_PROF_CNTS_COFF ".lprfc$M" 697#define INSTR_PROF_VALS_COFF ".lprfv$M" 698#define INSTR_PROF_VNODES_COFF ".lprfnd$M" 699#define INSTR_PROF_COVMAP_COFF ".lcovmap$M" 700#define INSTR_PROF_COVFUN_COFF ".lcovfun$M" 701#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" 702 703#ifdef _WIN32 704/* Runtime section names and name strings. */ 705#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF 706#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF 707#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF 708/* Array of pointers. Each pointer points to a list 709 * of value nodes associated with one value site. 710 */ 711#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF 712/* Value profile nodes section. */ 713#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF 714#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF 715#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF 716#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF 717#else 718/* Runtime section names and name strings. */ 719#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) 720#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) 721#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) 722/* Array of pointers. Each pointer points to a list 723 * of value nodes associated with one value site. 724 */ 725#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) 726/* Value profile nodes section. */ 727#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) 728#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) 729#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) 730/* Order file instrumentation. */ 731#define INSTR_PROF_ORDERFILE_SECT_NAME \ 732 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) 733#endif 734 735#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer 736#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ 737 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) 738#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx 739#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ 740 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) 741 742/* Macros to define start/stop section symbol for a given 743 * section on Linux. For instance 744 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will 745 * expand to __start___llvm_prof_data 746 */ 747#define INSTR_PROF_SECT_START(Sect) \ 748 INSTR_PROF_CONCAT(__start_,Sect) 749#define INSTR_PROF_SECT_STOP(Sect) \ 750 INSTR_PROF_CONCAT(__stop_,Sect) 751 752/* Value Profiling API linkage name. */ 753#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target 754#define INSTR_PROF_VALUE_PROF_FUNC_STR \ 755 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) 756#define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range 757#define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \ 758 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC) 759 760/* InstrProfile per-function control data alignment. */ 761#define INSTR_PROF_DATA_ALIGNMENT 8 762 763/* The data structure that represents a tracked value by the 764 * value profiler. 765 */ 766typedef struct InstrProfValueData { 767 /* Profiled value. */ 768 uint64_t Value; 769 /* Number of times the value appears in the training run. */ 770 uint64_t Count; 771} InstrProfValueData; 772 773#endif /* INSTR_PROF_DATA_INC */ 774 775#ifndef INSTR_ORDER_FILE_INC 776/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */ 777#define INSTR_ORDER_FILE_BUFFER_SIZE 131072 778#define INSTR_ORDER_FILE_BUFFER_BITS 17 779#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff 780#endif /* INSTR_ORDER_FILE_INC */ 781#else 782#undef INSTR_PROF_DATA_DEFINED 783#endif 784 785#undef COVMAP_V2_OR_V3 786