InstrProfData.inc revision 360784
1/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ 2|* 3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4|* See https://llvm.org/LICENSE.txt for license information. 5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6|* 7\*===----------------------------------------------------------------------===*/ 8/* 9 * This is the master file that defines all the data structure, signature, 10 * constant literals that are shared across profiling runtime library, 11 * compiler (instrumentation), and host tools (reader/writer). The entities 12 * defined in this file affect the profile runtime ABI, the raw profile format, 13 * or both. 14 * 15 * The file has two identical copies. The master copy lives in LLVM and 16 * the other one sits in compiler-rt/lib/profile directory. To make changes 17 * in this file, first modify the master copy and copy it over to compiler-rt. 18 * Testing of any change in this file can start only after the two copies are 19 * synced up. 20 * 21 * The first part of the file includes macros that defines types, names, and 22 * initializers for the member fields of the core data structures. The field 23 * declarations for one structure is enabled by defining the field activation 24 * macro associated with that structure. Only one field activation record 25 * can be defined at one time and the rest definitions will be filtered out by 26 * the preprocessor. 27 * 28 * Examples of how the template is used to instantiate structure definition: 29 * 1. To declare a structure: 30 * 31 * struct ProfData { 32 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 33 * Type Name; 34 * #include "llvm/ProfileData/InstrProfData.inc" 35 * }; 36 * 37 * 2. To construct LLVM type arrays for the struct type: 38 * 39 * Type *DataTypes[] = { 40 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 41 * LLVMType, 42 * #include "llvm/ProfileData/InstrProfData.inc" 43 * }; 44 * 45 * 4. To construct constant array for the initializers: 46 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 47 * Initializer, 48 * Constant *ConstantVals[] = { 49 * #include "llvm/ProfileData/InstrProfData.inc" 50 * }; 51 * 52 * 53 * The second part of the file includes definitions all other entities that 54 * are related to runtime ABI and format. When no field activation macro is 55 * defined, this file can be included to introduce the definitions. 56 * 57\*===----------------------------------------------------------------------===*/ 58 59/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in 60 * the compiler runtime. */ 61#ifndef INSTR_PROF_VISIBILITY 62#define INSTR_PROF_VISIBILITY 63#endif 64 65/* INSTR_PROF_DATA start. */ 66/* Definition of member fields of the per-function control structure. */ 67#ifndef INSTR_PROF_DATA 68#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) 69#else 70#define INSTR_PROF_DATA_DEFINED 71#endif 72INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 73 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 74 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName())))) 75INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 76 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 77 Inc->getHash()->getZExtValue())) 78INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ 79 ConstantExpr::getBitCast(CounterPtr, \ 80 llvm::Type::getInt64PtrTy(Ctx))) 81/* This is used to map function pointers for the indirect call targets to 82 * function name hashes during the conversion from raw to merged profile 83 * data. 84 */ 85INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \ 86 FunctionAddr) 87INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ 88 ValuesPtrExpr) 89INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ 90 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) 91INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ 92 ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) 93#undef INSTR_PROF_DATA 94/* INSTR_PROF_DATA end. */ 95 96 97/* This is an internal data structure used by value profiler. It 98 * is defined here to allow serialization code sharing by LLVM 99 * to be used in unit test. 100 * 101 * typedef struct ValueProfNode { 102 * // InstrProfValueData VData; 103 * uint64_t Value; 104 * uint64_t Count; 105 * struct ValueProfNode *Next; 106 * } ValueProfNode; 107 */ 108/* INSTR_PROF_VALUE_NODE start. */ 109#ifndef INSTR_PROF_VALUE_NODE 110#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) 111#else 112#define INSTR_PROF_DATA_DEFINED 113#endif 114INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \ 115 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 116INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \ 117 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 118INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ 119 ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0)) 120#undef INSTR_PROF_VALUE_NODE 121/* INSTR_PROF_VALUE_NODE end. */ 122 123/* INSTR_PROF_RAW_HEADER start */ 124/* Definition of member fields of the raw profile header data structure. */ 125#ifndef INSTR_PROF_RAW_HEADER 126#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) 127#else 128#define INSTR_PROF_DATA_DEFINED 129#endif 130INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) 131INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) 132INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) 133INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) 134INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) 135INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) 136INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) 137INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) 138INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) 139INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) 140#undef INSTR_PROF_RAW_HEADER 141/* INSTR_PROF_RAW_HEADER end */ 142 143/* VALUE_PROF_FUNC_PARAM start */ 144/* Definition of parameter types of the runtime API used to do value profiling 145 * for a given value site. 146 */ 147#ifndef VALUE_PROF_FUNC_PARAM 148#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) 149#define INSTR_PROF_COMMA 150#else 151#define INSTR_PROF_DATA_DEFINED 152#define INSTR_PROF_COMMA , 153#endif 154VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ 155 INSTR_PROF_COMMA 156VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA 157#ifndef VALUE_RANGE_PROF 158VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) 159#else /* VALUE_RANGE_PROF */ 160VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \ 161 INSTR_PROF_COMMA 162VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \ 163 INSTR_PROF_COMMA 164VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeLast, Type::getInt64Ty(Ctx)) \ 165 INSTR_PROF_COMMA 166VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) 167#endif /*VALUE_RANGE_PROF */ 168#undef VALUE_PROF_FUNC_PARAM 169#undef INSTR_PROF_COMMA 170/* VALUE_PROF_FUNC_PARAM end */ 171 172/* VALUE_PROF_KIND start */ 173#ifndef VALUE_PROF_KIND 174#define VALUE_PROF_KIND(Enumerator, Value, Descr) 175#else 176#define INSTR_PROF_DATA_DEFINED 177#endif 178/* For indirect function call value profiling, the addresses of the target 179 * functions are profiled by the instrumented code. The target addresses are 180 * written in the raw profile data and converted to target function name's MD5 181 * hash by the profile reader during deserialization. Typically, this happens 182 * when the raw profile data is read during profile merging. 183 * 184 * For this remapping the ProfData is used. ProfData contains both the function 185 * name hash and the function address. 186 */ 187VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") 188/* For memory intrinsic functions size profiling. */ 189VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") 190/* These two kinds must be the last to be 191 * declared. This is to make sure the string 192 * array created with the template can be 193 * indexed with the kind value. 194 */ 195VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") 196VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") 197 198#undef VALUE_PROF_KIND 199/* VALUE_PROF_KIND end */ 200 201/* COVMAP_FUNC_RECORD start */ 202/* Definition of member fields of the function record structure in coverage 203 * map. 204 */ 205#ifndef COVMAP_FUNC_RECORD 206#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) 207#else 208#define INSTR_PROF_DATA_DEFINED 209#endif 210#ifdef COVMAP_V1 211COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ 212 NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ 213 llvm::Type::getInt8PtrTy(Ctx))) 214COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ 215 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ 216 NameValue.size())) 217#else 218COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 219 llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 220 llvm::IndexedInstrProf::ComputeHash(NameValue))) 221#endif 222COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ 223 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\ 224 CoverageMapping.size())) 225COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 226 llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), FuncHash)) 227#undef COVMAP_FUNC_RECORD 228/* COVMAP_FUNC_RECORD end. */ 229 230/* COVMAP_HEADER start */ 231/* Definition of member fields of coverage map header. 232 */ 233#ifndef COVMAP_HEADER 234#define COVMAP_HEADER(Type, LLVMType, Name, Initializer) 235#else 236#define INSTR_PROF_DATA_DEFINED 237#endif 238COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \ 239 llvm::ConstantInt::get(Int32Ty, FunctionRecords.size())) 240COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ 241 llvm::ConstantInt::get(Int32Ty, FilenamesSize)) 242COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ 243 llvm::ConstantInt::get(Int32Ty, CoverageMappingSize)) 244COVMAP_HEADER(uint32_t, Int32Ty, Version, \ 245 llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion)) 246#undef COVMAP_HEADER 247/* COVMAP_HEADER end. */ 248 249 250#ifdef INSTR_PROF_SECT_ENTRY 251#define INSTR_PROF_DATA_DEFINED 252INSTR_PROF_SECT_ENTRY(IPSK_data, \ 253 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ 254 INSTR_PROF_DATA_COFF, "__DATA,") 255INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ 256 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ 257 INSTR_PROF_CNTS_COFF, "__DATA,") 258INSTR_PROF_SECT_ENTRY(IPSK_name, \ 259 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ 260 INSTR_PROF_NAME_COFF, "__DATA,") 261INSTR_PROF_SECT_ENTRY(IPSK_vals, \ 262 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ 263 INSTR_PROF_VALS_COFF, "__DATA,") 264INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ 265 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ 266 INSTR_PROF_VNODES_COFF, "__DATA,") 267INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ 268 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ 269 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") 270INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ 271 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ 272 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") 273 274#undef INSTR_PROF_SECT_ENTRY 275#endif 276 277 278#ifdef INSTR_PROF_VALUE_PROF_DATA 279#define INSTR_PROF_DATA_DEFINED 280 281#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255 282/*! 283 * This is the header of the data structure that defines the on-disk 284 * layout of the value profile data of a particular kind for one function. 285 */ 286typedef struct ValueProfRecord { 287 /* The kind of the value profile record. */ 288 uint32_t Kind; 289 /* 290 * The number of value profile sites. It is guaranteed to be non-zero; 291 * otherwise the record for this kind won't be emitted. 292 */ 293 uint32_t NumValueSites; 294 /* 295 * The first element of the array that stores the number of profiled 296 * values for each value site. The size of the array is NumValueSites. 297 * Since NumValueSites is greater than zero, there is at least one 298 * element in the array. 299 */ 300 uint8_t SiteCountArray[1]; 301 302 /* 303 * The fake declaration is for documentation purpose only. 304 * Align the start of next field to be on 8 byte boundaries. 305 uint8_t Padding[X]; 306 */ 307 308 /* The array of value profile data. The size of the array is the sum 309 * of all elements in SiteCountArray[]. 310 InstrProfValueData ValueData[]; 311 */ 312 313#ifdef __cplusplus 314 /*! 315 * Return the number of value sites. 316 */ 317 uint32_t getNumValueSites() const { return NumValueSites; } 318 /*! 319 * Read data from this record and save it to Record. 320 */ 321 void deserializeTo(InstrProfRecord &Record, 322 InstrProfSymtab *SymTab); 323 /* 324 * In-place byte swap: 325 * Do byte swap for this instance. \c Old is the original order before 326 * the swap, and \c New is the New byte order. 327 */ 328 void swapBytes(support::endianness Old, support::endianness New); 329#endif 330} ValueProfRecord; 331 332/*! 333 * Per-function header/control data structure for value profiling 334 * data in indexed format. 335 */ 336typedef struct ValueProfData { 337 /* 338 * Total size in bytes including this field. It must be a multiple 339 * of sizeof(uint64_t). 340 */ 341 uint32_t TotalSize; 342 /* 343 *The number of value profile kinds that has value profile data. 344 * In this implementation, a value profile kind is considered to 345 * have profile data if the number of value profile sites for the 346 * kind is not zero. More aggressively, the implementation can 347 * choose to check the actual data value: if none of the value sites 348 * has any profiled values, the kind can be skipped. 349 */ 350 uint32_t NumValueKinds; 351 352 /* 353 * Following are a sequence of variable length records. The prefix/header 354 * of each record is defined by ValueProfRecord type. The number of 355 * records is NumValueKinds. 356 * ValueProfRecord Record_1; 357 * ValueProfRecord Record_N; 358 */ 359 360#if __cplusplus 361 /*! 362 * Return the total size in bytes of the on-disk value profile data 363 * given the data stored in Record. 364 */ 365 static uint32_t getSize(const InstrProfRecord &Record); 366 /*! 367 * Return a pointer to \c ValueProfData instance ready to be streamed. 368 */ 369 static std::unique_ptr<ValueProfData> 370 serializeFrom(const InstrProfRecord &Record); 371 /*! 372 * Check the integrity of the record. 373 */ 374 Error checkIntegrity(); 375 /*! 376 * Return a pointer to \c ValueProfileData instance ready to be read. 377 * All data in the instance are properly byte swapped. The input 378 * data is assumed to be in little endian order. 379 */ 380 static Expected<std::unique_ptr<ValueProfData>> 381 getValueProfData(const unsigned char *SrcBuffer, 382 const unsigned char *const SrcBufferEnd, 383 support::endianness SrcDataEndianness); 384 /*! 385 * Swap byte order from \c Endianness order to host byte order. 386 */ 387 void swapBytesToHost(support::endianness Endianness); 388 /*! 389 * Swap byte order from host byte order to \c Endianness order. 390 */ 391 void swapBytesFromHost(support::endianness Endianness); 392 /*! 393 * Return the total size of \c ValueProfileData. 394 */ 395 uint32_t getSize() const { return TotalSize; } 396 /*! 397 * Read data from this data and save it to \c Record. 398 */ 399 void deserializeTo(InstrProfRecord &Record, 400 InstrProfSymtab *SymTab); 401 void operator delete(void *ptr) { ::operator delete(ptr); } 402#endif 403} ValueProfData; 404 405/* 406 * The closure is designed to abstact away two types of value profile data: 407 * - InstrProfRecord which is the primary data structure used to 408 * represent profile data in host tools (reader, writer, and profile-use) 409 * - value profile runtime data structure suitable to be used by C 410 * runtime library. 411 * 412 * Both sources of data need to serialize to disk/memory-buffer in common 413 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler 414 * writer to share the same format and code with indexed profile writer. 415 * 416 * For documentation of the member methods below, refer to corresponding methods 417 * in class InstrProfRecord. 418 */ 419typedef struct ValueProfRecordClosure { 420 const void *Record; 421 uint32_t (*GetNumValueKinds)(const void *Record); 422 uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); 423 uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); 424 uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); 425 426 /* 427 * After extracting the value profile data from the value profile record, 428 * this method is used to map the in-memory value to on-disk value. If 429 * the method is null, value will be written out untranslated. 430 */ 431 uint64_t (*RemapValueData)(uint32_t, uint64_t Value); 432 void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, 433 uint32_t S); 434 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); 435} ValueProfRecordClosure; 436 437INSTR_PROF_VISIBILITY ValueProfRecord * 438getFirstValueProfRecord(ValueProfData *VPD); 439INSTR_PROF_VISIBILITY ValueProfRecord * 440getValueProfRecordNext(ValueProfRecord *VPR); 441INSTR_PROF_VISIBILITY InstrProfValueData * 442getValueProfRecordValueData(ValueProfRecord *VPR); 443INSTR_PROF_VISIBILITY uint32_t 444getValueProfRecordHeaderSize(uint32_t NumValueSites); 445 446#undef INSTR_PROF_VALUE_PROF_DATA 447#endif /* INSTR_PROF_VALUE_PROF_DATA */ 448 449 450#ifdef INSTR_PROF_COMMON_API_IMPL 451#define INSTR_PROF_DATA_DEFINED 452#ifdef __cplusplus 453#define INSTR_PROF_INLINE inline 454#define INSTR_PROF_NULLPTR nullptr 455#else 456#define INSTR_PROF_INLINE 457#define INSTR_PROF_NULLPTR NULL 458#endif 459 460#ifndef offsetof 461#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) 462#endif 463 464/*! 465 * Return the \c ValueProfRecord header size including the 466 * padding bytes. 467 */ 468INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 469uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { 470 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + 471 sizeof(uint8_t) * NumValueSites; 472 /* Round the size to multiple of 8 bytes. */ 473 Size = (Size + 7) & ~7; 474 return Size; 475} 476 477/*! 478 * Return the total size of the value profile record including the 479 * header and the value data. 480 */ 481INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 482uint32_t getValueProfRecordSize(uint32_t NumValueSites, 483 uint32_t NumValueData) { 484 return getValueProfRecordHeaderSize(NumValueSites) + 485 sizeof(InstrProfValueData) * NumValueData; 486} 487 488/*! 489 * Return the pointer to the start of value data array. 490 */ 491INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 492InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { 493 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( 494 This->NumValueSites)); 495} 496 497/*! 498 * Return the total number of value data for \c This record. 499 */ 500INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 501uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { 502 uint32_t NumValueData = 0; 503 uint32_t I; 504 for (I = 0; I < This->NumValueSites; I++) 505 NumValueData += This->SiteCountArray[I]; 506 return NumValueData; 507} 508 509/*! 510 * Use this method to advance to the next \c This \c ValueProfRecord. 511 */ 512INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 513ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { 514 uint32_t NumValueData = getValueProfRecordNumValueData(This); 515 return (ValueProfRecord *)((char *)This + 516 getValueProfRecordSize(This->NumValueSites, 517 NumValueData)); 518} 519 520/*! 521 * Return the first \c ValueProfRecord instance. 522 */ 523INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 524ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { 525 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); 526} 527 528/* Closure based interfaces. */ 529 530/*! 531 * Return the total size in bytes of the on-disk value profile data 532 * given the data stored in Record. 533 */ 534INSTR_PROF_VISIBILITY uint32_t 535getValueProfDataSize(ValueProfRecordClosure *Closure) { 536 uint32_t Kind; 537 uint32_t TotalSize = sizeof(ValueProfData); 538 const void *Record = Closure->Record; 539 540 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 541 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); 542 if (!NumValueSites) 543 continue; 544 TotalSize += getValueProfRecordSize(NumValueSites, 545 Closure->GetNumValueData(Record, Kind)); 546 } 547 return TotalSize; 548} 549 550/*! 551 * Extract value profile data of a function for the profile kind \c ValueKind 552 * from the \c Closure and serialize the data into \c This record instance. 553 */ 554INSTR_PROF_VISIBILITY void 555serializeValueProfRecordFrom(ValueProfRecord *This, 556 ValueProfRecordClosure *Closure, 557 uint32_t ValueKind, uint32_t NumValueSites) { 558 uint32_t S; 559 const void *Record = Closure->Record; 560 This->Kind = ValueKind; 561 This->NumValueSites = NumValueSites; 562 InstrProfValueData *DstVD = getValueProfRecordValueData(This); 563 564 for (S = 0; S < NumValueSites; S++) { 565 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); 566 This->SiteCountArray[S] = ND; 567 Closure->GetValueForSite(Record, DstVD, ValueKind, S); 568 DstVD += ND; 569 } 570} 571 572/*! 573 * Extract value profile data of a function from the \c Closure 574 * and serialize the data into \c DstData if it is not NULL or heap 575 * memory allocated by the \c Closure's allocator method. If \c 576 * DstData is not null, the caller is expected to set the TotalSize 577 * in DstData. 578 */ 579INSTR_PROF_VISIBILITY ValueProfData * 580serializeValueProfDataFrom(ValueProfRecordClosure *Closure, 581 ValueProfData *DstData) { 582 uint32_t Kind; 583 uint32_t TotalSize = 584 DstData ? DstData->TotalSize : getValueProfDataSize(Closure); 585 586 ValueProfData *VPD = 587 DstData ? DstData : Closure->AllocValueProfData(TotalSize); 588 589 VPD->TotalSize = TotalSize; 590 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); 591 ValueProfRecord *VR = getFirstValueProfRecord(VPD); 592 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 593 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); 594 if (!NumValueSites) 595 continue; 596 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); 597 VR = getValueProfRecordNext(VR); 598 } 599 return VPD; 600} 601 602#undef INSTR_PROF_COMMON_API_IMPL 603#endif /* INSTR_PROF_COMMON_API_IMPL */ 604 605/*============================================================================*/ 606 607#ifndef INSTR_PROF_DATA_DEFINED 608 609#ifndef INSTR_PROF_DATA_INC 610#define INSTR_PROF_DATA_INC 611 612/* Helper macros. */ 613#define INSTR_PROF_SIMPLE_QUOTE(x) #x 614#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) 615#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y 616#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) 617 618/* Magic number to detect file format and endianness. 619 * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, 620 * so that utilities, like strings, don't grab it as a string. 129 is also 621 * invalid UTF-8, and high enough to be interesting. 622 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" 623 * for 32-bit platforms. 624 */ 625#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 626 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 627 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 628#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 629 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 630 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 631 632/* Raw profile format version (start from 1). */ 633#define INSTR_PROF_RAW_VERSION 5 634/* Indexed profile format version (start from 1). */ 635#define INSTR_PROF_INDEX_VERSION 5 636/* Coverage mapping format vresion (start from 0). */ 637#define INSTR_PROF_COVMAP_VERSION 2 638 639/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the 640 * version for other variants of profile. We set the lowest bit of the upper 8 641 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton 642 * generated profile, and 0 if this is a Clang FE generated profile. 643 * 1 in bit 57 indicates there are context-sensitive records in the profile. 644 */ 645#define VARIANT_MASKS_ALL 0xff00000000000000ULL 646#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) 647#define VARIANT_MASK_IR_PROF (0x1ULL << 56) 648#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) 649#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version 650#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime 651 652/* The variable that holds the name of the profile data 653 * specified via command line. */ 654#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename 655 656/* section name strings common to all targets other 657 than WIN32 */ 658#define INSTR_PROF_DATA_COMMON __llvm_prf_data 659#define INSTR_PROF_NAME_COMMON __llvm_prf_names 660#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts 661#define INSTR_PROF_VALS_COMMON __llvm_prf_vals 662#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds 663#define INSTR_PROF_COVMAP_COMMON __llvm_covmap 664#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile 665/* Windows section names. Because these section names contain dollar characters, 666 * they must be quoted. 667 */ 668#define INSTR_PROF_DATA_COFF ".lprfd$M" 669#define INSTR_PROF_NAME_COFF ".lprfn$M" 670#define INSTR_PROF_CNTS_COFF ".lprfc$M" 671#define INSTR_PROF_VALS_COFF ".lprfv$M" 672#define INSTR_PROF_VNODES_COFF ".lprfnd$M" 673#define INSTR_PROF_COVMAP_COFF ".lcovmap$M" 674#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" 675 676#ifdef _WIN32 677/* Runtime section names and name strings. */ 678#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF 679#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF 680#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF 681/* Array of pointers. Each pointer points to a list 682 * of value nodes associated with one value site. 683 */ 684#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF 685/* Value profile nodes section. */ 686#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF 687#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF 688#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF 689#else 690/* Runtime section names and name strings. */ 691#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) 692#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) 693#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) 694/* Array of pointers. Each pointer points to a list 695 * of value nodes associated with one value site. 696 */ 697#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) 698/* Value profile nodes section. */ 699#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) 700#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) 701/* Order file instrumentation. */ 702#define INSTR_PROF_ORDERFILE_SECT_NAME \ 703 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) 704#endif 705 706#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer 707#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ 708 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) 709#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx 710#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ 711 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) 712 713/* Macros to define start/stop section symbol for a given 714 * section on Linux. For instance 715 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will 716 * expand to __start___llvm_prof_data 717 */ 718#define INSTR_PROF_SECT_START(Sect) \ 719 INSTR_PROF_CONCAT(__start_,Sect) 720#define INSTR_PROF_SECT_STOP(Sect) \ 721 INSTR_PROF_CONCAT(__stop_,Sect) 722 723/* Value Profiling API linkage name. */ 724#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target 725#define INSTR_PROF_VALUE_PROF_FUNC_STR \ 726 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) 727#define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range 728#define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \ 729 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC) 730 731/* InstrProfile per-function control data alignment. */ 732#define INSTR_PROF_DATA_ALIGNMENT 8 733 734/* The data structure that represents a tracked value by the 735 * value profiler. 736 */ 737typedef struct InstrProfValueData { 738 /* Profiled value. */ 739 uint64_t Value; 740 /* Number of times the value appears in the training run. */ 741 uint64_t Count; 742} InstrProfValueData; 743 744#endif /* INSTR_PROF_DATA_INC */ 745 746#ifndef INSTR_ORDER_FILE_INC 747/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */ 748#define INSTR_ORDER_FILE_BUFFER_SIZE 131072 749#define INSTR_ORDER_FILE_BUFFER_BITS 17 750#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff 751#endif /* INSTR_ORDER_FILE_INC */ 752#else 753#undef INSTR_PROF_DATA_DEFINED 754#endif 755