InstrProfData.inc revision 355940
1/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ 2|* 3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4|* See https://llvm.org/LICENSE.txt for license information. 5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6|* 7\*===----------------------------------------------------------------------===*/ 8/* 9 * This is the master file that defines all the data structure, signature, 10 * constant literals that are shared across profiling runtime library, 11 * compiler (instrumentation), and host tools (reader/writer). The entities 12 * defined in this file affect the profile runtime ABI, the raw profile format, 13 * or both. 14 * 15 * The file has two identical copies. The master copy lives in LLVM and 16 * the other one sits in compiler-rt/lib/profile directory. To make changes 17 * in this file, first modify the master copy and copy it over to compiler-rt. 18 * Testing of any change in this file can start only after the two copies are 19 * synced up. 20 * 21 * The first part of the file includes macros that defines types, names, and 22 * initializers for the member fields of the core data structures. The field 23 * declarations for one structure is enabled by defining the field activation 24 * macro associated with that structure. Only one field activation record 25 * can be defined at one time and the rest definitions will be filtered out by 26 * the preprocessor. 27 * 28 * Examples of how the template is used to instantiate structure definition: 29 * 1. To declare a structure: 30 * 31 * struct ProfData { 32 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 33 * Type Name; 34 * #include "llvm/ProfileData/InstrProfData.inc" 35 * }; 36 * 37 * 2. To construct LLVM type arrays for the struct type: 38 * 39 * Type *DataTypes[] = { 40 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 41 * LLVMType, 42 * #include "llvm/ProfileData/InstrProfData.inc" 43 * }; 44 * 45 * 4. To construct constant array for the initializers: 46 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 47 * Initializer, 48 * Constant *ConstantVals[] = { 49 * #include "llvm/ProfileData/InstrProfData.inc" 50 * }; 51 * 52 * 53 * The second part of the file includes definitions all other entities that 54 * are related to runtime ABI and format. When no field activation macro is 55 * defined, this file can be included to introduce the definitions. 56 * 57\*===----------------------------------------------------------------------===*/ 58 59/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in 60 * the compiler runtime. */ 61#ifndef INSTR_PROF_VISIBILITY 62#define INSTR_PROF_VISIBILITY 63#endif 64 65/* INSTR_PROF_DATA start. */ 66/* Definition of member fields of the per-function control structure. */ 67#ifndef INSTR_PROF_DATA 68#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) 69#else 70#define INSTR_PROF_DATA_DEFINED 71#endif 72INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 73 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 74 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName())))) 75INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 76 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 77 Inc->getHash()->getZExtValue())) 78INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ 79 ConstantExpr::getBitCast(CounterPtr, \ 80 llvm::Type::getInt64PtrTy(Ctx))) 81/* This is used to map function pointers for the indirect call targets to 82 * function name hashes during the conversion from raw to merged profile 83 * data. 84 */ 85INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \ 86 FunctionAddr) 87INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ 88 ValuesPtrExpr) 89INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ 90 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) 91INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ 92 ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) 93#undef INSTR_PROF_DATA 94/* INSTR_PROF_DATA end. */ 95 96 97/* This is an internal data structure used by value profiler. It 98 * is defined here to allow serialization code sharing by LLVM 99 * to be used in unit test. 100 * 101 * typedef struct ValueProfNode { 102 * // InstrProfValueData VData; 103 * uint64_t Value; 104 * uint64_t Count; 105 * struct ValueProfNode *Next; 106 * } ValueProfNode; 107 */ 108/* INSTR_PROF_VALUE_NODE start. */ 109#ifndef INSTR_PROF_VALUE_NODE 110#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) 111#else 112#define INSTR_PROF_DATA_DEFINED 113#endif 114INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \ 115 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 116INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \ 117 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 118INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ 119 ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0)) 120#undef INSTR_PROF_VALUE_NODE 121/* INSTR_PROF_VALUE_NODE end. */ 122 123/* INSTR_PROF_RAW_HEADER start */ 124/* Definition of member fields of the raw profile header data structure. */ 125#ifndef INSTR_PROF_RAW_HEADER 126#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) 127#else 128#define INSTR_PROF_DATA_DEFINED 129#endif 130INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) 131INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) 132INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) 133INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) 134INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) 135INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) 136INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) 137INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) 138#undef INSTR_PROF_RAW_HEADER 139/* INSTR_PROF_RAW_HEADER end */ 140 141/* VALUE_PROF_FUNC_PARAM start */ 142/* Definition of parameter types of the runtime API used to do value profiling 143 * for a given value site. 144 */ 145#ifndef VALUE_PROF_FUNC_PARAM 146#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) 147#define INSTR_PROF_COMMA 148#else 149#define INSTR_PROF_DATA_DEFINED 150#define INSTR_PROF_COMMA , 151#endif 152VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ 153 INSTR_PROF_COMMA 154VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA 155#ifndef VALUE_RANGE_PROF 156VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) 157#else /* VALUE_RANGE_PROF */ 158VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \ 159 INSTR_PROF_COMMA 160VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \ 161 INSTR_PROF_COMMA 162VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeLast, Type::getInt64Ty(Ctx)) \ 163 INSTR_PROF_COMMA 164VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) 165#endif /*VALUE_RANGE_PROF */ 166#undef VALUE_PROF_FUNC_PARAM 167#undef INSTR_PROF_COMMA 168/* VALUE_PROF_FUNC_PARAM end */ 169 170/* VALUE_PROF_KIND start */ 171#ifndef VALUE_PROF_KIND 172#define VALUE_PROF_KIND(Enumerator, Value, Descr) 173#else 174#define INSTR_PROF_DATA_DEFINED 175#endif 176/* For indirect function call value profiling, the addresses of the target 177 * functions are profiled by the instrumented code. The target addresses are 178 * written in the raw profile data and converted to target function name's MD5 179 * hash by the profile reader during deserialization. Typically, this happens 180 * when the raw profile data is read during profile merging. 181 * 182 * For this remapping the ProfData is used. ProfData contains both the function 183 * name hash and the function address. 184 */ 185VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") 186/* For memory intrinsic functions size profiling. */ 187VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") 188/* These two kinds must be the last to be 189 * declared. This is to make sure the string 190 * array created with the template can be 191 * indexed with the kind value. 192 */ 193VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") 194VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") 195 196#undef VALUE_PROF_KIND 197/* VALUE_PROF_KIND end */ 198 199/* COVMAP_FUNC_RECORD start */ 200/* Definition of member fields of the function record structure in coverage 201 * map. 202 */ 203#ifndef COVMAP_FUNC_RECORD 204#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) 205#else 206#define INSTR_PROF_DATA_DEFINED 207#endif 208#ifdef COVMAP_V1 209COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ 210 NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ 211 llvm::Type::getInt8PtrTy(Ctx))) 212COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ 213 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ 214 NameValue.size())) 215#else 216COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 217 llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 218 llvm::IndexedInstrProf::ComputeHash(NameValue))) 219#endif 220COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ 221 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\ 222 CoverageMapping.size())) 223COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 224 llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), FuncHash)) 225#undef COVMAP_FUNC_RECORD 226/* COVMAP_FUNC_RECORD end. */ 227 228/* COVMAP_HEADER start */ 229/* Definition of member fields of coverage map header. 230 */ 231#ifndef COVMAP_HEADER 232#define COVMAP_HEADER(Type, LLVMType, Name, Initializer) 233#else 234#define INSTR_PROF_DATA_DEFINED 235#endif 236COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \ 237 llvm::ConstantInt::get(Int32Ty, FunctionRecords.size())) 238COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ 239 llvm::ConstantInt::get(Int32Ty, FilenamesSize)) 240COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ 241 llvm::ConstantInt::get(Int32Ty, CoverageMappingSize)) 242COVMAP_HEADER(uint32_t, Int32Ty, Version, \ 243 llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion)) 244#undef COVMAP_HEADER 245/* COVMAP_HEADER end. */ 246 247 248#ifdef INSTR_PROF_SECT_ENTRY 249#define INSTR_PROF_DATA_DEFINED 250INSTR_PROF_SECT_ENTRY(IPSK_data, \ 251 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ 252 INSTR_PROF_DATA_COFF, "__DATA,") 253INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ 254 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ 255 INSTR_PROF_CNTS_COFF, "__DATA,") 256INSTR_PROF_SECT_ENTRY(IPSK_name, \ 257 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ 258 INSTR_PROF_NAME_COFF, "__DATA,") 259INSTR_PROF_SECT_ENTRY(IPSK_vals, \ 260 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ 261 INSTR_PROF_VALS_COFF, "__DATA,") 262INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ 263 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ 264 INSTR_PROF_VNODES_COFF, "__DATA,") 265INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ 266 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ 267 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") 268INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ 269 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ 270 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") 271 272#undef INSTR_PROF_SECT_ENTRY 273#endif 274 275 276#ifdef INSTR_PROF_VALUE_PROF_DATA 277#define INSTR_PROF_DATA_DEFINED 278 279#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255 280/*! 281 * This is the header of the data structure that defines the on-disk 282 * layout of the value profile data of a particular kind for one function. 283 */ 284typedef struct ValueProfRecord { 285 /* The kind of the value profile record. */ 286 uint32_t Kind; 287 /* 288 * The number of value profile sites. It is guaranteed to be non-zero; 289 * otherwise the record for this kind won't be emitted. 290 */ 291 uint32_t NumValueSites; 292 /* 293 * The first element of the array that stores the number of profiled 294 * values for each value site. The size of the array is NumValueSites. 295 * Since NumValueSites is greater than zero, there is at least one 296 * element in the array. 297 */ 298 uint8_t SiteCountArray[1]; 299 300 /* 301 * The fake declaration is for documentation purpose only. 302 * Align the start of next field to be on 8 byte boundaries. 303 uint8_t Padding[X]; 304 */ 305 306 /* The array of value profile data. The size of the array is the sum 307 * of all elements in SiteCountArray[]. 308 InstrProfValueData ValueData[]; 309 */ 310 311#ifdef __cplusplus 312 /*! 313 * Return the number of value sites. 314 */ 315 uint32_t getNumValueSites() const { return NumValueSites; } 316 /*! 317 * Read data from this record and save it to Record. 318 */ 319 void deserializeTo(InstrProfRecord &Record, 320 InstrProfSymtab *SymTab); 321 /* 322 * In-place byte swap: 323 * Do byte swap for this instance. \c Old is the original order before 324 * the swap, and \c New is the New byte order. 325 */ 326 void swapBytes(support::endianness Old, support::endianness New); 327#endif 328} ValueProfRecord; 329 330/*! 331 * Per-function header/control data structure for value profiling 332 * data in indexed format. 333 */ 334typedef struct ValueProfData { 335 /* 336 * Total size in bytes including this field. It must be a multiple 337 * of sizeof(uint64_t). 338 */ 339 uint32_t TotalSize; 340 /* 341 *The number of value profile kinds that has value profile data. 342 * In this implementation, a value profile kind is considered to 343 * have profile data if the number of value profile sites for the 344 * kind is not zero. More aggressively, the implementation can 345 * choose to check the actual data value: if none of the value sites 346 * has any profiled values, the kind can be skipped. 347 */ 348 uint32_t NumValueKinds; 349 350 /* 351 * Following are a sequence of variable length records. The prefix/header 352 * of each record is defined by ValueProfRecord type. The number of 353 * records is NumValueKinds. 354 * ValueProfRecord Record_1; 355 * ValueProfRecord Record_N; 356 */ 357 358#if __cplusplus 359 /*! 360 * Return the total size in bytes of the on-disk value profile data 361 * given the data stored in Record. 362 */ 363 static uint32_t getSize(const InstrProfRecord &Record); 364 /*! 365 * Return a pointer to \c ValueProfData instance ready to be streamed. 366 */ 367 static std::unique_ptr<ValueProfData> 368 serializeFrom(const InstrProfRecord &Record); 369 /*! 370 * Check the integrity of the record. 371 */ 372 Error checkIntegrity(); 373 /*! 374 * Return a pointer to \c ValueProfileData instance ready to be read. 375 * All data in the instance are properly byte swapped. The input 376 * data is assumed to be in little endian order. 377 */ 378 static Expected<std::unique_ptr<ValueProfData>> 379 getValueProfData(const unsigned char *SrcBuffer, 380 const unsigned char *const SrcBufferEnd, 381 support::endianness SrcDataEndianness); 382 /*! 383 * Swap byte order from \c Endianness order to host byte order. 384 */ 385 void swapBytesToHost(support::endianness Endianness); 386 /*! 387 * Swap byte order from host byte order to \c Endianness order. 388 */ 389 void swapBytesFromHost(support::endianness Endianness); 390 /*! 391 * Return the total size of \c ValueProfileData. 392 */ 393 uint32_t getSize() const { return TotalSize; } 394 /*! 395 * Read data from this data and save it to \c Record. 396 */ 397 void deserializeTo(InstrProfRecord &Record, 398 InstrProfSymtab *SymTab); 399 void operator delete(void *ptr) { ::operator delete(ptr); } 400#endif 401} ValueProfData; 402 403/* 404 * The closure is designed to abstact away two types of value profile data: 405 * - InstrProfRecord which is the primary data structure used to 406 * represent profile data in host tools (reader, writer, and profile-use) 407 * - value profile runtime data structure suitable to be used by C 408 * runtime library. 409 * 410 * Both sources of data need to serialize to disk/memory-buffer in common 411 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler 412 * writer to share the same format and code with indexed profile writer. 413 * 414 * For documentation of the member methods below, refer to corresponding methods 415 * in class InstrProfRecord. 416 */ 417typedef struct ValueProfRecordClosure { 418 const void *Record; 419 uint32_t (*GetNumValueKinds)(const void *Record); 420 uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); 421 uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); 422 uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); 423 424 /* 425 * After extracting the value profile data from the value profile record, 426 * this method is used to map the in-memory value to on-disk value. If 427 * the method is null, value will be written out untranslated. 428 */ 429 uint64_t (*RemapValueData)(uint32_t, uint64_t Value); 430 void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, 431 uint32_t S); 432 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); 433} ValueProfRecordClosure; 434 435INSTR_PROF_VISIBILITY ValueProfRecord * 436getFirstValueProfRecord(ValueProfData *VPD); 437INSTR_PROF_VISIBILITY ValueProfRecord * 438getValueProfRecordNext(ValueProfRecord *VPR); 439INSTR_PROF_VISIBILITY InstrProfValueData * 440getValueProfRecordValueData(ValueProfRecord *VPR); 441INSTR_PROF_VISIBILITY uint32_t 442getValueProfRecordHeaderSize(uint32_t NumValueSites); 443 444#undef INSTR_PROF_VALUE_PROF_DATA 445#endif /* INSTR_PROF_VALUE_PROF_DATA */ 446 447 448#ifdef INSTR_PROF_COMMON_API_IMPL 449#define INSTR_PROF_DATA_DEFINED 450#ifdef __cplusplus 451#define INSTR_PROF_INLINE inline 452#define INSTR_PROF_NULLPTR nullptr 453#else 454#define INSTR_PROF_INLINE 455#define INSTR_PROF_NULLPTR NULL 456#endif 457 458#ifndef offsetof 459#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) 460#endif 461 462/*! 463 * Return the \c ValueProfRecord header size including the 464 * padding bytes. 465 */ 466INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 467uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { 468 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + 469 sizeof(uint8_t) * NumValueSites; 470 /* Round the size to multiple of 8 bytes. */ 471 Size = (Size + 7) & ~7; 472 return Size; 473} 474 475/*! 476 * Return the total size of the value profile record including the 477 * header and the value data. 478 */ 479INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 480uint32_t getValueProfRecordSize(uint32_t NumValueSites, 481 uint32_t NumValueData) { 482 return getValueProfRecordHeaderSize(NumValueSites) + 483 sizeof(InstrProfValueData) * NumValueData; 484} 485 486/*! 487 * Return the pointer to the start of value data array. 488 */ 489INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 490InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { 491 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( 492 This->NumValueSites)); 493} 494 495/*! 496 * Return the total number of value data for \c This record. 497 */ 498INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 499uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { 500 uint32_t NumValueData = 0; 501 uint32_t I; 502 for (I = 0; I < This->NumValueSites; I++) 503 NumValueData += This->SiteCountArray[I]; 504 return NumValueData; 505} 506 507/*! 508 * Use this method to advance to the next \c This \c ValueProfRecord. 509 */ 510INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 511ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { 512 uint32_t NumValueData = getValueProfRecordNumValueData(This); 513 return (ValueProfRecord *)((char *)This + 514 getValueProfRecordSize(This->NumValueSites, 515 NumValueData)); 516} 517 518/*! 519 * Return the first \c ValueProfRecord instance. 520 */ 521INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 522ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { 523 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); 524} 525 526/* Closure based interfaces. */ 527 528/*! 529 * Return the total size in bytes of the on-disk value profile data 530 * given the data stored in Record. 531 */ 532INSTR_PROF_VISIBILITY uint32_t 533getValueProfDataSize(ValueProfRecordClosure *Closure) { 534 uint32_t Kind; 535 uint32_t TotalSize = sizeof(ValueProfData); 536 const void *Record = Closure->Record; 537 538 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 539 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); 540 if (!NumValueSites) 541 continue; 542 TotalSize += getValueProfRecordSize(NumValueSites, 543 Closure->GetNumValueData(Record, Kind)); 544 } 545 return TotalSize; 546} 547 548/*! 549 * Extract value profile data of a function for the profile kind \c ValueKind 550 * from the \c Closure and serialize the data into \c This record instance. 551 */ 552INSTR_PROF_VISIBILITY void 553serializeValueProfRecordFrom(ValueProfRecord *This, 554 ValueProfRecordClosure *Closure, 555 uint32_t ValueKind, uint32_t NumValueSites) { 556 uint32_t S; 557 const void *Record = Closure->Record; 558 This->Kind = ValueKind; 559 This->NumValueSites = NumValueSites; 560 InstrProfValueData *DstVD = getValueProfRecordValueData(This); 561 562 for (S = 0; S < NumValueSites; S++) { 563 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); 564 This->SiteCountArray[S] = ND; 565 Closure->GetValueForSite(Record, DstVD, ValueKind, S); 566 DstVD += ND; 567 } 568} 569 570/*! 571 * Extract value profile data of a function from the \c Closure 572 * and serialize the data into \c DstData if it is not NULL or heap 573 * memory allocated by the \c Closure's allocator method. If \c 574 * DstData is not null, the caller is expected to set the TotalSize 575 * in DstData. 576 */ 577INSTR_PROF_VISIBILITY ValueProfData * 578serializeValueProfDataFrom(ValueProfRecordClosure *Closure, 579 ValueProfData *DstData) { 580 uint32_t Kind; 581 uint32_t TotalSize = 582 DstData ? DstData->TotalSize : getValueProfDataSize(Closure); 583 584 ValueProfData *VPD = 585 DstData ? DstData : Closure->AllocValueProfData(TotalSize); 586 587 VPD->TotalSize = TotalSize; 588 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); 589 ValueProfRecord *VR = getFirstValueProfRecord(VPD); 590 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 591 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); 592 if (!NumValueSites) 593 continue; 594 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); 595 VR = getValueProfRecordNext(VR); 596 } 597 return VPD; 598} 599 600#undef INSTR_PROF_COMMON_API_IMPL 601#endif /* INSTR_PROF_COMMON_API_IMPL */ 602 603/*============================================================================*/ 604 605#ifndef INSTR_PROF_DATA_DEFINED 606 607#ifndef INSTR_PROF_DATA_INC 608#define INSTR_PROF_DATA_INC 609 610/* Helper macros. */ 611#define INSTR_PROF_SIMPLE_QUOTE(x) #x 612#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) 613#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y 614#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) 615 616/* Magic number to detect file format and endianness. 617 * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, 618 * so that utilities, like strings, don't grab it as a string. 129 is also 619 * invalid UTF-8, and high enough to be interesting. 620 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" 621 * for 32-bit platforms. 622 */ 623#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 624 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 625 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 626#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 627 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 628 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 629 630/* Raw profile format version (start from 1). */ 631#define INSTR_PROF_RAW_VERSION 4 632/* Indexed profile format version (start from 1). */ 633#define INSTR_PROF_INDEX_VERSION 5 634/* Coverage mapping format vresion (start from 0). */ 635#define INSTR_PROF_COVMAP_VERSION 2 636 637/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the 638 * version for other variants of profile. We set the lowest bit of the upper 8 639 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton 640 * generated profile, and 0 if this is a Clang FE generated profile. 641 * 1 in bit 57 indicates there are context-sensitive records in the profile. 642 */ 643#define VARIANT_MASKS_ALL 0xff00000000000000ULL 644#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) 645#define VARIANT_MASK_IR_PROF (0x1ULL << 56) 646#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) 647#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version 648#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime 649 650/* The variable that holds the name of the profile data 651 * specified via command line. */ 652#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename 653 654/* section name strings common to all targets other 655 than WIN32 */ 656#define INSTR_PROF_DATA_COMMON __llvm_prf_data 657#define INSTR_PROF_NAME_COMMON __llvm_prf_names 658#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts 659#define INSTR_PROF_VALS_COMMON __llvm_prf_vals 660#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds 661#define INSTR_PROF_COVMAP_COMMON __llvm_covmap 662#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile 663/* Windows section names. Because these section names contain dollar characters, 664 * they must be quoted. 665 */ 666#define INSTR_PROF_DATA_COFF ".lprfd$M" 667#define INSTR_PROF_NAME_COFF ".lprfn$M" 668#define INSTR_PROF_CNTS_COFF ".lprfc$M" 669#define INSTR_PROF_VALS_COFF ".lprfv$M" 670#define INSTR_PROF_VNODES_COFF ".lprfnd$M" 671#define INSTR_PROF_COVMAP_COFF ".lcovmap$M" 672#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" 673 674#ifdef _WIN32 675/* Runtime section names and name strings. */ 676#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF 677#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF 678#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF 679/* Array of pointers. Each pointer points to a list 680 * of value nodes associated with one value site. 681 */ 682#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF 683/* Value profile nodes section. */ 684#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF 685#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF 686#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF 687#else 688/* Runtime section names and name strings. */ 689#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) 690#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) 691#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) 692/* Array of pointers. Each pointer points to a list 693 * of value nodes associated with one value site. 694 */ 695#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) 696/* Value profile nodes section. */ 697#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) 698#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) 699/* Order file instrumentation. */ 700#define INSTR_PROF_ORDERFILE_SECT_NAME \ 701 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) 702#endif 703 704#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer 705#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ 706 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) 707#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx 708#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ 709 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) 710 711/* Macros to define start/stop section symbol for a given 712 * section on Linux. For instance 713 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will 714 * expand to __start___llvm_prof_data 715 */ 716#define INSTR_PROF_SECT_START(Sect) \ 717 INSTR_PROF_CONCAT(__start_,Sect) 718#define INSTR_PROF_SECT_STOP(Sect) \ 719 INSTR_PROF_CONCAT(__stop_,Sect) 720 721/* Value Profiling API linkage name. */ 722#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target 723#define INSTR_PROF_VALUE_PROF_FUNC_STR \ 724 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) 725#define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range 726#define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \ 727 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC) 728 729/* InstrProfile per-function control data alignment. */ 730#define INSTR_PROF_DATA_ALIGNMENT 8 731 732/* The data structure that represents a tracked value by the 733 * value profiler. 734 */ 735typedef struct InstrProfValueData { 736 /* Profiled value. */ 737 uint64_t Value; 738 /* Number of times the value appears in the training run. */ 739 uint64_t Count; 740} InstrProfValueData; 741 742#endif /* INSTR_PROF_DATA_INC */ 743 744#ifndef INSTR_ORDER_FILE_INC 745// The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). 746#define INSTR_ORDER_FILE_BUFFER_SIZE 131072 747#define INSTR_ORDER_FILE_BUFFER_BITS 17 748#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff 749#endif /* INSTR_ORDER_FILE_INC */ 750#else 751#undef INSTR_PROF_DATA_DEFINED 752#endif 753