#ifndef INSTR_PROF_DATA
#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName()))))
INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
Inc->getHash()->getZExtValue()))
INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \
ConstantExpr::getBitCast(CounterPtr, \
llvm::Type::getInt64PtrTy(Ctx)))
INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \
FunctionAddr)
INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \
ConstantPointerNull::get(Int8PtrTy))
INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
ConstantArray::get(Int16ArrayTy, Int16ArrayVals))
#undef INSTR_PROF_DATA
#ifndef INSTR_PROF_RAW_HEADER
#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize)
INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize)
INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
INSTR_PROF_RAW_HEADER(uint64_t, ValueDataSize, ValueDataSize)
INSTR_PROF_RAW_HEADER(uint64_t, ValueDataDelta, (uintptr_t)ValueDataBegin)
#undef INSTR_PROF_RAW_HEADER
#ifndef VALUE_PROF_FUNC_PARAM
#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
#define INSTR_PROF_COMMA
#else
#define INSTR_PROF_DATA_DEFINED
#define INSTR_PROF_COMMA ,
#endif
VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
INSTR_PROF_COMMA
VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
#undef VALUE_PROF_FUNC_PARAM
#undef INSTR_PROF_COMMA
#ifndef VALUE_PROF_KIND
#define VALUE_PROF_KIND(Enumerator, Value)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0)
VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget)
VALUE_PROF_KIND(IPVK_Last, IPVK_IndirectCallTarget)
#undef VALUE_PROF_KIND
#ifndef COVMAP_FUNC_RECORD
#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
#ifdef COVMAP_V1
COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \
NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
llvm::Type::getInt8PtrTy(Ctx)))
COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
NameValue.size()))
#else
COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
llvm::IndexedInstrProf::ComputeHash(NameValue)))
#endif
COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\
CoverageMapping.size()))
COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), FuncHash))
#undef COVMAP_FUNC_RECORD
#ifndef COVMAP_HEADER
#define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
llvm::ConstantInt::get(Int32Ty, FunctionRecords.size()))
COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
llvm::ConstantInt::get(Int32Ty, FilenamesSize))
COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
COVMAP_HEADER(uint32_t, Int32Ty, Version, \
llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion))
#undef COVMAP_HEADER
#ifdef INSTR_PROF_VALUE_PROF_DATA
#define INSTR_PROF_DATA_DEFINED
#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
typedef struct ValueProfRecord {
uint32_t Kind;
uint32_t NumValueSites;
uint8_t SiteCountArray[1];
#ifdef __cplusplus
uint32_t getNumValueSites() const { return NumValueSites; }
void deserializeTo(InstrProfRecord &Record,
InstrProfRecord::ValueMapType *VMap);
void swapBytes(support::endianness Old, support::endianness New);
#endif
} ValueProfRecord;
typedef struct ValueProfData {
uint32_t TotalSize;
uint32_t NumValueKinds;
#if __cplusplus
static uint32_t getSize(const InstrProfRecord &Record);
static std::unique_ptr<ValueProfData>
serializeFrom(const InstrProfRecord &Record);
instrprof_error checkIntegrity();
static ErrorOr<std::unique_ptr<ValueProfData>>
getValueProfData(const unsigned char *SrcBuffer,
const unsigned char *const SrcBufferEnd,
support::endianness SrcDataEndianness);
void swapBytesToHost(support::endianness Endianness);
void swapBytesFromHost(support::endianness Endianness);
uint32_t getSize() const { return TotalSize; }
void deserializeTo(InstrProfRecord &Record,
InstrProfRecord::ValueMapType *VMap);
void operator delete(void *ptr) { ::operator delete(ptr); }
#endif
} ValueProfData;
typedef struct ValueProfRecordClosure {
const void *Record;
uint32_t (*GetNumValueKinds)(const void *Record);
uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
uint32_t S);
ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
} ValueProfRecordClosure;
typedef struct ValueProfRuntimeRecord {
const uint16_t *NumValueSites;
ValueProfNode **Nodes;
uint32_t NumValueKinds;
uint8_t *SiteCountArray[IPVK_Last + 1];
ValueProfNode **NodesKind[IPVK_Last + 1];
} ValueProfRuntimeRecord;
int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord,
const uint16_t *NumValueSites,
ValueProfNode **Nodes);
void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord);
uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record);
ValueProfData *
serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record,
ValueProfData *Dst);
uint32_t getNumValueKindsRT(const void *R);
#undef INSTR_PROF_VALUE_PROF_DATA
#endif
#ifdef INSTR_PROF_COMMON_API_IMPL
#define INSTR_PROF_DATA_DEFINED
#ifdef __cplusplus
#define INSTR_PROF_INLINE inline
#else
#define INSTR_PROF_INLINE
#endif
#ifndef offsetof
#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#endif
INSTR_PROF_INLINE
uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) {
uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
sizeof(uint8_t) * NumValueSites;
Size = (Size + 7) & ~7;
return Size;
}
INSTR_PROF_INLINE
uint32_t getValueProfRecordSize(uint32_t NumValueSites,
uint32_t NumValueData) {
return getValueProfRecordHeaderSize(NumValueSites) +
sizeof(InstrProfValueData) * NumValueData;
}
INSTR_PROF_INLINE
InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
This->NumValueSites));
}
INSTR_PROF_INLINE
uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
uint32_t NumValueData = 0;
uint32_t I;
for (I = 0; I < This->NumValueSites; I++)
NumValueData += This->SiteCountArray[I];
return NumValueData;
}
INSTR_PROF_INLINE
ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) {
uint32_t NumValueData = getValueProfRecordNumValueData(This);
return (ValueProfRecord *)((char *)This +
getValueProfRecordSize(This->NumValueSites,
NumValueData));
}
INSTR_PROF_INLINE
ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
}
uint32_t getValueProfDataSize(ValueProfRecordClosure *Closure) {
uint32_t Kind;
uint32_t TotalSize = sizeof(ValueProfData);
const void *Record = Closure->Record;
uint32_t NumValueKinds = Closure->GetNumValueKinds(Record);
if (NumValueKinds == 0)
return TotalSize;
for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
if (!NumValueSites)
continue;
TotalSize += getValueProfRecordSize(NumValueSites,
Closure->GetNumValueData(Record, Kind));
}
return TotalSize;
}
void serializeValueProfRecordFrom(ValueProfRecord *This,
ValueProfRecordClosure *Closure,
uint32_t ValueKind, uint32_t NumValueSites) {
uint32_t S;
const void *Record = Closure->Record;
This->Kind = ValueKind;
This->NumValueSites = NumValueSites;
InstrProfValueData *DstVD = getValueProfRecordValueData(This);
for (S = 0; S < NumValueSites; S++) {
uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
This->SiteCountArray[S] = ND;
Closure->GetValueForSite(Record, DstVD, ValueKind, S);
DstVD += ND;
}
}
ValueProfData *serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
ValueProfData *DstData) {
uint32_t Kind;
uint32_t TotalSize = getValueProfDataSize(Closure);
ValueProfData *VPD =
DstData ? DstData : Closure->AllocValueProfData(TotalSize);
VPD->TotalSize = TotalSize;
VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
ValueProfRecord *VR = getFirstValueProfRecord(VPD);
for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
if (!NumValueSites)
continue;
serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
VR = getValueProfRecordNext(VR);
}
return VPD;
}
int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord,
const uint16_t *NumValueSites,
ValueProfNode **Nodes) {
unsigned I, J, S = 0, NumValueKinds = 0;
RuntimeRecord->NumValueSites = NumValueSites;
RuntimeRecord->Nodes = Nodes;
for (I = 0; I <= IPVK_Last; I++) {
uint16_t N = NumValueSites[I];
if (!N) {
RuntimeRecord->SiteCountArray[I] = 0;
continue;
}
NumValueKinds++;
RuntimeRecord->SiteCountArray[I] = (uint8_t *)calloc(N, 1);
if (!RuntimeRecord->SiteCountArray[I])
return 1;
RuntimeRecord->NodesKind[I] = Nodes ? &Nodes[S] : NULL;
for (J = 0; J < N; J++) {
uint32_t C = 0;
ValueProfNode *Site = Nodes ? RuntimeRecord->NodesKind[I][J] : NULL;
while (Site) {
C++;
Site = Site->Next;
}
if (C > UCHAR_MAX)
C = UCHAR_MAX;
RuntimeRecord->SiteCountArray[I][J] = C;
}
S += N;
}
RuntimeRecord->NumValueKinds = NumValueKinds;
return 0;
}
void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord) {
unsigned I;
for (I = 0; I <= IPVK_Last; I++) {
if (RuntimeRecord->SiteCountArray[I])
free(RuntimeRecord->SiteCountArray[I]);
}
}
uint32_t getNumValueKindsRT(const void *R) {
return ((const ValueProfRuntimeRecord *)R)->NumValueKinds;
}
uint32_t getNumValueSitesRT(const void *R, uint32_t VK) {
return ((const ValueProfRuntimeRecord *)R)->NumValueSites[VK];
}
uint32_t getNumValueDataForSiteRT(const void *R, uint32_t VK, uint32_t S) {
const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
return Record->SiteCountArray[VK][S];
}
uint32_t getNumValueDataRT(const void *R, uint32_t VK) {
unsigned I, S = 0;
const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
if (Record->SiteCountArray[VK] == 0)
return 0;
for (I = 0; I < Record->NumValueSites[VK]; I++)
S += Record->SiteCountArray[VK][I];
return S;
}
void getValueForSiteRT(const void *R, InstrProfValueData *Dst, uint32_t VK,
uint32_t S) {
unsigned I, N = 0;
const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
N = getNumValueDataForSiteRT(R, VK, S);
if (N == 0)
return;
ValueProfNode *VNode = Record->NodesKind[VK][S];
for (I = 0; I < N; I++) {
Dst[I] = VNode->VData;
VNode = VNode->Next;
}
}
ValueProfData *allocValueProfDataRT(size_t TotalSizeInBytes) {
return (ValueProfData *)calloc(TotalSizeInBytes, 1);
}
static ValueProfRecordClosure RTRecordClosure = {0,
getNumValueKindsRT,
getNumValueSitesRT,
getNumValueDataRT,
getNumValueDataForSiteRT,
0,
getValueForSiteRT,
allocValueProfDataRT};
uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record) {
RTRecordClosure.Record = Record;
return getValueProfDataSize(&RTRecordClosure);
}
ValueProfData *
serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record,
ValueProfData *DstData) {
RTRecordClosure.Record = Record;
return serializeValueProfDataFrom(&RTRecordClosure, DstData);
}
#undef INSTR_PROF_COMMON_API_IMPL
#endif
#ifndef INSTR_PROF_DATA_DEFINED
#ifndef INSTR_PROF_DATA_INC_
#define INSTR_PROF_DATA_INC_
#define INSTR_PROF_SIMPLE_QUOTE(x) #x
#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
(uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
(uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
(uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
(uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
#define INSTR_PROF_RAW_VERSION 3
#define INSTR_PROF_INDEX_VERSION 4
#define INSTR_PROF_COVMAP_VERSION 1
#define VARIANT_MASKS_ALL 0xff00000000000000ULL
#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
#define IR_LEVEL_PROF_VERSION_VAR __llvm_profile_raw_version
#define INSTR_PROF_DATA_SECT_NAME __llvm_prf_data
#define INSTR_PROF_NAME_SECT_NAME __llvm_prf_names
#define INSTR_PROF_CNTS_SECT_NAME __llvm_prf_cnts
#define INSTR_PROF_COVMAP_SECT_NAME __llvm_covmap
#define INSTR_PROF_DATA_SECT_NAME_STR \
INSTR_PROF_QUOTE(INSTR_PROF_DATA_SECT_NAME)
#define INSTR_PROF_NAME_SECT_NAME_STR \
INSTR_PROF_QUOTE(INSTR_PROF_NAME_SECT_NAME)
#define INSTR_PROF_CNTS_SECT_NAME_STR \
INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME)
#define INSTR_PROF_COVMAP_SECT_NAME_STR \
INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_SECT_NAME)
#define INSTR_PROF_SECT_START(Sect) \
INSTR_PROF_CONCAT(__start_,Sect)
#define INSTR_PROF_SECT_STOP(Sect) \
INSTR_PROF_CONCAT(__stop_,Sect)
#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
#define INSTR_PROF_VALUE_PROF_FUNC_STR \
INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
#define INSTR_PROF_DATA_ALIGNMENT 8
typedef struct InstrProfValueData {
uint64_t Value;
uint64_t Count;
} InstrProfValueData;
typedef struct ValueProfNode {
InstrProfValueData VData;
struct ValueProfNode *Next;
} ValueProfNode;
#endif
#else
#undef INSTR_PROF_DATA_DEFINED
#endif