#ifndef LLVM_PROFILEDATA_INSTRPROF_H_
#define LLVM_PROFILEDATA_INSTRPROF_H_
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/MD5.h"
#include <cstdint>
#include <list>
#include <system_error>
#include <vector>
namespace llvm {
class Function;
class GlobalVariable;
class Module;
inline StringRef getInstrProfCountersSectionName(bool AddSegment) {
return AddSegment ? "__DATA,__llvm_prf_cnts" : "__llvm_prf_cnts";
}
inline StringRef getInstrProfNameSectionName(bool AddSegment) {
return AddSegment ? "__DATA,__llvm_prf_names" : "__llvm_prf_names";
}
inline StringRef getInstrProfDataSectionName(bool AddSegment) {
return AddSegment ? "__DATA,__llvm_prf_data" : "__llvm_prf_data";
}
inline StringRef getInstrProfCoverageSectionName(bool AddSegment) {
return AddSegment ? "__DATA,__llvm_covmap" : "__llvm_covmap";
}
inline StringRef getInstrProfNameVarPrefix() { return "__llvm_profile_name_"; }
inline StringRef getInstrProfDataVarPrefix() { return "__llvm_profile_data_"; }
inline StringRef getInstrProfCountersVarPrefix() {
return "__llvm_profile_counters_";
}
inline StringRef getInstrProfComdatPrefix() { return "__llvm_profile_vars_"; }
inline StringRef getCoverageMappingVarName() {
return "__llvm_coverage_mapping";
}
inline StringRef getInstrProfRegFuncsName() {
return "__llvm_profile_register_functions";
}
inline StringRef getInstrProfRegFuncName() {
return "__llvm_profile_register_function";
}
inline StringRef getInstrProfInitFuncName() { return "__llvm_profile_init"; }
inline StringRef getInstrProfRuntimeHookVarName() {
return "__llvm_profile_runtime";
}
inline StringRef getInstrProfRuntimeHookVarUseFuncName() {
return "__llvm_profile_runtime_user";
}
inline StringRef getInstrProfFileOverriderFuncName() {
return "__llvm_profile_override_default_filename";
}
std::string getPGOFuncName(const Function &F);
std::string getPGOFuncName(StringRef RawFuncName,
GlobalValue::LinkageTypes Linkage,
StringRef FileName);
GlobalVariable *createPGOFuncNameVar(Function &F, StringRef FuncName);
GlobalVariable *createPGOFuncNameVar(Module &M,
GlobalValue::LinkageTypes Linkage,
StringRef FuncName);
const std::error_category &instrprof_category();
enum class instrprof_error {
success = 0,
eof,
bad_magic,
bad_header,
unsupported_version,
unsupported_hash_type,
too_large,
truncated,
malformed,
unknown_function,
hash_mismatch,
count_mismatch,
counter_overflow,
value_site_count_mismatch
};
inline std::error_code make_error_code(instrprof_error E) {
return std::error_code(static_cast<int>(E), instrprof_category());
}
enum InstrProfValueKind : uint32_t {
IPVK_IndirectCallTarget = 0,
IPVK_First = IPVK_IndirectCallTarget,
IPVK_Last = IPVK_IndirectCallTarget
};
struct InstrProfStringTable {
StringSet<> StringValueSet;
InstrProfStringTable() { StringValueSet.clear(); }
const char *getStringData(StringRef Str) {
auto Result = StringValueSet.find(Str);
return (Result == StringValueSet.end()) ? nullptr : Result->first().data();
}
const char *insertString(StringRef Str) {
auto Result = StringValueSet.insert(Str);
return Result.first->first().data();
}
};
struct InstrProfValueData {
uint64_t Value;
uint64_t Count;
};
struct InstrProfValueSiteRecord {
std::list<InstrProfValueData> ValueData;
InstrProfValueSiteRecord() { ValueData.clear(); }
template <class InputIterator>
InstrProfValueSiteRecord(InputIterator F, InputIterator L)
: ValueData(F, L) {}
void sortByTargetValues() {
ValueData.sort(
[](const InstrProfValueData &left, const InstrProfValueData &right) {
return left.Value < right.Value;
});
}
void mergeValueData(InstrProfValueSiteRecord &Input) {
this->sortByTargetValues();
Input.sortByTargetValues();
auto I = ValueData.begin();
auto IE = ValueData.end();
for (auto J = Input.ValueData.begin(), JE = Input.ValueData.end(); J != JE;
++J) {
while (I != IE && I->Value < J->Value)
++I;
if (I != IE && I->Value == J->Value) {
I->Count += J->Count;
++I;
continue;
}
ValueData.insert(I, *J);
}
}
};
struct InstrProfRecord {
InstrProfRecord() {}
InstrProfRecord(StringRef Name, uint64_t Hash, std::vector<uint64_t> Counts)
: Name(Name), Hash(Hash), Counts(std::move(Counts)) {}
StringRef Name;
uint64_t Hash;
std::vector<uint64_t> Counts;
typedef std::vector<std::pair<uint64_t, const char *>> ValueMapType;
inline uint32_t getNumValueKinds() const;
inline uint32_t getNumValueSites(uint32_t ValueKind) const;
inline uint32_t getNumValueData(uint32_t ValueKind) const;
inline uint32_t getNumValueDataForSite(uint32_t ValueKind,
uint32_t Site) const;
inline std::unique_ptr<InstrProfValueData[]>
getValueForSite(uint32_t ValueKind, uint32_t Site) const;
inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites);
inline void addValueData(uint32_t ValueKind, uint32_t Site,
InstrProfValueData *VData, uint32_t N,
ValueMapType *HashKeys);
inline instrprof_error mergeValueProfData(uint32_t ValueKind,
InstrProfRecord &Src);
inline void updateStrings(InstrProfStringTable *StrTab);
private:
std::vector<InstrProfValueSiteRecord> IndirectCallSites;
const std::vector<InstrProfValueSiteRecord> &
getValueSitesForKind(uint32_t ValueKind) const {
switch (ValueKind) {
case IPVK_IndirectCallTarget:
return IndirectCallSites;
default:
llvm_unreachable("Unknown value kind!");
}
return IndirectCallSites;
}
std::vector<InstrProfValueSiteRecord> &
getValueSitesForKind(uint32_t ValueKind) {
return const_cast<std::vector<InstrProfValueSiteRecord> &>(
const_cast<const InstrProfRecord *>(this)
->getValueSitesForKind(ValueKind));
}
uint64_t remapValue(uint64_t Value, uint32_t ValueKind,
ValueMapType *HashKeys) {
if (!HashKeys)
return Value;
switch (ValueKind) {
case IPVK_IndirectCallTarget: {
auto Result =
std::lower_bound(HashKeys->begin(), HashKeys->end(), Value,
[](const std::pair<uint64_t, const char *> &LHS,
uint64_t RHS) { return LHS.first < RHS; });
assert(Result != HashKeys->end() &&
"Hash does not match any known keys\n");
Value = (uint64_t)Result->second;
break;
}
}
return Value;
}
};
uint32_t InstrProfRecord::getNumValueKinds() const {
uint32_t NumValueKinds = 0;
for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
NumValueKinds += !(getValueSitesForKind(Kind).empty());
return NumValueKinds;
}
uint32_t InstrProfRecord::getNumValueData(uint32_t ValueKind) const {
uint32_t N = 0;
const std::vector<InstrProfValueSiteRecord> &SiteRecords =
getValueSitesForKind(ValueKind);
for (auto &SR : SiteRecords) {
N += SR.ValueData.size();
}
return N;
}
uint32_t InstrProfRecord::getNumValueSites(uint32_t ValueKind) const {
return getValueSitesForKind(ValueKind).size();
}
uint32_t InstrProfRecord::getNumValueDataForSite(uint32_t ValueKind,
uint32_t Site) const {
return getValueSitesForKind(ValueKind)[Site].ValueData.size();
}
std::unique_ptr<InstrProfValueData[]>
InstrProfRecord::getValueForSite(uint32_t ValueKind, uint32_t Site) const {
uint32_t N = getNumValueDataForSite(ValueKind, Site);
if (N == 0)
return std::unique_ptr<InstrProfValueData[]>(nullptr);
std::unique_ptr<InstrProfValueData[]> VD(new InstrProfValueData[N]);
uint32_t I = 0;
for (auto V : getValueSitesForKind(ValueKind)[Site].ValueData) {
VD[I] = V;
I++;
}
assert(I == N);
return VD;
}
void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site,
InstrProfValueData *VData, uint32_t N,
ValueMapType *HashKeys) {
for (uint32_t I = 0; I < N; I++) {
VData[I].Value = remapValue(VData[I].Value, ValueKind, HashKeys);
}
std::vector<InstrProfValueSiteRecord> &ValueSites =
getValueSitesForKind(ValueKind);
if (N == 0)
ValueSites.push_back(InstrProfValueSiteRecord());
else
ValueSites.emplace_back(VData, VData + N);
}
void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) {
std::vector<InstrProfValueSiteRecord> &ValueSites =
getValueSitesForKind(ValueKind);
ValueSites.reserve(NumValueSites);
}
instrprof_error InstrProfRecord::mergeValueProfData(uint32_t ValueKind,
InstrProfRecord &Src) {
uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
if (ThisNumValueSites != OtherNumValueSites)
return instrprof_error::value_site_count_mismatch;
std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
getValueSitesForKind(ValueKind);
std::vector<InstrProfValueSiteRecord> &OtherSiteRecords =
Src.getValueSitesForKind(ValueKind);
for (uint32_t I = 0; I < ThisNumValueSites; I++)
ThisSiteRecords[I].mergeValueData(OtherSiteRecords[I]);
return instrprof_error::success;
}
void InstrProfRecord::updateStrings(InstrProfStringTable *StrTab) {
if (!StrTab)
return;
Name = StrTab->insertString(Name);
for (auto &VSite : IndirectCallSites)
for (auto &VData : VSite.ValueData)
VData.Value = (uint64_t)StrTab->insertString((const char *)VData.Value);
}
namespace IndexedInstrProf {
enum class HashT : uint32_t {
MD5,
Last = MD5
};
static inline uint64_t MD5Hash(StringRef Str) {
MD5 Hash;
Hash.update(Str);
llvm::MD5::MD5Result Result;
Hash.final(Result);
using namespace llvm::support;
return endian::read<uint64_t, little, unaligned>(Result);
}
static inline uint64_t ComputeHash(HashT Type, StringRef K) {
switch (Type) {
case HashT::MD5:
return IndexedInstrProf::MD5Hash(K);
}
llvm_unreachable("Unhandled hash type");
}
const uint64_t Magic = 0x8169666f72706cff; const uint64_t Version = 3;
const HashT HashType = HashT::MD5;
struct Header {
uint64_t Magic;
uint64_t Version;
uint64_t MaxFunctionCount;
uint64_t HashType;
uint64_t HashOffset;
};
inline support::endianness getHostEndianness() {
return sys::IsLittleEndianHost ? support::little : support::big;
}
struct ValueProfRecord {
uint32_t Kind;
uint32_t NumValueSites;
uint8_t SiteCountArray[1];
static uint32_t getHeaderSize(uint32_t NumValueSites);
static uint32_t getSize(uint32_t NumValueSites, uint32_t NumValueData);
uint32_t getSize() const { return getSize(NumValueSites, getNumValueData()); }
ValueProfRecord *getNext();
InstrProfValueData *getValueData();
uint32_t getNumValueSites() const { return NumValueSites; }
uint32_t getNumValueData() const;
void deserializeTo(InstrProfRecord &Record,
InstrProfRecord::ValueMapType *VMap);
void serializeFrom(const InstrProfRecord &Record, uint32_t ValueKind,
uint32_t NumValueSites);
void swapBytes(support::endianness Old, support::endianness New);
};
struct ValueProfData {
uint32_t TotalSize;
uint32_t NumValueKinds;
static uint32_t getSize(const InstrProfRecord &Record);
static std::unique_ptr<ValueProfData>
serializeFrom(const InstrProfRecord &Record);
static ErrorOr<std::unique_ptr<ValueProfData>>
getValueProfData(const unsigned char *D, const unsigned char *const BufferEnd,
support::endianness SrcDataEndianness);
void swapBytesToHost(support::endianness Endianness);
void swapBytesFromHost(support::endianness Endianness);
uint32_t getSize() const { return TotalSize; }
void deserializeTo(InstrProfRecord &Record,
InstrProfRecord::ValueMapType *VMap);
ValueProfRecord *getFirstValueProfRecord();
};
}
namespace RawInstrProf {
const uint64_t Version = 1;
template <class IntPtrT>
inline uint64_t getMagic();
template <>
inline uint64_t getMagic<uint64_t>() {
return uint64_t(255) << 56 | uint64_t('l') << 48 | uint64_t('p') << 40 |
uint64_t('r') << 32 | uint64_t('o') << 24 | uint64_t('f') << 16 |
uint64_t('r') << 8 | uint64_t(129);
}
template <>
inline uint64_t getMagic<uint32_t>() {
return uint64_t(255) << 56 | uint64_t('l') << 48 | uint64_t('p') << 40 |
uint64_t('r') << 32 | uint64_t('o') << 24 | uint64_t('f') << 16 |
uint64_t('R') << 8 | uint64_t(129);
}
template <class IntPtrT> struct ProfileData {
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
#include "llvm/ProfileData/InstrProfData.inc"
};
struct Header {
const uint64_t Magic;
const uint64_t Version;
const uint64_t DataSize;
const uint64_t CountersSize;
const uint64_t NamesSize;
const uint64_t CountersDelta;
const uint64_t NamesDelta;
};
}
namespace coverage {
LLVM_PACKED_START
template <class IntPtrT> struct CovMapFunctionRecord {
#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init) Type Name;
#include "llvm/ProfileData/InstrProfData.inc"
};
LLVM_PACKED_END
}
}
namespace std {
template <>
struct is_error_code_enum<llvm::instrprof_error> : std::true_type {};
}
#endif // LLVM_PROFILEDATA_INSTRPROF_H_