#include <sys/cdefs.h>
#include <osmemory>
#include <osutility>
#include <iolocks>
#include <IOKit/IOMemoryDescriptor.h>
#include <IOKit/IOLib.h>
#include "GTrace.hpp"
#include "GMetricTypes.h"
#if KERNEL
#include <IOKit/assert.h>
#include <mach/vm_param.h>
#include <mach/mach_types.h>
__BEGIN_DECLS
#include <machine/cpu_number.h>
__END_DECLS
#include <IOKit/graphics/IOGraphicsPrivate.h> // Debug logging macros
#else // !KERNEL
#include <mutex>
#include <stdlib.h>
#include <unistd.h>
#include <pthread.h>
#include <mach/mach_time.h>
#include <IOKit/IOReturn.h>
#include <pthread.h>
#define cpu_number() (-1)
#define D(categ, name, args...) do{}while(0) // Not needed in userland
#endif // KERNEL
#define DGT(args...) D(TRACE, "GTraceBuffer", args)
OSDefineMetaClassAndFinalStructors(GTraceBuffer, OSObject);
using iog::LockGuard;
using iog::OSUniqueObject;
extern GTraceBuffer::shared_type gGTraceArray[kGTraceMaximumBufferCount];
GTraceBuffer::shared_type gGTraceArray[kGTraceMaximumBufferCount];
namespace {
enum ThreadWakeups : bool {
kOneThreadWakeup = true,
kManyThreadWakeup = false,
};
struct Globals {
IOLock* fLock;
uint32_t fCurBufferID;
Globals() : fLock(IOLockAlloc())
{ DGT(" lckOK%d\n", static_cast<bool>(fLock)); }
~Globals() { IOLockFree(fLock); }
} sGlobals;
#define sLock (sGlobals.fLock)
#define sCurBufferID (sGlobals.fCurBufferID)
uint32_t nextPowerOf2(uint32_t x)
{
const uint64_t max32bit = UINT64_C(1) << 32;
assert(x <= (max32bit-1)/2);
const auto shift = __builtin_clz(x-1);
return (x <= 2) ? x : static_cast<uint32_t>(max32bit >> shift);
}
constexpr bool isPowerOf2(const uint64_t x) { return 0 == (x & (x-1)); }
template <typename T>
T bound(const T min, const T x, const T max)
{
assert(min <= max);
return (min > x) ? min : ((max < x) ? max : x);
}
uint64_t threadID()
{
uint64_t tid;
#if KERNEL
tid = thread_tid(current_thread());
#else
pthread_threadid_np(NULL, &tid);
#endif
return tid;
}
uint8_t cpu()
{
return static_cast<uint8_t>(cpu_number());
}
};
static_assert(isPowerOf2(kGTraceEntrySize), "Entry size not a power of two");
static_assert(isPowerOf2(kGTraceMinimumLineCount), "Min not a power of two");
static_assert(isPowerOf2(kGTraceMaximumLineCount), "Max not a power of two");
bool GTraceBuffer::init()
{
return super::init() && static_cast<bool>(sLock);
}
void GTraceBuffer::free()
{
DGT("[%d]\n", bufferID());
if (fBuffer) {
IODelete(fBuffer, GTraceEntry, fLineCount);
fBuffer = nullptr;
fLineCount = 0;
}
super::free();
}
GTraceBuffer::shared_type GTraceBuffer::make(
const char* decoderName, const char* bufferName,
const uint32_t lineCount, breadcrumb_func bcf, void* context)
{
shared_type ret(new GTraceBuffer);
if (static_cast<bool>(ret)) {
IOReturn err
= ret->init(decoderName, bufferName, lineCount, bcf, context);
if (!err) {
LockGuard<IOLock> locked(sLock);
for (int i = 0; i < kGTraceMaximumBufferCount; ++i) {
if ( !static_cast<bool>(gGTraceArray[i]) ) {
gGTraceArray[i] = ret; ret->fHeader.fBufferIndex = i;
DGT("[%d] (d=%s,b=%s,lc=%u, bcf=%s) {ind=%d}\n",
ret->bufferID(), decoderName, bufferName,
lineCount, static_cast<bool>(bcf) ? "yes" : "no", i);
return ret;
}
}
err = kIOReturnNoResources;
}
DGT("[%d] (d=%s,b=%s,lc=%u, %s bcf) -> %x\n",
ret->bufferID(), decoderName, bufferName, lineCount,
static_cast<bool>(bcf) ? "has" : "no", err);
IOLog("Failed %x to create "
"GTraceBuffer %s for %s decoder and count %d\n",
err, bufferName, decoderName, lineCount);
ret.reset(); }
return ret;
}
IOReturn GTraceBuffer::init(
const char* decoderName, const char* bufferName, const uint32_t count,
breadcrumb_func bcf, const void* context)
{
if (!init()) {
IOLog("GTraceBuffer init() failed\n");
return kIOReturnNoMemory;
}
if ( !static_cast<bool>(decoderName) || !*decoderName) {
IOLog("GTraceBuffer bad decoder name\n");
return kIOReturnBadArgument;
}
memset(&fHeader, '\0', sizeof(fHeader));
fHeader.fBufferIndex = static_cast<uint16_t>(-1);
const auto roundedCount = nextPowerOf2(count);
const auto lines = bound(
kGTraceMinimumLineCount, roundedCount, kGTraceMaximumLineCount);
fBuffer = IONew(GTraceEntry, lines);
if (fBuffer)
memset(fBuffer, '\0', lines * sizeof(GTraceEntry));
else
return kIOReturnNoMemory;
fLineCount = lines;
fLineMask = lines - 1; fNextLine = 0;
fBreadcrumbFunc = bcf;
fBCFContext = context;
fBCData = nullptr;
fBCActiveCount = 0;
fHeader.fVersion = GTRACE_REVISION;
fHeader.fCreationTime = mach_continuous_time();
if (bufferName)
GPACKSTRING(fHeader.fBufferName, bufferName);
GPACKSTRING(fHeader.fDecoderName, decoderName);
fHeader.fTokensMask = static_cast<uint16_t>(fLineMask);
{
LockGuard<IOLock> locked(sLock);
fHeader.fBufferID = sCurBufferID++;
}
DGT("[%d] {lc=%x lm=%x}\n", bufferID(), fLineCount, fLineMask);
return kIOReturnSuccess;
}
#define namecpy(dst, buf, len) \
snprintf(dst, len, "%.*s", \
static_cast<int>(sizeof(buf)), reinterpret_cast<const char*>(buf))
size_t GTraceBuffer::decoderName(char* outName, const int outLen) const
{
return namecpy(outName, fHeader.fDecoderName, outLen);
}
size_t GTraceBuffer::bufferName(char* outName, const int outLen) const
{
return namecpy(outName, fHeader.fBufferName, outLen);
}
#undef namecpy
GTraceEntry
GTraceBuffer::formatToken(const uint16_t line,
const uint64_t tag1, const uint64_t arg1,
const uint64_t tag2, const uint64_t arg2,
const uint64_t tag3, const uint64_t arg3,
const uint64_t tag4, const uint64_t arg4,
const uint64_t timestamp)
{
return GTraceEntry(timestamp, line, bufferID(),
cpu(), threadID(), 0,
MAKEGTRACETAG(tag1), MAKEGTRACEARG(arg1),
MAKEGTRACETAG(tag2), MAKEGTRACEARG(arg2),
MAKEGTRACETAG(tag3), MAKEGTRACEARG(arg3),
MAKEGTRACETAG(tag4), MAKEGTRACEARG(arg4));
}
void GTraceBuffer::recordToken(const GTraceEntry& entry)
{
fBuffer[getNextLine()] = entry;
}
namespace {
IOReturn fetchValidateAndMap(IOMemoryDescriptor* outDesc,
OSUniqueObject<IOMemoryMap>* mapP)
{
IOReturn err = kIOReturnSuccess;
const auto len = outDesc->getLength();
if (len < sizeof(IOGTraceBuffer))
err = kIOReturnBadArgument;
else if (len > INT_MAX)
err = kIOReturnBadArgument;
else {
OSUniqueObject<IOMemoryMap> map(outDesc->map());
if (static_cast<bool>(map))
*mapP = iog::move(map);
else
err = kIOReturnVMError;
}
return kIOReturnSuccess;
}
}
IOReturn GTraceBuffer::copyOut(
OSUniqueObject<IOMemoryMap> map, OSData* bcData) const
{
iog::locking_primitives::assertUnlocked(sLock);
GTraceHeader header = fHeader;
const auto outLen = static_cast<int>(map->getLength());
void* outP = reinterpret_cast<void*>(map->getVirtualAddress());
auto* outBufP = static_cast<IOGTraceBuffer*>(outP);
uint16_t bcTokens = 0;
int remaining = outLen - sizeof(IOGTraceBuffer);
assert(remaining >= 0);
const uint32_t bcDataLen
= static_cast<bool>(bcData) ? bcData->getLength() : 0;
const bool hasBCFunc = static_cast<bool>(fBreadcrumbFunc);
if (bcDataLen || hasBCFunc) {
const uint16_t kBCLen = static_cast<uint16_t>(
(remaining < kGTraceMaxBreadcrumbSize)
? remaining : kGTraceMaxBreadcrumbSize);
uint16_t bcCopied = 0;
if (bcDataLen) {
bcCopied = (bcDataLen < kBCLen) ? bcDataLen : kBCLen;
memcpy(&outBufP->fTokens[0], bcData->getBytesNoCopy(), bcCopied);
}
else {
assert(hasBCFunc);
void* buf = IOMalloc(kGTraceMaxBreadcrumbSize);
if (buf) {
void* context = const_cast<void*>(fBCFContext);
bcCopied = kBCLen;
IOReturn err = (*fBreadcrumbFunc)(context, buf, &bcCopied);
if (!err) {
if (bcCopied > kBCLen) {
err = kIOReturnInternalError;
} else if (bcCopied)
memcpy(&outBufP->fTokens[0], buf, bcCopied);
}
if (err) {
bcCopied = 0;
IOLog("GTrace[%u] bad breadcrumb %x, dropping\n",
bufferID(), err);
}
IOFree(buf, kGTraceMaxBreadcrumbSize);
}
}
bcTokens = (bcCopied + (kGTraceEntrySize - 1)) / kGTraceEntrySize;
header.fBreadcrumbTokens = bcTokens;
remaining -= bcTokens * kGTraceEntrySize; }
auto* const outTokensP = &outBufP->fTokens[bcTokens];
header.fTokenLine = nextLine();
if (remaining >= kGTraceEntrySize) {
const int outNumEntries = remaining / kGTraceEntrySize;
const int availLines = fLineCount;
const int copyEntries
= (availLines < outNumEntries) ? availLines : outNumEntries;
int curToken = 0;
for (int i = 0; i < copyEntries; i++) {
GTraceEntry entry = fBuffer[i];
if (!static_cast<bool>(entry.timestamp())) continue;
const auto& tags = entry.fArgsTag;
for (int j = 0; tags.tag() && j < GTraceEntry::ArgsTag::kNum; ++j) {
if (kGTRACE_ARGUMENT_POINTER & tags.tag(j)) {
vm_offset_t outArg = 0;
vm_kernel_addrperm_external(
static_cast<vm_offset_t>(entry.arg64(j)), &outArg);
entry.arg64(j) = outArg;
}
}
outTokensP[curToken++] = entry;
}
header.fTokensCopied = curToken;
}
header.fTokensCopied += bcTokens;
header.fBufferSize = kGTraceHeaderSize
+ header.fTokensCopied * sizeof(GTraceEntry);
outBufP->fHeader = header;
DGT("[%d] sz%d bc%d tkn%d\n", bufferID(),
header.fBufferSize, bcTokens, header.fTokensCopied - bcTokens);
return kIOReturnSuccess;
}
IOReturn
GTraceBuffer::fetch(const uint32_t index, IOMemoryDescriptor* outDesc)
{
if (index >= kGTraceMaximumBufferCount)
return kIOReturnNotFound;
shared_type traceBuffer;
bool releaseBuffer = false;
{
LockGuard<IOLock> locked(sLock);
auto& so = gGTraceArray[index]; if (!static_cast<bool>(so))
return kIOReturnNotFound;
DGT("[%d] cached buffer %u, uc=%ld\n",
so->bufferID(), index, so.use_count());
releaseBuffer = (1 == so.use_count()); traceBuffer = so; }
__unused const auto bufferID = traceBuffer->bufferID();
OSUniqueObject<IOMemoryMap> map;
IOReturn err = fetchValidateAndMap(outDesc, &map);
if (err) return err;
OSData* bcData = nullptr;
bool isBCActive = false;
{
LockGuard<IOLock> locked(sLock);
if (static_cast<bool>(traceBuffer->fBCData))
iog::swap(bcData, traceBuffer->fBCData);
auto& activeCount = traceBuffer->fBCActiveCount;
while ((isBCActive = static_cast<bool>(traceBuffer->fBreadcrumbFunc))) {
if (activeCount < 8) { ++activeCount;
break;
}
IOLockSleep(sLock, &activeCount, THREAD_UNINT);
}
}
err = traceBuffer->copyOut(iog::move(map), bcData);
OSSafeReleaseNULL(bcData);
{
LockGuard<IOLock> locked(sLock);
if (!err && releaseBuffer) {
traceBuffer = iog::move(gGTraceArray[index]);
assert(1 == traceBuffer.use_count());
}
if (isBCActive) {
auto& activeCount = traceBuffer->fBCActiveCount;
--activeCount;
IOLockWakeup(sLock, &activeCount, kManyThreadWakeup);
}
}
DGT("[%d] {tuc=%ld} -> %x\n", bufferID, traceBuffer.use_count(), err);
return err;
}
void GTraceBuffer::destroy(shared_type&& inBso)
{
DGT("[%d]\n", inBso->bufferID());
shared_type bso(iog::move(inBso));
breadcrumb_func breadcrumbFunc = nullptr;
const void* ccontext = nullptr; {
LockGuard<IOLock> locked(sLock);
while (bso->fBCActiveCount)
IOLockSleep(sLock, &bso->fBCActiveCount, THREAD_UNINT);
iog::swap(breadcrumbFunc, bso->fBreadcrumbFunc);
iog::swap(ccontext, bso->fBCFContext);
}
if (!static_cast<bool>(breadcrumbFunc))
return;
assert(!bso->fBCData);
void* buf = IOMalloc(kGTraceMaxBreadcrumbSize);
if (buf) {
void* context = const_cast<void*>(ccontext);
uint16_t bcCopied = kGTraceMaxBreadcrumbSize;
const IOReturn err = (*breadcrumbFunc)(context, buf, &bcCopied);
if (!err && bcCopied)
bso->fBCData = OSData::withBytes(buf, bcCopied);
IOFree(buf, kGTraceMaxBreadcrumbSize);
}
}
IOReturn
GTraceBuffer::fetch(GTraceBuffer::shared_type bso, IOMemoryDescriptor* outDesc)
{
OSUniqueObject<IOMemoryMap> map;
IOReturn err = fetchValidateAndMap(outDesc, &map);
if (!err)
err = bso->copyOut(iog::move(map), nullptr);
DGT("[%d] {buc=%ld} -> %x\n", bso->bufferID(), bso.use_count(), err);
return err;
}
GTraceBuffer* GTraceBuffer::makeArchaicCpp(
const char* decoderName, const char* bufferName,
const uint32_t lineCount, breadcrumb_func bcf, void* context)
{
shared_type bso = make(decoderName, bufferName, lineCount, bcf, context);
if (!static_cast<bool>(bso)) return nullptr;
bso->fArchaicCPPSharedObjectHack = bso;
return bso.get();
}
void
GTraceBuffer::destroyArchaicCpp(GTraceBuffer *buffer)
{
destroy(iog::move(buffer->fArchaicCPPSharedObjectHack)); }