// // GTrace.hpp // // Created by bparke on 3/17/17. // #ifndef GTrace_hpp #define GTrace_hpp #include <stdatomic.h> #include <libkern/c++/OSObject.h> #include <IOKit/IOLib.h> #include <IOKit/IOReturn.h> // Kernel clients that haven't enabled C++11 yet #define GTRACE_ARCHAIC_CPP (__cplusplus < 201103L) #if !GTRACE_ARCHAIC_CPP #include <osmemory> #endif #include "GTraceTypes.hpp" // TODO: Provide example of GTRACE() macro use. #define kGTraceMaxBreadcrumbSize (16 * 1024) // 16KiB #define kGTraceMinimumLineCount UINT32_C(1024) // @64b == 64k #define kGTraceMaximumLineCount UINT32_C(8192) // @64b == 512k #define kGTraceDefaultLineCount kGTraceMinimumLineCount // ---------------------------------------------------------------------------- // Main tracing Macros // ---------------------------------------------------------------------------- // Use this to encode t0, for automatic decoding and start/end pairing #define GTFuncTag(funcid, functype, tag0) \ MAKEGTRACETAG((((functype) & 0x3) << 10) | ((funcid) & 0x03FF) | (tag0)) #define GTRACERAW(tracer, t0, a0, t1, a1, t2, a2, t3, a3) do{ \ if (static_cast<bool>(tracer)){ \ (tracer)->recordToken(__LINE__, \ MAKEGTRACETAG(t0), MAKEGTRACEARG(a0), \ MAKEGTRACETAG(t1), MAKEGTRACEARG(a1), \ MAKEGTRACETAG(t2), MAKEGTRACEARG(a2), \ MAKEGTRACETAG(t3), MAKEGTRACEARG(a3)); \ } \ }while(0) // Convenience macro, used when all args are ordinary hex data fields. #define GTRACERAWNT(tracer, a0, a1, a2, a3) \ GTRACERAW(tracer, 0, a0, 0, a1, 0, a2, 0, a3) #define GTRACE(tracer, fid, ft, t0, a0, t1, a1, t2, a2, t3, a3) \ GTRACERAW(tracer, GTFuncTag(fid, ft, t0), a0, t1, a1, t2, a2, t3, a3) // Convenience macro, used when all as are ordinary hex data fields. #define GTRACE_NT(tracer, fid, ft, a0, a1, a2, a3) \ GTRACERAW(tracer, GTFuncTag(fid, ft, 0), a0, 0, a1, 0, a2, 0, a3) // GTrace for calls that are slow. The GTRACE_IFSLOW_START records the current // time, then GTRACE_IFSLOW_END will record an entry if the duration between // start and end is > the threshold. Note `delayat` is in absolute time units. #define GTRACE_IFSLOW_START(tracer, fid) do { \ const uint64_t _gtrace_ ## fid ## _start_ \ = ((static_cast<bool>(tracer)) ? mach_continuous_time() : 0) // Matches the do{ from GTRACE_IFSLOW_START, must be in same scope #define GTRACE_IFSLOW_END(tracer, fid, ft, t0, a0, t1, a1, t2, a2, delayat) \ if(static_cast<bool>(tracer)) { \ const uint64_t _gtrace_ifslow_now_ = mach_continuous_time(); \ const uint64_t _gtrace_delta_ \ = _gtrace_ifslow_now_ - _gtrace_ ## fid ## _start_; \ if(_gtrace_delta_ > (delayat)) \ (tracer)->recordToken(__LINE__, GTFuncTag(fid, ft, 0), \ _gtrace_delta_, t0, a0, t1, a1, t2, a2, \ _gtrace_ifslow_now_); \ } \ }while(0) // Create a pair of GTrace records but only record them if the time duration // was > than the absolute time threshold given in GTRACE_DEFER_END #define GTRACE_DEFER_START(tracer, t0, a0, t1, a1, t2, a2, t3, a3) \ do{ \ const GTraceEntry _gtrace_start_ = (static_cast<bool>(tracer)) \ ? (tracer)->formatToken(__LINE__, t0, a0, t1, a1, t2, a2, t3, a3) \ : GTraceEntry() // Matches the do{ from GTRACE_DEFER_START, must be in same block scope #define GTRACE_DEFER_END(tracer, t0, a0, t1, a1, t2, a2, t3, a3, delayat) \ if (static_cast<bool>(tracer)) { \ const uint64_t _gtrace_defer_now_ = mach_continuous_time(); \ const uint64_t _gtrace_delta_ \ = _gtrace_defer_now_ - _gtrace_start_.timestamp(); \ if(_gtrace_delta_ > (delayat)) { \ (tracer)->recordToken(_gtrace_start_); \ (tracer)->recordToken(__LINE__, t0, a0, t1, a1, \ t2, a2, t3, a3, _gtrace_defer_now_); \ } \ } \ }while(0) #if GTRACE_IMPL // Macro used by IOGraphicsFamily internally, not required for third party use. #define GTRACE_LOG_SYNCH(tracer, t0) do { \ if (static_cast<bool>(tracer)) { \ const uint32_t __gtrace_log_sync_ind__ = tracer->synchIndex(); \ tracer->recordToken(__LINE__, \ MAKEGTRACETAG(t0),static_cast<uint64_t>(__gtrace_log_sync_ind__), \ 0, 0, 0, 0, 0, 0); \ IOLog("GTrace synchronization point %x\n", __gtrace_log_sync_ind__); \ } \ }while(0) #endif class IOMemoryDescriptor; class IOMemoryMap; class GTraceBuffer final : public OSObject { OSDeclareFinalStructors(GTraceBuffer); using super = OSObject; public: // sizeP is an in/out variable, size will be <= 16KiB // The Breadcrumb function callout is a clean context that does not hold a // lock. using breadcrumb_func = IOReturn (*)(void* context, void* buffer, uint16_t* sizeP); #if !GTRACE_ARCHAIC_CPP using shared_type = iog::OSSharedObject<GTraceBuffer>; /*! @function make @abstract Make a shared object of a new GTrace buffer. @discussion Creates an optimised lockfree algorithm for storing 64byte records into a ring buffer. This buffer can be dumped by `iogdiagnose` and then decoded using /AppleInternal/AppleGraphicsControl/IOGDiagnoseDecode. In addition to the ring buffer the client can also provide a function that will copy up to 16KiB of additional data used by the client's decode module. It is much better if this func is LOCKFREE. If the function does not return within 1 second we may abort the thread reading the data and discard the buffer. Thus it is a data-looser to stall the fetch thread for too long. The breadcrumb_func is not called with GTraceBuffer locks held. @param bufferName: Max 32 byte C string copied into fetch header. @param decoderName: Max 32 byte C string copied into fetch header, used by IOGDiagnoseDecode to find a decode module in /AppleInternal/AppleGraphicsControl/IOGDiagnose_modules directory. @param lineCount: Number of lines to store in ring buffer. Line count will be rounded up to a power of two and is bounded by kGTraceMinimumLineCount and kGTraceMaximumLineCount. @param bcf: breadcrumb_func. May be null. Function to call when the buffer is being fetched, to be used by your decode module. See discussion. @param context: Context to pass to the bcf function. @result OSSharedObject<GTraceBuffer> with the new created buffer. An empty buffer on failure. */ static shared_type make( const char* decoderName, const char* bufferName, const uint32_t lineCount, breadcrumb_func bcf, void* context); /*! @function destroy @abstract Destroy a buffer shared object created with make. @discussion When a client is done with the gtrace buffer use destroy(iog::move(<your buffer>)) to destroy your shared object and stop further calls to the breadcrumb func. This will destroy your reference to the buffer. Before completing destruction the breadcrumb function if any will be called only once and then never called again after on return. The breadcrumb function is never called with any internal buffer locks held. The GTraceBuffer guarantees that no further calls to the breadcrumb function will occur once this call completes. Note it is not necessary to destroy every copy of the shared_type (OSSharedObject<GTraceBuffer>), rather only call destroy when you no longer wish any further breadcrumb callouts during teardown of your master objects, essentially it is symetric with make() calls. @param bso An rvalue reference to the buffer shared object. */ static void destroy(shared_type&& bso); #endif // !GTRACE_ARCHAIC_CPP #if GTRACE_ARCHAIC_CPP || GTRACE_IMPL /*! @function makeArchaicCpp @abstract Equivalent to make for archaic pre-C++11 projects. See make for more details. */ static GTraceBuffer* makeArchaicCpp( const char* decoderName, const char* bufferName, const uint32_t lineCount, breadcrumb_func bcf, void* context) __attribute__((deprecated ("Use C++11 and GTraceBuffer::make."))); /*! @function destroyArchaicCpp @abstract Equivalent to destroy for archaic pre-C++11 projects. See destroy for more details. @discussion Similar to destroy, destroyArchaicCpp releases one reference on the buffer. Clients must use destroyArchaicCpp once (mirroring the call to makeArchaicCpp). */ static void destroyArchaicCpp(GTraceBuffer *buffer) __attribute__((deprecated ("Use C++11 and GTraceBuffer::destroy."))); #endif /*! @function formatToken @abstract Format a token suitable for recording. @discussion GTrace supports tokenized tracing. formatToken is used to create a tokenized KTrace style entry suitable for inserting into the ring buffer. Carefully written for C++ return value optimisation. @param line The line number associated with the token @param tag1 An implementation specific tag associated with arg1 @param tag2 An implementation specific tag associated with arg2 @param tag3 An implementation specific tag associated with arg3 @param tag4 An implementation specific tag associated with arg4 @param arg1 Component/implementation specific value. @param arg2 Component/implementation specific value. @param arg3 Component/implementation specific value. @param arg4 Component/implementation specific value. @param timestamp Supplied timestamp or current MCT @result GTraceEntry structure filled in with given data. */ GTraceEntry formatToken(const uint16_t line, const uint64_t tag1, const uint64_t arg1, const uint64_t tag2, const uint64_t arg2, const uint64_t tag3, const uint64_t arg3, const uint64_t tag4, const uint64_t arg4, const uint64_t timestamp = mach_continuous_time()); /*! @function recordToken @abstract Add token data to token stream. @discussion recordToken() takes a formatted GTraceEntry and writes it to the ring buffer. @param entry a const GTraceEntry reference to be atomically added to token stream. */ void recordToken(const GTraceEntry& entry); /*! @function recordToken @abstract Add the token data to the token stream @discussion GTrace supports tokenized tracing. recordToken is used to insert a tokenized KTrace style trace into the trace buffer. @param line The line number associated with the token @param tag1 An implementation specific tag associated with arg1 @param tag2 An implementation specific tag associated with arg2 @param tag3 An implementation specific tag associated with arg3 @param tag4 An implementation specific tag associated with arg4 @param arg1 Component/implementation specific value. @param arg2 Component/implementation specific value. @param arg3 Component/implementation specific value. @param arg4 Component/implementation specific value. @param timestamp Supplied timestamp or current MCT */ void recordToken(const uint16_t line, const uint64_t tag1, const uint64_t arg1, const uint64_t tag2, const uint64_t arg2, const uint64_t tag3, const uint64_t arg3, const uint64_t tag4, const uint64_t arg4, const uint64_t timestamp = mach_continuous_time()) { recordToken(formatToken( line, tag1, arg1, tag2, arg2, tag3, arg3, tag4, arg4, timestamp)); } #if GTRACE_IMPL /*! @function synchIndex @abstract Publishes the current gtrace token index. @discussion Used by Decode to synchronize between os_log and GTrace. I have chosen to share the atomic next index operation though it may cause decode ambiguity. It is very light weight. */ inline uint32_t synchIndex() const { return nextLine(); } /*! @function fetch @abstract Copies token buffer into provided memory descriptor. @discussion Copies the tokens data from the internal buffers into the provided IOMemoryDescriptor, which must be prepared. Data is copied up to the buffer size or the internal buffer size (which ever is less). Once the data is copied a post processing step that obfuscates pointers is run. @param bso OSSharedObject<GTraceBuffer> (i.e. shared_type) of GTraceBuffer @param outDesc pointer to a prepared memory descriptor. @result kIOReturnSuccess if the copy was successful, else an error. */ static IOReturn fetch(shared_type bso, IOMemoryDescriptor* outDesc); /* Getters for constant header information */ size_t decoderName(char *name, const int len) const; // truncates size_t bufferName(char *name, const int len) const; // truncates uint32_t lineMask() const { return fLineMask; } uint32_t lineCount() const { return fLineCount; } uint32_t bufferID() const { return fHeader.fBufferID; } protected: // OSObject overrides bool init() APPLE_KEXT_OVERRIDE; void free() APPLE_KEXT_OVERRIDE; // Internal functions // See make() for details on arguments IOReturn init( const char* decoderName, const char* bufferName, const uint32_t lineCount, breadcrumb_func bcf, const void* context); inline uint32_t getNextLine(void) { return atomic_fetch_add(&fNextLine, 1) & fLineMask; } // Complicated const casting because atomic_load will not take a const // pointer, problem mixing C and C++, but C++ <atomic> doesn't exist // in the kernel yet and probably never will as it is deep STL. inline uint32_t nextLine() const { return atomic_load(&(const_cast<GTraceBuffer*>(this)->fNextLine)); } IOReturn copyOut( iog::OSUniqueObject<IOMemoryMap> map, OSData* bcData) const; // APIs for IODisplayWranglerUserClients.cpp, also used by unit tests friend class IOGDiagnosticUserClient; /*! @function fetch @abstract Copies the token buffer into the provided memory descriptor. IOGDiagnosticUserClient interface @discussion Copies the tokens data from the internal buffers into the provided IOMemoryDescriptor, which must be prepared. Data is copied up to the buffer size or the internal buffer size (which ever is less). Once the data is copied a post processing step that obfuscates pointers is run. Releases buffer if the globally cached OSSharedObject is unique, that is it has only one outstanding reference AND the complete buffer was sucessfully copied into outDesc. @param index Index of buffer in buffer pool cache @param outDesc pointer to a prepared memory descriptor. @result kIOReturnSuccess if the copy was successful, else an error. */ static IOReturn fetch(const uint32_t index, IOMemoryDescriptor* outDesc); private: // Header that is copied out on demand GTraceHeader fHeader; atomic_uint_fast32_t fNextLine; GTraceEntry* fBuffer; breadcrumb_func fBreadcrumbFunc; const void* fBCFContext; // Context to be passed to fBreadcrumbFunc OSData* fBCData; uint32_t fBCActiveCount; uint32_t fLineMask; uint32_t fLineCount; bool fWrapped; // Workaround for pre-C++11 clients, which can't see OSSharedObject. shared_type fArchaicCPPSharedObjectHack; #endif // GTRACE_IMPL }; #endif /* GTrace_h */