firehose_buffer_internal.h [plain text]
#ifndef __FIREHOSE_BUFFER_INTERNAL__
#define __FIREHOSE_BUFFER_INTERNAL__
#if BYTE_ORDER != LITTLE_ENDIAN
#error unsupported byte order
#endif
#ifndef KERNEL
#include <os/lock_private.h>
#endif
#define FIREHOSE_BUFFER_CHUNK_COUNT 64ul
#ifdef KERNEL
#define FIREHOSE_BUFFER_CHUNK_PREALLOCATED_COUNT 15
#else
#define FIREHOSE_BUFFER_CHUNK_PREALLOCATED_COUNT 4
#define FIREHOSE_BUFFER_MADVISE_CHUNK_COUNT 4
#endif
static const unsigned long firehose_stream_uses_io_bank =
(1UL << firehose_stream_persist) |
(1UL << firehose_stream_special);
typedef union {
#define FIREHOSE_BANK_SHIFT(bank) (16 * (bank))
#define FIREHOSE_BANK_INC(bank) (1ULL << FIREHOSE_BANK_SHIFT(bank))
#define FIREHOSE_BANK_UNAVAIL_BIT ((uint16_t)0x8000)
#define FIREHOSE_BANK_UNAVAIL_MASK(bank) (FIREHOSE_BANK_INC(bank) << 15)
uint64_t fbs_atomic_state;
struct {
uint16_t fbs_mem_bank;
uint16_t fbs_io_bank;
uint16_t fbs_max_ref;
uint16_t fbs_unused;
};
} firehose_bank_state_u;
#if __has_feature(c_static_assert)
_Static_assert(8 * offsetof(firehose_bank_state_u, fbs_mem_bank)
== FIREHOSE_BANK_SHIFT(0), "mem bank shift");
_Static_assert(8 * offsetof(firehose_bank_state_u, fbs_io_bank)
== FIREHOSE_BANK_SHIFT(1), "mem bank shift");
#endif
typedef struct firehose_buffer_bank_s {
firehose_bank_state_u volatile fbb_state;
uint64_t volatile fbb_metadata_bitmap;
uint64_t volatile fbb_mem_flushed;
uint64_t volatile fbb_mem_notifs;
uint64_t volatile fbb_mem_sync_pushes;
uint64_t volatile fbb_io_flushed;
uint64_t volatile fbb_io_notifs;
uint64_t volatile fbb_io_sync_pushes;
#define FIREHOSE_BUFFER_BANK_FLAG_LOW_MEMORY (1UL << 0)
#define FIREHOSE_BUFFER_BANK_FLAG_HIGH_RATE (1UL << 1)
unsigned long volatile fbb_flags;
uint64_t fbb_bitmap; firehose_bank_state_u fbb_limits; #ifdef KERNEL
uint32_t _fbb_unused;
#else
dispatch_unfair_lock_s fbb_lock;
#endif
} OS_ALIGNED(64) *firehose_buffer_bank_t;
typedef union {
uint64_t fss_atomic_state;
dispatch_gate_s fss_gate;
struct {
uint32_t fss_allocator;
#define FIREHOSE_STREAM_STATE_PRISTINE 0xffff
uint16_t fss_current;
uint16_t fss_generation;
};
} firehose_stream_state_u;
typedef struct firehose_buffer_stream_s {
firehose_stream_state_u fbs_state;
} OS_ALIGNED(128) *firehose_buffer_stream_t;
typedef union {
uint64_t frp_atomic_tail;
struct {
uint16_t frp_mem_tail;
uint16_t frp_mem_flushed;
uint16_t frp_io_tail;
uint16_t frp_io_flushed;
};
} firehose_ring_tail_u;
#define FIREHOSE_RING_POS_GEN_INC ((uint16_t)(FIREHOSE_BUFFER_CHUNK_COUNT))
#define FIREHOSE_RING_POS_IDX_MASK ((uint16_t)(FIREHOSE_RING_POS_GEN_INC - 1))
#define FIREHOSE_RING_POS_GEN_MASK ((uint16_t)~FIREHOSE_RING_POS_IDX_MASK)
typedef struct firehose_buffer_header_s {
uint16_t volatile fbh_mem_ring[FIREHOSE_BUFFER_CHUNK_COUNT];
uint16_t volatile fbh_io_ring[FIREHOSE_BUFFER_CHUNK_COUNT];
firehose_ring_tail_u volatile fbh_ring_tail OS_ALIGNED(64);
uint32_t fbh_spi_version;
uint16_t volatile fbh_ring_mem_head OS_ALIGNED(64);
uint16_t volatile fbh_ring_io_head OS_ALIGNED(64);
struct firehose_buffer_bank_s fbh_bank;
struct firehose_buffer_stream_s fbh_stream[_firehose_stream_max];
uint64_t fbh_uniquepid;
pid_t fbh_pid;
mach_port_t fbh_logd_port;
mach_port_t volatile fbh_sendp;
mach_port_t fbh_recvp;
#ifndef KERNEL
dispatch_once_t fbh_notifs_pred OS_ALIGNED(64);
dispatch_source_t fbh_notifs_source;
dispatch_unfair_lock_s fbh_logd_lock;
#endif
uint64_t fbh_unused[0];
} OS_ALIGNED(FIREHOSE_CHUNK_SIZE) *firehose_buffer_header_t;
union firehose_buffer_u {
struct firehose_buffer_header_s fb_header;
struct firehose_chunk_s fb_chunks[FIREHOSE_BUFFER_CHUNK_COUNT];
};
typedef struct firehose_tracepoint_query_s {
uint16_t pubsize;
uint16_t privsize;
firehose_stream_t stream;
bool is_bank_ok;
bool for_io;
uint64_t stamp;
} *firehose_tracepoint_query_t;
#ifndef FIREHOSE_SERVER
firehose_buffer_t
firehose_buffer_create(mach_port_t logd_port, uint64_t unique_pid,
unsigned long bank_flags);
firehose_tracepoint_t
firehose_buffer_tracepoint_reserve_slow(firehose_buffer_t fb,
firehose_tracepoint_query_t ask, uint8_t **privptr);
void
firehose_buffer_update_limits(firehose_buffer_t fb);
void
firehose_buffer_ring_enqueue(firehose_buffer_t fb, uint16_t ref);
void
firehose_buffer_force_connect(firehose_buffer_t fb);
#endif
#endif // __FIREHOSE_BUFFER_INTERNAL__