#if CONFIG_PGTRACE
#include <mach/mach_types.h>
#include <IOKit/IOLib.h>
#include <sys/msgbuf.h>
#include <sys/errno.h>
#include <arm64/pgtrace.h>
#include <libkern/OSDebug.h>
typedef struct {
queue_chain_t chain;
pmap_t pmap;
vm_offset_t start;
vm_offset_t end;
} probe_t;
#if CONFIG_PGTRACE_NONKEXT
#include "pgtrace_decoder.h"
#define RBUF_DEFAULT_SIZE 1024
#define RBUF_IDX(idx, mask) ((idx) & (mask))
#define MSG_MAX 130
typedef uint8_t RWLOCK;
typedef struct {
uint64_t id;
pgtrace_run_result_t res;
void *stack[PGTRACE_STACK_DEPTH];
} log_t;
static struct {
log_t *logs; uint32_t size; uint64_t rdidx, wridx; decl_simple_lock_data(, loglock);
uint64_t id;
uint32_t option;
uint32_t enabled;
uint32_t bytes;
queue_head_t probes;
lck_grp_t *lock_grp;
lck_grp_attr_t *lock_grp_attr;
lck_attr_t *lock_attr;
lck_mtx_t probelock;
} pgtrace = {};
void
pgtrace_init(void)
{
simple_lock_init(&pgtrace.loglock, 0);
pgtrace.lock_attr = lck_attr_alloc_init();
pgtrace.lock_grp_attr = lck_grp_attr_alloc_init();
pgtrace.lock_grp = lck_grp_alloc_init("pgtrace_lock", pgtrace.lock_grp_attr);
lck_mtx_init(&pgtrace.probelock, pgtrace.lock_grp, pgtrace.lock_attr);
queue_init(&pgtrace.probes);
pgtrace.size = RBUF_DEFAULT_SIZE;
pgtrace.logs = kalloc(RBUF_DEFAULT_SIZE * sizeof(log_t));
}
void
pgtrace_clear_probe(void)
{
probe_t *p, *next;
queue_head_t *q = &pgtrace.probes;
lck_mtx_lock(&pgtrace.probelock);
p = (probe_t *)queue_first(q);
while (!queue_end(q, (queue_entry_t)p)) {
next = (probe_t *)queue_next(&(p->chain));
queue_remove(q, p, probe_t *, chain);
kfree(p, sizeof(probe_t));
p = next;
}
lck_mtx_unlock(&pgtrace.probelock);
return;
}
int
pgtrace_add_probe(thread_t thread, vm_offset_t start, vm_offset_t end)
{
probe_t *p;
queue_head_t *q = &pgtrace.probes;
if (start > end) {
kprintf("%s Invalid start=%lx end=%lx\n", __func__, start, end);
return -1;
}
p = kalloc(sizeof(probe_t));
p->start = start;
p->end = end;
if (thread == NULL) {
p->pmap = NULL;
} else {
p->pmap = vm_map_pmap(thread->map);
}
lck_mtx_lock(&pgtrace.probelock);
queue_enter(q, p, probe_t *, chain);
lck_mtx_unlock(&pgtrace.probelock);
return 0;
}
void
pgtrace_start(void)
{
probe_t *p;
queue_head_t *q = &pgtrace.probes;
kprintf("%s\n", __func__);
if (pgtrace.enabled) {
return;
}
pgtrace.enabled = 1;
lck_mtx_lock(&pgtrace.probelock);
queue_iterate(q, p, probe_t *, chain) {
pmap_pgtrace_add_page(p->pmap, p->start, p->end);
}
lck_mtx_unlock(&pgtrace.probelock);
return;
}
void
pgtrace_stop(void)
{
probe_t *p;
queue_head_t *q = &pgtrace.probes;
kprintf("%s\n", __func__);
lck_mtx_lock(&pgtrace.probelock);
queue_iterate(q, p, probe_t *, chain) {
pmap_pgtrace_delete_page(p->pmap, p->start, p->end);
}
lck_mtx_unlock(&pgtrace.probelock);
pgtrace.enabled = 0;
}
uint32_t
pgtrace_get_size(void)
{
return pgtrace.size;
}
bool
pgtrace_set_size(uint32_t size)
{
log_t *old_buf, *new_buf;
uint32_t old_size, new_size = 1;
while (size > new_size) {
new_size <<= 1;
if (new_size > 0x100000) {
kprintf("%s: size=%x new_size=%x is too big\n", __func__, size, new_size);
return false;
}
}
new_buf = kalloc(new_size * sizeof(log_t));
if (new_buf == NULL) {
kprintf("%s: can't allocate new_size=%x\n entries", __func__, new_size);
return false;
}
pgtrace_stop();
simple_lock(&pgtrace.loglock);
old_buf = pgtrace.logs;
old_size = pgtrace.size;
pgtrace.logs = new_buf;
pgtrace.size = new_size;
pgtrace.rdidx = pgtrace.wridx = 0;
simple_unlock(&pgtrace.loglock);
if (old_buf) {
kfree(old_buf, old_size * sizeof(log_t));
}
return true;
}
void
pgtrace_clear_trace(void)
{
simple_lock(&pgtrace.loglock);
pgtrace.rdidx = pgtrace.wridx = 0;
simple_unlock(&pgtrace.loglock);
}
boolean_t
pgtrace_active(void)
{
return pgtrace.enabled > 0;
}
uint32_t
pgtrace_get_option(void)
{
return pgtrace.option;
}
void
pgtrace_set_option(uint32_t option)
{
pgtrace.option = option;
}
void
pgtrace_write_log(pgtrace_run_result_t res)
{
uint8_t i;
log_t log = {};
const char *rwmap[] = { "R", "W", "PREFETCH" };
log.id = pgtrace.id++;
log.res = res;
if (pgtrace.option & PGTRACE_OPTION_KPRINTF) {
char msg[MSG_MAX];
char *p;
p = msg;
snprintf(p, MSG_MAX, "%llu %s ", res.rr_time, rwmap[res.rr_rw]);
p += strlen(p);
for (i = 0; i < res.rr_num; i++) {
snprintf(p, MSG_MAX - (p - msg), "%lx=%llx ", res.rr_addrdata[i].ad_addr, res.rr_addrdata[i].ad_data);
p += strlen(p);
}
kprintf("%s %s\n", __func__, msg);
}
if (pgtrace.option & PGTRACE_OPTION_STACK) {
OSBacktrace(log.stack, PGTRACE_STACK_DEPTH);
}
pgtrace.bytes += sizeof(log);
simple_lock(&pgtrace.loglock);
pgtrace.logs[RBUF_IDX(pgtrace.wridx, pgtrace.size - 1)] = log;
if (RBUF_IDX(pgtrace.wridx, pgtrace.size - 1) == RBUF_IDX(pgtrace.rdidx, pgtrace.size - 1) &&
(pgtrace.wridx != pgtrace.rdidx)) {
pgtrace.rdidx++;
}
pgtrace.wridx++;
if (pgtrace.wridx == (pgtrace.rdidx + 1)) {
thread_wakeup(pgtrace.logs);
}
simple_unlock(&pgtrace.loglock);
return;
}
int64_t
pgtrace_read_log(uint8_t *buf, uint32_t size)
{
int total, front, back;
boolean_t ints;
wait_result_t wr;
if (pgtrace.enabled == FALSE) {
return -EINVAL;
}
total = size / sizeof(log_t);
if (buf && total == 0) {
return -EINVAL;
}
ints = ml_set_interrupts_enabled(FALSE);
simple_lock(&pgtrace.loglock);
if (pgtrace.rdidx == pgtrace.wridx) {
assert_wait(pgtrace.logs, THREAD_ABORTSAFE);
simple_unlock(&pgtrace.loglock);
ml_set_interrupts_enabled(ints);
wr = thread_block(NULL);
if (wr != THREAD_AWAKENED) {
return -EINTR;
}
ints = ml_set_interrupts_enabled(FALSE);
simple_lock(&pgtrace.loglock);
}
if ((pgtrace.rdidx + total) > pgtrace.wridx) {
total = (int)(pgtrace.wridx - pgtrace.rdidx);
}
if ((RBUF_IDX(pgtrace.rdidx, pgtrace.size - 1) + total) >= pgtrace.size) {
front = pgtrace.size - RBUF_IDX(pgtrace.rdidx, pgtrace.size - 1);
} else {
front = total;
}
memcpy(buf, &(pgtrace.logs[RBUF_IDX(pgtrace.rdidx, pgtrace.size - 1)]), front * sizeof(log_t));
back = total - front;
if (back) {
buf += front * sizeof(log_t);
memcpy(buf, pgtrace.logs, back * sizeof(log_t));
}
pgtrace.rdidx += total;
simple_unlock(&pgtrace.loglock);
ml_set_interrupts_enabled(ints);
return total * sizeof(log_t);
}
int
pgtrace_get_stats(pgtrace_stats_t *stats)
{
if (!stats) {
return -1;
}
stats->stat_logger.sl_bytes = pgtrace.bytes;
pgtrace_decoder_get_stats(stats);
return 0;
}
#else // CONFIG_PGTRACE_NONKEXT
static struct {
bool active;
decoder_t *decoder;
logger_t *logger;
queue_head_t probes;
lck_grp_t *lock_grp;
lck_grp_attr_t *lock_grp_attr;
lck_attr_t *lock_attr;
lck_mtx_t probelock;
} pgtrace = {};
int
pgtrace_decode_and_run(uint32_t inst, vm_offset_t fva, vm_map_offset_t *cva_page, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
vm_offset_t pa, cva;
pgtrace_instruction_info_t info;
vm_offset_t cva_front_page = cva_page[0];
vm_offset_t cva_cur_page = cva_page[1];
pgtrace.decoder->decode(inst, ss, &info);
if (info.addr == fva) {
cva = cva_cur_page + (fva & ARM_PGMASK);
} else {
cva = cva_front_page + (fva & ARM_PGMASK);
}
pa = mmu_kvtop(cva);
if (!pa) {
panic("%s: invalid address cva=%lx fva=%lx info.addr=%lx inst=%x", __func__, cva, fva, info.addr, inst);
}
absolutetime_to_nanoseconds(mach_absolute_time(), &res->rr_time);
pgtrace.decoder->run(inst, pa, cva, ss, res);
return 0;
}
int
pgtrace_write_log(pgtrace_run_result_t res)
{
pgtrace.logger->write(res);
return 0;
}
int
pgtrace_init(decoder_t *decoder, logger_t *logger)
{
kprintf("%s decoder=%p logger=%p\n", __func__, decoder, logger);
assert(decoder && logger);
if (decoder->magic != 0xfeedface || logger->magic != 0xfeedface ||
strcmp(decoder->arch, "arm64") != 0 || strcmp(logger->arch, "arm64") != 0) {
kprintf("%s:wrong decoder/logger magic=%llx/%llx arch=%s/%s", __func__, decoder->magic, logger->magic, decoder->arch, logger->arch);
return EINVAL;
}
pgtrace.lock_attr = lck_attr_alloc_init();
pgtrace.lock_grp_attr = lck_grp_attr_alloc_init();
pgtrace.lock_grp = lck_grp_alloc_init("pgtrace_lock", pgtrace.lock_grp_attr);
lck_mtx_init(&pgtrace.probelock, pgtrace.lock_grp, pgtrace.lock_attr);
queue_init(&pgtrace.probes);
pgtrace.decoder = decoder;
pgtrace.logger = logger;
return 0;
}
int
pgtrace_add_probe(thread_t thread, vm_offset_t start, vm_offset_t end)
{
probe_t *p;
queue_head_t *q = &pgtrace.probes;
kprintf("%s start=%lx end=%lx\n", __func__, start, end);
if (start > end) {
kprintf("%s Invalid start=%lx end=%lx\n", __func__, start, end);
return -1;
}
p = kalloc(sizeof(probe_t));
p->start = start;
p->end = end;
if (thread == NULL) {
p->pmap = NULL;
} else {
p->pmap = vm_map_pmap(thread->map);
}
lck_mtx_lock(&pgtrace.probelock);
queue_enter(q, p, probe_t *, chain);
lck_mtx_unlock(&pgtrace.probelock);
return 0;
}
void
pgtrace_clear_probe(void)
{
probe_t *p, *next;
queue_head_t *q = &pgtrace.probes;
kprintf("%s\n", __func__);
lck_mtx_lock(&pgtrace.probelock);
p = (probe_t *)queue_first(q);
while (!queue_end(q, (queue_entry_t)p)) {
next = (probe_t *)queue_next(&(p->chain));
queue_remove(q, p, probe_t *, chain);
kfree(p, sizeof(probe_t));
p = next;
}
lck_mtx_unlock(&pgtrace.probelock);
return;
}
void
pgtrace_start(void)
{
probe_t *p;
queue_head_t *q = &pgtrace.probes;
kprintf("%s\n", __func__);
if (pgtrace.active == true) {
return;
}
pgtrace.active = true;
lck_mtx_lock(&pgtrace.probelock);
queue_iterate(q, p, probe_t *, chain) {
pmap_pgtrace_add_page(p->pmap, p->start, p->end);
}
lck_mtx_unlock(&pgtrace.probelock);
return;
}
void
pgtrace_stop(void)
{
probe_t *p;
queue_head_t *q = &pgtrace.probes;
kprintf("%s\n", __func__);
lck_mtx_lock(&pgtrace.probelock);
queue_iterate(q, p, probe_t *, chain) {
pmap_pgtrace_delete_page(p->pmap, p->start, p->end);
}
lck_mtx_unlock(&pgtrace.probelock);
pgtrace.active = false;
}
bool
pgtrace_active(void)
{
return pgtrace.active;
}
#endif // CONFIG_PGTRACE_NONKEXT
#else
extern void pgtrace_stop(void);
extern void pgtrace_start(void);
extern void pgtrace_clear_probe(void);
extern void pgtrace_add_probe(void);
extern void pgtrace_init(void);
extern void pgtrace_active(void);
void
pgtrace_stop(void)
{
}
void
pgtrace_start(void)
{
}
void
pgtrace_clear_probe(void)
{
}
void
pgtrace_add_probe(void)
{
}
void
pgtrace_init(void)
{
}
void
pgtrace_active(void)
{
}
#endif