pgtrace.c   [plain text]

/*
 * Copyright (c) 2015 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#if CONFIG_PGTRACE
#include <mach/mach_types.h>
#include <IOKit/IOLib.h>
#include <sys/msgbuf.h>
#include <sys/errno.h>
#include <arm64/pgtrace.h>
#include <libkern/OSDebug.h>

typedef struct {
	queue_chain_t chain;

	pmap_t      pmap;
	vm_offset_t start;
	vm_offset_t end;
} probe_t;

#if CONFIG_PGTRACE_NONKEXT
#include "pgtrace_decoder.h"

//--------------------------------------------
// Macros
//
#define RBUF_DEFAULT_SIZE   1024
#define RBUF_IDX(idx, mask) ((idx) & (mask))
#define MSG_MAX             130

//--------------------------------------------
// Types
//
typedef uint8_t RWLOCK;

typedef struct {
	uint64_t                id;
	pgtrace_run_result_t    res;
	void                    *stack[PGTRACE_STACK_DEPTH];
} log_t;

//--------------------------------------------
// Statics
//
static struct {
	log_t           *logs;          // Protect
	uint32_t        size;           // Protect
	uint64_t        rdidx, wridx;   // Protect
	decl_simple_lock_data(, loglock);

	uint64_t id;
	uint32_t option;
	uint32_t enabled;
	uint32_t bytes;

	queue_head_t    probes;         // Protect

	lck_grp_t       *lock_grp;
	lck_grp_attr_t  *lock_grp_attr;
	lck_attr_t      *lock_attr;
	lck_mtx_t       probelock;
} pgtrace = {};

//--------------------------------------------
// Globals
//
void
pgtrace_init(void)
{
	simple_lock_init(&pgtrace.loglock, 0);

	pgtrace.lock_attr = lck_attr_alloc_init();
	pgtrace.lock_grp_attr = lck_grp_attr_alloc_init();
	pgtrace.lock_grp = lck_grp_alloc_init("pgtrace_lock", pgtrace.lock_grp_attr);

	lck_mtx_init(&pgtrace.probelock, pgtrace.lock_grp, pgtrace.lock_attr);

	queue_init(&pgtrace.probes);

	pgtrace.size = RBUF_DEFAULT_SIZE;
	pgtrace.logs = kalloc(RBUF_DEFAULT_SIZE * sizeof(log_t));
}

void
pgtrace_clear_probe(void)
{
	probe_t *p, *next;
	queue_head_t *q = &pgtrace.probes;

	lck_mtx_lock(&pgtrace.probelock);

	p = (probe_t *)queue_first(q);
	while (!queue_end(q, (queue_entry_t)p)) {
		next = (probe_t *)queue_next(&(p->chain));

		queue_remove(q, p, probe_t *, chain);
		kfree(p, sizeof(probe_t));

		p = next;
	}

	lck_mtx_unlock(&pgtrace.probelock);

	return;
}

int
pgtrace_add_probe(thread_t thread, vm_offset_t start, vm_offset_t end)
{
	probe_t *p;
	queue_head_t *q = &pgtrace.probes;

	if (start > end) {
		kprintf("%s Invalid start=%lx end=%lx\n", __func__, start, end);
		return -1;
	}

	p = kalloc(sizeof(probe_t));
	p->start = start;
	p->end = end;
	if (thread == NULL) {
		p->pmap = NULL;
	} else {
		p->pmap = vm_map_pmap(thread->map);
	}

	lck_mtx_lock(&pgtrace.probelock);
	queue_enter(q, p, probe_t *, chain);
	lck_mtx_unlock(&pgtrace.probelock);

	return 0;
}

void
pgtrace_start(void)
{
	probe_t *p;
	queue_head_t *q = &pgtrace.probes;

	kprintf("%s\n", __func__);

	if (pgtrace.enabled) {
		return;
	}

	pgtrace.enabled = 1;

	lck_mtx_lock(&pgtrace.probelock);

	queue_iterate(q, p, probe_t *, chain) {
		pmap_pgtrace_add_page(p->pmap, p->start, p->end);
	}

	lck_mtx_unlock(&pgtrace.probelock);

	return;
}

void
pgtrace_stop(void)
{
	probe_t *p;
	queue_head_t *q = &pgtrace.probes;

	kprintf("%s\n", __func__);

	lck_mtx_lock(&pgtrace.probelock);

	queue_iterate(q, p, probe_t *, chain) {
		pmap_pgtrace_delete_page(p->pmap, p->start, p->end);
	}

	lck_mtx_unlock(&pgtrace.probelock);

	pgtrace.enabled = 0;
}

uint32_t
pgtrace_get_size(void)
{
	return pgtrace.size;
}

bool
pgtrace_set_size(uint32_t size)
{
	log_t *old_buf, *new_buf;
	uint32_t old_size, new_size = 1;

	// round up to next power of 2
	while (size > new_size) {
		new_size <<= 1;
		if (new_size > 0x100000) {
			// over million entries
			kprintf("%s: size=%x new_size=%x is too big\n", __func__, size, new_size);
			return false;
		}
	}

	new_buf = kalloc(new_size * sizeof(log_t));
	if (new_buf == NULL) {
		kprintf("%s: can't allocate new_size=%x\n entries", __func__, new_size);
		return false;
	}

	pgtrace_stop();

	simple_lock(&pgtrace.loglock);
	old_buf = pgtrace.logs;
	old_size = pgtrace.size;
	pgtrace.logs = new_buf;
	pgtrace.size = new_size;
	pgtrace.rdidx = pgtrace.wridx = 0;
	simple_unlock(&pgtrace.loglock);

	if (old_buf) {
		kfree(old_buf, old_size * sizeof(log_t));
	}

	return true;
}

void
pgtrace_clear_trace(void)
{
	simple_lock(&pgtrace.loglock);
	pgtrace.rdidx = pgtrace.wridx = 0;
	simple_unlock(&pgtrace.loglock);
}

boolean_t
pgtrace_active(void)
{
	return pgtrace.enabled > 0;
}

uint32_t
pgtrace_get_option(void)
{
	return pgtrace.option;
}

void
pgtrace_set_option(uint32_t option)
{
	pgtrace.option = option;
}

// pgtrace_write_log() is in interrupt disabled context
void
pgtrace_write_log(pgtrace_run_result_t res)
{
	uint8_t i;
	log_t log = {};
	const char *rwmap[] = { "R", "W", "PREFETCH" };

	log.id = pgtrace.id++;
	log.res = res;

	if (pgtrace.option & PGTRACE_OPTION_KPRINTF) {
		char msg[MSG_MAX];
		char *p;

		p = msg;

		snprintf(p, MSG_MAX, "%llu %s ", res.rr_time, rwmap[res.rr_rw]);
		p += strlen(p);

		for (i = 0; i < res.rr_num; i++) {
			snprintf(p, MSG_MAX - (p - msg), "%lx=%llx ", res.rr_addrdata[i].ad_addr, res.rr_addrdata[i].ad_data);
			p += strlen(p);
		}

		kprintf("%s %s\n", __func__, msg);
	}

	if (pgtrace.option & PGTRACE_OPTION_STACK) {
		OSBacktrace(log.stack, PGTRACE_STACK_DEPTH);
	}

	pgtrace.bytes += sizeof(log);

	simple_lock(&pgtrace.loglock);

	pgtrace.logs[RBUF_IDX(pgtrace.wridx, pgtrace.size - 1)] = log;

	// Advance rdidx if ring is full
	if (RBUF_IDX(pgtrace.wridx, pgtrace.size - 1) == RBUF_IDX(pgtrace.rdidx, pgtrace.size - 1) &&
	    (pgtrace.wridx != pgtrace.rdidx)) {
		pgtrace.rdidx++;
	}
	pgtrace.wridx++;

	// Signal if ring was empty
	if (pgtrace.wridx == (pgtrace.rdidx + 1)) {
		thread_wakeup(pgtrace.logs);
	}

	simple_unlock(&pgtrace.loglock);

	return;
}

// pgtrace_read_log() is in user thread
int64_t
pgtrace_read_log(uint8_t *buf, uint32_t size)
{
	int total, front, back;
	boolean_t ints;
	wait_result_t wr;

	if (pgtrace.enabled == FALSE) {
		return -EINVAL;
	}

	total = size / sizeof(log_t);

	// Check if buf is too small
	if (buf && total == 0) {
		return -EINVAL;
	}

	ints = ml_set_interrupts_enabled(FALSE);
	simple_lock(&pgtrace.loglock);

	// Wait if ring is empty
	if (pgtrace.rdidx == pgtrace.wridx) {
		assert_wait(pgtrace.logs, THREAD_ABORTSAFE);

		simple_unlock(&pgtrace.loglock);
		ml_set_interrupts_enabled(ints);

		wr = thread_block(NULL);
		if (wr != THREAD_AWAKENED) {
			return -EINTR;
		}

		ints = ml_set_interrupts_enabled(FALSE);
		simple_lock(&pgtrace.loglock);
	}

	// Trim the size
	if ((pgtrace.rdidx + total) > pgtrace.wridx) {
		total = (int)(pgtrace.wridx - pgtrace.rdidx);
	}

	// Copy front
	if ((RBUF_IDX(pgtrace.rdidx, pgtrace.size - 1) + total) >= pgtrace.size) {
		front = pgtrace.size - RBUF_IDX(pgtrace.rdidx, pgtrace.size - 1);
	} else {
		front = total;
	}

	memcpy(buf, &(pgtrace.logs[RBUF_IDX(pgtrace.rdidx, pgtrace.size - 1)]), front * sizeof(log_t));

	// Copy back if any
	back = total - front;
	if (back) {
		buf += front * sizeof(log_t);
		memcpy(buf, pgtrace.logs, back * sizeof(log_t));
	}

	pgtrace.rdidx += total;

	simple_unlock(&pgtrace.loglock);
	ml_set_interrupts_enabled(ints);

	return total * sizeof(log_t);
}

int
pgtrace_get_stats(pgtrace_stats_t *stats)
{
	if (!stats) {
		return -1;
	}

	stats->stat_logger.sl_bytes = pgtrace.bytes;
	pgtrace_decoder_get_stats(stats);

	return 0;
}

#else // CONFIG_PGTRACE_NONKEXT

static struct {
	bool            active;
	decoder_t       *decoder;
	logger_t        *logger;
	queue_head_t    probes;

	lck_grp_t       *lock_grp;
	lck_grp_attr_t  *lock_grp_attr;
	lck_attr_t      *lock_attr;
	lck_mtx_t       probelock;
} pgtrace = {};

//------------------------------------
// functions for pmap fault handler
// - pgtrace_decode_and_run
// - pgtrace_write_log
//------------------------------------
int
pgtrace_decode_and_run(uint32_t inst, vm_offset_t fva, vm_map_offset_t *cva_page, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	vm_offset_t pa, cva;
	pgtrace_instruction_info_t info;
	vm_offset_t cva_front_page = cva_page[0];
	vm_offset_t cva_cur_page = cva_page[1];

	pgtrace.decoder->decode(inst, ss, &info);

	if (info.addr == fva) {
		cva = cva_cur_page + (fva & ARM_PGMASK);
	} else {
		// which means a front page is not a tracing page
		cva = cva_front_page + (fva & ARM_PGMASK);
	}

	pa = mmu_kvtop(cva);
	if (!pa) {
		panic("%s: invalid address cva=%lx fva=%lx info.addr=%lx inst=%x", __func__, cva, fva, info.addr, inst);
	}

	absolutetime_to_nanoseconds(mach_absolute_time(), &res->rr_time);

	pgtrace.decoder->run(inst, pa, cva, ss, res);

	return 0;
}

int
pgtrace_write_log(pgtrace_run_result_t res)
{
	pgtrace.logger->write(res);
	return 0;
}

//------------------------------------
// functions for kext
//  - pgtrace_init
//  - pgtrace_add_probe
//  - pgtrace_clear_probe
//  - pgtrace_start
//  - pgtrace_stop
//  - pgtrace_active
//------------------------------------
int
pgtrace_init(decoder_t *decoder, logger_t *logger)
{
	kprintf("%s decoder=%p logger=%p\n", __func__, decoder, logger);

	assert(decoder && logger);

	if (decoder->magic != 0xfeedface || logger->magic != 0xfeedface ||
	    strcmp(decoder->arch, "arm64") != 0 || strcmp(logger->arch, "arm64") != 0) {
		kprintf("%s:wrong decoder/logger magic=%llx/%llx arch=%s/%s", __func__, decoder->magic, logger->magic, decoder->arch, logger->arch);
		return EINVAL;
	}

	pgtrace.lock_attr = lck_attr_alloc_init();
	pgtrace.lock_grp_attr = lck_grp_attr_alloc_init();
	pgtrace.lock_grp = lck_grp_alloc_init("pgtrace_lock", pgtrace.lock_grp_attr);

	lck_mtx_init(&pgtrace.probelock, pgtrace.lock_grp, pgtrace.lock_attr);

	queue_init(&pgtrace.probes);
	pgtrace.decoder = decoder;
	pgtrace.logger = logger;

	return 0;
}

int
pgtrace_add_probe(thread_t thread, vm_offset_t start, vm_offset_t end)
{
	probe_t *p;
	queue_head_t *q = &pgtrace.probes;

	kprintf("%s start=%lx end=%lx\n", __func__, start, end);

	if (start > end) {
		kprintf("%s Invalid start=%lx end=%lx\n", __func__, start, end);
		return -1;
	}

	p = kalloc(sizeof(probe_t));
	p->start = start;
	p->end = end;
	if (thread == NULL) {
		p->pmap = NULL;
	} else {
		p->pmap = vm_map_pmap(thread->map);
	}

	lck_mtx_lock(&pgtrace.probelock);
	queue_enter(q, p, probe_t *, chain);
	lck_mtx_unlock(&pgtrace.probelock);

	return 0;
}

void
pgtrace_clear_probe(void)
{
	probe_t *p, *next;
	queue_head_t *q = &pgtrace.probes;

	kprintf("%s\n", __func__);

	lck_mtx_lock(&pgtrace.probelock);

	p = (probe_t *)queue_first(q);
	while (!queue_end(q, (queue_entry_t)p)) {
		next = (probe_t *)queue_next(&(p->chain));

		queue_remove(q, p, probe_t *, chain);
		kfree(p, sizeof(probe_t));

		p = next;
	}

	lck_mtx_unlock(&pgtrace.probelock);

	return;
}

void
pgtrace_start(void)
{
	probe_t *p;
	queue_head_t *q = &pgtrace.probes;

	kprintf("%s\n", __func__);

	if (pgtrace.active == true) {
		return;
	}

	pgtrace.active = true;

	lck_mtx_lock(&pgtrace.probelock);

	queue_iterate(q, p, probe_t *, chain) {
		pmap_pgtrace_add_page(p->pmap, p->start, p->end);
	}

	lck_mtx_unlock(&pgtrace.probelock);

	return;
}

void
pgtrace_stop(void)
{
	probe_t *p;
	queue_head_t *q = &pgtrace.probes;

	kprintf("%s\n", __func__);

	lck_mtx_lock(&pgtrace.probelock);

	queue_iterate(q, p, probe_t *, chain) {
		pmap_pgtrace_delete_page(p->pmap, p->start, p->end);
	}

	lck_mtx_unlock(&pgtrace.probelock);

	pgtrace.active = false;
}

bool
pgtrace_active(void)
{
	return pgtrace.active;
}
#endif // CONFIG_PGTRACE_NONKEXT
#else
// empty funcs for release kernel
extern void pgtrace_stop(void);
extern void pgtrace_start(void);
extern void pgtrace_clear_probe(void);
extern void pgtrace_add_probe(void);
extern void pgtrace_init(void);
extern void pgtrace_active(void);
void
pgtrace_stop(void)
{
}
void
pgtrace_start(void)
{
}
void
pgtrace_clear_probe(void)
{
}
void
pgtrace_add_probe(void)
{
}
void
pgtrace_init(void)
{
}
void
pgtrace_active(void)
{
}
#endif