#include <kern/kalloc.h>
#include <kern/kern_types.h>
#include <kern/locks.h>
#include <kern/misc_protos.h>
#include <kern/task.h>
#include <kern/thread.h>
#include <kern/zalloc.h>
#include <machine/machine_cpu.h>
#include <pmc/pmc.h>
#include <libkern/OSAtomic.h>
#if defined(__i386__) || defined(__x86_64__)
#include <i386/mp.h>
#endif
#if CONFIG_COUNTERS
#undef DEBUG_COUNTERS
typedef uint8_t pmc_state_event_t;
#define PMC_STATE_EVENT_START 0
#define PMC_STATE_EVENT_STOP 1
#define PMC_STATE_EVENT_FREE 2
#define PMC_STATE_EVENT_INTERRUPT 3
#define PMC_STATE_EVENT_END_OF_INTERRUPT 4
#define PMC_STATE_EVENT_CONTEXT_IN 5
#define PMC_STATE_EVENT_CONTEXT_OUT 6
#define PMC_STATE_EVENT_LOAD_FINISHED 7
#define PMC_STATE_EVENT_STORE_FINISHED 8
#define PMC_SPIN_THRESHOLD 10
#define PMC_SPIN_TIMEOUT_US 10
uint64_t pmc_spin_timeout_count = 0;
#ifdef DEBUG_COUNTERS
# include <pexpert/pexpert.h>
# define COUNTER_DEBUG(...) \
do { \
kprintf("[%s:%s][%u] ", __FILE__, __PRETTY_FUNCTION__, cpu_number()); \
kprintf(__VA_ARGS__); \
} while(0)
# define PRINT_PERF_MON(x) \
do { \
kprintf("perfmon: %p (obj: %p refCt: %u switchable: %u)\n", \
x, x->object, x->useCount, \
(x->methods.flags & PERFMON_FLAG_SUPPORTS_CONTEXT_SWITCHING) ? \
1 : 0); \
} while(0)
static const char const * pmc_state_state_name(pmc_state_t state) {
switch (PMC_STATE_STATE(state)) {
case PMC_STATE_STATE_INVALID:
return "INVALID";
case PMC_STATE_STATE_STOP:
return "STOP";
case PMC_STATE_STATE_CAN_RUN:
return "CAN_RUN";
case PMC_STATE_STATE_LOAD:
return "LOAD";
case PMC_STATE_STATE_RUN:
return "RUN";
case PMC_STATE_STATE_STORE:
return "STORE";
case PMC_STATE_STATE_INTERRUPT:
return "INTERRUPT";
case PMC_STATE_STATE_DEALLOC:
return "DEALLOC";
default:
return "UNKNOWN";
}
}
static const char const * pmc_state_event_name(pmc_state_event_t event) {
switch (event) {
case PMC_STATE_EVENT_START:
return "START";
case PMC_STATE_EVENT_STOP:
return "STOP";
case PMC_STATE_EVENT_FREE:
return "FREE";
case PMC_STATE_EVENT_INTERRUPT:
return "INTERRUPT";
case PMC_STATE_EVENT_END_OF_INTERRUPT:
return "END OF INTERRUPT";
case PMC_STATE_EVENT_CONTEXT_IN:
return "CONTEXT IN";
case PMC_STATE_EVENT_CONTEXT_OUT:
return "CONTEXT OUT";
case PMC_STATE_EVENT_LOAD_FINISHED:
return "LOAD_FINISHED";
case PMC_STATE_EVENT_STORE_FINISHED:
return "STORE_FINISHED";
default:
return "UNKNOWN";
}
}
# define PMC_STATE_FORMAT "<%s, %u, %s%s%s>"
# define PMC_STATE_ARGS(x) pmc_state_state_name(x), PMC_STATE_CONTEXT_COUNT(x), ((PMC_STATE_FLAGS(x) & PMC_STATE_FLAGS_INTERRUPTING) ? "I" : ""), \
((PMC_STATE_FLAGS(x) & PMC_STATE_FLAGS_STOPPING) ? "S" : ""), ((PMC_STATE_FLAGS(x) & PMC_STATE_FLAGS_DEALLOCING) ? "D" : "")
#else
# define COUNTER_DEBUG(...)
# define PRINT_PERF_MON(x)
# define PMC_STATE_FORMAT
# define PMC_STATE_ARGS(x)
#endif
struct pmc_config {
pmc_config_object_t object;
volatile pmc_interrupt_method_t method;
uint64_t interrupt_after_value;
void *refCon;
};
static zone_t perf_small_zone = NULL;
#define MAX_PERF_SMALLS (256 + 8196 + 8196)
#define PERF_SMALL_UNIT_SZ (MAX(MAX(sizeof(struct perf_monitor), \
sizeof(struct pmc_reservation)), sizeof(struct pmc_config)))
static zone_t perf_big_zone = NULL;
#define MAX_PERF_BIGS (1024)
#define PERF_BIG_UNIT_SZ (sizeof(struct pmc))
static lck_grp_t *pmc_lock_grp = LCK_GRP_NULL;
static lck_grp_attr_t *pmc_lock_grp_attr;
static lck_attr_t *pmc_lock_attr;
static lck_mtx_t cpu_monitor_queue_mutex;
static lck_spin_t perf_monitor_queue_spin;
static lck_spin_t perf_counters_queue_spin;
static lck_spin_t reservations_spin;
static queue_head_t **cpu_monitor_queues = NULL;
static queue_head_t *perf_monitors_queue = NULL;
static volatile uint32_t perf_monitors_count = 0U;
static queue_head_t *perf_counters_queue = NULL;
static volatile uint32_t perf_counters_count = 0U;
static queue_head_t *system_reservations = NULL;
static volatile uint32_t system_reservation_count = 0U;
static queue_head_t *task_reservations = NULL;
static volatile uint32_t task_reservation_count = 0U;
static queue_head_t *thread_reservations = NULL;
static volatile uint32_t thread_reservation_count = 0U;
#if XNU_KERNEL_PRIVATE
static void init_pmc_locks(void) {
pmc_lock_attr = lck_attr_alloc_init();
assert(pmc_lock_attr);
pmc_lock_grp_attr = lck_grp_attr_alloc_init();
assert(pmc_lock_grp_attr);
pmc_lock_grp = lck_grp_alloc_init("pmc", pmc_lock_grp_attr);
assert(pmc_lock_grp);
lck_spin_init(&perf_monitor_queue_spin, pmc_lock_grp, pmc_lock_attr);
lck_spin_init(&perf_counters_queue_spin, pmc_lock_grp, pmc_lock_attr);
lck_spin_init(&reservations_spin, pmc_lock_grp, pmc_lock_attr);
lck_mtx_init(&cpu_monitor_queue_mutex, pmc_lock_grp, pmc_lock_attr);
}
static void init_pmc_zones(void) {
perf_small_zone = zinit(PERF_SMALL_UNIT_SZ,
MAX_PERF_SMALLS * PERF_SMALL_UNIT_SZ, MAX_PERF_SMALLS,
"pmc.small zone");
assert(perf_small_zone);
perf_big_zone = zinit(PERF_BIG_UNIT_SZ,
MAX_PERF_BIGS * PERF_BIG_UNIT_SZ, MAX_PERF_BIGS,
"pmc.big zone");
assert(perf_big_zone);
}
static void init_pmc_queues(void) {
perf_monitors_queue = (queue_head_t*)kalloc(sizeof(queue_head_t));
assert(perf_monitors_queue);
queue_init(perf_monitors_queue);
perf_counters_queue = (queue_head_t*)kalloc(sizeof(queue_head_t));
assert(perf_counters_queue);
queue_init(perf_counters_queue);
system_reservations = (queue_head_t*)kalloc(sizeof(queue_t));
assert(system_reservations);
queue_init(system_reservations);
task_reservations = (queue_head_t*)kalloc(sizeof(queue_head_t));
assert(task_reservations);
queue_init(task_reservations);
thread_reservations = (queue_head_t*)kalloc(sizeof(queue_head_t));
assert(thread_reservations);
queue_init(thread_reservations);
}
__private_extern__
void pmc_bootstrap(void) {
init_pmc_zones();
init_pmc_locks();
init_pmc_queues();
}
#endif
static perf_monitor_t perf_monitor_alloc(void) {
return (perf_monitor_t)zalloc(perf_small_zone);
}
static void perf_monitor_free(void *pm) {
zfree(perf_small_zone, pm);
}
static void perf_monitor_init(perf_monitor_t pm, int cpu) {
assert(pm);
pm->object = NULL;
bzero(&(pm->methods), sizeof(perf_monitor_methods_t));
pm->useCount = 1;
pm->reservedCounters = 0;
pm->cpu = cpu;
pm->link.next = pm->link.prev = (queue_entry_t)NULL;
pm->cpu_link.next = pm->cpu_link.prev = (queue_entry_t)NULL;
}
static void perf_monitor_dequeue(perf_monitor_t pm) {
lck_spin_lock(&perf_monitor_queue_spin);
if (pm->methods.flags & PERFMON_FLAG_REQUIRES_IDLE_NOTIFICATIONS) {
queue_remove(cpu_monitor_queues[pm->cpu], pm, perf_monitor_t, cpu_link);
}
queue_remove(perf_monitors_queue, pm, perf_monitor_t, link);
perf_monitors_count--;
lck_spin_unlock(&perf_monitor_queue_spin);
}
static void perf_monitor_enqueue(perf_monitor_t pm) {
lck_mtx_lock(&cpu_monitor_queue_mutex);
lck_spin_lock(&perf_monitor_queue_spin);
if (pm->cpu >= 0) {
if (!cpu_monitor_queues) {
uint32_t max_cpus;
queue_head_t **queues;
uint32_t i;
lck_spin_unlock(&perf_monitor_queue_spin);
max_cpus = ml_get_max_cpus();
queues = (queue_head_t**)kalloc(sizeof(queue_head_t*) * max_cpus);
assert(queues);
for (i = 0; i < max_cpus; i++) {
queue_head_t *queue = (queue_head_t*)kalloc(sizeof(queue_head_t));
assert(queue);
queue_init(queue);
queues[i] = queue;
}
lck_spin_lock(&perf_monitor_queue_spin);
cpu_monitor_queues = queues;
}
queue_enter(cpu_monitor_queues[pm->cpu], pm, perf_monitor_t, cpu_link);
}
queue_enter(perf_monitors_queue, pm, perf_monitor_t, link);
perf_monitors_count++;
lck_spin_unlock(&perf_monitor_queue_spin);
lck_mtx_unlock(&cpu_monitor_queue_mutex);
}
static void perf_monitor_reference(perf_monitor_t pm) {
assert(pm);
OSIncrementAtomic(&(pm->useCount));
}
static void perf_monitor_deallocate(perf_monitor_t pm) {
assert(pm);
if(1 == OSDecrementAtomic(&(pm->useCount))) {
perf_monitor_free(pm);
}
}
static perf_monitor_t perf_monitor_find(perf_monitor_object_t monitor) {
assert(monitor);
perf_monitor_t element = NULL;
perf_monitor_t found = NULL;
lck_spin_lock(&perf_monitor_queue_spin);
queue_iterate(perf_monitors_queue, element, perf_monitor_t, link) {
if(element->object == monitor) {
perf_monitor_reference(element);
found = element;
break;
}
}
lck_spin_unlock(&perf_monitor_queue_spin);
return found;
}
static void perf_monitor_add_pmc(perf_monitor_t pm, pmc_t pmc __unused) {
assert(pm);
assert(pmc);
perf_monitor_reference(pm);
}
static void perf_monitor_remove_pmc(perf_monitor_t pm, pmc_t pmc __unused) {
assert(pm);
assert(pmc);
perf_monitor_deallocate(pm);
}
static pmc_t pmc_alloc(void) {
return (pmc_t)zalloc(perf_big_zone);
}
static void pmc_free(void *pmc) {
zfree(perf_big_zone, pmc);
}
static void pmc_init(pmc_t pmc) {
assert(pmc);
pmc->object = NULL;
pmc->monitor = NULL;
bzero(&pmc->methods, sizeof(pmc_methods_t));
pmc->useCount = 1;
}
static void pmc_reference(pmc_t pmc) {
assert(pmc);
OSIncrementAtomic(&(pmc->useCount));
}
static void pmc_deallocate(pmc_t pmc) {
assert(pmc);
if(1 == OSDecrementAtomic(&(pmc->useCount))) {
pmc_free(pmc);
}
}
static void pmc_dequeue(pmc_t pmc) {
lck_spin_lock(&perf_counters_queue_spin);
queue_remove(perf_counters_queue, pmc, pmc_t, link);
perf_counters_count--;
lck_spin_unlock(&perf_counters_queue_spin);
}
static void pmc_enqueue(pmc_t pmc) {
lck_spin_lock(&perf_counters_queue_spin);
queue_enter(perf_counters_queue, pmc, pmc_t, link);
perf_counters_count++;
lck_spin_unlock(&perf_counters_queue_spin);
}
static pmc_t pmc_find(pmc_object_t object) {
assert(object);
lck_spin_lock(&perf_counters_queue_spin);
pmc_t element = NULL;
pmc_t found = NULL;
queue_iterate(perf_counters_queue, element, pmc_t, link) {
if(element->object == object) {
pmc_reference(element);
found = element;
break;
}
}
lck_spin_unlock(&perf_counters_queue_spin);
return found;
}
static pmc_config_t pmc_config_alloc(pmc_t pmc __unused) {
return (pmc_config_t)zalloc(perf_small_zone);
}
static void pmc_config_free(pmc_t pmc, pmc_config_t config) {
assert(pmc);
assert(config);
if(config->object) {
pmc->methods.free_config(pmc->object, config->object);
config->object = NULL;
}
zfree(perf_small_zone, config);
}
static kern_return_t pmc_open(pmc_t pmc) {
assert(pmc);
assert(pmc->object);
assert(pmc->open_object);
return pmc->methods.open(pmc->object, pmc->open_object);
}
static kern_return_t pmc_close(pmc_t pmc) {
assert(pmc);
assert(pmc->object);
assert(pmc->open_object);
return pmc->methods.close(pmc->object, pmc->open_object);
}
static kern_return_t pmc_internal_reservation_set_pmc(pmc_reservation_t resv, pmc_t pmc);
static void pmc_internal_reservation_store(pmc_reservation_t reservation);
static void pmc_internal_reservation_load(pmc_reservation_t reservation);
static pmc_reservation_t reservation_alloc(void) {
return (pmc_reservation_t)zalloc(perf_small_zone);
}
static void reservation_free(pmc_reservation_t resv) {
if(resv->config) {
assert(resv->pmc);
pmc_free_config(resv->pmc, resv->config);
resv->config = NULL;
}
(void)pmc_internal_reservation_set_pmc(resv, NULL);
zfree(perf_small_zone, resv);
}
static void reservation_init(pmc_reservation_t resv) {
assert(resv);
resv->pmc = NULL;
resv->config = NULL;
resv->value = 0ULL;
resv->flags = 0U;
resv->state = PMC_STATE(PMC_STATE_STATE_STOP, 0, 0);
resv->active_last_context_in = 0U;
resv->task = TASK_NULL;
}
static kern_return_t pmc_internal_reservation_set_pmc(pmc_reservation_t resv, pmc_t pmc) {
assert(resv);
if(resv->pmc) {
(void)pmc_close(resv->pmc);
pmc_deallocate(resv->pmc);
resv->pmc = NULL;
}
resv->pmc = pmc;
if(resv->pmc) {
pmc_reference(resv->pmc);
if(KERN_SUCCESS != pmc_open(resv->pmc)) {
pmc_deallocate(resv->pmc);
resv->pmc = NULL;
return KERN_FAILURE;
}
}
return KERN_SUCCESS;
}
static void pmc_internal_reservation_enqueue(queue_t queue, pmc_reservation_t resv) {
assert(queue);
assert(resv);
queue_enter(queue, resv, pmc_reservation_t, link);
}
static void pmc_internal_reservation_dequeue(queue_t queue, pmc_reservation_t resv) {
assert(queue);
assert(resv);
queue_remove(queue, resv, pmc_reservation_t, link);
}
static boolean_t pmc_internal_reservation_matches_context(pmc_reservation_t resv) {
boolean_t ret = FALSE;
assert(resv);
if(PMC_FLAG_IS_SYSTEM_SCOPE(resv->flags)) {
ret = TRUE;
} else if(PMC_FLAG_IS_TASK_SCOPE(resv->flags)) {
if(current_task() == resv->task) {
ret = TRUE;
}
} else if(PMC_FLAG_IS_THREAD_SCOPE(resv->flags)) {
if(current_thread() == resv->thread) {
ret = TRUE;
}
}
return ret;
}
static uint32_t pmc_accessible_core_count(pmc_t pmc) {
assert(pmc);
uint32_t *cores = NULL;
size_t coreCt = 0UL;
if(KERN_SUCCESS != pmc->methods.accessible_cores(pmc->object,
&cores, &coreCt)) {
coreCt = 0U;
}
return (uint32_t)coreCt;
}
static boolean_t pmc_internal_reservation_queue_contains_pmc(queue_t queue, pmc_reservation_t resv) {
assert(queue);
assert(resv);
boolean_t ret = FALSE;
pmc_reservation_t tmp = NULL;
queue_iterate(queue, tmp, pmc_reservation_t, link) {
if(tmp->pmc == resv->pmc) {
switch(PMC_FLAG_SCOPE(tmp->flags)) {
case PMC_FLAG_SCOPE_SYSTEM:
ret = TRUE;
break;
case PMC_FLAG_SCOPE_THREAD:
ret = (PMC_FLAG_SCOPE(resv->flags) != PMC_FLAG_SCOPE_THREAD) ||
(tmp->thread == resv->thread);
if(!ret) {
if(1 != pmc_accessible_core_count(tmp->pmc)) {
ret = TRUE;
}
}
break;
case PMC_FLAG_SCOPE_TASK:
ret = (PMC_FLAG_SCOPE(resv->flags) != PMC_FLAG_SCOPE_TASK) ||
(tmp->task == resv->task);
if(!ret) {
if(1 != pmc_accessible_core_count(tmp->pmc)) {
ret = TRUE;
}
}
break;
}
if(ret) break;
}
}
return ret;
}
static boolean_t pmc_internal_reservation_validate_for_pmc(pmc_reservation_t resv) {
assert(resv);
boolean_t ret = TRUE;
if(pmc_internal_reservation_queue_contains_pmc(system_reservations, resv) ||
pmc_internal_reservation_queue_contains_pmc(task_reservations, resv) ||
pmc_internal_reservation_queue_contains_pmc(thread_reservations, resv)) {
ret = FALSE;
}
return ret;
}
static void pmc_internal_update_thread_flag(thread_t thread, boolean_t newFlag) {
assert(thread);
pmc_reservation_t tmp = NULL;
if(!newFlag) {
lck_spin_lock(&reservations_spin);
queue_iterate(thread_reservations, tmp, pmc_reservation_t, link) {
if(tmp->thread == thread) {
newFlag = TRUE;
break;
}
}
lck_spin_unlock(&reservations_spin);
}
if(newFlag) {
OSBitOrAtomic(THREAD_PMC_FLAG, &thread->t_chud);
} else {
OSBitAndAtomic(~(THREAD_PMC_FLAG), &thread->t_chud);
}
}
static void pmc_internal_update_task_flag(task_t task, boolean_t newFlag) {
assert(task);
thread_t thread = NULL;
if(newFlag) {
OSBitOrAtomic(TASK_PMC_FLAG, &task->t_chud);
} else {
OSBitAndAtomic(~(TASK_PMC_FLAG), &task->t_chud);
}
task_lock(task);
queue_iterate(&task->threads, thread, thread_t, task_threads) {
pmc_internal_update_thread_flag(thread, newFlag);
}
task_unlock(task);
}
static boolean_t pmc_internal_reservation_add(pmc_reservation_t resv) {
assert(resv);
boolean_t ret = FALSE;
lck_spin_lock(&reservations_spin);
if(pmc_internal_reservation_validate_for_pmc(resv)) {
switch(PMC_FLAG_SCOPE(resv->flags)) {
case PMC_FLAG_SCOPE_SYSTEM:
pmc_internal_reservation_enqueue(system_reservations, resv);
system_reservation_count++;
lck_spin_unlock(&reservations_spin);
break;
case PMC_FLAG_SCOPE_TASK:
assert(resv->task);
pmc_internal_reservation_enqueue(task_reservations, resv);
task_reservation_count++;
lck_spin_unlock(&reservations_spin);
pmc_internal_update_task_flag(resv->task, TRUE);
break;
case PMC_FLAG_SCOPE_THREAD:
assert(resv->thread);
pmc_internal_reservation_enqueue(thread_reservations, resv);
thread_reservation_count++;
lck_spin_unlock(&reservations_spin);
pmc_internal_update_thread_flag(resv->thread, TRUE);
break;
}
ret = TRUE;
} else {
lck_spin_unlock(&reservations_spin);
}
return ret;
}
static void pmc_internal_reservation_broadcast(pmc_reservation_t reservation, void (*action_func)(void *)) {
uint32_t * cores;
size_t core_cnt;
if (KERN_SUCCESS == pmc_get_accessible_core_list(reservation->pmc, &cores, &core_cnt)) {
boolean_t intrs_enabled = ml_set_interrupts_enabled(FALSE);
if (core_cnt == 1 && cores[0] == (uint32_t)cpu_number()) {
action_func(reservation);
} else {
#if defined(__i386__) || defined(__x86_64__)
size_t ii;
cpumask_t mask = 0;
if (core_cnt > 0) {
for (ii = 0; ii < core_cnt; ii++) {
mask |= cpu_to_cpumask(cores[ii]);
}
} else {
mask = CPUMASK_ALL;
}
mp_cpus_call(mask, ASYNC, action_func, reservation);
#else
#error pmc_reservation_interrupt needs an inter-processor method invocation mechanism for this architecture
#endif
}
ml_set_interrupts_enabled(intrs_enabled);
}
}
static void pmc_internal_reservation_remove(pmc_reservation_t resv) {
assert(resv);
lck_spin_lock(&reservations_spin);
switch(PMC_FLAG_SCOPE(resv->flags)) {
case PMC_FLAG_SCOPE_SYSTEM:
pmc_internal_reservation_dequeue(system_reservations, resv);
system_reservation_count--;
lck_spin_unlock(&reservations_spin);
break;
case PMC_FLAG_SCOPE_TASK:
pmc_internal_reservation_dequeue(task_reservations, resv);
task_reservation_count--;
lck_spin_unlock(&reservations_spin);
pmc_internal_update_task_flag(resv->task, FALSE);
break;
case PMC_FLAG_SCOPE_THREAD:
pmc_internal_reservation_dequeue(thread_reservations, resv);
thread_reservation_count--;
lck_spin_unlock(&reservations_spin);
pmc_internal_update_thread_flag(resv->thread, FALSE);
break;
}
}
static uint32_t pmc_internal_reservation_next_state(uint32_t current_state, pmc_state_event_t event) {
uint32_t new_state = PMC_STATE(PMC_STATE_STATE_INVALID, 0, 0);
switch (event) {
case PMC_STATE_EVENT_START:
switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_STOPPING):
new_state = PMC_STATE_MODIFY(current_state, 0, 0, PMC_STATE_FLAGS_STOPPING);
break;
case PMC_STATE(PMC_STATE_STATE_STOP, 0, 0):
if (PMC_STATE_CONTEXT_COUNT(current_state) == 0) {
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_CAN_RUN, 0, 0, 0);
}
break;
}
break;
case PMC_STATE_EVENT_STOP:
switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
case PMC_STATE(PMC_STATE_STATE_CAN_RUN, 0, 0):
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STOP, 0, 0, 0);
break;
case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, 0):
case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
case PMC_STATE(PMC_STATE_STATE_RUN, 0, 0):
case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
new_state = PMC_STATE_MODIFY(current_state, 0, PMC_STATE_FLAGS_STOPPING, 0);
break;
case PMC_STATE(PMC_STATE_STATE_STOP, 0, 0):
if (PMC_STATE_CONTEXT_COUNT(current_state) > 0) {
new_state = PMC_STATE_MODIFY(current_state, 0, PMC_STATE_FLAGS_STOPPING, 0);
}
break;
}
break;
case PMC_STATE_EVENT_FREE:
switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
case PMC_STATE(PMC_STATE_STATE_CAN_RUN, 0, 0):
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, 0, 0);
break;
case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_STOPPING):
new_state = PMC_STATE_MODIFY(current_state, 0, PMC_STATE_FLAGS_DEALLOCING, PMC_STATE_FLAGS_STOPPING);
break;
case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, 0):
case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
case PMC_STATE(PMC_STATE_STATE_RUN, 0, 0):
case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
new_state = PMC_STATE_MODIFY(current_state, 0, PMC_STATE_FLAGS_DEALLOCING, 0);
break;
case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_STOPPING):
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, PMC_STATE_FLAGS_DEALLOCING, PMC_STATE_FLAGS_STOPPING);
break;
case PMC_STATE(PMC_STATE_STATE_STOP, 0, 0):
if (PMC_STATE_CONTEXT_COUNT(current_state) > 0) {
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, PMC_STATE_FLAGS_DEALLOCING, 0);
} else {
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, 0, 0);
}
break;
}
break;
case PMC_STATE_EVENT_INTERRUPT:
switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
case PMC_STATE(PMC_STATE_STATE_RUN, 0, 0):
case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
new_state = PMC_STATE_MODIFY(current_state, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING, 0);
break;
}
break;
case PMC_STATE_EVENT_END_OF_INTERRUPT:
switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, PMC_STATE_FLAGS_DEALLOCING):
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, 0, PMC_STATE_FLAGS_DEALLOCING);
break;
case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, PMC_STATE_FLAGS_STOPPING):
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STOP, 0, 0, PMC_STATE_FLAGS_STOPPING);
break;
case PMC_STATE(PMC_STATE_STATE_INTERRUPT, 0, 0):
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_CAN_RUN, 0, 0, 0);
break;
}
break;
case PMC_STATE_EVENT_CONTEXT_IN:
switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
case PMC_STATE(PMC_STATE_STATE_CAN_RUN, 0, 0):
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_LOAD, 1, 0, 0);
break;
case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
case PMC_STATE(PMC_STATE_STATE_RUN, 0, 0):
case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
new_state = PMC_STATE_MODIFY(current_state, 1, 0, 0);
break;
case PMC_STATE(PMC_STATE_STATE_STOP, 0, 0):
if (PMC_STATE_CONTEXT_COUNT(current_state) > 0) {
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_LOAD, 1, 0, 0);
}
break;
}
break;
case PMC_STATE_EVENT_CONTEXT_OUT:
switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
case PMC_STATE(PMC_STATE_STATE_DEALLOC, 0, PMC_STATE_FLAGS_DEALLOCING):
if (PMC_STATE_CONTEXT_COUNT(current_state) > 1) {
new_state = PMC_STATE_MODIFY(current_state, -1, 0, PMC_STATE_FLAGS_DEALLOCING);
} else {
new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
}
break;
case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_DEALLOCING):
case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
if (PMC_STATE_CONTEXT_COUNT(current_state) > 1) {
new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
}
break;
case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_DEALLOCING):
case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_RUN, 0, PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_RUN, 0, 0):
if (PMC_STATE_CONTEXT_COUNT(current_state) == 1) {
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STORE, -1, 0, 0);
} else {
new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
}
break;
case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
if (PMC_STATE_CONTEXT_COUNT(current_state) == 1) {
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_INTERRUPT, -1, 0, PMC_STATE_FLAGS_INTERRUPTING);
} else {
new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
}
break;
case PMC_STATE(PMC_STATE_STATE_STOP, 0, PMC_STATE_FLAGS_STOPPING):
if (PMC_STATE_CONTEXT_COUNT(current_state) == 1) {
new_state = PMC_STATE_MODIFY(current_state, -1, 0, PMC_STATE_FLAGS_STOPPING);
} else {
new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
}
break;
case PMC_STATE(PMC_STATE_STATE_STOP, 0, 0):
if (PMC_STATE_CONTEXT_COUNT(current_state) > 0) {
if (PMC_STATE_CONTEXT_COUNT(current_state) == 1) {
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_CAN_RUN, -1, 0, 0);
} else {
new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
}
}
break;
case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_DEALLOCING):
case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
if (PMC_STATE_CONTEXT_COUNT(current_state) > 0) {
new_state = PMC_STATE_MODIFY(current_state, -1, 0, 0);
}
break;
}
break;
case PMC_STATE_EVENT_LOAD_FINISHED:
switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_DEALLOCING):
case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
case PMC_STATE(PMC_STATE_STATE_LOAD, 0, PMC_STATE_FLAGS_STOPPING):
if (PMC_STATE_CONTEXT_COUNT(current_state) > 1) {
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_RUN, -1, 0, 0);
} else {
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STORE, -1, 0, 0);
}
break;
case PMC_STATE(PMC_STATE_STATE_LOAD, 0, 0):
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_RUN, 0, 0, 0);
break;
}
break;
case PMC_STATE_EVENT_STORE_FINISHED:
switch (current_state & ~(PMC_STATE_CONTEXT_COUNT_MASK)) {
case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_DEALLOCING):
if (PMC_STATE_CONTEXT_COUNT(current_state) == 0) {
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, 0, PMC_STATE_FLAGS_DEALLOCING);
} else {
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_DEALLOC, 0, 0, 0);
}
break;
case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_DEALLOCING):
case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_INTERRUPTING | PMC_STATE_FLAGS_STOPPING):
if (PMC_STATE_CONTEXT_COUNT(current_state) == 0) {
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_INTERRUPT, 0, 0, PMC_STATE_FLAGS_INTERRUPTING);
} else {
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STOP, 0, 0, 0);
}
break;
case PMC_STATE(PMC_STATE_STATE_STORE, 0, PMC_STATE_FLAGS_STOPPING):
if (PMC_STATE_CONTEXT_COUNT(current_state) == 0) {
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STOP, 0, 0, PMC_STATE_FLAGS_STOPPING);
} else {
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_STOP, 0, 0, 0);
}
break;
case PMC_STATE(PMC_STATE_STATE_STORE, 0, 0):
if (PMC_STATE_CONTEXT_COUNT(current_state) == 0) {
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_CAN_RUN, 0, 0, 0);
} else {
new_state = PMC_STATE_MOVE(current_state, PMC_STATE_STATE_LOAD, 0, 0, 0);
}
break;
}
break;
}
return new_state;
}
static uint32_t pmc_internal_reservation_move_for_event(pmc_reservation_t reservation, pmc_state_event_t event, pmc_state_t *old_state_out) {
pmc_state_t oldState;
pmc_state_t newState;
assert(reservation);
do {
oldState = reservation->state;
newState = pmc_internal_reservation_next_state(oldState, event);
} while (newState != PMC_STATE_INVALID && !OSCompareAndSwap(oldState, newState, &(reservation->state)));
if (newState != PMC_STATE_INVALID) {
COUNTER_DEBUG("Moved reservation %p from state "PMC_STATE_FORMAT" to state "PMC_STATE_FORMAT" for event %s\n", reservation, PMC_STATE_ARGS(oldState), PMC_STATE_ARGS(newState), pmc_state_event_name(event));
} else {
COUNTER_DEBUG("No valid moves for reservation %p in state "PMC_STATE_FORMAT" for event %s\n", reservation, PMC_STATE_ARGS(oldState), pmc_state_event_name(event));
}
if (old_state_out != NULL) {
*old_state_out = oldState;
}
return newState;
}
static void pmc_internal_reservation_context_out(pmc_reservation_t reservation) {
assert(reservation);
pmc_state_t newState;
pmc_state_t oldState;
OSBitAndAtomic(~(1U << cpu_number()), &(reservation->active_last_context_in));
if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_CONTEXT_OUT, &oldState))) {
return;
}
if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_STORE && PMC_STATE_STATE(oldState) != PMC_STATE_STATE_STORE) {
pmc_internal_reservation_store(reservation);
} else if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_DEALLOC && PMC_STATE_CONTEXT_COUNT(newState) == 0 && PMC_STATE_FLAGS(newState) == 0) {
thread_wakeup((event_t)reservation);
}
}
static void pmc_internal_reservation_context_in(pmc_reservation_t reservation) {
assert(reservation);
pmc_state_t oldState;
pmc_state_t newState;
if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_CONTEXT_IN, &oldState))) {
return;
}
OSBitOrAtomic(1U << cpu_number(), &(reservation->active_last_context_in));
if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_LOAD && PMC_STATE_STATE(oldState) != PMC_STATE_STATE_LOAD) {
pmc_internal_reservation_load(reservation);
}
}
static void pmc_internal_reservation_store(pmc_reservation_t reservation) {
assert(reservation);
assert(PMC_STATE_STATE(reservation->state) == PMC_STATE_STATE_STORE);
assert(reservation->pmc);
assert(reservation->config);
pmc_state_t newState;
kern_return_t ret = KERN_SUCCESS;
pmc_t store_pmc = reservation->pmc;
pmc_object_t store_pmc_obj = store_pmc->object;
perf_monitor_t store_pm = store_pmc->monitor;
ret = store_pm->methods.disable_counters(store_pm->object, &store_pmc_obj, 1);
if(KERN_SUCCESS != ret) {
COUNTER_DEBUG(" [error] disable_counters: 0x%x\n", ret);
return;
}
ret = store_pmc->methods.disable(store_pmc_obj);
if(KERN_SUCCESS != ret) {
COUNTER_DEBUG(" [error] disable: 0x%x\n", ret);
}
ret = store_pmc->methods.get_count(store_pmc_obj, &reservation->value);
if(KERN_SUCCESS != ret) {
COUNTER_DEBUG(" [error] get_count: 0x%x\n", ret);
return;
}
if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_STORE_FINISHED, NULL))) {
return;
}
if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_LOAD) {
pmc_internal_reservation_load(reservation);
} else if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_DEALLOC && PMC_STATE_CONTEXT_COUNT(newState) == 0 && PMC_STATE_FLAGS(newState) == 0) {
thread_wakeup((event_t)reservation);
}
}
static void pmc_internal_reservation_load(pmc_reservation_t reservation) {
assert(reservation);
assert(PMC_STATE_STATE(reservation->state) == PMC_STATE_STATE_LOAD);
pmc_state_t newState;
kern_return_t ret = KERN_SUCCESS;
assert(reservation->pmc);
assert(reservation->config);
pmc_t load_pmc = reservation->pmc;
pmc_object_t load_pmc_obj = load_pmc->object;
perf_monitor_t load_pm = load_pmc->monitor;
ret = load_pmc->methods.set_config(load_pmc_obj, reservation->config->object);
if(KERN_SUCCESS != ret) {
COUNTER_DEBUG(" [error] set_config: 0x%x\n", ret);
return;
}
ret = load_pmc->methods.set_count(load_pmc_obj, reservation->value);
if(KERN_SUCCESS != ret) {
COUNTER_DEBUG(" [error] set_count: 0x%x\n", ret);
return;
}
ret = load_pmc->methods.enable(load_pmc_obj);
if(KERN_SUCCESS != ret) {
COUNTER_DEBUG(" [error] enable: 0x%x\n", ret);
return;
}
ret = load_pm->methods.enable_counters(load_pm->object, &load_pmc_obj, 1);
if(KERN_SUCCESS != ret) {
COUNTER_DEBUG(" [error] enable_counters: 0x%x\n", ret);
return;
}
if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_LOAD_FINISHED, NULL))) {
return;
}
if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_STORE) {
pmc_internal_reservation_store(reservation);
}
}
static inline boolean_t pmc_accessible_from_core(pmc_t pmc, uint32_t logicalCore) {
boolean_t ret = FALSE;
assert(pmc);
ret = pmc->methods.accessible_from_core(pmc->object, logicalCore);
return ret;
}
static void pmc_internal_reservation_start_cpu(void * arg) {
pmc_reservation_t reservation = (pmc_reservation_t)arg;
assert(reservation);
if (pmc_internal_reservation_matches_context(reservation)) {
uint32_t oldMask = OSBitOrAtomic(1U << cpu_number(), &(reservation->active_last_context_in));
if ((oldMask & (1U << cpu_number())) == 0) {
COUNTER_DEBUG("Starting already in-context reservation %p for cpu %d\n", reservation, cpu_number());
pmc_internal_reservation_context_in(reservation);
}
}
}
static void pmc_internal_reservation_stop_cpu(void * arg) {
pmc_reservation_t reservation = (pmc_reservation_t)arg;
assert(reservation);
if (pmc_internal_reservation_matches_context(reservation)) {
COUNTER_DEBUG("Stopping in-context reservation %p for cpu %d\n", reservation, cpu_number());
pmc_internal_reservation_context_out(reservation);
}
}
static void pmc_reservation_interrupt(void *target, void *refCon) {
pmc_reservation_t reservation = (pmc_reservation_t)target;
pmc_state_t newState;
uint64_t timeout;
uint32_t spins;
assert(reservation);
if (PMC_STATE_INVALID == pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_INTERRUPT, NULL)) {
return;
}
pmc_internal_reservation_broadcast(reservation, pmc_internal_reservation_stop_cpu);
nanoseconds_to_absolutetime(PMC_SPIN_TIMEOUT_US * 1000, &timeout);
timeout += mach_absolute_time();
spins = 0;
while (PMC_STATE_STATE(reservation->state) != PMC_STATE_STATE_INTERRUPT) {
if (++spins > PMC_SPIN_THRESHOLD) {
if (mach_absolute_time() > timeout) {
pmc_spin_timeout_count++;
assert(0);
}
}
cpu_pause();
}
assert(reservation->config);
assert(reservation->config->method);
#if DEBUG_COUNTERS
uint64_t start = mach_absolute_time();
#endif
(void)reservation->config->method(reservation, refCon);
#if DEBUG_COUNTERS
uint64_t end = mach_absolute_time();
if((end - start) > 5000ULL) {
kprintf("%s - user method %p took %llu ns\n", __FUNCTION__,
reservation->config->method, (end - start));
}
#endif
if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_END_OF_INTERRUPT, NULL))) {
return;
}
if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_CAN_RUN) {
pmc_internal_reservation_broadcast(reservation, pmc_internal_reservation_start_cpu);
} else if (PMC_STATE_STATE(newState) == PMC_STATE_STATE_DEALLOC && PMC_STATE_CONTEXT_COUNT(newState) == 0 && PMC_STATE_FLAGS(newState) == 0) {
thread_wakeup((event_t)reservation);
}
}
#if 0
#pragma mark -
#pragma mark IOProfileFamily private KPI
#endif
kern_return_t perf_monitor_register(perf_monitor_object_t monitor,
perf_monitor_methods_t *methods) {
int cpu = -1;
COUNTER_DEBUG("registering perf monitor %p\n", monitor);
if(!monitor || !methods) {
return KERN_INVALID_ARGUMENT;
}
if(MACH_PERFMON_METHODS_VERSION != methods->perf_monitor_methods_version) {
return KERN_INVALID_ARGUMENT;
}
if (methods->flags & PERFMON_FLAG_REQUIRES_IDLE_NOTIFICATIONS) {
uint32_t *cores;
size_t core_cnt;
if (KERN_SUCCESS == methods->accessible_cores(monitor, &cores, &core_cnt)) {
if ((core_cnt == 1) && (cores[0] < (uint32_t)ml_get_max_cpus())) {
cpu = cores[0];
} else {
return KERN_INVALID_ARGUMENT;
}
}
}
if(!methods->accessible_cores |
!methods->enable_counters || !methods->disable_counters ||
!methods->on_idle || !methods->on_idle_exit) {
return KERN_INVALID_ARGUMENT;
}
perf_monitor_t dupe = perf_monitor_find(monitor);
if(dupe) {
COUNTER_DEBUG("Duplicate registration for %p\n", monitor);
perf_monitor_deallocate(dupe);
return KERN_FAILURE;
}
perf_monitor_t pm = perf_monitor_alloc();
if(!pm) {
return KERN_RESOURCE_SHORTAGE;
}
perf_monitor_init(pm, cpu);
pm->object = monitor;
memcpy(&(pm->methods), methods, sizeof(perf_monitor_methods_t));
perf_monitor_enqueue(pm);
PRINT_PERF_MON(pm);
return KERN_SUCCESS;
}
kern_return_t perf_monitor_unregister(perf_monitor_object_t monitor) {
kern_return_t ret = KERN_FAILURE;
COUNTER_DEBUG("unregistering perf monitor %p\n", monitor);
if(!monitor) {
return KERN_INVALID_ARGUMENT;
}
perf_monitor_t pm = perf_monitor_find(monitor);
if(pm) {
perf_monitor_dequeue(pm);
perf_monitor_deallocate(pm);
perf_monitor_deallocate(pm);
ret = KERN_SUCCESS;
} else {
COUNTER_DEBUG("could not find a registered pm that matches!\n");
}
return ret;
}
kern_return_t pmc_register(perf_monitor_object_t monitor, pmc_object_t pmc_object,
pmc_methods_t *methods, void *object) {
COUNTER_DEBUG("%p %p\n", monitor, pmc_object);
if(!monitor || !pmc_object || !methods || !object) {
return KERN_INVALID_ARGUMENT;
}
if(MACH_PMC_METHODS_VERSION != methods->pmc_methods_version) {
COUNTER_DEBUG("version mismatch\n");
return KERN_INVALID_ARGUMENT;
}
if(!methods->create_config ||
!methods->free_config ||
!methods->config_set_value ||
!methods->config_set_threshold ||
!methods->config_set_handler ||
!methods->set_config ||
!methods->get_monitor ||
!methods->get_name ||
!methods->accessible_from_core ||
!methods->accessible_cores ||
!methods->get_count ||
!methods->set_count ||
!methods->disable ||
!methods->enable ||
!methods->open ||
!methods->close) {
return KERN_INVALID_ARGUMENT;
}
perf_monitor_t pm = perf_monitor_find(monitor);
if(!pm) {
COUNTER_DEBUG("Could not find perf monitor for %p\n", monitor);
return KERN_INVALID_ARGUMENT;
}
pmc_t pmc = pmc_alloc();
if(!pmc) {
perf_monitor_deallocate(pm);
return KERN_RESOURCE_SHORTAGE;
}
pmc_init(pmc);
pmc->object = pmc_object;
pmc->open_object = object;
memcpy(&(pmc->methods), methods, sizeof(pmc_methods_t));
pmc->monitor = pm;
perf_monitor_add_pmc(pmc->monitor, pmc);
pmc_enqueue(pmc);
perf_monitor_deallocate(pm);
return KERN_SUCCESS;
}
kern_return_t pmc_unregister(perf_monitor_object_t monitor, pmc_object_t pmc_object) {
COUNTER_DEBUG("%p %p\n", monitor, pmc_object);
if(!monitor || !pmc_object) {
return KERN_INVALID_ARGUMENT;
}
pmc_t pmc = pmc_find(pmc_object);
if(!pmc) {
COUNTER_DEBUG("Could not find a matching pmc.\n");
return KERN_FAILURE;
}
pmc_dequeue(pmc);
perf_monitor_remove_pmc(pmc->monitor, pmc);
pmc_deallocate(pmc);
pmc_deallocate(pmc);
return KERN_SUCCESS;
}
static void perf_monitor_reservation_add(perf_monitor_t monitor) {
assert(monitor);
OSIncrementAtomic(&(monitor->reservedCounters));
}
static void perf_monitor_reservation_remove(perf_monitor_t monitor) {
assert(monitor);
OSDecrementAtomic(&(monitor->reservedCounters));
}
#if 0
#pragma mark -
#pragma mark KPI
#endif
kern_return_t pmc_create_config(pmc_t pmc, pmc_config_t *config) {
pmc_config_t tmp = NULL;
if(!pmc || !config) {
return KERN_INVALID_ARGUMENT;
}
pmc_reference(pmc);
tmp = pmc_config_alloc(pmc);
if(tmp) {
tmp->object = pmc->methods.create_config(pmc->object);
if(!tmp->object) {
pmc_config_free(pmc, tmp);
tmp = NULL;
} else {
tmp->interrupt_after_value = 0ULL;
tmp->method = NULL;
tmp->refCon = NULL;
}
}
pmc_deallocate(pmc);
if(!tmp) {
return KERN_RESOURCE_SHORTAGE;
}
*config = tmp;
return KERN_SUCCESS;
}
void pmc_free_config(pmc_t pmc, pmc_config_t config) {
assert(pmc);
assert(config);
pmc_reference(pmc);
pmc_config_free(pmc, config);
pmc_deallocate(pmc);
}
kern_return_t pmc_config_set_value(pmc_t pmc, pmc_config_t config,
uint8_t id, uint64_t value) {
kern_return_t ret = KERN_INVALID_ARGUMENT;
if(!pmc || !config) {
return ret;
}
pmc_reference(pmc);
ret = pmc->methods.config_set_value(config->object, id, value);
pmc_deallocate(pmc);
return ret;
}
kern_return_t pmc_config_set_interrupt_threshold(pmc_t pmc, pmc_config_t config,
uint64_t threshold, pmc_interrupt_method_t method, void *refCon) {
kern_return_t ret = KERN_INVALID_ARGUMENT;
if(!config || !pmc) {
return ret;
}
assert(config);
assert(pmc);
pmc_reference(pmc);
do {
config->interrupt_after_value = threshold;
config->method = method;
config->refCon = refCon;
ret = KERN_SUCCESS;
}while(0);
pmc_deallocate(pmc);
return ret;
}
kern_return_t pmc_get_pmc_list(pmc_t **pmcs, size_t *pmcCount) {
pmc_t *array = NULL;
pmc_t pmc = NULL;
size_t count = 0UL;
do {
vm_size_t size = perf_counters_count;
array = (pmc_t *)kalloc(sizeof(pmc_t) * size);
if(!array) {
return KERN_RESOURCE_SHORTAGE;
}
lck_spin_lock(&perf_counters_queue_spin);
if(size != perf_counters_count) {
lck_spin_unlock(&perf_counters_queue_spin);
kfree(array, sizeof(pmc_t) * size);
array = NULL;
}
}while(!array);
queue_iterate(perf_counters_queue, pmc, pmc_t, link) {
array[count++] = pmc;
}
lck_spin_unlock(&perf_counters_queue_spin);
*pmcs = array;
*pmcCount = count;
return KERN_SUCCESS;
}
void pmc_free_pmc_list(pmc_t *pmcs, size_t pmcCount) {
if(pmcs && pmcCount) {
COUNTER_DEBUG("pmcs: %p pmcCount: %lu\n", pmcs, pmcCount);
kfree(pmcs, pmcCount * sizeof(pmc_t));
}
}
kern_return_t pmc_find_by_name(const char *name, pmc_t **pmcs, size_t *pmcCount) {
kern_return_t ret = KERN_INVALID_ARGUMENT;
if(!name || !pmcs || !pmcCount) {
return ret;
}
pmc_t *list = NULL;
size_t count = 0UL;
if(KERN_SUCCESS == (ret = pmc_get_pmc_list(&list, &count))) {
size_t matchCount = 0UL, ii = 0UL, swapPtr = 0UL;
size_t len = strlen(name);
for(ii = 0UL; ii < count; ii++) {
const char *pmcName = pmc_get_name(list[ii]);
if(strlen(pmcName) < len) {
continue;
}
if(0 == strncmp(name, pmcName, len)) {
pmc_t temp = list[ii];
list[ii] = list[swapPtr];
list[swapPtr] = temp;
swapPtr++;
matchCount++;
}
}
if(matchCount) {
pmc_t *result = (pmc_t *)kalloc(sizeof(pmc_t) * matchCount);
if(result) {
memcpy(result, list, sizeof(pmc_t) * matchCount);
ret = KERN_SUCCESS;
}
pmc_free_pmc_list(list, count);
if(!result) {
*pmcs = NULL;
*pmcCount = 0UL;
return KERN_RESOURCE_SHORTAGE;
}
*pmcs = result;
*pmcCount = matchCount;
} else {
*pmcs = NULL;
*pmcCount = 0UL;
}
}
return ret;
}
const char *pmc_get_name(pmc_t pmc) {
assert(pmc);
const char *name = pmc->methods.get_name(pmc->object);
return name;
}
kern_return_t pmc_get_accessible_core_list(pmc_t pmc, uint32_t **logicalCores,
size_t *logicalCoreCt) {
kern_return_t ret = KERN_INVALID_ARGUMENT;
if(!pmc || !logicalCores || !logicalCoreCt) {
return ret;
}
ret = pmc->methods.accessible_cores(pmc->object, logicalCores, logicalCoreCt);
return ret;
}
static boolean_t pmc_reservation_setup_pmi(pmc_reservation_t resv, pmc_config_t config) {
assert(resv);
assert(resv->pmc);
assert(config);
assert(config->object);
if(config->interrupt_after_value && config->method) {
kern_return_t ret = resv->pmc->methods.config_set_threshold(config->object,
config->interrupt_after_value);
if(KERN_SUCCESS != ret) {
COUNTER_DEBUG("Failed to set threshold for pmc %p\n", resv->pmc);
return FALSE;
}
if(KERN_SUCCESS != resv->pmc->methods.config_set_handler(config->object,
(void *)resv, &pmc_reservation_interrupt, config->refCon)) {
COUNTER_DEBUG("Failed to set handler for pmc %p\n", resv->pmc);
return FALSE;
}
}
return TRUE;
}
kern_return_t pmc_reserve(pmc_t pmc, pmc_config_t config,
pmc_reservation_t *reservation) {
if(!pmc || !config || !reservation) {
return KERN_INVALID_ARGUMENT;
}
pmc_reservation_t resv = reservation_alloc();
if(!resv) {
return KERN_RESOURCE_SHORTAGE;
}
reservation_init(resv);
resv->flags |= PMC_FLAG_SCOPE_SYSTEM;
resv->config = config;
if(KERN_SUCCESS != pmc_internal_reservation_set_pmc(resv, pmc)) {
resv->config = NULL;
return KERN_FAILURE;
}
if(!pmc_internal_reservation_add(resv) || !pmc_reservation_setup_pmi(resv, config)) {
resv->config = NULL;
reservation_free(resv);
return KERN_FAILURE;
}
perf_monitor_reservation_add(pmc->monitor);
*reservation = resv;
return KERN_SUCCESS;
}
kern_return_t pmc_reserve_task(pmc_t pmc, pmc_config_t config,
task_t task, pmc_reservation_t *reservation) {
if(!pmc || !config || !reservation || !task) {
return KERN_INVALID_ARGUMENT;
}
if (!(pmc->monitor->methods.flags & PERFMON_FLAG_SUPPORTS_CONTEXT_SWITCHING)) {
COUNTER_DEBUG("pmc %p cannot be context switched!\n", pmc);
return KERN_INVALID_ARGUMENT;
}
pmc_reservation_t resv = reservation_alloc();
if(!resv) {
return KERN_RESOURCE_SHORTAGE;
}
reservation_init(resv);
resv->flags |= PMC_FLAG_SCOPE_TASK;
resv->task = task;
resv->config = config;
if(KERN_SUCCESS != pmc_internal_reservation_set_pmc(resv, pmc)) {
resv->config = NULL;
return KERN_FAILURE;
}
if(!pmc_internal_reservation_add(resv) || !pmc_reservation_setup_pmi(resv, config)) {
resv->config = NULL;
reservation_free(resv);
return KERN_FAILURE;
}
perf_monitor_reservation_add(pmc->monitor);
*reservation = resv;
return KERN_SUCCESS;
}
kern_return_t pmc_reserve_thread(pmc_t pmc, pmc_config_t config,
thread_t thread, pmc_reservation_t *reservation) {
if(!pmc || !config || !reservation || !thread) {
return KERN_INVALID_ARGUMENT;
}
if (!(pmc->monitor->methods.flags & PERFMON_FLAG_SUPPORTS_CONTEXT_SWITCHING)) {
COUNTER_DEBUG("pmc %p cannot be context switched!\n", pmc);
return KERN_INVALID_ARGUMENT;
}
pmc_reservation_t resv = reservation_alloc();
if(!resv) {
return KERN_RESOURCE_SHORTAGE;
}
reservation_init(resv);
resv->flags |= PMC_FLAG_SCOPE_THREAD;
resv->thread = thread;
resv->config = config;
if(KERN_SUCCESS != pmc_internal_reservation_set_pmc(resv, pmc)) {
resv->config = NULL;
return KERN_FAILURE;
}
if(!pmc_internal_reservation_add(resv) || !pmc_reservation_setup_pmi(resv, config)) {
resv->config = NULL;
reservation_free(resv);
return KERN_FAILURE;
}
perf_monitor_reservation_add(pmc->monitor);
*reservation = resv;
return KERN_SUCCESS;
}
kern_return_t pmc_reservation_start(pmc_reservation_t reservation) {
pmc_state_t newState;
if(!reservation) {
return KERN_INVALID_ARGUMENT;
}
if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_START, NULL))) {
return KERN_FAILURE;
}
if (PMC_STATE_STATE(newState) != PMC_STATE_STATE_INTERRUPT) {
pmc_internal_reservation_broadcast(reservation, pmc_internal_reservation_start_cpu);
}
return KERN_SUCCESS;
}
kern_return_t pmc_reservation_stop(pmc_reservation_t reservation) {
pmc_state_t newState;
if(!reservation) {
return KERN_INVALID_ARGUMENT;
}
if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_STOP, NULL))) {
return KERN_FAILURE;
}
if (PMC_STATE_STATE(newState) != PMC_STATE_STATE_INTERRUPT && PMC_STATE_STATE(newState) != PMC_STATE_STATE_STOP) {
pmc_internal_reservation_broadcast(reservation, pmc_internal_reservation_stop_cpu);
}
return KERN_SUCCESS;
}
kern_return_t pmc_reservation_read(pmc_reservation_t reservation, uint64_t *value) {
kern_return_t ret = KERN_FAILURE;
uint64_t timeout;
uint32_t spins;
if(!reservation || !value) {
return KERN_INVALID_ARGUMENT;
}
nanoseconds_to_absolutetime(PMC_SPIN_TIMEOUT_US * 1000, &timeout);
timeout += mach_absolute_time();
spins = 0;
do {
uint32_t state = reservation->state;
if((PMC_STATE_STATE(state) == PMC_STATE_STATE_RUN)) {
assert(reservation->pmc);
ret = reservation->pmc->methods.get_count(reservation->pmc->object, value);
break;
} else if ((PMC_STATE_STATE(state) == PMC_STATE_STATE_STORE) ||
(PMC_STATE_STATE(state) == PMC_STATE_STATE_LOAD)) {
if (++spins > PMC_SPIN_THRESHOLD) {
if (mach_absolute_time() > timeout) {
pmc_spin_timeout_count++;
assert(0);
}
}
cpu_pause();
} else {
break;
}
} while (1);
if(KERN_SUCCESS != ret) {
*value = reservation->value;
}
return KERN_SUCCESS;
}
kern_return_t pmc_reservation_write(pmc_reservation_t reservation, uint64_t value) {
kern_return_t ret = KERN_FAILURE;
uint64_t timeout;
uint32_t spins;
if(!reservation) {
return KERN_INVALID_ARGUMENT;
}
nanoseconds_to_absolutetime(PMC_SPIN_TIMEOUT_US * 1000, &timeout);
timeout += mach_absolute_time();
spins = 0;
do {
uint32_t state = reservation->state;
if((PMC_STATE_STATE(state) == PMC_STATE_STATE_RUN)) {
assert(reservation->pmc);
ret = reservation->pmc->methods.set_count(reservation->pmc->object, value);
break;
} else if ((PMC_STATE_STATE(state) == PMC_STATE_STATE_STORE) ||
(PMC_STATE_STATE(state) == PMC_STATE_STATE_LOAD)) {
if (++spins > PMC_SPIN_THRESHOLD) {
if (mach_absolute_time() > timeout) {
pmc_spin_timeout_count++;
assert(0);
}
}
cpu_pause();
} else {
break;
}
} while (1);
if(KERN_SUCCESS != ret) {
reservation->value = value;
}
return KERN_SUCCESS;
}
kern_return_t pmc_reservation_free(pmc_reservation_t reservation) {
pmc_state_t newState;
if(!reservation) {
return KERN_INVALID_ARGUMENT;
}
perf_monitor_reservation_remove(reservation->pmc->monitor);
if (PMC_STATE_INVALID == (newState = pmc_internal_reservation_move_for_event(reservation, PMC_STATE_EVENT_FREE, NULL))) {
return KERN_FAILURE;
}
if (PMC_STATE_STATE(newState) != PMC_STATE_STATE_DEALLOC) {
pmc_internal_reservation_broadcast(reservation, pmc_internal_reservation_stop_cpu);
}
while (!(PMC_STATE_STATE(reservation->state) == PMC_STATE_STATE_DEALLOC && PMC_STATE_CONTEXT_COUNT(reservation->state) == 0 && PMC_STATE_FLAGS(reservation->state) == 0)) {
assert_wait((event_t)reservation, THREAD_UNINT);
thread_block(THREAD_CONTINUE_NULL);
}
pmc_internal_reservation_remove(reservation);
reservation_free(reservation);
return KERN_SUCCESS;
}
boolean_t pmc_idle(void) {
perf_monitor_t monitor = NULL;
queue_head_t *cpu_queue;
lck_spin_lock(&perf_monitor_queue_spin);
if (cpu_monitor_queues) {
cpu_queue = cpu_monitor_queues[cpu_number()];
queue_iterate(cpu_queue, monitor, perf_monitor_t, cpu_link) {
perf_monitor_methods_t *methods = &(monitor->methods);
if ((methods->flags & PERFMON_FLAG_ALWAYS_ACTIVE) || (monitor->reservedCounters)) {
methods->on_idle(monitor->object);
}
}
}
lck_spin_unlock(&perf_monitor_queue_spin);
return TRUE;
}
boolean_t pmc_idle_exit(void) {
perf_monitor_t monitor = NULL;
queue_head_t *cpu_queue;
lck_spin_lock(&perf_monitor_queue_spin);
if (cpu_monitor_queues) {
cpu_queue = cpu_monitor_queues[cpu_number()];
queue_iterate(cpu_queue, monitor, perf_monitor_t, cpu_link) {
perf_monitor_methods_t *methods = &(monitor->methods);
if ((methods->flags & PERFMON_FLAG_ALWAYS_ACTIVE) || (monitor->reservedCounters)) {
methods->on_idle_exit(monitor->object);
}
}
}
lck_spin_unlock(&perf_monitor_queue_spin);
return TRUE;
}
boolean_t pmc_context_switch(thread_t oldThread, thread_t newThread) {
pmc_reservation_t resv = NULL;
uint32_t cpuNum = cpu_number();
lck_spin_lock(&reservations_spin);
if (thread_reservation_count) {
queue_iterate(thread_reservations, resv, pmc_reservation_t, link) {
if ((oldThread == resv->thread) && pmc_accessible_from_core(resv->pmc, cpuNum)) {
(void)pmc_internal_reservation_context_out(resv);
}
}
}
if (task_reservation_count) {
queue_iterate(task_reservations, resv, pmc_reservation_t, link) {
if ((resv->task == oldThread->task) && pmc_accessible_from_core(resv->pmc, cpuNum)) {
(void)pmc_internal_reservation_context_out(resv);
}
}
}
if (thread_reservation_count) {
queue_iterate(thread_reservations, resv, pmc_reservation_t, link) {
if ((resv->thread == newThread) && pmc_accessible_from_core(resv->pmc, cpuNum)) {
(void)pmc_internal_reservation_context_in(resv);
}
}
}
if (task_reservation_count) {
queue_iterate(task_reservations, resv, pmc_reservation_t, link) {
if ((resv->task == newThread->task) && pmc_accessible_from_core(resv->pmc, cpuNum)) {
(void)pmc_internal_reservation_context_in(resv);
}
}
}
lck_spin_unlock(&reservations_spin);
return TRUE;
}
#else
#if 0
#pragma mark -
#pragma mark Dummy functions
#endif
kern_return_t perf_monitor_register(perf_monitor_object_t monitor __unused,
perf_monitor_methods_t *methods __unused) {
return KERN_FAILURE;
}
kern_return_t perf_monitor_unregister(perf_monitor_object_t monitor __unused) {
return KERN_FAILURE;
}
kern_return_t pmc_register(perf_monitor_object_t monitor __unused,
pmc_object_t pmc __unused, pmc_methods_t *methods __unused, void *object __unused) {
return KERN_FAILURE;
}
kern_return_t pmc_unregister(perf_monitor_object_t monitor __unused,
pmc_object_t pmc __unused) {
return KERN_FAILURE;
}
kern_return_t pmc_create_config(pmc_t pmc __unused,
pmc_config_t *config __unused) {
return KERN_FAILURE;
}
void pmc_free_config(pmc_t pmc __unused, pmc_config_t config __unused) {
}
kern_return_t pmc_config_set_value(pmc_t pmc __unused,
pmc_config_t config __unused, uint8_t id __unused,
uint64_t value __unused) {
return KERN_FAILURE;
}
kern_return_t pmc_config_set_interrupt_threshold(pmc_t pmc __unused,
pmc_config_t config __unused, uint64_t threshold __unused,
pmc_interrupt_method_t method __unused, void *refCon __unused) {
return KERN_FAILURE;
}
kern_return_t pmc_get_pmc_list(pmc_t **pmcs __unused, size_t *pmcCount __unused) {
return KERN_FAILURE;
}
void pmc_free_pmc_list(pmc_t *pmcs __unused, size_t pmcCount __unused) {
}
kern_return_t pmc_find_by_name(const char *name __unused, pmc_t **pmcs __unused,
size_t *pmcCount __unused) {
return KERN_FAILURE;
}
const char *pmc_get_name(pmc_t pmc __unused) {
return "";
}
kern_return_t pmc_get_accessible_core_list(pmc_t pmc __unused,
uint32_t **logicalCores __unused, size_t *logicalCoreCt __unused) {
return KERN_FAILURE;
}
kern_return_t pmc_reserve(pmc_t pmc __unused,
pmc_config_t config __unused, pmc_reservation_t *reservation __unused) {
return KERN_FAILURE;
}
kern_return_t pmc_reserve_task(pmc_t pmc __unused,
pmc_config_t config __unused, task_t task __unused,
pmc_reservation_t *reservation __unused) {
return KERN_FAILURE;
}
kern_return_t pmc_reserve_thread(pmc_t pmc __unused,
pmc_config_t config __unused, thread_t thread __unused,
pmc_reservation_t *reservation __unused) {
return KERN_FAILURE;
}
kern_return_t pmc_reservation_start(pmc_reservation_t reservation __unused) {
return KERN_FAILURE;
}
kern_return_t pmc_reservation_stop(pmc_reservation_t reservation __unused) {
return KERN_FAILURE;
}
kern_return_t pmc_reservation_read(pmc_reservation_t reservation __unused,
uint64_t *value __unused) {
return KERN_FAILURE;
}
kern_return_t pmc_reservation_write(pmc_reservation_t reservation __unused,
uint64_t value __unused) {
return KERN_FAILURE;
}
kern_return_t pmc_reservation_free(pmc_reservation_t reservation __unused) {
return KERN_FAILURE;
}
#endif