#include <mach/mach_types.h>
#include <machine/machine_routines.h>
#include <kern/processor.h>
#include <kern/kalloc.h>
#include <kern/thread.h>
#include <sys/errno.h>
#include <arm/cpu_data_internal.h>
#include <arm/cpu_internal.h>
#include <kern/kpc.h>
#ifdef ARMA7
static uint32_t saved_PMXEVTYPER[MAX_CPUS][KPC_ARM_TOTAL_COUNT];
static uint32_t saved_PMCNTENSET[MAX_CPUS];
static uint64_t saved_counter[MAX_CPUS][KPC_ARM_TOTAL_COUNT];
static uint32_t saved_PMOVSR[MAX_CPUS];
static uint32_t kpc_configured = 0;
static uint32_t kpc_xcall_sync;
static uint64_t kpc_running_cfg_pmc_mask = 0;
static uint32_t kpc_running_classes = 0;
static uint32_t kpc_reload_sync;
static uint32_t kpc_enabled_counters = 0;
static int first_time = 1;
static boolean_t
enable_counter(uint32_t counter)
{
boolean_t enabled;
uint32_t PMCNTENSET;
uint32_t mask = (counter == 0) ? (1 << 31) : (1 << (counter - 1));
__asm__ volatile("mrc p15, 0, %0, c9, c12, 1;" : "=r" (PMCNTENSET));
enabled = (PMCNTENSET & mask);
if (!enabled) {
__asm__ volatile("mcr p15, 0, %0, c9, c14, 1;" : : "r" (mask));
__asm__ volatile("mcr p15, 0, %0, c9, c12, 1;" : : "r" (mask));
kpc_enabled_counters++;
if (kpc_enabled_counters == 1) {
uint32_t PMCR = 1;
__asm__ volatile("mcr p15, 0, %0, c9, c12, 0;" : : "r" (PMCR));
}
}
return enabled;
}
static boolean_t
disable_counter(uint32_t counter)
{
boolean_t enabled;
uint32_t PMCNTENCLR;
uint32_t mask = (counter == 0) ? (1 << 31) : (1 << (counter - 1));
__asm__ volatile("mrc p15, 0, %0, c9, c12, 2;" : "=r" (PMCNTENCLR));
enabled = (PMCNTENCLR & mask);
if (enabled) {
__asm__ volatile("mcr p15, 0, %0, c9, c12, 2;" : : "r" (mask));
__asm__ volatile("mcr p15, 0, %0, c9, c14, 2;" : : "r" (mask));
kpc_enabled_counters--;
if (kpc_enabled_counters == 0) {
uint32_t PMCR = 0;
__asm__ volatile("mcr p15, 0, %0, c9, c12, 0;" : : "r" (PMCR));
}
}
return enabled;
}
static uint64_t
read_counter(uint32_t counter)
{
uint32_t low = 0;
switch (counter) {
case 0:
__asm__ volatile("mrc p15, 0, %0, c9, c13, 0;" : "=r" (low));
break;
case 1:
case 2:
case 3:
case 4:
__asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter - 1));
__asm__ volatile("mrc p15, 0, %0, c9, c13, 2;" : "=r" (low));
break;
default:
break;
}
return (uint64_t)low;
}
static void
write_counter(uint32_t counter, uint64_t value)
{
uint32_t low = value & 0xFFFFFFFF;
switch (counter) {
case 0:
__asm__ volatile("mcr p15, 0, %0, c9, c13, 0;" : : "r" (low));
break;
case 1:
case 2:
case 3:
case 4:
__asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter - 1));
__asm__ volatile("mcr p15, 0, %0, c9, c13, 2;" : : "r" (low));
break;
default:
break;
}
}
static uint64_t
kpc_reload_counter(int ctr)
{
uint64_t old = read_counter(ctr);
write_counter(ctr, FIXED_RELOAD(ctr));
return old;
}
static void
set_running_fixed(boolean_t on)
{
int i;
boolean_t enabled;
int n = KPC_ARM_FIXED_COUNT;
enabled = ml_set_interrupts_enabled(FALSE);
for( i = 0; i < n; i++ ) {
if (on) {
enable_counter(i);
} else {
disable_counter(i);
}
}
ml_set_interrupts_enabled(enabled);
}
static void
set_running_configurable(uint64_t target_mask, uint64_t state_mask)
{
uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count();
boolean_t enabled;
enabled = ml_set_interrupts_enabled(FALSE);
for (uint32_t i = 0; i < cfg_count; ++i) {
if (((1ULL << i) & target_mask) == 0)
continue;
assert(kpc_controls_counter(offset + i));
if ((1ULL << i) & state_mask) {
enable_counter(offset + i);
} else {
disable_counter(offset + i);
}
}
ml_set_interrupts_enabled(enabled);
}
void kpc_pmi_handler(cpu_id_t source);
void
kpc_pmi_handler(cpu_id_t source)
{
uint64_t extra;
int ctr;
int enabled;
enabled = ml_set_interrupts_enabled(FALSE);
if (source != getCpuDatap()->interrupt_nub) {
panic("pmi from IOCPU %p delivered to IOCPU %p", source, getCpuDatap()->interrupt_nub);
}
for (ctr = 0;
ctr < (KPC_ARM_FIXED_COUNT + KPC_ARM_CONFIGURABLE_COUNT);
ctr++)
{
uint32_t PMOVSR;
uint32_t mask;
if (ctr == 0) {
mask = 1 << 31;
} else {
mask = 1 << (ctr - 1);
}
__asm__ volatile("mrc p15, 0, %0, c9, c12, 3;" : "=r" (PMOVSR));
if (PMOVSR & mask) {
extra = kpc_reload_counter(ctr);
FIXED_SHADOW(ctr)
+= (kpc_fixed_max() - FIXED_RELOAD(ctr) + 1 ) + extra;
if (FIXED_ACTIONID(ctr))
kpc_sample_kperf(FIXED_ACTIONID(ctr));
__asm__ volatile("mcr p15, 0, %0, c9, c12, 3;" : : "r" (mask));
}
}
ml_set_interrupts_enabled(enabled);
}
static void
kpc_set_running_xcall( void *vstate )
{
struct kpc_running_remote *mp_config = (struct kpc_running_remote*) vstate;
assert(mp_config);
if (kpc_controls_fixed_counters())
set_running_fixed(mp_config->classes & KPC_CLASS_FIXED_MASK);
set_running_configurable(mp_config->cfg_target_mask,
mp_config->cfg_state_mask);
if (hw_atomic_sub(&kpc_xcall_sync, 1) == 0) {
thread_wakeup((event_t) &kpc_xcall_sync);
}
}
static uint64_t
get_counter_config(uint32_t counter)
{
uint32_t config = 0;
switch (counter) {
case 0:
counter = 31;
__asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter));
__asm__ volatile("mcr p15, 0, %0, c9, c13, 1;" : "=r" (config));
break;
case 1:
case 2:
case 3:
case 4:
counter -= 1;
__asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter));
__asm__ volatile("mrc p15, 0, %0, c9, c13, 1;" : "=r" (config));
break;
default:
break;
}
return config;
}
static void
set_counter_config(uint32_t counter, uint64_t config)
{
switch (counter) {
case 0:
__asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (31));
__asm__ volatile("mcr p15, 0, %0, c9, c13, 1;" : : "r" (config & 0xFFFFFFFF));
break;
case 1:
case 2:
case 3:
case 4:
__asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter - 1));
__asm__ volatile("mcr p15, 0, %0, c9, c13, 1;" : : "r" (config & 0xFFFFFFFF));
break;
default:
break;
}
}
void
kpc_arch_init(void)
{
uint32_t PMCR;
uint32_t event_counters;
__asm__ volatile("mrc p15, 0, %0, c9, c12, 0;" : "=r" (PMCR));
event_counters = (PMCR >> 11) & 0x1F;
assert(event_counters >= KPC_ARM_CONFIGURABLE_COUNT);
}
uint32_t
kpc_get_classes(void)
{
return KPC_CLASS_FIXED_MASK | KPC_CLASS_CONFIGURABLE_MASK;
}
uint32_t
kpc_fixed_count(void)
{
return KPC_ARM_FIXED_COUNT;
}
uint32_t
kpc_configurable_count(void)
{
return KPC_ARM_CONFIGURABLE_COUNT;
}
uint32_t
kpc_fixed_config_count(void)
{
return KPC_ARM_FIXED_COUNT;
}
uint32_t
kpc_configurable_config_count(uint64_t pmc_mask)
{
assert(kpc_popcount(pmc_mask) <= kpc_configurable_count());
return kpc_popcount(pmc_mask);
}
int
kpc_get_fixed_config(kpc_config_t *configv)
{
configv[0] = get_counter_config(0);
return 0;
}
uint64_t
kpc_fixed_max(void)
{
return (1ULL << KPC_ARM_COUNTER_WIDTH) - 1;
}
uint64_t
kpc_configurable_max(void)
{
return (1ULL << KPC_ARM_COUNTER_WIDTH) - 1;
}
int
kpc_get_configurable_counters(uint64_t *counterv, uint64_t pmc_mask)
{
uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count();
assert(counterv);
for (uint32_t i = 0; i < cfg_count; ++i) {
uint32_t PMOVSR;
uint32_t mask;
uint64_t ctr;
if (((1ULL << i) & pmc_mask) == 0)
continue;
ctr = read_counter(i + offset);
mask = 1 << i;
__asm__ volatile("mrc p15, 0, %0, c9, c12, 3;" : "=r" (PMOVSR));
if (PMOVSR & mask) {
ctr = CONFIGURABLE_SHADOW(i) +
(kpc_configurable_max() - CONFIGURABLE_RELOAD(i) + 1 ) +
ctr;
} else {
ctr = CONFIGURABLE_SHADOW(i) +
(ctr - CONFIGURABLE_RELOAD(i));
}
*counterv++ = ctr;
}
return 0;
}
int
kpc_get_fixed_counters(uint64_t *counterv)
{
uint32_t PMOVSR;
uint32_t mask;
uint64_t ctr;
mask = 1 << 31;
__asm__ volatile("mrc p15, 0, %0, c9, c12, 3;" : "=r" (PMOVSR));
ctr = read_counter(0);
if (PMOVSR & mask) {
ctr = FIXED_SHADOW(0) +
(kpc_fixed_max() - FIXED_RELOAD(0) + 1 ) +
(ctr & 0xFFFFFFFF);
} else {
ctr = FIXED_SHADOW(0) +
(ctr - FIXED_RELOAD(0));
}
counterv[0] = ctr;
return 0;
}
boolean_t
kpc_is_running_fixed(void)
{
return (kpc_running_classes & KPC_CLASS_FIXED_MASK) == KPC_CLASS_FIXED_MASK;
}
boolean_t
kpc_is_running_configurable(uint64_t pmc_mask)
{
assert(kpc_popcount(pmc_mask) <= kpc_configurable_count());
return ((kpc_running_classes & KPC_CLASS_CONFIGURABLE_MASK) == KPC_CLASS_CONFIGURABLE_MASK) &&
((kpc_running_cfg_pmc_mask & pmc_mask) == pmc_mask);
}
int
kpc_set_running_arch(struct kpc_running_remote *mp_config)
{
unsigned int cpu;
assert(mp_config);
if (first_time) {
kprintf( "kpc: setting PMI handler\n" );
PE_cpu_perfmon_interrupt_install_handler(kpc_pmi_handler);
for (cpu = 0; cpu < real_ncpus; cpu++)
PE_cpu_perfmon_interrupt_enable(cpu_datap(cpu)->cpu_id,
TRUE);
first_time = 0;
}
cpu_broadcast_xcall(&kpc_xcall_sync, TRUE, kpc_set_running_xcall,
mp_config);
kpc_running_cfg_pmc_mask = mp_config->cfg_state_mask;
kpc_running_classes = mp_config->classes;
kpc_configured = 1;
return 0;
}
static void
save_regs(void)
{
int i;
int cpuid = current_processor()->cpu_id;
uint32_t PMCR = 0;
__asm__ volatile("dmb ish");
__asm__ volatile("mcr p15, 0, %0, c9, c12, 0;" : : "r" (PMCR));
__asm__ volatile("mrc p15, 0, %0, c9, c12, 1;" : "=r" (saved_PMCNTENSET[cpuid]));
__asm__ volatile("mrc p15, 0, %0, c9, c12, 3;" : "=r" (saved_PMOVSR[cpuid]));
__asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (31));
__asm__ volatile("mrc p15, 0, %0, c9, c13, 1;" : "=r" (saved_PMXEVTYPER[cpuid][0]));
for (i = 0; i < 4; i++) {
__asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (i));
__asm__ volatile("mrc p15, 0, %0, c9, c13, 1;" : "=r" (saved_PMXEVTYPER[cpuid][i + 1]));
}
for (i=0; i < 5; i++) {
saved_counter[cpuid][i] = read_counter(i);
}
}
static void
restore_regs(void)
{
int i;
int cpuid = current_processor()->cpu_id;
uint64_t extra;
uint32_t PMCR = 1;
for (i = 0; i < 5; i++) {
uint32_t mask;
if (i == 0) {
mask = 1 << 31;
} else {
mask = 1 << (i - 1);
}
if (saved_PMOVSR[cpuid] & mask) {
extra = kpc_reload_counter(i);
FIXED_SHADOW(i)
+= (kpc_fixed_max() - FIXED_RELOAD(i) + 1 ) + extra;
if (FIXED_ACTIONID(i))
kpc_sample_kperf(FIXED_ACTIONID(i));
} else {
write_counter(i, saved_counter[cpuid][i]);
}
}
__asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (31));
__asm__ volatile("mcr p15, 0, %0, c9, c13, 1;" : : "r" (saved_PMXEVTYPER[cpuid][0]));
for (i = 0; i < 4; i++) {
__asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (i));
__asm__ volatile("mcr p15, 0, %0, c9, c13, 1;" : : "r" (saved_PMXEVTYPER[cpuid][i + 1]));
}
__asm__ volatile("mcr p15, 0, %0, c9, c12, 1;" : : "r" (saved_PMCNTENSET[cpuid]));
__asm__ volatile("mcr p15, 0, %0, c9, c12, 0;" : : "r" (PMCR));
}
static void
kpc_set_reload_xcall(void *vmp_config)
{
struct kpc_config_remote *mp_config = vmp_config;
uint32_t classes = 0, count = 0, offset = kpc_fixed_count();
uint64_t *new_period = NULL, max = kpc_configurable_max();
boolean_t enabled;
assert(mp_config);
assert(mp_config->configv);
classes = mp_config->classes;
new_period = mp_config->configv;
enabled = ml_set_interrupts_enabled(FALSE);
if ((classes & KPC_CLASS_FIXED_MASK) && kpc_controls_fixed_counters()) {
kpc_get_fixed_counters(&FIXED_SHADOW(0));
count = kpc_fixed_count();
for (uint32_t i = 0; i < count; ++i) {
if (*new_period == 0)
*new_period = kpc_fixed_max();
FIXED_RELOAD(i) = max - *new_period;
kpc_reload_counter(i);
new_period++;
}
}
if (classes & KPC_CLASS_CONFIGURABLE_MASK) {
uint64_t all_cfg_mask = (1ULL << kpc_configurable_count()) - 1;
kpc_get_configurable_counters(&CONFIGURABLE_SHADOW(0), all_cfg_mask);
count = kpc_configurable_count();
for (uint32_t i = 0; i < count; ++i) {
if (((1ULL << i) & mp_config->pmc_mask) == 0)
continue;
if (*new_period == 0)
*new_period = kpc_configurable_max();
CONFIGURABLE_RELOAD(i) = max - *new_period;
kpc_reload_counter(offset + i);
new_period++;
}
}
ml_set_interrupts_enabled(enabled);
if (hw_atomic_sub(&kpc_reload_sync, 1) == 0)
thread_wakeup((event_t) &kpc_reload_sync);
}
int
kpc_set_period_arch(struct kpc_config_remote *mp_config)
{
cpu_broadcast_xcall(&kpc_reload_sync, TRUE, kpc_set_reload_xcall, mp_config);
kpc_configured = 1;
return 0;
}
int
kpc_get_configurable_config(kpc_config_t *configv, uint64_t pmc_mask)
{
uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count();
assert(configv);
for (uint32_t i = 0; i < cfg_count; ++i)
if ((1ULL << i) & pmc_mask)
*configv++ = get_counter_config(i + offset);
return 0;
}
static int
kpc_set_configurable_config(kpc_config_t *configv, uint64_t pmc_mask)
{
uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count();
boolean_t enabled;
assert(configv);
enabled = ml_set_interrupts_enabled(FALSE);
for (uint32_t i = 0; i < cfg_count; ++i) {
if (((1ULL << i) & pmc_mask) == 0)
continue;
assert(kpc_controls_counter(i + offset));
set_counter_config(i + offset, *configv++);
}
ml_set_interrupts_enabled(enabled);
return 0;
}
static uint32_t kpc_config_sync;
static void
kpc_set_config_xcall(void *vmp_config)
{
struct kpc_config_remote *mp_config = vmp_config;
kpc_config_t *new_config = NULL;
uint32_t classes = 0ULL;
assert(mp_config);
assert(mp_config->configv);
classes = mp_config->classes;
new_config = mp_config->configv;
if (classes & KPC_CLASS_CONFIGURABLE_MASK) {
kpc_set_configurable_config(new_config, mp_config->pmc_mask);
new_config += kpc_popcount(mp_config->pmc_mask);
}
if (hw_atomic_sub(&kpc_config_sync, 1) == 0)
thread_wakeup((event_t) &kpc_config_sync);
}
int
kpc_set_config_arch(struct kpc_config_remote *mp_config)
{
cpu_broadcast_xcall(&kpc_config_sync, TRUE, kpc_set_config_xcall, mp_config);
kpc_configured = 1;
return 0;
}
void
kpc_idle(void)
{
if (kpc_configured)
save_regs();
}
void
kpc_idle_exit(void)
{
if (kpc_configured)
restore_regs();
}
static uint32_t kpc_xread_sync;
static void
kpc_get_curcpu_counters_xcall(void *args)
{
struct kpc_get_counters_remote *handler = args;
int offset=0, r=0;
assert(handler);
assert(handler->buf);
offset = cpu_number() * handler->buf_stride;
r = kpc_get_curcpu_counters(handler->classes, NULL, &handler->buf[offset]);
hw_atomic_add(&(handler->nb_counters), r);
if (hw_atomic_sub(&kpc_xread_sync, 1) == 0)
thread_wakeup((event_t) &kpc_xread_sync);
}
int
kpc_get_all_cpus_counters(uint32_t classes, int *curcpu, uint64_t *buf)
{
int enabled = 0;
struct kpc_get_counters_remote hdl = {
.classes = classes, .nb_counters = 0,
.buf_stride = kpc_get_counter_count(classes),
.buf = buf
};
assert(buf);
enabled = ml_set_interrupts_enabled(FALSE);
if (curcpu)
*curcpu = current_processor()->cpu_id;
cpu_broadcast_xcall(&kpc_xread_sync, TRUE, kpc_get_curcpu_counters_xcall, &hdl);
ml_set_interrupts_enabled(enabled);
return hdl.nb_counters;
}
int
kpc_get_pmu_version(void)
{
return KPC_PMU_ARM_V2;
}
int
kpc_set_sw_inc( uint32_t mask )
{
__asm__ volatile("mcr p15, 0, %0, c9, c12, 4;" : : "r" (mask));
return 0;
}
#else
void
kpc_arch_init(void)
{
}
uint32_t
kpc_get_classes(void)
{
return 0;
}
uint32_t
kpc_fixed_count(void)
{
return 0;
}
uint32_t
kpc_configurable_count(void)
{
return 0;
}
uint32_t
kpc_fixed_config_count(void)
{
return 0;
}
uint32_t
kpc_configurable_config_count(uint64_t pmc_mask __unused)
{
return 0;
}
int
kpc_get_fixed_config(kpc_config_t *configv __unused)
{
return 0;
}
uint64_t
kpc_fixed_max(void)
{
return 0;
}
uint64_t
kpc_configurable_max(void)
{
return 0;
}
int
kpc_get_configurable_config(kpc_config_t *configv __unused, uint64_t pmc_mask __unused)
{
return ENOTSUP;
}
int
kpc_get_configurable_counters(uint64_t *counterv __unused, uint64_t pmc_mask __unused)
{
return ENOTSUP;
}
int
kpc_get_fixed_counters(uint64_t *counterv __unused)
{
return 0;
}
boolean_t
kpc_is_running_fixed(void)
{
return FALSE;
}
boolean_t
kpc_is_running_configurable(uint64_t pmc_mask __unused)
{
return FALSE;
}
int
kpc_set_running_arch(struct kpc_running_remote *mp_config __unused)
{
return ENOTSUP;
}
int
kpc_set_period_arch(struct kpc_config_remote *mp_config __unused)
{
return ENOTSUP;
}
int
kpc_set_config_arch(struct kpc_config_remote *mp_config __unused)
{
return ENOTSUP;
}
void
kpc_idle(void)
{
}
void
kpc_idle_exit(void)
{
}
int
kpc_get_all_cpus_counters(uint32_t classes, int *curcpu, uint64_t *buf)
{
#pragma unused(classes)
#pragma unused(curcpu)
#pragma unused(buf)
return 0;
}
int
kpc_set_sw_inc( uint32_t mask __unused )
{
return ENOTSUP;
}
int
kpc_get_pmu_version(void)
{
return KPC_PMU_ERROR;
}
#endif
uint32_t
kpc_rawpmu_config_count(void)
{
return 0;
}
int
kpc_get_rawpmu_config(__unused kpc_config_t *configv)
{
return 0;
}