monotonic_x86_64.c [plain text]
#include <i386/cpu_data.h>
#include <i386/cpuid.h>
#include <i386/lapic.h>
#include <i386/mp.h>
#include <i386/proc_reg.h>
#include <kern/assert.h>
#include <kern/monotonic.h>
#include <x86_64/monotonic.h>
#include <sys/errno.h>
#include <sys/monotonic.h>
#ifndef __has_builtin
#define __has_builtin(x) 0
#endif
#if !__has_builtin(__builtin_ia32_rdpmc)
#error requires __builtin_ia32_rdpmc builtin
#endif
#pragma mark core counters
bool mt_core_supported = false;
#define PMC_FIXED_RD(CTR) ((UINT64_C(1) << 30) | (CTR))
#define PMC_FIXED_WR(CTR) (MSR_IA32_PERF_FIXED_CTR0 + (CTR))
#define PMC0_RD PMC_FIXED_RD(0)
#define PMC0_WR PMC_FIXED_WR(0)
#define PMC1_RD PMC_FIXED_RD(1)
#define PMC1_WR PMC_FIXED_WR(1)
#define PMC2_RD PMC_FIXED_RD(2)
#define PMC2_WR PMC_FIXED_WR(2)
struct mt_cpu *
mt_cur_cpu(void)
{
return ¤t_cpu_datap()->cpu_monotonic;
}
uint64_t
mt_core_snap(unsigned int ctr)
{
if (!mt_core_supported) {
return 0;
}
switch (ctr) {
case 0:
return __builtin_ia32_rdpmc(PMC0_RD);
case 1:
return __builtin_ia32_rdpmc(PMC1_RD);
case 2:
return __builtin_ia32_rdpmc(PMC2_RD);
default:
panic("monotonic: invalid core counter read: %u", ctr);
__builtin_unreachable();
}
}
void
mt_core_set_snap(unsigned int ctr, uint64_t count)
{
if (!mt_core_supported) {
return;
}
switch (ctr) {
case 0:
wrmsr64(PMC0_WR, count);
break;
case 1:
wrmsr64(PMC1_WR, count);
break;
case 2:
wrmsr64(PMC2_WR, count);
break;
default:
panic("monotonic: invalid core counter write: %u", ctr);
__builtin_unreachable();
}
}
#define FIXED_CTR_CTRL 0x38d
#define FIXED_CTR_CTRL_INIT (0x888)
#define FIXED_CTR_CTRL_ENABLE (0x333)
#define GLOBAL_CTRL 0x38f
#define GLOBAL_CTRL_FIXED_EN (((UINT64_C(1) << 3) - 1) << 32)
#define GLOBAL_STATUS 0x38e
#define CTR_MAX ((UINT64_C(1) << 48) - 1)
#define CTR_FIX_POS(CTR) ((UINT64_C(1) << (CTR)) << 32)
#define GLOBAL_OVF 0x390
static void
core_down(cpu_data_t *cpu)
{
if (!mt_core_supported) {
return;
}
assert(ml_get_interrupts_enabled() == FALSE);
wrmsr64(GLOBAL_CTRL, 0);
mt_mtc_update_fixed_counts(&cpu->cpu_monotonic, NULL, NULL);
}
static void
core_up(cpu_data_t *cpu)
{
struct mt_cpu *mtc;
if (!mt_core_supported) {
return;
}
assert(ml_get_interrupts_enabled() == FALSE);
mtc = &cpu->cpu_monotonic;
for (int i = 0; i < MT_CORE_NFIXED; i++) {
mt_core_set_snap(i, mtc->mtc_snaps[i]);
}
wrmsr64(FIXED_CTR_CTRL, FIXED_CTR_CTRL_INIT | FIXED_CTR_CTRL_ENABLE);
wrmsr64(GLOBAL_CTRL, GLOBAL_CTRL_FIXED_EN);
}
void
mt_cpu_down(cpu_data_t *cpu)
{
core_down(cpu);
}
void
mt_cpu_up(cpu_data_t *cpu)
{
boolean_t intrs_en;
intrs_en = ml_set_interrupts_enabled(FALSE);
core_up(cpu);
ml_set_interrupts_enabled(intrs_en);
}
static int
mt_pmi_x86_64(x86_saved_state_t *state)
{
uint64_t status;
struct mt_cpu *mtc;
assert(ml_get_interrupts_enabled() == FALSE);
mtc = mt_cur_cpu();
status = rdmsr64(GLOBAL_STATUS);
(void)atomic_fetch_add_explicit(&mt_pmis, 1, memory_order_relaxed);
for (unsigned int i = 0; i < MT_CORE_NFIXED; i++) {
if (status & CTR_FIX_POS(i)) {
uint64_t prior = CTR_MAX - mtc->mtc_snaps[i];
assert(prior <= CTR_MAX);
prior += 1;
uint64_t delta = mt_mtc_update_count(mtc, i);
mtc->mtc_counts[i] += delta;
if (mt_microstackshots && mt_microstackshot_ctr == i) {
x86_saved_state64_t *state64 = saved_state64(state);
bool user_mode = (state64->isf.cs & 0x3) ? true : false;
KDBG_RELEASE(KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_DEBUG, 1),
mt_microstackshot_ctr, user_mode);
mt_microstackshot_pmi_handler(user_mode, mt_microstackshot_ctx);
} else if (mt_debug) {
KDBG(KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_DEBUG, 2),
mt_microstackshot_ctr, i);
}
mtc->mtc_snaps[i] = mt_core_reset_values[i];
mt_core_set_snap(i, mt_core_reset_values[i]);
}
}
if (status & ((UINT64_C(1) << 4) - 1)) {
extern void kpc_pmi_handler(x86_saved_state_t *state);
kpc_pmi_handler(state);
}
return 0;
}
static void
mt_microstackshot_start_remote(__unused void *arg)
{
struct mt_cpu *mtc = mt_cur_cpu();
wrmsr64(FIXED_CTR_CTRL, FIXED_CTR_CTRL_INIT);
for (int i = 0; i < MT_CORE_NFIXED; i++) {
uint64_t delta = mt_mtc_update_count(mtc, i);
mtc->mtc_counts[i] += delta;
mt_core_set_snap(i, mt_core_reset_values[i]);
mtc->mtc_snaps[i] = mt_core_reset_values[i];
}
wrmsr64(FIXED_CTR_CTRL, FIXED_CTR_CTRL_INIT | FIXED_CTR_CTRL_ENABLE);
}
int
mt_microstackshot_start_arch(uint64_t period)
{
if (!mt_core_supported) {
return ENOTSUP;
}
mt_core_reset_values[mt_microstackshot_ctr] = CTR_MAX - period;
mp_cpus_call(CPUMASK_ALL, ASYNC, mt_microstackshot_start_remote,
NULL);
return 0;
}
void
mt_early_init(void)
{
i386_cpu_info_t *info = cpuid_info();
if (info->cpuid_arch_perf_leaf.version >= 2) {
lapic_set_pmi_func((i386_intr_func_t)mt_pmi_x86_64);
mt_core_supported = true;
}
}
static int
core_init(__unused mt_device_t dev)
{
return ENOTSUP;
}
#pragma mark common hooks
struct mt_device mt_devices[] = {
[0] = {
.mtd_name = "core",
.mtd_init = core_init
}
};
static_assert(
(sizeof(mt_devices) / sizeof(mt_devices[0])) == MT_NDEVS,
"MT_NDEVS macro should be same as the length of mt_devices");