#ifdef KERNEL
#ifndef _KERNEL
#define _KERNEL
#endif
#endif
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/errno.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <sys/conf.h>
#include <sys/fcntl.h>
#include <miscfs/devfs/devfs.h>
#include <sys/dtrace.h>
#include <sys/dtrace_impl.h>
#include <sys/dtrace_glue.h>
#include <sys/lockstat.h>
#include <kern/processor.h>
#define membar_producer dtrace_membar_producer
#define PROBE_ARGS0(a, b, c, d, e) "\000"
#define PROBE_ARGS1(a, b, c, d, e) a "\000"
#define PROBE_ARGS2(a, b, c, d, e) a "\000" b "\000"
#define PROBE_ARGS3(a, b, c, d, e) a "\000" b "\000" c "\000"
#define PROBE_ARGS4(a, b, c, d, e) a "\000" b "\000" c "\000" d "\000"
#define PROBE_ARGS5(a, b, c, d, e) a "\000" b "\000" c "\000" d "\000" e "\000"
#define PROBE_ARGS_(a, b, c, d, e, n, ...) PROBE_ARGS##n(a, b, c, d, e)
#define PROBE_ARGS(...) PROBE_ARGS_(__VA_ARGS__, 5, 4, 3, 2, 1, 0)
#define LOCKSTAT_PROBE(func, name, probe, ...) \
{func, name, probe, DTRACE_IDNONE, PROBE_ARGS(__VA_ARGS__)}
#if defined(__x86_64__)
#define NOP 0x90
#define RET 0xc3
#define LOCKSTAT_AFRAMES 1
#elif defined(__arm__)
#define NOP 0xE1A00000
#define BXLR 0xE12FFF1E
#define LOCKSTAT_AFRAMES 2
#elif defined(__arm64__)
#define NOP 0xD503201F
#define RET 0xD65f03c0
#define LOCKSTAT_AFRAMES 2
#else
#error "not ported to this architecture"
#endif
typedef struct lockstat_probe {
const char *lsp_func;
const char *lsp_name;
int lsp_probe;
dtrace_id_t lsp_id;
const char *lsp_args;
} lockstat_probe_t;
lockstat_probe_t lockstat_probes[] =
{
#if defined(__x86_64__)
LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_ACQUIRE, "lck_mtx_t"),
LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_SPIN, LS_LCK_MTX_LOCK_SPIN, "lck_mtx_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_BLOCK, LS_LCK_MTX_LOCK_BLOCK, "lck_mtx_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_MTX_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_LOCK_ACQUIRE, "lck_mtx_t"),
LOCKSTAT_PROBE(LS_LCK_MTX_TRY_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, "lck_mtx_t"),
LOCKSTAT_PROBE(LS_LCK_MTX_UNLOCK, LSA_RELEASE, LS_LCK_MTX_UNLOCK_RELEASE, "lck_mtx_t"),
LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ACQUIRE, LS_LCK_MTX_EXT_LOCK_ACQUIRE, "lck_mtx_t"),
LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_SPIN, LS_LCK_MTX_EXT_LOCK_SPIN, "lck_mtx_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_BLOCK, LS_LCK_MTX_EXT_LOCK_BLOCK, "lck_mtx_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_MTX_EXT_UNLOCK, LSA_RELEASE, LS_LCK_MTX_EXT_UNLOCK_RELEASE, "lck_mtx_t"),
LOCKSTAT_PROBE(LS_LCK_MTX_LOCK_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_SPIN_ACQUIRE, "lck_mtx_t"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_SPIN, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_EXCL_SPIN, "lck_rw_t", "uint64_t", "int"),
LOCKSTAT_PROBE(LS_LCK_RW_DONE, LSR_RELEASE, LS_LCK_RW_DONE_RELEASE, "lck_rw_t", "_Bool"),
LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_UPGRADE, LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, "lck_rw_t", "_Bool"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, "lck_rw_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSR_DOWNGRADE, LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, "lck_rw_t"),
LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_ACQUIRE, LS_LCK_SPIN_LOCK_ACQUIRE, "hw_lock_t"),
LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_SPIN, LS_LCK_SPIN_LOCK_SPIN, "hw_lock_t", "uint64_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_SPIN_UNLOCK, LSS_RELEASE, LS_LCK_SPIN_UNLOCK_RELEASE, "hw_lock_t"),
#elif defined(__arm__) || defined(__arm64__)
LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_ACQUIRE, "lck_mtx_t"),
LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_BLOCK, LS_LCK_MTX_LOCK_BLOCK, "lck_mtx_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_MTX_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_LOCK_ACQUIRE, "lck_mtx_t"),
LOCKSTAT_PROBE(LS_LCK_MTX_UNLOCK, LSA_RELEASE, LS_LCK_MTX_UNLOCK_RELEASE, "lck_mtx_t"),
LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ACQUIRE, LS_LCK_MTX_EXT_LOCK_ACQUIRE, "lck_mtx_t"),
LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_BLOCK, LS_LCK_MTX_EXT_LOCK_BLOCK, "lck_mtx_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_SPIN, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_EXCL_SPIN, "lck_rw_t", "uint64_t", "int"),
LOCKSTAT_PROBE(LS_LCK_RW_DONE, LSR_RELEASE, LS_LCK_RW_DONE_RELEASE, "lck_rw_t", "_Bool"),
LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_UPGRADE, LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, "lck_rw_t", "_Bool"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, "lck_rw_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"),
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSR_DOWNGRADE, LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, "lck_rw_t"),
LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_ACQUIRE, LS_LCK_SPIN_LOCK_ACQUIRE, "hw_lock_t"),
LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_SPIN, LS_LCK_SPIN_LOCK_SPIN, "hw_lock_t", "uint64_t", "uint64_t"),
LOCKSTAT_PROBE(LS_LCK_SPIN_UNLOCK, LSS_RELEASE, LS_LCK_SPIN_UNLOCK_RELEASE, "hw_lock_t"),
#endif
#ifdef LATER
LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSA_ILK_SPIN, LS_LCK_RW_LOCK_EXCL_TO_SHARED_ILK_SPIN),
LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ILK_SPIN, LS_LCK_MTX_LOCK_ILK_SPIN),
LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ILK_SPIN, LS_LCK_MTX_EXT_LOCK_ILK_SPIN),
LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSA_ILK_SPIN, LS_LCK_RW_TRY_LOCK_EXCL_ILK_SPIN),
LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSA_SPIN, LS_LCK_RW_TRY_LOCK_SHARED_SPIN)
#endif
{ NULL, NULL, 0, 0, NULL}
};
dtrace_id_t lockstat_probemap[LS_NPROBES];
#if CONFIG_DTRACE
#if defined(__x86_64__)
extern void lck_mtx_lock_lockstat_patch_point(void);
extern void lck_mtx_try_lock_lockstat_patch_point(void);
extern void lck_mtx_try_lock_spin_lockstat_patch_point(void);
extern void lck_mtx_unlock_lockstat_patch_point(void);
extern void lck_mtx_lock_ext_lockstat_patch_point(void);
extern void lck_mtx_ext_unlock_lockstat_patch_point(void);
extern void lck_mtx_lock_spin_lockstat_patch_point(void);
#endif
#if defined (__arm__)
#endif
#if defined (__arm64__)
#endif
#endif
typedef struct lockstat_assembly_probe {
int lsap_probe;
vm_offset_t * lsap_patch_point;
} lockstat_assembly_probe_t;
lockstat_assembly_probe_t assembly_probes[] =
{
#if CONFIG_DTRACE
#if defined(__x86_64__)
{ LS_LCK_MTX_LOCK_ACQUIRE, (vm_offset_t *) lck_mtx_lock_lockstat_patch_point },
{ LS_LCK_MTX_TRY_LOCK_ACQUIRE, (vm_offset_t *) lck_mtx_try_lock_lockstat_patch_point },
{ LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, (vm_offset_t *) lck_mtx_try_lock_spin_lockstat_patch_point },
{ LS_LCK_MTX_UNLOCK_RELEASE, (vm_offset_t *) lck_mtx_unlock_lockstat_patch_point },
{ LS_LCK_MTX_EXT_LOCK_ACQUIRE, (vm_offset_t *) lck_mtx_lock_ext_lockstat_patch_point },
{ LS_LCK_MTX_EXT_UNLOCK_RELEASE, (vm_offset_t *) lck_mtx_ext_unlock_lockstat_patch_point },
{ LS_LCK_MTX_LOCK_SPIN_ACQUIRE, (vm_offset_t *) lck_mtx_lock_spin_lockstat_patch_point },
#endif
#endif
{ LS_LCK_INVALID, NULL }
};
static
void lockstat_hot_patch(boolean_t active, int ls_probe)
{
#pragma unused(active)
int i;
for (i = 0; assembly_probes[i].lsap_patch_point; i++) {
if (ls_probe == assembly_probes[i].lsap_probe)
#if defined(__x86_64__)
{
uint8_t instr;
instr = (active ? NOP : RET );
(void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point),
sizeof(instr));
}
#elif defined (__arm__)
{
uint32_t instr;
instr = (active ? NOP : BXLR );
(void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point),
sizeof(instr));
}
#elif defined (__arm64__)
{
uint32_t instr;
instr = (active ? NOP : RET );
(void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point),
sizeof(instr));
}
#endif
}
}
void (*lockstat_probe)(dtrace_id_t, uint64_t, uint64_t,
uint64_t, uint64_t, uint64_t);
void
lockstat_probe_wrapper(int probe, uintptr_t lp, int rwflag)
{
dtrace_id_t id;
id = lockstat_probemap[probe];
if (id != 0)
{
(*lockstat_probe)(id, (uintptr_t)lp, (uint64_t)rwflag, 0,0,0);
}
}
static dev_info_t *lockstat_devi;
static dtrace_provider_id_t lockstat_id;
static int
lockstat_enable(void *arg, dtrace_id_t id, void *parg)
{
#pragma unused(arg)
lockstat_probe_t *probe = parg;
ASSERT(!lockstat_probemap[probe->lsp_probe]);
lockstat_probemap[probe->lsp_probe] = id;
membar_producer();
lockstat_hot_patch(TRUE, probe->lsp_probe);
membar_producer();
return(0);
}
static void
lockstat_disable(void *arg, dtrace_id_t id, void *parg)
{
#pragma unused(arg, id)
lockstat_probe_t *probe = parg;
int i;
ASSERT(lockstat_probemap[probe->lsp_probe]);
lockstat_probemap[probe->lsp_probe] = 0;
lockstat_hot_patch(FALSE, probe->lsp_probe);
membar_producer();
for (i = 0; i < LS_NPROBES; i++) {
if (lockstat_probemap[i]) {
return;
}
}
}
static void
lockstat_provide(void *arg, const dtrace_probedesc_t *desc)
{
#pragma unused(arg, desc)
int i = 0;
for (i = 0; lockstat_probes[i].lsp_func != NULL; i++) {
lockstat_probe_t *probe = &lockstat_probes[i];
if (dtrace_probe_lookup(lockstat_id, "mach_kernel",
probe->lsp_func, probe->lsp_name) != 0)
continue;
ASSERT(!probe->lsp_id);
probe->lsp_id = dtrace_probe_create(lockstat_id,
"mach_kernel", probe->lsp_func, probe->lsp_name,
LOCKSTAT_AFRAMES, probe);
}
}
static void
lockstat_destroy(void *arg, dtrace_id_t id, void *parg)
{
#pragma unused(arg, id)
lockstat_probe_t *probe = parg;
ASSERT(!lockstat_probemap[probe->lsp_probe]);
probe->lsp_id = 0;
}
static void
lockstat_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc)
{
#pragma unused(arg, id)
lockstat_probe_t *probe = parg;
const char* argdesc = probe->lsp_args;
int narg = 0;
desc->dtargd_native[0] = '\0';
desc->dtargd_xlate[0] = '\0';
while(argdesc[0] != '\0') {
if (narg == desc->dtargd_ndx) {
strlcpy(desc->dtargd_native, argdesc, DTRACE_ARGTYPELEN);
return;
}
argdesc += strlen(argdesc) + 1;
narg++;
}
desc->dtargd_ndx = DTRACE_ARGNONE;
}
static dtrace_pattr_t lockstat_attr = {
{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
};
static dtrace_pops_t lockstat_pops = {
lockstat_provide,
NULL,
lockstat_enable,
lockstat_disable,
NULL,
NULL,
lockstat_getargdesc,
NULL,
NULL,
lockstat_destroy
};
static int
lockstat_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
{
switch (cmd) {
case DDI_ATTACH:
break;
case DDI_RESUME:
return (DDI_SUCCESS);
default:
return (DDI_FAILURE);
}
if (ddi_create_minor_node(devi, "lockstat", S_IFCHR, 0,
DDI_PSEUDO, 0) == DDI_FAILURE ||
dtrace_register("lockstat", &lockstat_attr, DTRACE_PRIV_KERNEL,
NULL, &lockstat_pops, NULL, &lockstat_id) != 0) {
ddi_remove_minor_node(devi, NULL);
return (DDI_FAILURE);
}
lockstat_probe = dtrace_probe;
membar_producer();
ddi_report_dev(devi);
lockstat_devi = devi;
return (DDI_SUCCESS);
}
d_open_t _lockstat_open;
int _lockstat_open(dev_t dev, int flags, int devtype, struct proc *p)
{
#pragma unused(dev,flags,devtype,p)
return 0;
}
#define LOCKSTAT_MAJOR -24
static struct cdevsw lockstat_cdevsw =
{
_lockstat_open,
eno_opcl,
eno_rdwrt,
eno_rdwrt,
eno_ioctl,
(stop_fcn_t *)nulldev,
(reset_fcn_t *)nulldev,
NULL,
eno_select,
eno_mmap,
eno_strat,
eno_getc,
eno_putc,
0
};
static int gLockstatInited = 0;
void lockstat_init( void );
void lockstat_init( void )
{
if (0 == gLockstatInited)
{
int majdevno = cdevsw_add(LOCKSTAT_MAJOR, &lockstat_cdevsw);
if (majdevno < 0) {
printf("lockstat_init: failed to allocate a major number!\n");
gLockstatInited = 0;
return;
}
lockstat_attach( (dev_info_t *)(uintptr_t)majdevno, DDI_ATTACH );
gLockstatInited = 1;
} else
panic("lockstat_init: called twice!\n");
}
#undef LOCKSTAT_MAJOR