#include <sys/types.h>
#include <sys/time.h>
#include <sys/errno.h>
#include <sys/stat.h>
#include <sys/conf.h>
#include <sys/systm.h>
#include <sys/kauth.h>
#include <sys/fasttrap.h>
#include <sys/fasttrap_impl.h>
#include <sys/fasttrap_isa.h>
#include <sys/dtrace.h>
#include <sys/dtrace_impl.h>
#include <sys/proc.h>
#include <miscfs/devfs/devfs.h>
#include <sys/proc_internal.h>
#include <sys/dtrace_glue.h>
#include <sys/dtrace_ptss.h>
#include <kern/zalloc.h>
#define proc_t struct proc
__private_extern__
void
qsort(void *a, size_t n, size_t es, int (*cmp)(const void *, const void *));
static dev_info_t *fasttrap_devi;
static dtrace_meta_provider_id_t fasttrap_meta_id;
static thread_call_t fasttrap_timeout;
static lck_mtx_t fasttrap_cleanup_mtx;
static uint_t fasttrap_cleanup_work;
static volatile uint64_t fasttrap_mod_gen;
#if !defined(__APPLE__)
#define FASTTRAP_MAX_DEFAULT 2500000
#endif
static uint32_t fasttrap_max;
static uint32_t fasttrap_total;
#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000
#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100
#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100
fasttrap_hash_t fasttrap_tpoints;
static fasttrap_hash_t fasttrap_provs;
static fasttrap_hash_t fasttrap_procs;
static uint64_t fasttrap_pid_count;
static lck_mtx_t fasttrap_count_mtx;
#define FASTTRAP_ENABLE_FAIL 1
#define FASTTRAP_ENABLE_PARTIAL 2
static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
#if defined(__APPLE__)
static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, fasttrap_provider_type_t, const char *,
const dtrace_pattr_t *);
#endif
static void fasttrap_provider_retire(pid_t, const char *, int);
static void fasttrap_provider_free(fasttrap_provider_t *);
static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
static void fasttrap_proc_release(fasttrap_proc_t *);
#define FASTTRAP_PROVS_INDEX(pid, name) \
((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
#if defined(__APPLE__)
struct zone *fasttrap_tracepoint_t_zone;
#define FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS 4
struct zone *fasttrap_probe_t_zones[FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS];
static const char *fasttrap_probe_t_zone_names[FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS] = {
"",
"dtrace.fasttrap_probe_t[1]",
"dtrace.fasttrap_probe_t[2]",
"dtrace.fasttrap_probe_t[3]"
};
lck_grp_t* fasttrap_lck_grp;
lck_grp_attr_t* fasttrap_lck_grp_attr;
lck_attr_t* fasttrap_lck_attr;
#endif
static int
fasttrap_highbit(ulong_t i)
{
int h = 1;
if (i == 0)
return (0);
#ifdef _LP64
if (i & 0xffffffff00000000ul) {
h += 32; i >>= 32;
}
#endif
if (i & 0xffff0000) {
h += 16; i >>= 16;
}
if (i & 0xff00) {
h += 8; i >>= 8;
}
if (i & 0xf0) {
h += 4; i >>= 4;
}
if (i & 0xc) {
h += 2; i >>= 2;
}
if (i & 0x2) {
h += 1;
}
return (h);
}
static uint_t
fasttrap_hash_str(const char *p)
{
unsigned int g;
uint_t hval = 0;
while (*p) {
hval = (hval << 4) + *p++;
if ((g = (hval & 0xf0000000)) != 0)
hval ^= g >> 24;
hval &= ~g;
}
return (hval);
}
void
fasttrap_sigtrap(proc_t *p, uthread_t t, user_addr_t pc)
{
#pragma unused(p, t, pc)
#if 0
sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
sqp->sq_info.si_signo = SIGTRAP;
sqp->sq_info.si_code = TRAP_DTRACE;
sqp->sq_info.si_addr = (caddr_t)pc;
mutex_enter(&p->p_lock);
sigaddqa(p, t, sqp);
mutex_exit(&p->p_lock);
if (t != NULL)
aston(t);
#endif
printf("fasttrap_sigtrap called with no implementation.\n");
}
static void
fasttrap_mod_barrier(uint64_t gen)
{
unsigned int i;
if (gen < fasttrap_mod_gen)
return;
fasttrap_mod_gen++;
for (i = 0; i < NCPU; i++) {
lck_mtx_lock(&cpu_core[i].cpuc_pid_lock);
lck_mtx_unlock(&cpu_core[i].cpuc_pid_lock);
}
}
static void
fasttrap_pid_cleanup_cb(void *ignored, void* ignored2)
{
#pragma unused(ignored, ignored2)
fasttrap_provider_t **fpp, *fp;
fasttrap_bucket_t *bucket;
dtrace_provider_id_t provid;
unsigned int i, later = 0;
static volatile int in = 0;
ASSERT(in == 0);
in = 1;
lck_mtx_lock(&fasttrap_cleanup_mtx);
while (fasttrap_cleanup_work) {
fasttrap_cleanup_work = 0;
lck_mtx_unlock(&fasttrap_cleanup_mtx);
later = 0;
for (i = 0; i < fasttrap_provs.fth_nent; i++) {
bucket = &fasttrap_provs.fth_table[i];
lck_mtx_lock(&bucket->ftb_mtx);
fpp = (fasttrap_provider_t **)&bucket->ftb_data;
while ((fp = *fpp) != NULL) {
if (!fp->ftp_marked) {
fpp = &fp->ftp_next;
continue;
}
lck_mtx_lock(&fp->ftp_mtx);
if (fp->ftp_ccount != 0 ||
fp->ftp_mcount != 0) {
fp->ftp_marked = 0;
lck_mtx_unlock(&fp->ftp_mtx);
continue;
}
if (!fp->ftp_retired || fp->ftp_rcount != 0)
fp->ftp_marked = 0;
lck_mtx_unlock(&fp->ftp_mtx);
provid = fp->ftp_provid;
if (dtrace_unregister(provid) != 0) {
if (fasttrap_total > fasttrap_max / 2)
(void) dtrace_condense(provid);
later += fp->ftp_marked;
fpp = &fp->ftp_next;
} else {
*fpp = fp->ftp_next;
fasttrap_provider_free(fp);
}
}
lck_mtx_unlock(&bucket->ftb_mtx);
}
lck_mtx_lock(&fasttrap_cleanup_mtx);
}
ASSERT(fasttrap_timeout != 0);
if (fasttrap_timeout != (thread_call_t)1)
thread_call_free(fasttrap_timeout);
if (later > 0 && fasttrap_timeout != (thread_call_t)1)
fasttrap_timeout = dtrace_timeout(&fasttrap_pid_cleanup_cb, NULL, NANOSEC / SEC);
else if (later > 0)
fasttrap_cleanup_work = 1;
else
fasttrap_timeout = 0;
lck_mtx_unlock(&fasttrap_cleanup_mtx);
in = 0;
}
static void
fasttrap_pid_cleanup(void)
{
lck_mtx_lock(&fasttrap_cleanup_mtx);
fasttrap_cleanup_work = 1;
if (fasttrap_timeout == 0)
fasttrap_timeout = dtrace_timeout(&fasttrap_pid_cleanup_cb, NULL, NANOSEC / MILLISEC);
lck_mtx_unlock(&fasttrap_cleanup_mtx);
}
static void
fasttrap_fork(proc_t *p, proc_t *cp)
{
pid_t ppid = p->p_pid;
unsigned int i;
ASSERT(current_proc() == p);
lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED);
ASSERT(p->p_dtrace_count > 0);
ASSERT(cp->p_dtrace_count == 0);
if (cp != sprlock(cp->p_pid)) {
printf("fasttrap_fork: sprlock(%d) returned a differt proc\n", cp->p_pid);
return;
}
proc_unlock(cp);
for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
fasttrap_tracepoint_t *tp;
fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
lck_mtx_lock(&bucket->ftb_mtx);
for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
if (tp->ftt_pid == ppid &&
tp->ftt_proc->ftpc_acount != 0) {
fasttrap_tracepoint_remove(cp, tp);
ASSERT(tp->ftt_proc->ftpc_acount != 0);
}
}
lck_mtx_unlock(&bucket->ftb_mtx);
}
dtrace_ptss_fork(p, cp);
proc_lock(cp);
sprunlock(cp);
}
static void
fasttrap_exec_exit(proc_t *p)
{
ASSERT(p == current_proc());
lck_mtx_assert(&p->p_mlock, LCK_MTX_ASSERT_OWNED);
lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED);
proc_unlock(p);
fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
#if defined(__APPLE__)
fasttrap_provider_retire(p->p_pid, FASTTRAP_OBJC_NAME, 0);
fasttrap_provider_retire(p->p_pid, FASTTRAP_ONESHOT_NAME, 0);
#endif
lck_mtx_lock(&p->p_dtrace_sprlock);
dtrace_ptss_exec_exit(p);
lck_mtx_unlock(&p->p_dtrace_sprlock);
proc_lock(p);
}
static void
fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc)
{
#pragma unused(arg, desc)
}
static int
fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
{
fasttrap_tracepoint_t *tp, *new_tp = NULL;
fasttrap_bucket_t *bucket;
fasttrap_id_t *id;
pid_t pid;
user_addr_t pc;
ASSERT(index < probe->ftp_ntps);
pid = probe->ftp_pid;
pc = probe->ftp_tps[index].fit_tp->ftt_pc;
id = &probe->ftp_tps[index].fit_id;
ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
fasttrap_mod_barrier(probe->ftp_gen);
bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
again:
lck_mtx_lock(&bucket->ftb_mtx);
for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
tp->ftt_proc->ftpc_acount == 0)
continue;
ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
switch (id->fti_ptype) {
case DTFTP_ENTRY:
case DTFTP_OFFSETS:
case DTFTP_IS_ENABLED:
id->fti_next = tp->ftt_ids;
dtrace_membar_producer();
tp->ftt_ids = id;
dtrace_membar_producer();
break;
case DTFTP_RETURN:
case DTFTP_POST_OFFSETS:
id->fti_next = tp->ftt_retids;
dtrace_membar_producer();
tp->ftt_retids = id;
dtrace_membar_producer();
break;
default:
ASSERT(0);
}
lck_mtx_unlock(&bucket->ftb_mtx);
if (new_tp != NULL) {
new_tp->ftt_ids = NULL;
new_tp->ftt_retids = NULL;
}
return (0);
}
if (new_tp != NULL) {
int rc = 0;
new_tp->ftt_next = bucket->ftb_data;
dtrace_membar_producer();
bucket->ftb_data = new_tp;
dtrace_membar_producer();
lck_mtx_unlock(&bucket->ftb_mtx);
if (fasttrap_tracepoint_install(p, new_tp) != 0)
rc = FASTTRAP_ENABLE_PARTIAL;
p->p_dtrace_count++;
return (rc);
}
lck_mtx_unlock(&bucket->ftb_mtx);
new_tp = probe->ftp_tps[index].fit_tp;
ASSERT(new_tp->ftt_pid == pid);
ASSERT(new_tp->ftt_pc == pc);
ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
ASSERT(new_tp->ftt_ids == NULL);
ASSERT(new_tp->ftt_retids == NULL);
switch (id->fti_ptype) {
case DTFTP_ENTRY:
case DTFTP_OFFSETS:
case DTFTP_IS_ENABLED:
id->fti_next = NULL;
new_tp->ftt_ids = id;
break;
case DTFTP_RETURN:
case DTFTP_POST_OFFSETS:
id->fti_next = NULL;
new_tp->ftt_retids = id;
break;
default:
ASSERT(0);
}
if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
goto again;
new_tp->ftt_ids = NULL;
new_tp->ftt_retids = NULL;
return (FASTTRAP_ENABLE_FAIL);
}
static void
fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
{
fasttrap_bucket_t *bucket;
fasttrap_provider_t *provider = probe->ftp_prov;
fasttrap_tracepoint_t **pp, *tp;
fasttrap_id_t *id, **idp;
pid_t pid;
user_addr_t pc;
ASSERT(index < probe->ftp_ntps);
pid = probe->ftp_pid;
pc = probe->ftp_tps[index].fit_tp->ftt_pc;
id = &probe->ftp_tps[index].fit_id;
ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
lck_mtx_lock(&bucket->ftb_mtx);
for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
tp->ftt_proc == provider->ftp_proc)
break;
}
ASSERT(tp != NULL);
switch (id->fti_ptype) {
case DTFTP_ENTRY:
case DTFTP_OFFSETS:
case DTFTP_IS_ENABLED:
ASSERT(tp->ftt_ids != NULL);
idp = &tp->ftt_ids;
break;
case DTFTP_RETURN:
case DTFTP_POST_OFFSETS:
ASSERT(tp->ftt_retids != NULL);
idp = &tp->ftt_retids;
break;
default:
idp = NULL;
ASSERT(0);
}
while ((*idp)->fti_probe != probe) {
idp = &(*idp)->fti_next;
ASSERT(*idp != NULL);
}
id = *idp;
*idp = id->fti_next;
dtrace_membar_producer();
ASSERT(id->fti_probe == probe);
if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
if (tp == probe->ftp_tps[index].fit_tp) {
fasttrap_probe_t *tmp_probe;
fasttrap_tracepoint_t **tmp_tp;
uint_t tmp_index;
if (tp->ftt_ids != NULL) {
tmp_probe = tp->ftt_ids->fti_probe;
tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
} else {
tmp_probe = tp->ftt_retids->fti_probe;
tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
}
ASSERT(*tmp_tp != NULL);
ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
ASSERT((*tmp_tp)->ftt_ids == NULL);
ASSERT((*tmp_tp)->ftt_retids == NULL);
probe->ftp_tps[index].fit_tp = *tmp_tp;
*tmp_tp = tp;
}
lck_mtx_unlock(&bucket->ftb_mtx);
probe->ftp_gen = fasttrap_mod_gen;
return;
}
lck_mtx_unlock(&bucket->ftb_mtx);
if (p != NULL) {
if (fasttrap_tracepoint_remove(p, tp) != 0)
fasttrap_sigtrap(p, NULL, pc);
p->p_dtrace_count--;
}
lck_mtx_lock(&bucket->ftb_mtx);
pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
ASSERT(*pp != NULL);
while (*pp != tp) {
pp = &(*pp)->ftt_next;
ASSERT(*pp != NULL);
}
*pp = tp->ftt_next;
dtrace_membar_producer();
lck_mtx_unlock(&bucket->ftb_mtx);
probe->ftp_gen = fasttrap_mod_gen;
}
static void
fasttrap_enable_callbacks(void)
{
lck_mtx_lock(&fasttrap_count_mtx);
if (fasttrap_pid_count == 0) {
ASSERT(dtrace_pid_probe_ptr == NULL);
ASSERT(dtrace_return_probe_ptr == NULL);
dtrace_pid_probe_ptr = &fasttrap_pid_probe;
dtrace_return_probe_ptr = &fasttrap_return_probe;
}
ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
fasttrap_pid_count++;
lck_mtx_unlock(&fasttrap_count_mtx);
}
static void
fasttrap_disable_callbacks(void)
{
lck_mtx_lock(&fasttrap_count_mtx);
ASSERT(fasttrap_pid_count > 0);
fasttrap_pid_count--;
if (fasttrap_pid_count == 0) {
cpu_t *cur, *cpu = CPU;
for (cur = cpu->cpu_next; cur != cpu; cur = cur->cpu_next) {
lck_rw_lock_exclusive(&cur->cpu_ft_lock);
}
dtrace_pid_probe_ptr = NULL;
dtrace_return_probe_ptr = NULL;
for (cur = cpu->cpu_next; cur != cpu; cur = cur->cpu_next) {
lck_rw_unlock_exclusive(&cur->cpu_ft_lock);
}
}
lck_mtx_unlock(&fasttrap_count_mtx);
}
static void
fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
{
#pragma unused(arg, id)
fasttrap_probe_t *probe = parg;
proc_t *p;
int i, rc;
ASSERT(probe != NULL);
ASSERT(!probe->ftp_enabled);
ASSERT(id == probe->ftp_id);
lck_mtx_lock(&probe->ftp_prov->ftp_mtx);
probe->ftp_prov->ftp_rcount++;
lck_mtx_unlock(&probe->ftp_prov->ftp_mtx);
if (probe->ftp_prov->ftp_retired)
return;
if ((p = sprlock(probe->ftp_pid)) == PROC_NULL) {
#if defined(__APPLE__)
return;
#else
if ((curproc->p_flag & SFORKING) == 0)
return;
lck_mtx_lock(&pidlock);
p = prfind(probe->ftp_pid);
ASSERT(p != NULL);
ASSERT(p->p_stat == SIDL);
lck_mtx_lock(&p->p_lock);
lck_mtx_unlock(&pidlock);
sprlock_proc(p);
#endif
}
if (p->p_dtrace_ptss_pages == NULL) {
dtrace_ptss_enable(p);
}
proc_unlock(p);
fasttrap_enable_callbacks();
for (i = 0; i < (int)probe->ftp_ntps; i++) {
if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
if (rc == FASTTRAP_ENABLE_FAIL)
i--;
else
ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
while (i >= 0) {
fasttrap_tracepoint_disable(p, probe, i);
i--;
}
proc_lock(p);
sprunlock(p);
fasttrap_disable_callbacks();
return;
}
}
proc_lock(p);
sprunlock(p);
probe->ftp_enabled = 1;
}
static void
fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
{
#pragma unused(arg, id)
fasttrap_probe_t *probe = parg;
fasttrap_provider_t *provider = probe->ftp_prov;
proc_t *p;
int i, whack = 0;
ASSERT(id == probe->ftp_id);
if ((p = sprlock(probe->ftp_pid)) != PROC_NULL) {
proc_unlock(p);
}
lck_mtx_lock(&provider->ftp_mtx);
if (probe->ftp_enabled) {
for (i = 0; i < (int)probe->ftp_ntps; i++) {
fasttrap_tracepoint_disable(p, probe, i);
}
}
ASSERT(provider->ftp_rcount > 0);
provider->ftp_rcount--;
if (p != NULL) {
if (provider->ftp_retired && !provider->ftp_marked)
whack = provider->ftp_marked = 1;
lck_mtx_unlock(&provider->ftp_mtx);
proc_lock(p);
sprunlock(p);
} else {
if (provider->ftp_rcount == 0 && !provider->ftp_marked)
whack = provider->ftp_marked = 1;
lck_mtx_unlock(&provider->ftp_mtx);
}
if (whack)
fasttrap_pid_cleanup();
if (!probe->ftp_enabled)
return;
probe->ftp_enabled = 0;
fasttrap_disable_callbacks();
}
static void
fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
dtrace_argdesc_t *desc)
{
#pragma unused(arg, id)
fasttrap_probe_t *probe = parg;
char *str;
int i, ndx;
desc->dtargd_native[0] = '\0';
desc->dtargd_xlate[0] = '\0';
if (probe->ftp_prov->ftp_retired != 0 ||
desc->dtargd_ndx >= probe->ftp_nargs) {
desc->dtargd_ndx = DTRACE_ARGNONE;
return;
}
ndx = (probe->ftp_argmap != NULL) ?
probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
str = probe->ftp_ntypes;
for (i = 0; i < ndx; i++) {
str += strlen(str) + 1;
}
(void) strlcpy(desc->dtargd_native, str, sizeof(desc->dtargd_native));
if (probe->ftp_xtypes == NULL)
return;
str = probe->ftp_xtypes;
for (i = 0; i < desc->dtargd_ndx; i++) {
str += strlen(str) + 1;
}
(void) strlcpy(desc->dtargd_xlate, str, sizeof(desc->dtargd_xlate));
}
static void
fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
{
#pragma unused(arg, id)
fasttrap_probe_t *probe = parg;
unsigned int i;
ASSERT(probe != NULL);
ASSERT(!probe->ftp_enabled);
ASSERT(fasttrap_total >= probe->ftp_ntps);
atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
#if !defined(__APPLE__)
size_t size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
#endif
if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
fasttrap_mod_barrier(probe->ftp_gen);
for (i = 0; i < probe->ftp_ntps; i++) {
#if !defined(__APPLE__)
kmem_free(probe->ftp_tps[i].fit_tp, sizeof (fasttrap_tracepoint_t));
#else
zfree(fasttrap_tracepoint_t_zone, probe->ftp_tps[i].fit_tp);
#endif
}
#if !defined(__APPLE__)
kmem_free(probe, size);
#else
if (probe->ftp_ntps < FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS) {
zfree(fasttrap_probe_t_zones[probe->ftp_ntps], probe);
} else {
size_t size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
kmem_free(probe, size);
}
#endif
}
static const dtrace_pattr_t pid_attr = {
{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
};
static dtrace_pops_t pid_pops = {
fasttrap_pid_provide,
NULL,
fasttrap_pid_enable,
fasttrap_pid_disable,
NULL,
NULL,
fasttrap_pid_getargdesc,
fasttrap_pid_getarg,
NULL,
fasttrap_pid_destroy
};
static dtrace_pops_t usdt_pops = {
fasttrap_pid_provide,
NULL,
fasttrap_pid_enable,
fasttrap_pid_disable,
NULL,
NULL,
fasttrap_pid_getargdesc,
fasttrap_usdt_getarg,
NULL,
fasttrap_pid_destroy
};
static fasttrap_proc_t *
fasttrap_proc_lookup(pid_t pid)
{
fasttrap_bucket_t *bucket;
fasttrap_proc_t *fprc, *new_fprc;
bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
lck_mtx_lock(&bucket->ftb_mtx);
for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
lck_mtx_lock(&fprc->ftpc_mtx);
lck_mtx_unlock(&bucket->ftb_mtx);
fprc->ftpc_rcount++;
atomic_add_64(&fprc->ftpc_acount, 1);
ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
lck_mtx_unlock(&fprc->ftpc_mtx);
return (fprc);
}
}
lck_mtx_unlock(&bucket->ftb_mtx);
new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
ASSERT(new_fprc != NULL);
new_fprc->ftpc_pid = pid;
new_fprc->ftpc_rcount = 1;
new_fprc->ftpc_acount = 1;
lck_mtx_lock(&bucket->ftb_mtx);
for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
lck_mtx_lock(&fprc->ftpc_mtx);
lck_mtx_unlock(&bucket->ftb_mtx);
fprc->ftpc_rcount++;
atomic_add_64(&fprc->ftpc_acount, 1);
ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
lck_mtx_unlock(&fprc->ftpc_mtx);
kmem_free(new_fprc, sizeof (fasttrap_proc_t));
return (fprc);
}
}
#if defined(__APPLE__)
lck_mtx_init(&new_fprc->ftpc_mtx, fasttrap_lck_grp, fasttrap_lck_attr);
#endif
new_fprc->ftpc_next = bucket->ftb_data;
bucket->ftb_data = new_fprc;
lck_mtx_unlock(&bucket->ftb_mtx);
return (new_fprc);
}
static void
fasttrap_proc_release(fasttrap_proc_t *proc)
{
fasttrap_bucket_t *bucket;
fasttrap_proc_t *fprc, **fprcp;
pid_t pid = proc->ftpc_pid;
lck_mtx_lock(&proc->ftpc_mtx);
ASSERT(proc->ftpc_rcount != 0);
ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
if (--proc->ftpc_rcount != 0) {
lck_mtx_unlock(&proc->ftpc_mtx);
return;
}
lck_mtx_unlock(&proc->ftpc_mtx);
ASSERT(proc->ftpc_acount == 0);
bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
lck_mtx_lock(&bucket->ftb_mtx);
fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
while ((fprc = *fprcp) != NULL) {
if (fprc == proc)
break;
fprcp = &fprc->ftpc_next;
}
ASSERT(fprc != NULL);
*fprcp = fprc->ftpc_next;
lck_mtx_unlock(&bucket->ftb_mtx);
#if defined(__APPLE__)
lck_mtx_destroy(&fprc->ftpc_mtx, fasttrap_lck_grp);
#endif
kmem_free(fprc, sizeof (fasttrap_proc_t));
}
#if defined(__APPLE__)
static fasttrap_provider_t *
fasttrap_provider_lookup(pid_t pid, fasttrap_provider_type_t provider_type, const char *name,
const dtrace_pattr_t *pattr)
#endif
{
fasttrap_provider_t *fp, *new_fp = NULL;
fasttrap_bucket_t *bucket;
char provname[DTRACE_PROVNAMELEN];
proc_t *p;
cred_t *cred;
ASSERT(strlen(name) < sizeof (fp->ftp_name));
ASSERT(pattr != NULL);
bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
lck_mtx_lock(&bucket->ftb_mtx);
for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
if (fp->ftp_pid == pid &&
#if defined(__APPLE__)
fp->ftp_provider_type == provider_type &&
#endif
strncmp(fp->ftp_name, name, sizeof(fp->ftp_name)) == 0 &&
!fp->ftp_retired) {
lck_mtx_lock(&fp->ftp_mtx);
lck_mtx_unlock(&bucket->ftb_mtx);
return (fp);
}
}
lck_mtx_unlock(&bucket->ftb_mtx);
if ((p = proc_find(pid)) == NULL) {
return NULL;
}
proc_lock(p);
if (p->p_lflag & (P_LINVFORK | P_LEXIT)) {
proc_unlock(p);
proc_rele(p);
return (NULL);
}
p->p_dtrace_probes++;
#if !defined(__APPLE__)
mutex_enter(&p->p_crlock);
crhold(p->p_cred);
cred = p->p_cred;
mutex_exit(&p->p_crlock);
mutex_exit(&p->p_lock);
#else
crhold(p->p_ucred);
cred = p->p_ucred;
proc_unlock(p);
proc_rele(p);
#endif
new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
ASSERT(new_fp != NULL);
new_fp->ftp_pid = pid;
new_fp->ftp_proc = fasttrap_proc_lookup(pid);
#if defined(__APPLE__)
new_fp->ftp_provider_type = provider_type;
lck_mtx_init(&new_fp->ftp_mtx, fasttrap_lck_grp, fasttrap_lck_attr);
lck_mtx_init(&new_fp->ftp_cmtx, fasttrap_lck_grp, fasttrap_lck_attr);
#endif
ASSERT(new_fp->ftp_proc != NULL);
lck_mtx_lock(&bucket->ftb_mtx);
for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
if (fp->ftp_pid == pid && strncmp(fp->ftp_name, name, sizeof(fp->ftp_name)) == 0 &&
!fp->ftp_retired) {
lck_mtx_lock(&fp->ftp_mtx);
lck_mtx_unlock(&bucket->ftb_mtx);
fasttrap_provider_free(new_fp);
crfree(cred);
return (fp);
}
}
(void) strlcpy(new_fp->ftp_name, name, sizeof(new_fp->ftp_name));
if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
(int)sizeof (provname) ||
dtrace_register(provname, pattr,
DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
&new_fp->ftp_provid) != 0) {
lck_mtx_unlock(&bucket->ftb_mtx);
fasttrap_provider_free(new_fp);
crfree(cred);
return (NULL);
}
new_fp->ftp_next = bucket->ftb_data;
bucket->ftb_data = new_fp;
lck_mtx_lock(&new_fp->ftp_mtx);
lck_mtx_unlock(&bucket->ftb_mtx);
crfree(cred);
return (new_fp);
}
static void
fasttrap_provider_free(fasttrap_provider_t *provider)
{
pid_t pid = provider->ftp_pid;
proc_t *p;
ASSERT(provider->ftp_rcount == 0);
ASSERT(provider->ftp_ccount == 0);
ASSERT(provider->ftp_mcount == 0);
if (!provider->ftp_retired) {
atomic_add_64(&provider->ftp_proc->ftpc_acount, -1);
ASSERT(provider->ftp_proc->ftpc_acount <
provider->ftp_proc->ftpc_rcount);
}
fasttrap_proc_release(provider->ftp_proc);
#if defined(__APPLE__)
lck_mtx_destroy(&provider->ftp_mtx, fasttrap_lck_grp);
lck_mtx_destroy(&provider->ftp_cmtx, fasttrap_lck_grp);
#endif
kmem_free(provider, sizeof (fasttrap_provider_t));
if ((p = proc_find(pid)) == NULL) {
return;
}
proc_lock(p);
p->p_dtrace_probes--;
proc_unlock(p);
proc_rele(p);
}
static void
fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
{
fasttrap_provider_t *fp;
fasttrap_bucket_t *bucket;
dtrace_provider_id_t provid;
ASSERT(strlen(name) < sizeof (fp->ftp_name));
bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
lck_mtx_lock(&bucket->ftb_mtx);
for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
if (fp->ftp_pid == pid && strncmp(fp->ftp_name, name, sizeof(fp->ftp_name)) == 0 &&
!fp->ftp_retired)
break;
}
if (fp == NULL) {
lck_mtx_unlock(&bucket->ftb_mtx);
return;
}
lck_mtx_lock(&fp->ftp_mtx);
ASSERT(!mprov || fp->ftp_mcount > 0);
if (mprov && --fp->ftp_mcount != 0) {
lck_mtx_unlock(&fp->ftp_mtx);
lck_mtx_unlock(&bucket->ftb_mtx);
return;
}
atomic_add_64(&fp->ftp_proc->ftpc_acount, -1);
ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
fp->ftp_retired = 1;
fp->ftp_marked = 1;
provid = fp->ftp_provid;
lck_mtx_unlock(&fp->ftp_mtx);
dtrace_invalidate(provid);
lck_mtx_unlock(&bucket->ftb_mtx);
fasttrap_pid_cleanup();
}
static int
fasttrap_uint32_cmp(const void *ap, const void *bp)
{
return (*(const uint32_t *)ap - *(const uint32_t *)bp);
}
static int
fasttrap_uint64_cmp(const void *ap, const void *bp)
{
return (*(const uint64_t *)ap - *(const uint64_t *)bp);
}
static int
fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
{
fasttrap_provider_t *provider;
fasttrap_probe_t *pp;
fasttrap_tracepoint_t *tp;
const char *name;
unsigned int i, aframes, whack;
if (pdata->ftps_noffs == 0)
return (EINVAL);
#if defined(__APPLE__)
switch (pdata->ftps_probe_type) {
#endif
case DTFTP_ENTRY:
name = "entry";
aframes = FASTTRAP_ENTRY_AFRAMES;
break;
case DTFTP_RETURN:
name = "return";
aframes = FASTTRAP_RETURN_AFRAMES;
break;
case DTFTP_OFFSETS:
aframes = 0;
name = NULL;
break;
default:
return (EINVAL);
}
#if defined(__APPLE__)
const char* provider_name;
switch (pdata->ftps_provider_type) {
case DTFTP_PROVIDER_PID:
provider_name = FASTTRAP_PID_NAME;
break;
case DTFTP_PROVIDER_OBJC:
provider_name = FASTTRAP_OBJC_NAME;
break;
case DTFTP_PROVIDER_ONESHOT:
provider_name = FASTTRAP_ONESHOT_NAME;
break;
default:
return (EINVAL);
}
if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, pdata->ftps_provider_type,
provider_name, &pid_attr)) == NULL)
return (ESRCH);
#endif
provider->ftp_ccount++;
lck_mtx_unlock(&provider->ftp_mtx);
lck_mtx_lock(&provider->ftp_cmtx);
if (name == NULL) {
for (i = 0; i < pdata->ftps_noffs; i++) {
char name_str[17];
(void) snprintf(name_str, sizeof(name_str), "%llx",
(uint64_t)pdata->ftps_offs[i]);
if (dtrace_probe_lookup(provider->ftp_provid,
pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
continue;
atomic_add_32(&fasttrap_total, 1);
if (fasttrap_total > fasttrap_max) {
atomic_add_32(&fasttrap_total, -1);
goto no_mem;
}
#if !defined(__APPLE__)
pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
ASSERT(pp != NULL);
#else
pp = zalloc(fasttrap_probe_t_zones[1]);
bzero(pp, sizeof (fasttrap_probe_t));
#endif
pp->ftp_prov = provider;
pp->ftp_faddr = pdata->ftps_pc;
pp->ftp_fsize = pdata->ftps_size;
pp->ftp_pid = pdata->ftps_pid;
pp->ftp_ntps = 1;
#if !defined(__APPLE__)
tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
#else
tp = zalloc(fasttrap_tracepoint_t_zone);
bzero(tp, sizeof (fasttrap_tracepoint_t));
#endif
tp->ftt_proc = provider->ftp_proc;
tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
tp->ftt_pid = pdata->ftps_pid;
pp->ftp_tps[0].fit_tp = tp;
pp->ftp_tps[0].fit_id.fti_probe = pp;
#if defined(__APPLE__)
pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_probe_type;
#endif
pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
pdata->ftps_mod, pdata->ftps_func, name_str,
FASTTRAP_OFFSET_AFRAMES, pp);
}
} else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
pdata->ftps_func, name) == 0) {
atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
if (fasttrap_total > fasttrap_max) {
atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
goto no_mem;
}
qsort(pdata->ftps_offs, pdata->ftps_noffs,
sizeof (uint64_t), fasttrap_uint64_cmp);
for (i = 1; i < pdata->ftps_noffs; i++) {
if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
continue;
atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
goto no_mem;
}
ASSERT(pdata->ftps_noffs > 0);
#if !defined(__APPLE__)
pp = kmem_zalloc(offsetof(fasttrap_probe_t,
ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
ASSERT(pp != NULL);
#else
if (pdata->ftps_noffs < FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS) {
pp = zalloc(fasttrap_probe_t_zones[pdata->ftps_noffs]);
bzero(pp, offsetof(fasttrap_probe_t, ftp_tps[pdata->ftps_noffs]));
} else {
pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
}
#endif
pp->ftp_prov = provider;
pp->ftp_faddr = pdata->ftps_pc;
pp->ftp_fsize = pdata->ftps_size;
pp->ftp_pid = pdata->ftps_pid;
pp->ftp_ntps = pdata->ftps_noffs;
for (i = 0; i < pdata->ftps_noffs; i++) {
#if !defined(__APPLE__)
tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
#else
tp = zalloc(fasttrap_tracepoint_t_zone);
bzero(tp, sizeof (fasttrap_tracepoint_t));
#endif
tp->ftt_proc = provider->ftp_proc;
tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
tp->ftt_pid = pdata->ftps_pid;
pp->ftp_tps[i].fit_tp = tp;
pp->ftp_tps[i].fit_id.fti_probe = pp;
#if defined(__APPLE__)
pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_probe_type;
#endif
}
pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
}
lck_mtx_unlock(&provider->ftp_cmtx);
lck_mtx_lock(&provider->ftp_mtx);
provider->ftp_ccount--;
whack = provider->ftp_retired;
lck_mtx_unlock(&provider->ftp_mtx);
if (whack)
fasttrap_pid_cleanup();
return (0);
no_mem:
lck_mtx_unlock(&provider->ftp_cmtx);
lck_mtx_lock(&provider->ftp_mtx);
provider->ftp_ccount--;
provider->ftp_marked = 1;
lck_mtx_unlock(&provider->ftp_mtx);
fasttrap_pid_cleanup();
return (ENOMEM);
}
static void *
fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
{
#pragma unused(arg)
fasttrap_provider_t *provider;
if (strlen(dhpv->dthpv_provname) + 10 >=
sizeof (provider->ftp_name)) {
cmn_err(CE_WARN, "failed to instantiate provider %s: "
"name too long to accomodate pid", dhpv->dthpv_provname);
return (NULL);
}
if (strncmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME, sizeof(FASTTRAP_PID_NAME)) == 0) {
cmn_err(CE_WARN, "failed to instantiate provider %s: "
"%s is an invalid name", dhpv->dthpv_provname,
FASTTRAP_PID_NAME);
return (NULL);
}
#if defined(__APPLE__)
if (strncmp(dhpv->dthpv_provname, FASTTRAP_OBJC_NAME, sizeof(FASTTRAP_OBJC_NAME)) == 0) {
cmn_err(CE_WARN, "failed to instantiate provider %s: "
"%s is an invalid name", dhpv->dthpv_provname,
FASTTRAP_OBJC_NAME);
return (NULL);
}
if (strncmp(dhpv->dthpv_provname, FASTTRAP_ONESHOT_NAME, sizeof(FASTTRAP_ONESHOT_NAME)) == 0) {
cmn_err(CE_WARN, "failed to instantiate provider %s: "
"%s is an invalid name", dhpv->dthpv_provname,
FASTTRAP_ONESHOT_NAME);
return (NULL);
}
#endif
if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
#if defined(__APPLE__)
if ((provider = fasttrap_provider_lookup(pid, DTFTP_PROVIDER_USDT, dhpv->dthpv_provname,
&dhpv->dthpv_pattr)) == NULL) {
cmn_err(CE_WARN, "failed to instantiate provider %s for "
"process %u", dhpv->dthpv_provname, (uint_t)pid);
return (NULL);
}
#endif
provider->ftp_mcount++;
lck_mtx_unlock(&provider->ftp_mtx);
return (provider);
}
static void
fasttrap_meta_create_probe(void *arg, void *parg,
dtrace_helper_probedesc_t *dhpb)
{
#pragma unused(arg)
fasttrap_provider_t *provider = parg;
fasttrap_probe_t *pp;
fasttrap_tracepoint_t *tp;
unsigned int i, j;
uint32_t ntps;
ASSERT(provider->ftp_mcount > 0);
qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
fasttrap_uint32_cmp);
for (i = 1; i < dhpb->dthpb_noffs; i++) {
if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
return;
}
qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
fasttrap_uint32_cmp);
for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
return;
}
lck_mtx_lock(&provider->ftp_cmtx);
#if !defined(__APPLE__)
if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod,
dhpb->dthpb_func, dhpb->dthpb_name) != 0) {
lck_mtx_unlock(&provider->ftp_cmtx);
return;
}
#endif
ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
ASSERT(ntps > 0);
atomic_add_32(&fasttrap_total, ntps);
if (fasttrap_total > fasttrap_max) {
atomic_add_32(&fasttrap_total, -ntps);
lck_mtx_unlock(&provider->ftp_cmtx);
return;
}
#if !defined(__APPLE__)
pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
ASSERT(pp != NULL);
#else
if (ntps < FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS) {
pp = zalloc(fasttrap_probe_t_zones[ntps]);
bzero(pp, offsetof(fasttrap_probe_t, ftp_tps[ntps]));
} else {
pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
}
#endif
pp->ftp_prov = provider;
pp->ftp_pid = provider->ftp_pid;
pp->ftp_ntps = ntps;
pp->ftp_nargs = dhpb->dthpb_xargc;
pp->ftp_xtypes = dhpb->dthpb_xtypes;
pp->ftp_ntypes = dhpb->dthpb_ntypes;
for (i = 0; i < dhpb->dthpb_noffs; i++) {
#if !defined(__APPLE__)
tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
#else
tp = zalloc(fasttrap_tracepoint_t_zone);
bzero(tp, sizeof (fasttrap_tracepoint_t));
#endif
tp->ftt_proc = provider->ftp_proc;
#if defined(__APPLE__)
#if defined(__i386__) || defined(__x86_64__)
tp->ftt_pc = dhpb->dthpb_base + (int64_t)dhpb->dthpb_offs[i] - 1;
#elif defined(__ppc__)
tp->ftt_pc = dhpb->dthpb_base + (int64_t)dhpb->dthpb_offs[i];
#else
#error "Architecture not supported"
#endif
#else
tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
#endif
tp->ftt_pid = provider->ftp_pid;
pp->ftp_tps[i].fit_tp = tp;
pp->ftp_tps[i].fit_id.fti_probe = pp;
#ifdef __sparc
pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
#else
pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
#endif
}
for (j = 0; i < ntps; i++, j++) {
#if !defined(__APPLE__)
tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
#else
tp = zalloc(fasttrap_tracepoint_t_zone);
bzero(tp, sizeof (fasttrap_tracepoint_t));
#endif
tp->ftt_proc = provider->ftp_proc;
#if defined(__APPLE__)
#if defined(__i386__) || defined(__x86_64__)
tp->ftt_pc = dhpb->dthpb_base + (int64_t)dhpb->dthpb_enoffs[j] + 2;
#elif defined(__ppc__)
tp->ftt_pc = dhpb->dthpb_base + (int64_t)dhpb->dthpb_enoffs[j];
#else
#error "Architecture not supported"
#endif
#else
tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
#endif
tp->ftt_pid = provider->ftp_pid;
pp->ftp_tps[i].fit_tp = tp;
pp->ftp_tps[i].fit_id.fti_probe = pp;
pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
}
for (i = 0; i < dhpb->dthpb_xargc; i++) {
if (dhpb->dthpb_args[i] != i) {
pp->ftp_argmap = dhpb->dthpb_args;
break;
}
}
pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
lck_mtx_unlock(&provider->ftp_cmtx);
}
static void
fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
{
#pragma unused(arg)
fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
}
static dtrace_mops_t fasttrap_mops = {
fasttrap_meta_create_probe,
fasttrap_meta_provide,
fasttrap_meta_remove
};
static int
fasttrap_ioctl(dev_t dev, u_long cmd, user_addr_t arg, int md, cred_t *cr, int *rv)
{
#pragma unused(dev, md, rv)
if (!dtrace_attached())
return (EAGAIN);
if (cmd == FASTTRAPIOC_MAKEPROBE) {
fasttrap_probe_spec_t *probe;
uint64_t noffs;
size_t size, i;
int ret;
char *c;
if (copyin(arg + __offsetof(fasttrap_probe_spec_t, ftps_noffs), &noffs,
sizeof (probe->ftps_noffs)))
return (EFAULT);
if (noffs == 0)
return (EINVAL);
if (noffs > ((1024 * 1024) - sizeof(fasttrap_probe_spec_t)) / sizeof(probe->ftps_offs[0]))
return (ENOMEM);
size = sizeof (fasttrap_probe_spec_t) +
sizeof (probe->ftps_offs[0]) * (noffs - 1);
probe = kmem_alloc(size, KM_SLEEP);
if (copyin(arg, probe, size) != 0) {
kmem_free(probe, size);
return (EFAULT);
}
for (i = 0, c = &probe->ftps_func[0]; i < sizeof(probe->ftps_func) && *c != '\0'; i++, c++) {
if (*c < 0x20 || 0x7f <= *c) {
ret = EINVAL;
goto err;
}
}
if (*c != '\0') {
ret = EINVAL;
goto err;
}
for (i = 0, c = &probe->ftps_mod[0]; i < sizeof(probe->ftps_mod) && *c != '\0'; i++, c++) {
if (*c < 0x20 || 0x7f <= *c) {
ret = EINVAL;
goto err;
}
}
if (*c != '\0') {
ret = EINVAL;
goto err;
}
if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
proc_t *p;
pid_t pid = probe->ftps_pid;
if ((p = proc_find(pid)) == PROC_NULL || p->p_stat == SIDL) {
if (p != PROC_NULL)
proc_rele(p);
return (ESRCH);
}
proc_rele(p);
}
ret = fasttrap_add_probe(probe);
err:
kmem_free(probe, size);
return (ret);
} else if (cmd == FASTTRAPIOC_GETINSTR) {
fasttrap_instr_query_t instr;
fasttrap_tracepoint_t *tp;
uint_t index;
if (copyin(arg, &instr, sizeof (instr)) != 0)
return (EFAULT);
if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
proc_t *p;
pid_t pid = instr.ftiq_pid;
if ((p = proc_find(pid)) == NULL || p->p_stat == SIDL) {
if (p != PROC_NULL)
proc_rele(p);
return (ESRCH);
}
proc_rele(p);
}
index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
lck_mtx_lock(&fasttrap_tpoints.fth_table[index].ftb_mtx);
tp = fasttrap_tpoints.fth_table[index].ftb_data;
while (tp != NULL) {
if (instr.ftiq_pid == tp->ftt_pid &&
instr.ftiq_pc == tp->ftt_pc &&
tp->ftt_proc->ftpc_acount != 0)
break;
tp = tp->ftt_next;
}
if (tp == NULL) {
lck_mtx_unlock(&fasttrap_tpoints.fth_table[index].ftb_mtx);
return (ENOENT);
}
bcopy(&tp->ftt_instr, &instr.ftiq_instr,
sizeof (instr.ftiq_instr));
lck_mtx_unlock(&fasttrap_tpoints.fth_table[index].ftb_mtx);
if (copyout(&instr, arg, sizeof (instr)) != 0)
return (EFAULT);
return (0);
}
return (EINVAL);
}
static int
fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
{
ulong_t nent;
switch (cmd) {
case DDI_ATTACH:
break;
case DDI_RESUME:
return (DDI_SUCCESS);
default:
return (DDI_FAILURE);
}
ddi_report_dev(devi);
fasttrap_devi = devi;
dtrace_fasttrap_fork_ptr = &fasttrap_fork;
dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit;
dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit;
#if !defined(__APPLE__)
fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
"fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
#else
fasttrap_max = (sane_size >> 28) * 100000;
if (fasttrap_max == 0)
fasttrap_max = 50000;
#endif
fasttrap_total = 0;
nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
"fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
if (nent <= 0 || nent > 0x1000000)
nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
if ((nent & (nent - 1)) == 0)
fasttrap_tpoints.fth_nent = nent;
else
fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
ASSERT(fasttrap_tpoints.fth_nent > 0);
fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
sizeof (fasttrap_bucket_t), KM_SLEEP);
ASSERT(fasttrap_tpoints.fth_table != NULL);
#if defined(__APPLE__)
unsigned int i;
for (i=0; i<fasttrap_tpoints.fth_nent; i++) {
lck_mtx_init(&fasttrap_tpoints.fth_table[i].ftb_mtx, fasttrap_lck_grp, fasttrap_lck_attr);
}
#endif
nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
if ((nent & (nent - 1)) == 0)
fasttrap_provs.fth_nent = nent;
else
fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
ASSERT(fasttrap_provs.fth_nent > 0);
fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
sizeof (fasttrap_bucket_t), KM_SLEEP);
ASSERT(fasttrap_provs.fth_table != NULL);
#if defined(__APPLE__)
for (i=0; i<fasttrap_provs.fth_nent; i++) {
lck_mtx_init(&fasttrap_provs.fth_table[i].ftb_mtx, fasttrap_lck_grp, fasttrap_lck_attr);
}
#endif
nent = FASTTRAP_PROCS_DEFAULT_SIZE;
if ((nent & (nent - 1)) == 0)
fasttrap_procs.fth_nent = nent;
else
fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
ASSERT(fasttrap_procs.fth_nent > 0);
fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
sizeof (fasttrap_bucket_t), KM_SLEEP);
ASSERT(fasttrap_procs.fth_table != NULL);
#if defined(__APPLE__)
for (i=0; i<fasttrap_procs.fth_nent; i++) {
lck_mtx_init(&fasttrap_procs.fth_table[i].ftb_mtx, fasttrap_lck_grp, fasttrap_lck_attr);
}
#endif
(void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
&fasttrap_meta_id);
return (DDI_SUCCESS);
}
static int
_fasttrap_open(dev_t dev, int flags, int devtype, struct proc *p)
{
#pragma unused(dev, flags, devtype, p)
return 0;
}
static int
_fasttrap_ioctl(dev_t dev, u_long cmd, caddr_t data, int fflag, struct proc *p)
{
int err, rv = 0;
user_addr_t uaddrp;
if (proc_is64bit(p))
uaddrp = *(user_addr_t *)data;
else
uaddrp = (user_addr_t) *(uint32_t *)data;
err = fasttrap_ioctl(dev, cmd, uaddrp, fflag, CRED(), &rv);
if (err != 0) {
ASSERT( (err & 0xfffff000) == 0 );
return (err & 0xfff);
} else if (rv != 0) {
ASSERT( (rv & 0xfff00000) == 0 );
return (((rv & 0xfffff) << 12));
} else
return 0;
}
static int gFasttrapInited = 0;
#define FASTTRAP_MAJOR -24
static struct cdevsw fasttrap_cdevsw =
{
_fasttrap_open,
eno_opcl,
eno_rdwrt,
eno_rdwrt,
_fasttrap_ioctl,
(stop_fcn_t *)nulldev,
(reset_fcn_t *)nulldev,
NULL,
eno_select,
eno_mmap,
eno_strat,
eno_getc,
eno_putc,
0
};
void fasttrap_init(void);
void
fasttrap_init( void )
{
if (0 == gFasttrapInited) {
int majdevno = cdevsw_add(FASTTRAP_MAJOR, &fasttrap_cdevsw);
if (majdevno < 0) {
printf("fasttrap_init: failed to allocate a major number!\n");
return;
}
dev_t device = makedev( (uint32_t)majdevno, 0 );
if (NULL == devfs_make_node( device, DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666, "fasttrap", 0 )) {
return;
}
fasttrap_tracepoint_t_zone = zinit(sizeof(fasttrap_tracepoint_t),
1024 * sizeof(fasttrap_tracepoint_t),
sizeof(fasttrap_tracepoint_t),
"dtrace.fasttrap_tracepoint_t");
int i;
for (i=1; i<FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS; i++) {
size_t zone_element_size = offsetof(fasttrap_probe_t, ftp_tps[i]);
fasttrap_probe_t_zones[i] = zinit(zone_element_size,
1024 * zone_element_size,
zone_element_size,
fasttrap_probe_t_zone_names[i]);
}
fasttrap_lck_attr = lck_attr_alloc_init();
fasttrap_lck_grp_attr= lck_grp_attr_alloc_init();
fasttrap_lck_grp = lck_grp_alloc_init("fasttrap", fasttrap_lck_grp_attr);
lck_mtx_init(&fasttrap_cleanup_mtx, fasttrap_lck_grp, fasttrap_lck_attr);
lck_mtx_init(&fasttrap_count_mtx, fasttrap_lck_grp, fasttrap_lck_attr);
if (DDI_FAILURE == fasttrap_attach((dev_info_t *)(uintptr_t)device, 0 )) {
printf("fasttrap_init: Call to fasttrap_attach failed.\n");
return;
}
gFasttrapInited = 1;
}
}
#undef FASTTRAP_MAJOR