#include <mach/host_priv.h>
#include <mach/host_special_ports.h>
#include <mach/mach_types.h>
#include <mach/telemetry_notification_server.h>
#include <kern/assert.h>
#include <kern/clock.h>
#include <kern/debug.h>
#include <kern/host.h>
#include <kern/kalloc.h>
#include <kern/kern_types.h>
#include <kern/locks.h>
#include <kern/misc_protos.h>
#include <kern/sched.h>
#include <kern/sched_prim.h>
#include <kern/telemetry.h>
#include <kern/timer_call.h>
#include <pexpert/pexpert.h>
#include <vm/vm_kern.h>
#include <vm/vm_shared_region.h>
#include <kperf/kperf.h>
#include <kperf/context.h>
#include <kperf/callstack.h>
#include <sys/kdebug.h>
#include <uuid/uuid.h>
#include <kdp/kdp_dyld.h>
#define TELEMETRY_DEBUG 0
extern int proc_pid(void *);
extern char *proc_name_address(void *p);
extern uint64_t proc_uniqueid(void *p);
extern uint64_t proc_was_throttled(void *p);
extern uint64_t proc_did_throttle(void *p);
extern uint64_t get_dispatchqueue_serialno_offset_from_proc(void *p);
extern int proc_selfpid(void);
void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags);
#define TELEMETRY_DEFAULT_SAMPLE_RATE (1)
#define TELEMETRY_DEFAULT_BUFFER_SIZE (16*1024)
#define TELEMETRY_MAX_BUFFER_SIZE (64*1024)
#define TELEMETRY_DEFAULT_NOTIFY_LEEWAY (4*1024) // Userland gets 4k of leeway to collect data after notification
uint32_t telemetry_sample_rate = 0;
volatile boolean_t telemetry_needs_record = FALSE;
volatile boolean_t telemetry_needs_timer_arming_record = FALSE;
boolean_t telemetry_sample_all_tasks = FALSE;
uint32_t telemetry_active_tasks = 0;
uint32_t telemetry_timestamp = 0;
vm_offset_t telemetry_buffer = 0;
uint32_t telemetry_buffer_size = 0;
uint32_t telemetry_buffer_current_position = 0;
uint32_t telemetry_buffer_end_point = 0; int telemetry_bytes_since_last_mark = -1; int telemetry_buffer_notify_at = 0;
lck_grp_t telemetry_lck_grp;
lck_mtx_t telemetry_mtx;
#define TELEMETRY_LOCK() do { lck_mtx_lock(&telemetry_mtx); } while(0)
#define TELEMETRY_TRY_SPIN_LOCK() lck_mtx_try_lock_spin(&telemetry_mtx)
#define TELEMETRY_UNLOCK() do { lck_mtx_unlock(&telemetry_mtx); } while(0)
void telemetry_init(void)
{
kern_return_t ret;
uint32_t telemetry_notification_leeway;
lck_grp_init(&telemetry_lck_grp, "telemetry group", LCK_GRP_ATTR_NULL);
lck_mtx_init(&telemetry_mtx, &telemetry_lck_grp, LCK_ATTR_NULL);
if (!PE_parse_boot_argn("telemetry_buffer_size", &telemetry_buffer_size, sizeof(telemetry_buffer_size))) {
telemetry_buffer_size = TELEMETRY_DEFAULT_BUFFER_SIZE;
}
if (telemetry_buffer_size > TELEMETRY_MAX_BUFFER_SIZE)
telemetry_buffer_size = TELEMETRY_MAX_BUFFER_SIZE;
ret = kmem_alloc(kernel_map, &telemetry_buffer, telemetry_buffer_size);
if (ret != KERN_SUCCESS) {
kprintf("Telemetry: Allocation failed: %d\n", ret);
return;
}
if (!PE_parse_boot_argn("telemetry_notification_leeway", &telemetry_notification_leeway, sizeof(telemetry_notification_leeway))) {
telemetry_notification_leeway = TELEMETRY_DEFAULT_NOTIFY_LEEWAY;
}
if (telemetry_notification_leeway >= telemetry_buffer_size) {
printf("telemetry: nonsensical telemetry_notification_leeway boot-arg %d changed to %d\n",
telemetry_notification_leeway, TELEMETRY_DEFAULT_NOTIFY_LEEWAY);
telemetry_notification_leeway = TELEMETRY_DEFAULT_NOTIFY_LEEWAY;
}
telemetry_buffer_notify_at = telemetry_buffer_size - telemetry_notification_leeway;
if (!PE_parse_boot_argn("telemetry_sample_rate", &telemetry_sample_rate, sizeof(telemetry_sample_rate))) {
telemetry_sample_rate = TELEMETRY_DEFAULT_SAMPLE_RATE;
}
if (!PE_parse_boot_argn("telemetry_sample_all_tasks", &telemetry_sample_all_tasks, sizeof(telemetry_sample_all_tasks))) {
telemetry_sample_all_tasks = TRUE;
}
kprintf("Telemetry: Sampling %stasks once per %u second%s\n",
(telemetry_sample_all_tasks) ? "all " : "",
telemetry_sample_rate, telemetry_sample_rate == 1 ? "" : "s");
}
void
telemetry_global_ctl(int enable_disable)
{
if (enable_disable == 1) {
telemetry_sample_all_tasks = TRUE;
} else {
telemetry_sample_all_tasks = FALSE;
}
}
void
telemetry_task_ctl(task_t task, uint32_t reasons, int enable_disable)
{
task_lock(task);
telemetry_task_ctl_locked(task, reasons, enable_disable);
task_unlock(task);
}
void
telemetry_task_ctl_locked(task_t task, uint32_t reasons, int enable_disable)
{
uint32_t origflags;
assert((reasons != 0) && ((reasons | TF_TELEMETRY) == TF_TELEMETRY));
task_lock_assert_owned(task);
origflags = task->t_flags;
if (enable_disable == 1) {
task->t_flags |= reasons;
if ((origflags & TF_TELEMETRY) == 0) {
OSIncrementAtomic(&telemetry_active_tasks);
#if TELEMETRY_DEBUG
printf("%s: telemetry OFF -> ON (%d active)\n", proc_name_address(task->bsd_info), telemetry_active_tasks);
#endif
}
} else {
task->t_flags &= ~reasons;
if (((origflags & TF_TELEMETRY) != 0) && ((task->t_flags & TF_TELEMETRY) == 0)) {
OSDecrementAtomic(&telemetry_active_tasks);
#if TELEMETRY_DEBUG
printf("%s: telemetry ON -> OFF (%d active)\n", proc_name_address(task->bsd_info), telemetry_active_tasks);
#endif
}
}
}
static boolean_t
telemetry_is_active(thread_t thread)
{
if (telemetry_sample_all_tasks == TRUE) {
return (TRUE);
}
if ((telemetry_active_tasks > 0) && ((thread->task->t_flags & TF_TELEMETRY) != 0)) {
return (TRUE);
}
return (FALSE);
}
int telemetry_timer_event(__unused uint64_t deadline, __unused uint64_t interval, __unused uint64_t leeway)
{
if (telemetry_needs_timer_arming_record == TRUE) {
telemetry_needs_timer_arming_record = FALSE;
telemetry_take_sample(current_thread(), kTimerArmingRecord | kUserMode);
}
return (0);
}
void telemetry_mark_curthread(boolean_t interrupted_userspace)
{
thread_t thread = current_thread();
if (telemetry_is_active(thread) == FALSE) {
return;
}
telemetry_needs_record = FALSE;
thread_ast_set(thread, interrupted_userspace ? AST_TELEMETRY_USER : AST_TELEMETRY_KERNEL);
ast_propagate(thread->ast);
}
void compute_telemetry(void *arg __unused)
{
if (telemetry_sample_all_tasks || (telemetry_active_tasks > 0)) {
if ((++telemetry_timestamp) % telemetry_sample_rate == 0) {
if (TELEMETRY_TRY_SPIN_LOCK()) {
telemetry_needs_record = TRUE;
telemetry_needs_timer_arming_record = TRUE;
TELEMETRY_UNLOCK();
}
}
}
}
static void
telemetry_notify_user(void)
{
mach_port_t user_port;
uint32_t flags = 0;
int error;
error = host_get_telemetry_port(host_priv_self(), &user_port);
if ((error != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) {
return;
}
telemetry_notification(user_port, flags);
}
void telemetry_ast(thread_t thread, boolean_t interrupted_userspace)
{
uint8_t microsnapshot_flags = kInterruptRecord;
if (interrupted_userspace)
microsnapshot_flags |= kUserMode;
telemetry_take_sample(thread, microsnapshot_flags);
}
void telemetry_take_sample(thread_t thread, uint8_t microsnapshot_flags)
{
task_t task;
void *p;
struct kperf_context ctx;
struct callstack cs;
uint32_t btcount, bti;
struct micro_snapshot *msnap;
struct task_snapshot *tsnap;
struct thread_snapshot *thsnap;
clock_sec_t secs;
clock_usec_t usecs;
vm_size_t framesize;
uint32_t current_record_start;
uint32_t tmp = 0;
boolean_t notify = FALSE;
if (thread == THREAD_NULL)
return;
task = thread->task;
if ((task == TASK_NULL) || (task == kernel_task))
return;
KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_RECORD) | DBG_FUNC_START, microsnapshot_flags, telemetry_bytes_since_last_mark, 0, 0, 0);
p = get_bsdtask_info(task);
ctx.cur_thread = thread;
ctx.cur_pid = proc_pid(p);
kperf_ucallstack_sample(&cs, &ctx);
if (!(cs.flags & CALLSTACK_VALID))
return;
int shared_cache_uuid_valid = 0;
uint64_t shared_cache_base_address;
struct _dyld_cache_header shared_cache_header;
uint64_t shared_cache_slide;
int sc_header_uuid_offset = (char *)&shared_cache_header.uuid - (char *)&shared_cache_header;
vm_shared_region_t sr = vm_shared_region_get(task);
if (sr != NULL) {
if ((vm_shared_region_start_address(sr, &shared_cache_base_address) == KERN_SUCCESS) &&
(copyin(shared_cache_base_address + sc_header_uuid_offset, (char *)&shared_cache_header.uuid,
sizeof (shared_cache_header.uuid)) == 0)) {
shared_cache_uuid_valid = 1;
shared_cache_slide = vm_shared_region_get_slide(sr);
}
vm_shared_region_deallocate(sr);
}
uint32_t uuid_info_count = 0;
mach_vm_address_t uuid_info_addr = 0;
if (task_has_64BitAddr(task)) {
struct user64_dyld_all_image_infos task_image_infos;
if (copyin(task->all_image_info_addr, (char *)&task_image_infos, sizeof(task_image_infos)) == 0) {
uuid_info_count = (uint32_t)task_image_infos.uuidArrayCount;
uuid_info_addr = task_image_infos.uuidArray;
}
} else {
struct user32_dyld_all_image_infos task_image_infos;
if (copyin(task->all_image_info_addr, (char *)&task_image_infos, sizeof(task_image_infos)) == 0) {
uuid_info_count = task_image_infos.uuidArrayCount;
uuid_info_addr = task_image_infos.uuidArray;
}
}
if (!uuid_info_addr) {
uuid_info_count = 0;
}
uint32_t uuid_info_size = (uint32_t)(task_has_64BitAddr(thread->task) ? sizeof(struct user64_dyld_uuid_info) : sizeof(struct user32_dyld_uuid_info));
uint32_t uuid_info_array_size = uuid_info_count * uuid_info_size;
char *uuid_info_array = NULL;
if (uuid_info_count > 0) {
if ((uuid_info_array = (char *)kalloc(uuid_info_array_size)) == NULL) {
return;
}
if (copyin(uuid_info_addr, uuid_info_array, uuid_info_array_size) != 0) {
kfree(uuid_info_array, uuid_info_array_size);
uuid_info_array = NULL;
uuid_info_array_size = 0;
}
}
uint64_t dqserialnum = 0;
int dqserialnum_valid = 0;
uint64_t dqkeyaddr = thread_dispatchqaddr(thread);
if (dqkeyaddr != 0) {
uint64_t dqaddr = 0;
uint64_t dq_serialno_offset = get_dispatchqueue_serialno_offset_from_proc(task->bsd_info);
if ((copyin(dqkeyaddr, (char *)&dqaddr, (task_has_64BitAddr(task) ? 8 : 4)) == 0) &&
(dqaddr != 0) && (dq_serialno_offset != 0)) {
uint64_t dqserialnumaddr = dqaddr + dq_serialno_offset;
if (copyin(dqserialnumaddr, (char *)&dqserialnum, (task_has_64BitAddr(task) ? 8 : 4)) == 0) {
dqserialnum_valid = 1;
}
}
}
clock_get_calendar_microtime(&secs, &usecs);
TELEMETRY_LOCK();
copytobuffer:
current_record_start = telemetry_buffer_current_position;
if ((telemetry_buffer_size - telemetry_buffer_current_position) < sizeof(struct micro_snapshot)) {
telemetry_buffer_end_point = current_record_start;
telemetry_buffer_current_position = 0;
goto copytobuffer;
}
msnap = (struct micro_snapshot *)(uintptr_t)(telemetry_buffer + telemetry_buffer_current_position);
msnap->snapshot_magic = STACKSHOT_MICRO_SNAPSHOT_MAGIC;
msnap->ms_flags = microsnapshot_flags;
msnap->ms_opaque_flags = 0;
msnap->ms_cpu = 0;
msnap->ms_time = secs;
msnap->ms_time_microsecs = usecs;
telemetry_buffer_current_position += sizeof(struct micro_snapshot);
if ((telemetry_buffer_size - telemetry_buffer_current_position) < sizeof(struct task_snapshot)) {
telemetry_buffer_end_point = current_record_start;
telemetry_buffer_current_position = 0;
goto copytobuffer;
}
tsnap = (struct task_snapshot *)(uintptr_t)(telemetry_buffer + telemetry_buffer_current_position);
bzero(tsnap, sizeof(*tsnap));
tsnap->snapshot_magic = STACKSHOT_TASK_SNAPSHOT_MAGIC;
tsnap->pid = proc_pid(p);
tsnap->uniqueid = proc_uniqueid(p);
tsnap->user_time_in_terminated_threads = task->total_user_time;
tsnap->system_time_in_terminated_threads = task->total_system_time;
tsnap->suspend_count = task->suspend_count;
tsnap->task_size = pmap_resident_count(task->map->pmap);
tsnap->faults = task->faults;
tsnap->pageins = task->pageins;
tsnap->cow_faults = task->cow_faults;
tsnap->was_throttled = (uint32_t) proc_was_throttled(p);
tsnap->did_throttle = (uint32_t) proc_did_throttle(p);
if (task->t_flags & TF_TELEMETRY) {
tsnap->ss_flags |= kTaskRsrcFlagged;
}
if (task->effective_policy.darwinbg == 1) {
tsnap->ss_flags |= kTaskDarwinBG;
}
proc_get_darwinbgstate(task, &tmp);
if (task->requested_policy.t_role == TASK_FOREGROUND_APPLICATION) {
tsnap->ss_flags |= kTaskIsForeground;
}
if (tmp & PROC_FLAG_ADAPTIVE_IMPORTANT) {
tsnap->ss_flags |= kTaskIsBoosted;
}
if (tmp & PROC_FLAG_SUPPRESSED) {
tsnap->ss_flags |= kTaskIsSuppressed;
}
tsnap->latency_qos = task_grab_latency_qos(task);
strlcpy(tsnap->p_comm, proc_name_address(p), sizeof(tsnap->p_comm));
if (task_has_64BitAddr(thread->task)) {
tsnap->ss_flags |= kUser64_p;
}
if (shared_cache_uuid_valid) {
tsnap->shared_cache_slide = shared_cache_slide;
bcopy(shared_cache_header.uuid, tsnap->shared_cache_identifier, sizeof (shared_cache_header.uuid));
}
telemetry_buffer_current_position += sizeof(struct task_snapshot);
if ((telemetry_buffer_size - telemetry_buffer_current_position) < uuid_info_array_size) {
telemetry_buffer_end_point = current_record_start;
telemetry_buffer_current_position = 0;
goto copytobuffer;
}
if (uuid_info_array_size > 0) {
bcopy(uuid_info_array, (char *)(telemetry_buffer + telemetry_buffer_current_position), uuid_info_array_size);
tsnap->nloadinfos = uuid_info_count;
}
telemetry_buffer_current_position += uuid_info_array_size;
if ((telemetry_buffer_size - telemetry_buffer_current_position) < sizeof(struct thread_snapshot)) {
telemetry_buffer_end_point = current_record_start;
telemetry_buffer_current_position = 0;
goto copytobuffer;
}
thsnap = (struct thread_snapshot *)(uintptr_t)(telemetry_buffer + telemetry_buffer_current_position);
bzero(thsnap, sizeof(*thsnap));
thsnap->snapshot_magic = STACKSHOT_THREAD_SNAPSHOT_MAGIC;
thsnap->thread_id = thread_tid(thread);
thsnap->state = thread->state;
thsnap->priority = thread->priority;
thsnap->sched_pri = thread->sched_pri;
thsnap->sched_flags = thread->sched_flags;
thsnap->ss_flags |= kStacksPCOnly;
if (thread->effective_policy.darwinbg) {
thsnap->ss_flags |= kThreadDarwinBG;
}
thsnap->user_time = timer_grab(&thread->user_timer);
uint64_t tval = timer_grab(&thread->system_timer);
if (thread->precise_user_kernel_time) {
thsnap->system_time = tval;
} else {
thsnap->user_time += tval;
thsnap->system_time = 0;
}
telemetry_buffer_current_position += sizeof(struct thread_snapshot);
if (dqserialnum_valid) {
if ((telemetry_buffer_size - telemetry_buffer_current_position) < sizeof(dqserialnum)) {
telemetry_buffer_end_point = current_record_start;
telemetry_buffer_current_position = 0;
goto copytobuffer;
}
thsnap->ss_flags |= kHasDispatchSerial;
bcopy(&dqserialnum, (char *)telemetry_buffer + telemetry_buffer_current_position, sizeof (dqserialnum));
telemetry_buffer_current_position += sizeof (dqserialnum);
}
if (task_has_64BitAddr(task)) {
framesize = 8;
thsnap->ss_flags |= kUser64_p;
} else {
framesize = 4;
}
btcount = cs.nframes;
if ((telemetry_buffer_size - telemetry_buffer_current_position)/framesize < btcount) {
telemetry_buffer_end_point = current_record_start;
telemetry_buffer_current_position = 0;
goto copytobuffer;
}
for (bti=0; bti < btcount; bti++, telemetry_buffer_current_position += framesize) {
if (framesize == 8) {
*(uint64_t *)(uintptr_t)(telemetry_buffer + telemetry_buffer_current_position) = cs.frames[bti];
} else {
*(uint32_t *)(uintptr_t)(telemetry_buffer + telemetry_buffer_current_position) = (uint32_t)cs.frames[bti];
}
}
if (telemetry_buffer_end_point < telemetry_buffer_current_position) {
telemetry_buffer_end_point = telemetry_buffer_current_position;
}
thsnap->nuser_frames = btcount;
telemetry_bytes_since_last_mark += (telemetry_buffer_current_position - current_record_start);
if (telemetry_bytes_since_last_mark > telemetry_buffer_notify_at) {
notify = TRUE;
}
TELEMETRY_UNLOCK();
KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_RECORD) | DBG_FUNC_END, notify, telemetry_bytes_since_last_mark, telemetry_buffer_current_position, telemetry_buffer_end_point, 0);
if (notify) {
telemetry_notify_user();
}
if (uuid_info_array != NULL) {
kfree(uuid_info_array, uuid_info_array_size);
}
}
#if TELEMETRY_DEBUG
static void
log_telemetry_output(vm_offset_t buf, uint32_t pos, uint32_t sz)
{
struct micro_snapshot *p;
uint32_t offset;
printf("Copying out %d bytes of telemetry at offset %d\n", sz, pos);
buf += pos;
for (offset = 0; offset < sz; offset++) {
p = (struct micro_snapshot *)(buf + offset);
if (p->snapshot_magic == STACKSHOT_MICRO_SNAPSHOT_MAGIC) {
printf("telemetry timestamp: %lld\n", p->ms_time);
}
}
}
#endif
int telemetry_gather(user_addr_t buffer, uint32_t *length, boolean_t mark)
{
int result = 0;
uint32_t oldest_record_offset;
KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_GATHER) | DBG_FUNC_START, mark, telemetry_bytes_since_last_mark, 0, 0, 0);
TELEMETRY_LOCK();
if (telemetry_buffer == 0) {
*length = 0;
goto out;
}
if (*length < telemetry_buffer_size) {
result = KERN_NO_SPACE;
goto out;
}
oldest_record_offset = telemetry_buffer_current_position;
do {
if ((oldest_record_offset == telemetry_buffer_size) ||
(oldest_record_offset == telemetry_buffer_end_point)) {
if (*(uint32_t *)(uintptr_t)(telemetry_buffer) == 0) {
*length = 0;
goto out;
}
oldest_record_offset = 0;
assert(*(uint32_t *)(uintptr_t)(telemetry_buffer) == STACKSHOT_MICRO_SNAPSHOT_MAGIC);
break;
}
if (*(uint32_t *)(uintptr_t)(telemetry_buffer + oldest_record_offset) == STACKSHOT_MICRO_SNAPSHOT_MAGIC)
break;
oldest_record_offset++;
} while (oldest_record_offset != telemetry_buffer_current_position);
if (oldest_record_offset != 0) {
#if TELEMETRY_DEBUG
log_telemetry_output(telemetry_buffer, oldest_record_offset,
telemetry_buffer_end_point - oldest_record_offset);
#endif
if ((result = copyout((void *)(telemetry_buffer + oldest_record_offset), buffer,
telemetry_buffer_end_point - oldest_record_offset)) != 0) {
*length = 0;
goto out;
}
*length = telemetry_buffer_end_point - oldest_record_offset;
} else {
*length = 0;
}
#if TELEMETRY_DEBUG
log_telemetry_output(telemetry_buffer, 0, telemetry_buffer_current_position);
#endif
if ((result = copyout((void *)telemetry_buffer, buffer + *length,
telemetry_buffer_current_position)) != 0) {
*length = 0;
goto out;
}
*length += (uint32_t)telemetry_buffer_current_position;
out:
if (mark && (*length > 0)) {
telemetry_bytes_since_last_mark = 0;
}
TELEMETRY_UNLOCK();
KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_STACKSHOT, MICROSTACKSHOT_GATHER) | DBG_FUNC_END, telemetry_buffer_current_position, *length, telemetry_buffer_end_point, 0, 0);
return (result);
}
#define BOOTPROFILE_MAX_BUFFER_SIZE (64*1024*1024)
vm_offset_t bootprofile_buffer = 0;
uint32_t bootprofile_buffer_size = 0;
uint32_t bootprofile_buffer_current_position = 0;
uint32_t bootprofile_interval_ms = 0;
uint64_t bootprofile_interval_abs = 0;
uint64_t bootprofile_next_deadline = 0;
uint32_t bootprofile_all_procs = 0;
char bootprofile_proc_name[17];
lck_grp_t bootprofile_lck_grp;
lck_mtx_t bootprofile_mtx;
static timer_call_data_t bootprofile_timer_call_entry;
#define BOOTPROFILE_LOCK() do { lck_mtx_lock(&bootprofile_mtx); } while(0)
#define BOOTPROFILE_TRY_SPIN_LOCK() lck_mtx_try_lock_spin(&bootprofile_mtx)
#define BOOTPROFILE_UNLOCK() do { lck_mtx_unlock(&bootprofile_mtx); } while(0)
static void bootprofile_timer_call(
timer_call_param_t param0,
timer_call_param_t param1);
extern int
stack_snapshot_from_kernel(int pid, void *buf, uint32_t size, uint32_t flags, unsigned *retbytes);
void bootprofile_init(void)
{
kern_return_t ret;
lck_grp_init(&bootprofile_lck_grp, "bootprofile group", LCK_GRP_ATTR_NULL);
lck_mtx_init(&bootprofile_mtx, &bootprofile_lck_grp, LCK_ATTR_NULL);
if (!PE_parse_boot_argn("bootprofile_buffer_size", &bootprofile_buffer_size, sizeof(bootprofile_buffer_size))) {
bootprofile_buffer_size = 0;
}
if (bootprofile_buffer_size > BOOTPROFILE_MAX_BUFFER_SIZE)
bootprofile_buffer_size = BOOTPROFILE_MAX_BUFFER_SIZE;
if (!PE_parse_boot_argn("bootprofile_interval_ms", &bootprofile_interval_ms, sizeof(bootprofile_interval_ms))) {
bootprofile_interval_ms = 0;
}
if (!PE_parse_boot_argn("bootprofile_proc_name", &bootprofile_proc_name, sizeof(bootprofile_proc_name))) {
bootprofile_all_procs = 1;
bootprofile_proc_name[0] = '\0';
}
clock_interval_to_absolutetime_interval(bootprofile_interval_ms, NSEC_PER_MSEC, &bootprofile_interval_abs);
if ((bootprofile_buffer_size == 0) || (bootprofile_interval_abs == 0)) {
return;
}
ret = kmem_alloc(kernel_map, &bootprofile_buffer, bootprofile_buffer_size);
if (ret != KERN_SUCCESS) {
kprintf("Boot profile: Allocation failed: %d\n", ret);
return;
}
kprintf("Boot profile: Sampling %s once per %u ms\n", bootprofile_all_procs ? "all procs" : bootprofile_proc_name, bootprofile_interval_ms);
timer_call_setup(&bootprofile_timer_call_entry,
bootprofile_timer_call,
NULL);
bootprofile_next_deadline = mach_absolute_time() + bootprofile_interval_abs;
timer_call_enter_with_leeway(&bootprofile_timer_call_entry,
NULL,
bootprofile_next_deadline,
0,
TIMER_CALL_SYS_NORMAL,
FALSE);
}
static void bootprofile_timer_call(
timer_call_param_t param0 __unused,
timer_call_param_t param1 __unused)
{
unsigned retbytes = 0;
int pid_to_profile = -1;
if (!BOOTPROFILE_TRY_SPIN_LOCK()) {
goto reprogram;
}
if (!bootprofile_all_procs) {
if ((current_task() != NULL) && (current_task()->bsd_info != NULL) &&
(0 == strncmp(bootprofile_proc_name, proc_name_address(current_task()->bsd_info), 17))) {
pid_to_profile = proc_selfpid();
}
else {
BOOTPROFILE_UNLOCK();
goto reprogram;
}
}
if (bootprofile_buffer_current_position < bootprofile_buffer_size) {
stack_snapshot_from_kernel(
pid_to_profile,
(void *)(bootprofile_buffer + bootprofile_buffer_current_position),
bootprofile_buffer_size - bootprofile_buffer_current_position,
STACKSHOT_SAVE_LOADINFO | STACKSHOT_SAVE_KEXT_LOADINFO | STACKSHOT_GET_GLOBAL_MEM_STATS,
&retbytes
);
bootprofile_buffer_current_position += retbytes;
}
BOOTPROFILE_UNLOCK();
if ((retbytes == 0) || (bootprofile_buffer_current_position == bootprofile_buffer_size)) {
return;
}
reprogram:
if (bootprofile_interval_abs == 0) {
return;
}
clock_deadline_for_periodic_event(bootprofile_interval_abs,
mach_absolute_time(),
&bootprofile_next_deadline);
timer_call_enter_with_leeway(&bootprofile_timer_call_entry,
NULL,
bootprofile_next_deadline,
0,
TIMER_CALL_SYS_NORMAL,
FALSE);
}
int bootprofile_gather(user_addr_t buffer, uint32_t *length)
{
int result = 0;
BOOTPROFILE_LOCK();
if (bootprofile_buffer == 0) {
*length = 0;
goto out;
}
if (*length < bootprofile_buffer_current_position) {
result = KERN_NO_SPACE;
goto out;
}
if ((result = copyout((void *)bootprofile_buffer, buffer,
bootprofile_buffer_current_position)) != 0) {
*length = 0;
goto out;
}
*length = bootprofile_buffer_current_position;
bootprofile_interval_abs = 0;
out:
BOOTPROFILE_UNLOCK();
return (result);
}