#define MACH__POSIX_C_SOURCE_PRIVATE 1
#include <arm/caches_internal.h>
#include <arm/proc_reg.h>
#include <kern/thread.h>
#include <mach/thread_status.h>
#if __has_include(<ptrauth.h>)
#include <ptrauth.h>
#endif
#include <stdarg.h>
#include <string.h>
#include <sys/malloc.h>
#include <sys/time.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/proc_internal.h>
#include <sys/kauth.h>
#include <sys/dtrace.h>
#include <sys/dtrace_impl.h>
#include <libkern/OSAtomic.h>
#include <kern/simple_lock.h>
#include <kern/sched_prim.h>
#include <kern/thread_call.h>
#include <kern/task.h>
#include <miscfs/devfs/devfs.h>
#include <mach/vm_param.h>
extern struct arm_saved_state *find_kern_regs(thread_t);
extern dtrace_id_t dtrace_probeid_error;
typedef arm_saved_state_t savearea_t;
extern lck_attr_t *dtrace_lck_attr;
extern lck_grp_t *dtrace_lck_grp;
struct frame {
struct frame *backchain;
uintptr_t retaddr;
};
inline void
dtrace_membar_producer(void)
{
#if __ARM_SMP__
__asm__ volatile("dmb ish" : : : "memory");
#else
__asm__ volatile("nop" : : : "memory");
#endif
}
inline void
dtrace_membar_consumer(void)
{
#if __ARM_SMP__
__asm__ volatile("dmb ish" : : : "memory");
#else
__asm__ volatile("nop" : : : "memory");
#endif
}
int
dtrace_getipl(void)
{
return (ml_at_interrupt_context() ? 1 : 0);
}
#if __ARM_SMP__
decl_lck_mtx_data(static, dt_xc_lock);
static uint32_t dt_xc_sync;
typedef struct xcArg {
processorid_t cpu;
dtrace_xcall_t f;
void *arg;
} xcArg_t;
static void
xcRemote(void *foo)
{
xcArg_t *pArg = (xcArg_t *) foo;
if (pArg->cpu == CPU->cpu_id || pArg->cpu == DTRACE_CPUALL)
(pArg->f) (pArg->arg);
if (hw_atomic_sub(&dt_xc_sync, 1) == 0)
thread_wakeup((event_t) &dt_xc_sync);
}
#endif
void
dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg)
{
#if __ARM_SMP__
lck_mtx_lock(&dt_xc_lock);
xcArg_t xcArg;
xcArg.cpu = cpu;
xcArg.f = f;
xcArg.arg = arg;
cpu_broadcast_xcall(&dt_xc_sync, TRUE, xcRemote, (void*) &xcArg);
lck_mtx_unlock(&dt_xc_lock);
return;
#else
#pragma unused(cpu)
ASSERT(cpu == CPU->cpu_id || cpu == DTRACE_CPUALL);
(*f)(arg);
return;
#endif
}
void
dtrace_isa_init(void)
{
lck_mtx_init(&dt_xc_lock, dtrace_lck_grp, dtrace_lck_attr);
return;
}
#define ARM_FP 7
#define ARM_SP 13
#define ARM_LR 14
#define ARM_PC 15
#define ARM_CPSR 16
#define ARM64_FP 29
#define ARM64_LR 30
#define ARM64_SP 31
#define ARM64_PC 32
#define ARM64_CPSR 33
uint64_t
dtrace_getreg(struct regs * savearea, uint_t reg)
{
struct arm_saved_state *regs = (struct arm_saved_state *) savearea;
if (regs == NULL) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
return (0);
}
if (is_saved_state32(regs)) {
switch (reg) {
case ARM64_FP:
reg = ARM_FP;
break;
case ARM64_SP:
reg = ARM_SP;
break;
case ARM64_LR:
reg = ARM_LR;
break;
case ARM64_PC:
reg = ARM_PC;
break;
case ARM64_CPSR:
reg = ARM_CPSR;
break;
}
}
if (!check_saved_state_reglimit(regs, reg)) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
return (0);
}
return ((uint64_t)get_saved_state_reg(regs, reg));
}
#define RETURN_OFFSET 4
#define RETURN_OFFSET64 8
static int
dtrace_getustack_common(uint64_t * pcstack, int pcstack_limit, user_addr_t pc,
user_addr_t sp)
{
int ret = 0;
boolean_t is64bit = proc_is64bit_data(current_proc());
ASSERT(pcstack == NULL || pcstack_limit > 0);
while (pc != 0) {
ret++;
if (pcstack != NULL) {
*pcstack++ = (uint64_t) pc;
pcstack_limit--;
if (pcstack_limit <= 0)
break;
}
if (sp == 0)
break;
if (is64bit) {
pc = dtrace_fuword64((sp + RETURN_OFFSET64));
sp = dtrace_fuword64(sp);
} else {
pc = dtrace_fuword32((sp + RETURN_OFFSET));
sp = dtrace_fuword32(sp);
}
}
return (ret);
}
void
dtrace_getupcstack(uint64_t * pcstack, int pcstack_limit)
{
thread_t thread = current_thread();
savearea_t *regs;
user_addr_t pc, sp, fp;
volatile uint16_t *flags = (volatile uint16_t *) & cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
int n;
if (*flags & CPU_DTRACE_FAULT)
return;
if (pcstack_limit <= 0)
return;
if (thread == NULL)
goto zero;
regs = (savearea_t *) find_user_regs(thread);
if (regs == NULL)
goto zero;
*pcstack++ = (uint64_t)dtrace_proc_selfpid();
pcstack_limit--;
if (pcstack_limit <= 0)
return;
pc = get_saved_state_pc(regs);
sp = get_saved_state_sp(regs);
fp = get_saved_state_fp(regs);
if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
*pcstack++ = (uint64_t) pc;
pcstack_limit--;
if (pcstack_limit <= 0)
return;
pc = get_saved_state_lr(regs);
}
n = dtrace_getustack_common(pcstack, pcstack_limit, pc, fp);
ASSERT(n >= 0);
ASSERT(n <= pcstack_limit);
pcstack += n;
pcstack_limit -= n;
zero:
while (pcstack_limit-- > 0)
*pcstack++ = 0ULL;
}
int
dtrace_getustackdepth(void)
{
thread_t thread = current_thread();
savearea_t *regs;
user_addr_t pc, sp, fp;
int n = 0;
if (thread == NULL)
return 0;
if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT))
return (-1);
regs = (savearea_t *) find_user_regs(thread);
if (regs == NULL)
return 0;
pc = get_saved_state_pc(regs);
sp = get_saved_state_sp(regs);
fp = get_saved_state_fp(regs);
if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
n++;
pc = get_saved_state_lr(regs);
}
n += dtrace_getustack_common(NULL, 0, pc, fp);
return (n);
}
void
dtrace_getufpstack(uint64_t * pcstack, uint64_t * fpstack, int pcstack_limit)
{
thread_t thread = current_thread();
boolean_t is64bit = proc_is64bit_data(current_proc());
savearea_t *regs;
user_addr_t pc, sp;
volatile uint16_t *flags = (volatile uint16_t *) & cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
#if 0
uintptr_t oldcontext;
size_t s1, s2;
#endif
if (*flags & CPU_DTRACE_FAULT)
return;
if (pcstack_limit <= 0)
return;
if (thread == NULL)
goto zero;
regs = (savearea_t *) find_user_regs(thread);
if (regs == NULL)
goto zero;
*pcstack++ = (uint64_t)dtrace_proc_selfpid();
pcstack_limit--;
if (pcstack_limit <= 0)
return;
pc = get_saved_state_pc(regs);
sp = get_saved_state_lr(regs);
#if 0
oldcontext = lwp->lwp_oldcontext;
if (p->p_model == DATAMODEL_NATIVE) {
s1 = sizeof(struct frame) + 2 * sizeof(long);
s2 = s1 + sizeof(siginfo_t);
} else {
s1 = sizeof(struct frame32) + 3 * sizeof(int);
s2 = s1 + sizeof(siginfo32_t);
}
#endif
if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
*pcstack++ = (uint64_t) pc;
*fpstack++ = 0;
pcstack_limit--;
if (pcstack_limit <= 0)
return;
if (is64bit)
pc = dtrace_fuword64(sp);
else
pc = dtrace_fuword32(sp);
}
while (pc != 0 && sp != 0) {
*pcstack++ = (uint64_t) pc;
*fpstack++ = sp;
pcstack_limit--;
if (pcstack_limit <= 0)
break;
#if 0
if (oldcontext == sp + s1 || oldcontext == sp + s2) {
if (p->p_model == DATAMODEL_NATIVE) {
ucontext_t *ucp = (ucontext_t *) oldcontext;
greg_t *gregs = ucp->uc_mcontext.gregs;
sp = dtrace_fulword(&gregs[REG_FP]);
pc = dtrace_fulword(&gregs[REG_PC]);
oldcontext = dtrace_fulword(&ucp->uc_link);
} else {
ucontext_t *ucp = (ucontext_t *) oldcontext;
greg_t *gregs = ucp->uc_mcontext.gregs;
sp = dtrace_fuword32(&gregs[EBP]);
pc = dtrace_fuword32(&gregs[EIP]);
oldcontext = dtrace_fuword32(&ucp->uc_link);
}
} else
#endif
{
if (is64bit) {
pc = dtrace_fuword64((sp + RETURN_OFFSET64));
sp = dtrace_fuword64(sp);
} else {
pc = dtrace_fuword32((sp + RETURN_OFFSET));
sp = dtrace_fuword32(sp);
}
}
#if 0
if (*flags & CPU_DTRACE_FAULT) {
*flags &= ~CPU_DTRACE_FAULT;
break;
}
#endif
}
zero:
while (pcstack_limit-- > 0)
*pcstack++ = 0ULL;
}
void
dtrace_getpcstack(pc_t * pcstack, int pcstack_limit, int aframes,
uint32_t * intrpc)
{
struct frame *fp = (struct frame *) __builtin_frame_address(0);
struct frame *nextfp, *minfp, *stacktop;
int depth = 0;
int on_intr;
int last = 0;
uintptr_t pc;
uintptr_t caller = CPU->cpu_dtrace_caller;
if ((on_intr = CPU_ON_INTR(CPU)) != 0)
stacktop = (struct frame *) dtrace_get_cpu_int_stack_top();
else
stacktop = (struct frame *) (dtrace_get_kernel_stack(current_thread()) + kernel_stack_size);
minfp = fp;
aframes++;
if (intrpc != NULL && depth < pcstack_limit)
pcstack[depth++] = (pc_t) intrpc;
while (depth < pcstack_limit) {
nextfp = *(struct frame **) fp;
pc = *(uintptr_t *) (((uintptr_t) fp) + RETURN_OFFSET64);
if (nextfp <= minfp || nextfp >= stacktop) {
if (on_intr) {
arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread());
if (arm_kern_regs) {
nextfp = (struct frame *)(saved_state64(arm_kern_regs)->fp);
{
vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());
minfp = (struct frame *)kstack_base;
stacktop = (struct frame *)(kstack_base + kernel_stack_size);
}
on_intr = 0;
if (nextfp <= minfp || nextfp >= stacktop) {
last = 1;
}
} else {
last = 1;
}
} else {
{
last = 1;
}
}
}
if (aframes > 0) {
if (--aframes == 0 && caller != (uintptr_t)NULL) {
ASSERT(depth < pcstack_limit);
pcstack[depth++] = (pc_t) caller;
caller = (uintptr_t)NULL;
}
} else {
if (depth < pcstack_limit)
pcstack[depth++] = (pc_t) pc;
}
if (last) {
while (depth < pcstack_limit)
pcstack[depth++] = (pc_t) NULL;
return;
}
fp = nextfp;
minfp = fp;
}
}
int
dtrace_instr_size(uint32_t instr, int thumb_mode)
{
if (thumb_mode) {
uint16_t instr16 = *(uint16_t*) &instr;
if (((instr16 >> 11) & 0x1F) > 0x1C)
return 4;
else
return 2;
} else {
return 4;
}
}
uint64_t
dtrace_getarg(int arg, int aframes, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
{
#pragma unused(arg, aframes)
uint64_t val = 0;
struct frame *fp = (struct frame *)__builtin_frame_address(0);
uintptr_t *stack;
uintptr_t pc;
int i;
int inreg = 7;
for (i = 1; i <= aframes; ++i) {
fp = fp->backchain;
#if __has_feature(ptrauth_returns)
pc = (uintptr_t)ptrauth_strip((void*)fp->retaddr, ptrauth_key_return_address);
#else
pc = fp->retaddr;
#endif
if (dtrace_invop_callsite_pre != NULL
&& pc > (uintptr_t) dtrace_invop_callsite_pre
&& pc <= (uintptr_t) dtrace_invop_callsite_post)
{
fp = fp->backchain;
fp = fp->backchain;
fp = fp->backchain;
arm_saved_state_t *tagged_regs = (arm_saved_state_t*) ((void*) &fp[1]);
arm_saved_state64_t *saved_state = saved_state64(tagged_regs);
if (arg <= inreg) {
stack = (uintptr_t*) &saved_state->x[0];
} else {
fp = (struct frame*) saved_state->sp;
stack = (uintptr_t*) &fp[1];
arg -= (inreg + 1);
}
goto load;
}
}
arg++;
if (arg <= inreg) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
return (0);
}
arg -= (inreg + 1);
stack = (uintptr_t*) &fp[1];
load:
if (dtrace_canload((uint64_t)(stack + arg), sizeof(uint64_t),
mstate, vstate)) {
val = dtrace_load64((uint64_t)(stack + arg));
}
return (val);
}
void
dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
int fltoffs, int fault, uint64_t illval)
{
state->dts_arg_error_illval = illval;
dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fltoffs, fault );
}
void
dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
{
func(0x0, VM_MIN_KERNEL_ADDRESS);
if (VM_MAX_KERNEL_ADDRESS < ~(uintptr_t)0)
func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0);
}
void dtrace_flush_caches(void)
{
FlushPoU_Dcache();
InvalidatePoU_Icache();
}