#include <mach/mach_types.h>
#include <kern/timer_queue.h>
#include <kern/timer_call.h>
#include <kern/clock.h>
#include <kern/thread.h>
#include <kern/processor.h>
#include <kern/macro_help.h>
#include <kern/spl.h>
#include <kern/timer_queue.h>
#include <kern/pms.h>
#include <machine/commpage.h>
#include <machine/machine_routines.h>
#include <sys/kdebug.h>
#include <i386/cpu_data.h>
#include <i386/cpu_topology.h>
#include <i386/cpu_threads.h>
uint32_t spurious_timers;
void
timer_intr(int user_mode,
uint64_t rip)
{
uint64_t abstime;
rtclock_timer_t *mytimer;
cpu_data_t *pp;
int64_t latency;
uint64_t pmdeadline;
boolean_t timer_processed = FALSE;
pp = current_cpu_datap();
SCHED_STATS_TIMER_POP(current_processor());
abstime = mach_absolute_time();
mytimer = &pp->rtclock_timer;
if ((timer_processed = ((mytimer->deadline <= abstime) ||
(abstime >= (mytimer->queue.earliest_soft_deadline))))) {
latency = (int64_t) (abstime - MAX(mytimer->deadline,
mytimer->when_set));
if (latency < 0) {
TCOAL_DEBUG(0xEEEE0000, abstime, mytimer->queue.earliest_soft_deadline, abstime - mytimer->queue.earliest_soft_deadline, 0, 0);
latency = 0;
}
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
DECR_TRAP_LATENCY | DBG_FUNC_NONE,
-latency,
((user_mode != 0) ? rip : VM_KERNEL_UNSLIDE(rip)),
user_mode, 0, 0);
mytimer->has_expired = TRUE;
mytimer->deadline = timer_queue_expire(&mytimer->queue, abstime);
mytimer->has_expired = FALSE;
abstime = mach_absolute_time();
mytimer->when_set = abstime;
}
if ((pmdeadline = pmCPUGetDeadline(pp)) && (pmdeadline <= abstime)) {
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
DECR_PM_DEADLINE | DBG_FUNC_START,
0, 0, 0, 0, 0);
pmCPUDeadline(pp);
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
DECR_PM_DEADLINE | DBG_FUNC_END,
0, 0, 0, 0, 0);
timer_processed = TRUE;
}
x86_lcpu()->rtcDeadline = EndOfAllTime;
timer_resync_deadlines();
if (__improbable(timer_processed == FALSE))
spurious_timers++;
}
void timer_set_deadline(uint64_t deadline)
{
rtclock_timer_t *mytimer;
spl_t s;
cpu_data_t *pp;
s = splclock();
pp = current_cpu_datap();
mytimer = &pp->rtclock_timer;
mytimer->deadline = deadline;
mytimer->when_set = mach_absolute_time();
timer_resync_deadlines();
splx(s);
}
void
timer_resync_deadlines(void)
{
uint64_t deadline = EndOfAllTime;
uint64_t pmdeadline;
rtclock_timer_t *mytimer;
spl_t s = splclock();
cpu_data_t *pp;
uint32_t decr;
pp = current_cpu_datap();
if (!pp->cpu_running)
return;
mytimer = &pp->rtclock_timer;
if (!mytimer->has_expired &&
0 < mytimer->deadline && mytimer->deadline < EndOfAllTime)
deadline = mytimer->deadline;
pmdeadline = pmCPUGetDeadline(pp);
if (0 < pmdeadline && pmdeadline < deadline)
deadline = pmdeadline;
decr = (uint32_t) setPop(deadline);
if (decr != 0 && deadline != pmdeadline) {
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
DECR_SET_DEADLINE | DBG_FUNC_NONE,
decr, 2,
deadline,
mytimer->queue.count, 0);
}
splx(s);
}
void
timer_queue_expire_local(
__unused void *arg)
{
rtclock_timer_t *mytimer;
uint64_t abstime;
cpu_data_t *pp;
pp = current_cpu_datap();
mytimer = &pp->rtclock_timer;
abstime = mach_absolute_time();
mytimer->has_expired = TRUE;
mytimer->deadline = timer_queue_expire(&mytimer->queue, abstime);
mytimer->has_expired = FALSE;
mytimer->when_set = mach_absolute_time();
timer_resync_deadlines();
}
void
timer_queue_expire_rescan(
__unused void *arg)
{
rtclock_timer_t *mytimer;
uint64_t abstime;
cpu_data_t *pp;
assert(ml_get_interrupts_enabled() == FALSE);
pp = current_cpu_datap();
mytimer = &pp->rtclock_timer;
abstime = mach_absolute_time();
mytimer->has_expired = TRUE;
mytimer->deadline = timer_queue_expire_with_options(&mytimer->queue, abstime, TRUE);
mytimer->has_expired = FALSE;
mytimer->when_set = mach_absolute_time();
timer_resync_deadlines();
}
timer_coalescing_priority_params_t tcoal_prio_params =
{
0, -5, 3, 3, 3,
0ULL, 100*NSEC_PER_MSEC, NSEC_PER_MSEC, NSEC_PER_MSEC, NSEC_PER_MSEC,
{3, 2, 1, -2, -15, -15},
{1*NSEC_PER_MSEC, 5*NSEC_PER_MSEC, 20*NSEC_PER_MSEC, 75*NSEC_PER_MSEC,
10*NSEC_PER_SEC, 10*NSEC_PER_SEC},
{FALSE, FALSE, FALSE, FALSE, TRUE, TRUE}
};
#define TIMER_RESORT_THRESHOLD_ABSTIME (50 * NSEC_PER_MSEC)
#if TCOAL_PRIO_STATS
int32_t nc_tcl, rt_tcl, bg_tcl, kt_tcl, fp_tcl, ts_tcl, qos_tcl;
#define TCOAL_PRIO_STAT(x) (x++)
#else
#define TCOAL_PRIO_STAT(x)
#endif
static boolean_t tcoal_qos_adjust(thread_t t, int32_t *tshift, uint64_t *tmax, boolean_t *pratelimited) {
uint32_t latency_qos;
boolean_t adjusted = FALSE;
task_t ctask = t->task;
if (ctask) {
latency_qos = proc_get_effective_task_policy(ctask, TASK_POLICY_LATENCY_QOS);
assert(latency_qos <= NUM_LATENCY_QOS_TIERS);
if (latency_qos) {
*tshift = tcoal_prio_params.latency_qos_scale[latency_qos - 1];
*tmax = tcoal_prio_params.latency_qos_ns_max[latency_qos - 1];
*pratelimited = tcoal_prio_params.latency_tier_rate_limited[latency_qos - 1];
adjusted = TRUE;
}
}
return adjusted;
}
static void
timer_compute_leeway(thread_t cthread, int32_t urgency, int32_t *tshift, uint64_t *tmax, boolean_t *pratelimited) {
int16_t tpri = cthread->sched_pri;
if ((urgency & TIMER_CALL_USER_MASK) != 0) {
if (tpri >= BASEPRI_RTQUEUES ||
urgency == TIMER_CALL_USER_CRITICAL) {
*tshift = tcoal_prio_params.timer_coalesce_rt_shift;
*tmax = tcoal_prio_params.timer_coalesce_rt_ns_max;
TCOAL_PRIO_STAT(rt_tcl);
} else if ((urgency == TIMER_CALL_USER_BACKGROUND) ||
proc_get_effective_thread_policy(cthread, TASK_POLICY_DARWIN_BG)) {
if (tcoal_qos_adjust(cthread, tshift, tmax, pratelimited)) {
if (*tmax > tcoal_prio_params.timer_coalesce_bg_ns_max) {
return;
} else {
*pratelimited = FALSE;
}
}
*tshift = tcoal_prio_params.timer_coalesce_bg_shift;
*tmax = tcoal_prio_params.timer_coalesce_bg_ns_max;
TCOAL_PRIO_STAT(bg_tcl);
} else if (tpri >= MINPRI_KERNEL) {
*tshift = tcoal_prio_params.timer_coalesce_kt_shift;
*tmax = tcoal_prio_params.timer_coalesce_kt_ns_max;
TCOAL_PRIO_STAT(kt_tcl);
} else if (cthread->sched_mode == TH_MODE_FIXED) {
*tshift = tcoal_prio_params.timer_coalesce_fp_shift;
*tmax = tcoal_prio_params.timer_coalesce_fp_ns_max;
TCOAL_PRIO_STAT(fp_tcl);
} else if (tcoal_qos_adjust(cthread, tshift, tmax, pratelimited)) {
TCOAL_PRIO_STAT(qos_tcl);
} else if (cthread->sched_mode == TH_MODE_TIMESHARE) {
*tshift = tcoal_prio_params.timer_coalesce_ts_shift;
*tmax = tcoal_prio_params.timer_coalesce_ts_ns_max;
TCOAL_PRIO_STAT(ts_tcl);
} else {
TCOAL_PRIO_STAT(nc_tcl);
}
} else if (urgency == TIMER_CALL_SYS_BACKGROUND) {
*tshift = tcoal_prio_params.timer_coalesce_bg_shift;
*tmax = tcoal_prio_params.timer_coalesce_bg_ns_max;
TCOAL_PRIO_STAT(bg_tcl);
} else {
*tshift = tcoal_prio_params.timer_coalesce_kt_shift;
*tmax = tcoal_prio_params.timer_coalesce_kt_ns_max;
TCOAL_PRIO_STAT(kt_tcl);
}
}
int timer_user_idle_level;
uint64_t
timer_call_slop(uint64_t deadline, uint64_t now, uint32_t flags, thread_t cthread, boolean_t *pratelimited)
{
int32_t tcs_shift = 0;
uint64_t tcs_ns_max = 0;
uint64_t adjval;
uint32_t urgency = (flags & TIMER_CALL_URGENCY_MASK);
if (mach_timer_coalescing_enabled &&
(deadline > now) && (urgency != TIMER_CALL_SYS_CRITICAL)) {
timer_compute_leeway(cthread, urgency, &tcs_shift, &tcs_ns_max, pratelimited);
if (tcs_shift >= 0)
adjval = MIN((deadline - now) >> tcs_shift, tcs_ns_max);
else
adjval = MIN((deadline - now) << (-tcs_shift), tcs_ns_max);
adjval += (adjval * timer_user_idle_level) >> 7;
return adjval;
} else {
return 0;
}
}
boolean_t
timer_resort_threshold(uint64_t skew) {
if (skew >= TIMER_RESORT_THRESHOLD_ABSTIME)
return TRUE;
else
return FALSE;
}
int
ml_timer_get_user_idle_level(void) {
return timer_user_idle_level;
}
kern_return_t ml_timer_set_user_idle_level(int ilevel) {
boolean_t do_reeval = FALSE;
if ((ilevel < 0) || (ilevel > 128))
return KERN_INVALID_ARGUMENT;
if (ilevel < timer_user_idle_level) {
do_reeval = TRUE;
}
timer_user_idle_level = ilevel;
if (do_reeval)
ml_timer_evaluate();
return KERN_SUCCESS;
}
mpqueue_head_t *
timer_queue_assign(
uint64_t deadline)
{
cpu_data_t *cdp = current_cpu_datap();
mpqueue_head_t *queue;
if (cdp->cpu_running) {
queue = &cdp->rtclock_timer.queue;
if (deadline < cdp->rtclock_timer.deadline)
timer_set_deadline(deadline);
}
else
queue = &cpu_datap(master_cpu)->rtclock_timer.queue;
return (queue);
}
void
timer_queue_cancel(
mpqueue_head_t *queue,
uint64_t deadline,
uint64_t new_deadline)
{
if (queue == ¤t_cpu_datap()->rtclock_timer.queue) {
if (deadline < new_deadline)
timer_set_deadline(new_deadline);
}
}
uint32_t
timer_queue_migrate_cpu(int target_cpu)
{
cpu_data_t *target_cdp = cpu_datap(target_cpu);
cpu_data_t *cdp = current_cpu_datap();
int ntimers_moved;
assert(!ml_get_interrupts_enabled());
assert(target_cpu != cdp->cpu_number);
assert(target_cpu == master_cpu);
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
DECR_TIMER_MIGRATE | DBG_FUNC_START,
target_cpu,
cdp->rtclock_timer.deadline, (cdp->rtclock_timer.deadline >>32),
0, 0);
ntimers_moved = timer_queue_migrate(&cdp->rtclock_timer.queue,
&target_cdp->rtclock_timer.queue);
if (ntimers_moved > 0) {
cdp->rtclock_timer.deadline = EndOfAllTime;
setPop(EndOfAllTime);
}
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
DECR_TIMER_MIGRATE | DBG_FUNC_END,
target_cpu, ntimers_moved, 0, 0, 0);
return ntimers_moved;
}
mpqueue_head_t *
timer_queue_cpu(int cpu)
{
return &cpu_datap(cpu)->rtclock_timer.queue;
}
void
timer_call_cpu(int cpu, void (*fn)(void *), void *arg)
{
mp_cpus_call(cpu_to_cpumask(cpu), SYNC, fn, arg);
}
void
timer_call_nosync_cpu(int cpu, void (*fn)(void *), void *arg)
{
mp_cpus_call(cpu_to_cpumask(cpu), NOSYNC, fn, arg);
}