#include <mach/boolean.h>
#include <mach/thread_switch.h>
#include <ipc/ipc_port.h>
#include <ipc/ipc_space.h>
#include <kern/ipc_kobject.h>
#include <kern/processor.h>
#include <kern/sched.h>
#include <kern/sched_prim.h>
#include <kern/spl.h>
#include <kern/task.h>
#include <kern/thread.h>
#include <kern/ast.h>
#include <mach/policy.h>
#include <kern/syscall_subr.h>
#include <mach/mach_host_server.h>
#include <mach/mach_syscalls.h>
#include <kern/sf.h>
#include <kern/mk_sp.h>
#include <kern/misc_protos.h>
#include <kern/spl.h>
#include <kern/sched.h>
#include <kern/sched_prim.h>
#include <kern/assert.h>
#include <kern/thread.h>
#include <mach/mach_host_server.h>
void _mk_sp_thread_depress_priority(
sf_object_t policy,
mach_msg_timeout_t depress_time);
#include <mach/thread_act_server.h>
#include <mach/host_priv_server.h>
sp_ops_t mk_sp_ops = {
_mk_sp_thread_update_mpri,
_mk_sp_thread_unblock,
_mk_sp_thread_done,
_mk_sp_thread_begin,
_mk_sp_thread_dispatch,
_mk_sp_thread_attach,
_mk_sp_thread_detach,
_mk_sp_thread_processor,
_mk_sp_thread_processor_set,
_mk_sp_thread_setup,
_mk_sp_swtch_pri,
_mk_sp_thread_switch,
_mk_sp_thread_depress_abort,
_mk_sp_thread_depress_timeout,
_mk_sp_thread_runnable,
};
kern_return_t thread_policy_common(
thread_t thread,
int policy,
int data,
processor_set_t pset);
sf_return_t
_mk_sp_thread_update_mpri(
sf_object_t policy,
thread_t thread)
{
if (thread->sched_stamp != sched_tick)
update_priority(thread);
return(SF_SUCCESS);
}
sf_return_t
_mk_sp_thread_unblock(
sf_object_t policy,
thread_t thread)
{
thread->sp_state = MK_SP_RUNNABLE;
if (!(thread->state&TH_IDLE))
thread_setrun(thread, TRUE, TAIL_Q);
return(SF_SUCCESS);
}
sf_return_t
_mk_sp_thread_done(
sf_object_t policy,
thread_t old_thread)
{
processor_t myprocessor = cpu_to_processor(cpu_number());
old_thread->unconsumed_quantum = myprocessor->quantum;
if (old_thread->state & TH_WAIT)
old_thread->sp_state = MK_SP_BLOCKED;
return(SF_SUCCESS);
}
sf_return_t
_mk_sp_thread_begin(
sf_object_t policy,
thread_t thread)
{
processor_t myprocessor = cpu_to_processor(cpu_number());
processor_set_t pset;
pset = myprocessor->processor_set;
assert(thread->sp_state == MK_SP_RUNNABLE);
if (thread->policy & (POLICY_RR|POLICY_FIFO))
myprocessor->quantum = thread->unconsumed_quantum;
else
myprocessor->quantum = (thread->bound_processor ?
min_quantum : pset->set_quantum);
return(SF_SUCCESS);
}
sf_return_t
_mk_sp_thread_dispatch(
sf_object_t policy,
thread_t old_thread)
{
if (old_thread->sp_state & MK_SP_RUNNABLE) {
if (old_thread->reason & AST_QUANTUM) {
thread_setrun(old_thread, FALSE, TAIL_Q);
old_thread->unconsumed_quantum = min_quantum;
}
else
thread_setrun(old_thread, FALSE, HEAD_Q);
}
if (old_thread->sp_state & MK_SP_ATTACHED) {
old_thread->sp_state = MK_SP_RUNNABLE;
thread_setrun(old_thread, FALSE, TAIL_Q);
}
return(SF_SUCCESS);
}
sf_return_t
_mk_sp_thread_attach(
sf_object_t policy,
thread_t thread)
{
thread->sp_state = MK_SP_ATTACHED;
thread->max_priority = thread->priority = BASEPRI_DEFAULT;
thread->depress_priority = -1;
thread->cpu_usage = 0;
thread->sched_usage = 0;
thread->sched_stamp = 0;
thread->unconsumed_quantum = min_quantum;
thread->policy = policy->policy_id;
return(SF_SUCCESS);
}
sf_return_t
_mk_sp_thread_detach(
sf_object_t policy,
thread_t thread)
{
struct run_queue *rq;
assert(thread->policy == policy->policy_id);
if (thread->runq != RUN_QUEUE_NULL) {
rq = rem_runq(thread);
if (rq == RUN_QUEUE_NULL) {
panic("mk_sp_thread_detach: missed thread");
}
}
if (thread->depress_priority >= 0) {
thread->priority = thread->depress_priority;
thread->depress_priority = -1;
if (thread_call_cancel(&thread->depress_timer))
thread_call_enter(&thread->depress_timer);
}
thread->policy = POLICY_NULL;
return(SF_SUCCESS);
}
sf_return_t
_mk_sp_thread_processor(
sf_object_t policy,
thread_t *thread,
processor_t processor)
{
return(SF_FAILURE);
}
sf_return_t
_mk_sp_thread_processor_set(
sf_object_t policy,
thread_t thread,
processor_set_t processor_set)
{
pset_add_thread(processor_set, thread);
return(SF_SUCCESS);
}
sf_return_t
_mk_sp_thread_setup(
sf_object_t policy,
thread_t thread)
{
if (thread->state & TH_WAIT)
thread->sp_state = MK_SP_BLOCKED;
thread->sched_stamp = sched_tick;
compute_priority(thread, TRUE);
return(SF_SUCCESS);
}
kern_return_t
thread_priority_internal(
thread_t thread,
int priority)
{
kern_return_t result = KERN_SUCCESS;
spl_t s;
s = splsched();
thread_lock(thread);
if (priority > thread->max_priority)
priority = thread->max_priority;
if (thread->depress_priority >= 0)
thread->depress_priority = priority;
else {
thread->priority = priority;
compute_priority(thread, TRUE);
if (thread == current_thread())
ast_on(AST_BLOCK);
}
thread_unlock(thread);
splx(s);
return (result);
}
kern_return_t
thread_policy_common(
thread_t thread,
integer_t policy,
integer_t data,
processor_set_t pset)
{
kern_return_t result = KERN_SUCCESS;
register int temp;
spl_t s;
if ( thread == THREAD_NULL ||
invalid_policy(policy) )
return(KERN_INVALID_ARGUMENT);
s = splsched();
thread_lock(thread);
if (policy != thread->policy) {
if ( pset == PROCESSOR_SET_NULL &&
(thread->processor_set->policies & policy) == 0 )
result = KERN_FAILURE;
else {
if (pset != thread->processor_set)
result = KERN_FAILURE;
else {
thread->policy = policy;
compute_priority(thread, TRUE);
}
}
}
thread_unlock(thread);
splx(s);
return (result);
}
kern_return_t
thread_set_policy(
thread_act_t thr_act,
processor_set_t pset,
policy_t policy,
policy_base_t base,
mach_msg_type_number_t base_count,
policy_limit_t limit,
mach_msg_type_number_t limit_count)
{
thread_t thread;
int max, bas, dat, incr;
kern_return_t result = KERN_SUCCESS;
if ( thr_act == THR_ACT_NULL ||
pset == PROCESSOR_SET_NULL )
return (KERN_INVALID_ARGUMENT);
thread = act_lock_thread(thr_act);
if (thread == THREAD_NULL) {
act_unlock_thread(thr_act);
return(KERN_INVALID_ARGUMENT);
}
if (pset != thread->processor_set) {
act_unlock_thread(thr_act);
return(KERN_FAILURE);
}
switch (policy) {
case POLICY_RR:
{
policy_rr_base_t rr_base = (policy_rr_base_t) base;
policy_rr_limit_t rr_limit = (policy_rr_limit_t) limit;
if ( base_count != POLICY_RR_BASE_COUNT ||
limit_count != POLICY_RR_LIMIT_COUNT ) {
result = KERN_INVALID_ARGUMENT;
break;
}
dat = rr_base->quantum;
bas = rr_base->base_priority;
max = rr_limit->max_priority;
if (invalid_pri(bas) || invalid_pri(max)) {
result = KERN_INVALID_ARGUMENT;
break;
}
break;
}
case POLICY_FIFO:
{
policy_fifo_base_t fifo_base = (policy_fifo_base_t) base;
policy_fifo_limit_t fifo_limit = (policy_fifo_limit_t) limit;
if ( base_count != POLICY_FIFO_BASE_COUNT ||
limit_count != POLICY_FIFO_LIMIT_COUNT) {
result = KERN_INVALID_ARGUMENT;
break;
}
dat = 0;
bas = fifo_base->base_priority;
max = fifo_limit->max_priority;
if (invalid_pri(bas) || invalid_pri(max)) {
result = KERN_INVALID_ARGUMENT;
break;
}
break;
}
case POLICY_TIMESHARE:
{
policy_timeshare_base_t ts_base = (policy_timeshare_base_t) base;
policy_timeshare_limit_t ts_limit =
(policy_timeshare_limit_t) limit;
if ( base_count != POLICY_TIMESHARE_BASE_COUNT ||
limit_count != POLICY_TIMESHARE_LIMIT_COUNT ) {
result = KERN_INVALID_ARGUMENT;
break;
}
dat = 0;
bas = ts_base->base_priority;
max = ts_limit->max_priority;
if (invalid_pri(bas) || invalid_pri(max)) {
result = KERN_INVALID_ARGUMENT;
break;
}
break;
}
default:
result = KERN_INVALID_POLICY;
}
if (result != KERN_SUCCESS) {
act_unlock_thread(thr_act);
return(result);
}
result = thread_priority_internal(thread, bas);
if (result == KERN_SUCCESS)
result = thread_policy_common(thread, policy, dat, pset);
act_unlock_thread(thr_act);
return(result);
}
kern_return_t
thread_policy(
thread_act_t thr_act,
policy_t policy,
policy_base_t base,
mach_msg_type_number_t count,
boolean_t set_limit)
{
thread_t thread;
processor_set_t pset;
kern_return_t result = KERN_SUCCESS;
policy_limit_t limit;
int limcount;
policy_rr_limit_data_t rr_limit;
policy_fifo_limit_data_t fifo_limit;
policy_timeshare_limit_data_t ts_limit;
if (thr_act == THR_ACT_NULL)
return (KERN_INVALID_ARGUMENT);
thread = act_lock_thread(thr_act);
pset = thread->processor_set;
if ( thread == THREAD_NULL ||
pset == PROCESSOR_SET_NULL ){
act_unlock_thread(thr_act);
return(KERN_INVALID_ARGUMENT);
}
if ( invalid_policy(policy) ||
(pset->policies & policy) == 0 ) {
act_unlock_thread(thr_act);
return(KERN_INVALID_POLICY);
}
if (set_limit) {
switch (policy) {
case POLICY_RR:
{
policy_rr_base_t rr_base;
if (count != POLICY_RR_BASE_COUNT) {
result = KERN_INVALID_ARGUMENT;
break;
}
limcount = POLICY_RR_LIMIT_COUNT;
rr_base = (policy_rr_base_t) base;
rr_limit.max_priority = rr_base->base_priority;
limit = (policy_limit_t) &rr_limit;
break;
}
case POLICY_FIFO:
{
policy_fifo_base_t fifo_base;
if (count != POLICY_FIFO_BASE_COUNT) {
result = KERN_INVALID_ARGUMENT;
break;
}
limcount = POLICY_FIFO_LIMIT_COUNT;
fifo_base = (policy_fifo_base_t) base;
fifo_limit.max_priority = fifo_base->base_priority;
limit = (policy_limit_t) &fifo_limit;
break;
}
case POLICY_TIMESHARE:
{
policy_timeshare_base_t ts_base;
if (count != POLICY_TIMESHARE_BASE_COUNT) {
result = KERN_INVALID_ARGUMENT;
break;
}
limcount = POLICY_TIMESHARE_LIMIT_COUNT;
ts_base = (policy_timeshare_base_t) base;
ts_limit.max_priority = ts_base->base_priority;
limit = (policy_limit_t) &ts_limit;
break;
}
default:
result = KERN_INVALID_POLICY;
break;
}
}
else {
switch (policy) {
case POLICY_RR:
{
policy_rr_base_t rr_base;
if (count != POLICY_RR_BASE_COUNT) {
result = KERN_INVALID_ARGUMENT;
break;
}
limcount = POLICY_RR_LIMIT_COUNT;
rr_base = (policy_rr_base_t) base;
if (rr_base->base_priority > thread->max_priority) {
result = KERN_POLICY_LIMIT;
break;
}
rr_limit.max_priority = thread->max_priority;
limit = (policy_limit_t) &rr_limit;
break;
}
case POLICY_FIFO:
{
policy_fifo_base_t fifo_base;
if (count != POLICY_FIFO_BASE_COUNT) {
result = KERN_INVALID_ARGUMENT;
break;
}
limcount = POLICY_FIFO_LIMIT_COUNT;
fifo_base = (policy_fifo_base_t) base;
if (fifo_base->base_priority > thread->max_priority) {
result = KERN_POLICY_LIMIT;
break;
}
fifo_limit.max_priority = thread->max_priority;
limit = (policy_limit_t) &fifo_limit;
break;
}
case POLICY_TIMESHARE:
{
policy_timeshare_base_t ts_base;
if (count != POLICY_TIMESHARE_BASE_COUNT) {
result = KERN_INVALID_ARGUMENT;
break;
}
limcount = POLICY_TIMESHARE_LIMIT_COUNT;
ts_base = (policy_timeshare_base_t) base;
if (ts_base->base_priority > thread->max_priority) {
result = KERN_POLICY_LIMIT;
break;
}
ts_limit.max_priority = thread->max_priority;
limit = (policy_limit_t) &ts_limit;
break;
}
default:
result = KERN_INVALID_POLICY;
break;
}
}
act_unlock_thread(thr_act);
if (result == KERN_SUCCESS)
result = thread_set_policy(thr_act, pset,
policy, base, count, limit, limcount);
return(result);
}
shift_data_t wait_shift[32] = {
{1,1},{1,3},{1,-3},{2,-7},{3,5},{3,-5},{4,-8},{5,7},
{5,-7},{6,-10},{7,10},{7,-9},{8,-11},{9,12},{9,-11},{10,-13},
{11,14},{11,-13},{12,-15},{13,17},{13,-15},{14,-17},{15,19},{16,18},
{16,-19},{17,22},{18,20},{18,-20},{19,26},{20,22},{20,-22},{21,-27}};
#ifdef PRI_SHIFT_2
#if PRI_SHIFT_2 > 0
#define do_priority_computation(thread, pri) \
MACRO_BEGIN \
(pri) = (thread)->priority \
- ((thread)->sched_usage >> (PRI_SHIFT + SCHED_SHIFT)) \
- ((thread)->sched_usage >> (PRI_SHIFT_2 + SCHED_SHIFT)); \
if ((pri) < MINPRI_STANDARD) \
(pri) = MINPRI_STANDARD; \
else \
if ((pri) > MAXPRI_STANDARD) \
(pri) = MAXPRI_STANDARD; \
MACRO_END
#else
#define do_priority_computation(thread, pri) \
MACRO_BEGIN \
(pri) = (thread)->priority \
- ((thread)->sched_usage >> (PRI_SHIFT + SCHED_SHIFT)) \
+ ((thread)->sched_usage >> (SCHED_SHIFT - PRI_SHIFT_2)); \
if ((pri) < MINPRI_STANDARD) \
(pri) = MINPRI_STANDARD; \
else \
if ((pri) > MAXPRI_STANDARD) \
(pri) = MAXPRI_STANDARD; \
MACRO_END
#endif
#else
#define do_priority_computation(thread, pri) \
MACRO_BEGIN \
(pri) = (thread)->priority \
- ((thread)->sched_usage >> (PRI_SHIFT + SCHED_SHIFT)); \
if ((pri) < MINPRI_STANDARD) \
(pri) = MINPRI_STANDARD; \
else \
if ((pri) > MAXPRI_STANDARD) \
(pri) = MAXPRI_STANDARD; \
MACRO_END
#endif
void
compute_priority(
register thread_t thread,
boolean_t resched)
{
register int pri;
if (thread->policy == POLICY_TIMESHARE) {
do_priority_computation(thread, pri);
if (thread->depress_priority < 0)
set_pri(thread, pri, resched);
else
thread->depress_priority = pri;
}
else
set_pri(thread, thread->priority, resched);
}
void
compute_my_priority(
register thread_t thread)
{
register int pri;
do_priority_computation(thread, pri);
assert(thread->runq == RUN_QUEUE_NULL);
thread->sched_pri = pri;
}
#if DEBUG
struct mk_sp_usage {
natural_t cpu_delta, sched_delta;
natural_t sched_tick, ticks;
natural_t cpu_usage, sched_usage,
aged_cpu, aged_sched;
thread_t thread;
} idled_info, loaded_info;
#endif
void
update_priority(
register thread_t thread)
{
register unsigned int ticks;
register shift_t shiftp;
ticks = sched_tick - thread->sched_stamp;
assert(ticks != 0);
thread->sched_stamp += ticks;
thread_timer_delta(thread);
if (ticks > 30) {
thread->cpu_usage = 0;
thread->sched_usage = 0;
}
else {
#if DEBUG
struct mk_sp_usage *sp_usage;
#endif
thread->cpu_usage += thread->cpu_delta;
thread->sched_usage += thread->sched_delta;
#if DEBUG
if (thread->state & TH_IDLE)
sp_usage = &idled_info;
else
if (thread == loaded_info.thread)
sp_usage = &loaded_info;
else
sp_usage = NULL;
if (sp_usage != NULL) {
sp_usage->cpu_delta = thread->cpu_delta;
sp_usage->sched_delta = thread->sched_delta;
sp_usage->sched_tick = thread->sched_stamp;
sp_usage->ticks = ticks;
sp_usage->cpu_usage = thread->cpu_usage;
sp_usage->sched_usage = thread->sched_usage;
sp_usage->thread = thread;
}
#endif
shiftp = &wait_shift[ticks];
if (shiftp->shift2 > 0) {
thread->cpu_usage =
(thread->cpu_usage >> shiftp->shift1) +
(thread->cpu_usage >> shiftp->shift2);
thread->sched_usage =
(thread->sched_usage >> shiftp->shift1) +
(thread->sched_usage >> shiftp->shift2);
}
else {
thread->cpu_usage =
(thread->cpu_usage >> shiftp->shift1) -
(thread->cpu_usage >> -(shiftp->shift2));
thread->sched_usage =
(thread->sched_usage >> shiftp->shift1) -
(thread->sched_usage >> -(shiftp->shift2));
}
#if DEBUG
if (sp_usage != NULL) {
sp_usage->aged_cpu = thread->cpu_usage;
sp_usage->aged_sched = thread->sched_usage;
}
#endif
}
thread->cpu_delta = 0;
thread->sched_delta = 0;
if ( thread->policy == POLICY_TIMESHARE &&
thread->depress_priority < 0 ) {
register int new_pri;
run_queue_t runq;
do_priority_computation(thread, new_pri);
if (new_pri != thread->sched_pri) {
runq = rem_runq(thread);
thread->sched_pri = new_pri;
if (runq != RUN_QUEUE_NULL)
thread_setrun(thread, TRUE, TAIL_Q);
}
}
}
void
_mk_sp_swtch_pri(
sf_object_t policy,
int pri)
{
register thread_t self = current_thread();
extern natural_t min_quantum_ms;
#ifdef lint
pri++;
#endif
_mk_sp_thread_depress_priority(policy, min_quantum_ms);
thread_block((void (*)(void)) 0);
_mk_sp_thread_depress_abort(policy, self);
}
void
_mk_sp_thread_switch_continue(void)
{
thread_t self = current_thread();
int wait_result = self->wait_result;
int option = self->saved.swtch.option;
sf_object_t policy = self->saved.swtch.policy;
if (option == SWITCH_OPTION_WAIT && wait_result != THREAD_TIMED_OUT)
thread_cancel_timer();
else if (option == SWITCH_OPTION_DEPRESS)
_mk_sp_thread_depress_abort(policy, self);
thread_syscall_return(KERN_SUCCESS);
}
kern_return_t
_mk_sp_thread_switch(
sf_object_t policy,
thread_act_t hint_act,
int option,
mach_msg_timeout_t option_time)
{
register thread_t self = current_thread();
register processor_t myprocessor;
int s;
if (hint_act != THR_ACT_NULL) {
register thread_t thread = act_lock_thread(hint_act);
if ( thread != THREAD_NULL &&
thread != self &&
thread->top_act == hint_act ) {
s = splsched();
thread_lock(thread);
if ( thread->processor_set == self->processor_set &&
rem_runq(thread) != RUN_QUEUE_NULL ) {
if (thread->policy & (POLICY_FIFO|POLICY_RR)) {
myprocessor = current_processor();
myprocessor->quantum = thread->unconsumed_quantum;
myprocessor->first_quantum = TRUE;
}
thread_unlock(thread);
act_unlock_thread(hint_act);
act_deallocate(hint_act);
if (option == SWITCH_OPTION_WAIT)
assert_wait_timeout(option_time, THREAD_ABORTSAFE);
else if (option == SWITCH_OPTION_DEPRESS)
_mk_sp_thread_depress_priority(policy, option_time);
self->saved.swtch.policy = policy;
self->saved.swtch.option = option;
thread_run(self, _mk_sp_thread_switch_continue, thread);
splx(s);
goto out;
}
thread_unlock(thread);
splx(s);
}
act_unlock_thread(hint_act);
act_deallocate(hint_act);
}
mp_disable_preemption();
myprocessor = current_processor();
if ( option != SWITCH_OPTION_NONE ||
myprocessor->processor_set->runq.count > 0 ||
myprocessor->runq.count > 0 ) {
myprocessor->first_quantum = FALSE;
mp_enable_preemption();
if (option == SWITCH_OPTION_WAIT)
assert_wait_timeout(option_time, THREAD_ABORTSAFE);
else if (option == SWITCH_OPTION_DEPRESS)
_mk_sp_thread_depress_priority(policy, option_time);
self->saved.swtch.policy = policy;
self->saved.swtch.option = option;
thread_block(_mk_sp_thread_switch_continue);
}
else
mp_enable_preemption();
out:
if (option == SWITCH_OPTION_WAIT)
thread_cancel_timer();
else if (option == SWITCH_OPTION_DEPRESS)
_mk_sp_thread_depress_abort(policy, self);
return (KERN_SUCCESS);
}
void
_mk_sp_thread_depress_priority(
sf_object_t policy,
mach_msg_timeout_t interval)
{
register thread_t self = current_thread();
AbsoluteTime deadline;
boolean_t release = FALSE;
spl_t s;
s = splsched();
thread_lock(self);
if (self->policy == policy->policy_id) {
if (self->depress_priority < 0)
self->depress_priority = self->priority;
else if (thread_call_cancel(&self->depress_timer))
release = TRUE;
self->sched_pri = self->priority = DEPRESSPRI;
if (interval != 0) {
clock_interval_to_deadline(
interval, 1000*NSEC_PER_USEC, &deadline);
thread_call_enter_delayed(&self->depress_timer, deadline);
if (!release)
self->ref_count++;
else
release = FALSE;
}
}
thread_unlock(self);
splx(s);
if (release)
thread_deallocate(self);
}
void
_mk_sp_thread_depress_timeout(
sf_object_t policy,
register thread_t thread)
{
spl_t s;
s = splsched();
thread_lock(thread);
if (thread->policy == policy->policy_id) {
if ( thread->depress_priority >= 0 &&
!thread_call_is_delayed(&thread->depress_timer, NULL) ) {
thread->priority = thread->depress_priority;
thread->depress_priority = -1;
compute_priority(thread, FALSE);
}
else
if (thread->depress_priority == -2) {
thread->depress_priority = -1;
}
}
thread_unlock(thread);
splx(s);
}
kern_return_t
_mk_sp_thread_depress_abort(
sf_object_t policy,
register thread_t thread)
{
kern_return_t result = KERN_SUCCESS;
boolean_t release = FALSE;
spl_t s;
s = splsched();
thread_lock(thread);
if (thread->policy == policy->policy_id) {
if (thread->depress_priority >= 0) {
if (thread_call_cancel(&thread->depress_timer))
release = TRUE;
thread->priority = thread->depress_priority;
thread->depress_priority = -1;
compute_priority(thread, FALSE);
}
else
result = KERN_NOT_DEPRESSED;
}
thread_unlock(thread);
splx(s);
if (release)
thread_deallocate(thread);
return (result);
}
boolean_t
_mk_sp_thread_runnable(
sf_object_t policy,
thread_t thread)
{
return (thread->sp_state == MK_SP_RUNNABLE);
}