#include <mach/boolean.h>
#include <mach/thread_switch.h>
#include <ipc/ipc_port.h>
#include <ipc/ipc_space.h>
#include <kern/counters.h>
#include <kern/ipc_kobject.h>
#include <kern/processor.h>
#include <kern/sched.h>
#include <kern/sched_prim.h>
#include <kern/spl.h>
#include <kern/task.h>
#include <kern/thread.h>
#include <kern/policy_internal.h>
#include <mach/policy.h>
#include <kern/syscall_subr.h>
#include <mach/mach_host_server.h>
#include <mach/mach_syscalls.h>
#include <sys/kdebug.h>
#include <kern/ast.h>
static void thread_depress_abstime(uint64_t interval);
static void thread_depress_ms(mach_msg_timeout_t interval);
kern_return_t
pfz_exit(
__unused struct pfz_exit_args *args)
{
return KERN_SUCCESS;
}
static void
swtch_continue(void)
{
processor_t myprocessor;
boolean_t result;
disable_preemption();
myprocessor = current_processor();
result = SCHED(thread_should_yield)(myprocessor, current_thread());
enable_preemption();
ml_delay_on_yield();
thread_syscall_return(result);
}
boolean_t
swtch(
__unused struct swtch_args *args)
{
processor_t myprocessor;
disable_preemption();
myprocessor = current_processor();
if (!SCHED(thread_should_yield)(myprocessor, current_thread())) {
mp_enable_preemption();
return FALSE;
}
enable_preemption();
counter(c_swtch_block++);
thread_yield_with_continuation((thread_continue_t)swtch_continue, NULL);
}
static void
swtch_pri_continue(void)
{
processor_t myprocessor;
boolean_t result;
thread_depress_abort(current_thread());
disable_preemption();
myprocessor = current_processor();
result = SCHED(thread_should_yield)(myprocessor, current_thread());
mp_enable_preemption();
ml_delay_on_yield();
thread_syscall_return(result);
}
boolean_t
swtch_pri(
__unused struct swtch_pri_args *args)
{
processor_t myprocessor;
disable_preemption();
myprocessor = current_processor();
if (!SCHED(thread_should_yield)(myprocessor, current_thread())) {
mp_enable_preemption();
return FALSE;
}
enable_preemption();
counter(c_swtch_pri_block++);
thread_depress_abstime(thread_depress_time);
thread_yield_with_continuation((thread_continue_t)swtch_pri_continue, NULL);
}
static void
thread_switch_continue(void *parameter, __unused int ret)
{
thread_t self = current_thread();
int option = (int)(intptr_t)parameter;
if (option == SWITCH_OPTION_DEPRESS || option == SWITCH_OPTION_OSLOCK_DEPRESS) {
thread_depress_abort(self);
}
ml_delay_on_yield();
thread_syscall_return(KERN_SUCCESS);
}
kern_return_t
thread_switch(
struct thread_switch_args *args)
{
thread_t thread = THREAD_NULL;
thread_t self = current_thread();
mach_port_name_t thread_name = args->thread_name;
int option = args->option;
mach_msg_timeout_t option_time = args->option_time;
uint32_t scale_factor = NSEC_PER_MSEC;
boolean_t depress_option = FALSE;
boolean_t wait_option = FALSE;
wait_interrupt_t interruptible = THREAD_ABORTSAFE;
port_to_thread_options_t ptt_options = PORT_TO_THREAD_NOT_CURRENT_THREAD;
switch (option) {
case SWITCH_OPTION_NONE:
break;
case SWITCH_OPTION_WAIT:
wait_option = TRUE;
break;
case SWITCH_OPTION_DEPRESS:
depress_option = TRUE;
break;
case SWITCH_OPTION_DISPATCH_CONTENTION:
scale_factor = NSEC_PER_USEC;
wait_option = TRUE;
interruptible |= THREAD_WAIT_NOREPORT;
break;
case SWITCH_OPTION_OSLOCK_DEPRESS:
depress_option = TRUE;
interruptible |= THREAD_WAIT_NOREPORT;
ptt_options |= PORT_TO_THREAD_IN_CURRENT_TASK;
break;
case SWITCH_OPTION_OSLOCK_WAIT:
wait_option = TRUE;
interruptible |= THREAD_WAIT_NOREPORT;
ptt_options |= PORT_TO_THREAD_IN_CURRENT_TASK;
break;
default:
return KERN_INVALID_ARGUMENT;
}
if (thread_name != MACH_PORT_NULL) {
thread = port_name_to_thread(thread_name, ptt_options);
}
if (option == SWITCH_OPTION_OSLOCK_DEPRESS || option == SWITCH_OPTION_OSLOCK_WAIT) {
if (thread != THREAD_NULL) {
int new_policy = proc_get_effective_thread_policy(self, TASK_POLICY_IO);
set_thread_iotier_override(thread, new_policy);
}
}
if (thread != THREAD_NULL) {
spl_t s = splsched();
thread_t pulled_thread = thread_run_queue_remove_for_handoff(thread);
KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_THREAD_SWITCH) | DBG_FUNC_NONE,
thread_tid(thread), thread->state,
pulled_thread ? TRUE : FALSE, 0, 0);
if (pulled_thread != THREAD_NULL) {
thread_deallocate_safe(thread);
if (wait_option) {
assert_wait_timeout((event_t)assert_wait_timeout, interruptible,
option_time, scale_factor);
} else if (depress_option) {
thread_depress_ms(option_time);
}
thread_run(self, thread_switch_continue, (void *)(intptr_t)option, pulled_thread);
__builtin_unreachable();
}
splx(s);
thread_deallocate(thread);
}
if (wait_option) {
assert_wait_timeout((event_t)assert_wait_timeout, interruptible, option_time, scale_factor);
} else {
disable_preemption();
bool should_yield = SCHED(thread_should_yield)(current_processor(), current_thread());
enable_preemption();
if (should_yield == false) {
return KERN_SUCCESS;
}
if (depress_option) {
thread_depress_ms(option_time);
}
}
thread_yield_with_continuation(thread_switch_continue, (void *)(intptr_t)option);
__builtin_unreachable();
}
void
thread_yield_with_continuation(
thread_continue_t continuation,
void *parameter)
{
assert(continuation);
thread_block_reason(continuation, parameter, AST_YIELD);
__builtin_unreachable();
}
static wait_result_t
thread_handoff_internal(thread_t thread, thread_continue_t continuation,
void *parameter)
{
thread_t deallocate_thread = THREAD_NULL;
thread_t self = current_thread();
if (thread != THREAD_NULL) {
spl_t s = splsched();
thread_t pulled_thread = thread_run_queue_remove_for_handoff(thread);
KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_THREAD_SWITCH) | DBG_FUNC_NONE,
thread_tid(thread), thread->state,
pulled_thread ? TRUE : FALSE, 0, 0);
if (pulled_thread != THREAD_NULL) {
if (continuation == NULL) {
thread_deallocate_safe(thread);
}
int result = thread_run(self, continuation, parameter, pulled_thread);
splx(s);
return result;
}
splx(s);
deallocate_thread = thread;
thread = THREAD_NULL;
}
int result = thread_block_parameter(continuation, parameter);
if (deallocate_thread != THREAD_NULL) {
thread_deallocate(deallocate_thread);
}
return result;
}
void
thread_handoff_parameter(thread_t thread, thread_continue_t continuation,
void *parameter)
{
thread_handoff_internal(thread, continuation, parameter);
panic("NULL continuation passed to %s", __func__);
__builtin_unreachable();
}
wait_result_t
thread_handoff_deallocate(thread_t thread)
{
return thread_handoff_internal(thread, NULL, NULL);
}
void
thread_depress_abstime(uint64_t interval)
{
thread_t self = current_thread();
spl_t s = splsched();
thread_lock(self);
assert((self->sched_flags & TH_SFLAG_DEPRESS) == 0);
if ((self->sched_flags & TH_SFLAG_POLLDEPRESS) == 0) {
self->sched_flags |= TH_SFLAG_DEPRESS;
thread_recompute_sched_pri(self, SETPRI_LAZY);
if (interval != 0) {
uint64_t deadline;
clock_absolutetime_interval_to_deadline(interval, &deadline);
if (!timer_call_enter(&self->depress_timer, deadline, TIMER_CALL_USER_CRITICAL)) {
self->depress_timer_active++;
}
}
}
thread_unlock(self);
splx(s);
}
void
thread_depress_ms(mach_msg_timeout_t interval)
{
uint64_t abstime;
clock_interval_to_absolutetime_interval(interval, NSEC_PER_MSEC, &abstime);
thread_depress_abstime(abstime);
}
void
thread_depress_expire(void *p0,
__unused void *p1)
{
thread_t thread = (thread_t)p0;
spl_t s = splsched();
thread_lock(thread);
assert((thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != TH_SFLAG_DEPRESSED_MASK);
if (--thread->depress_timer_active == 0) {
thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
if ((thread->state & TH_RUN) == TH_RUN) {
thread->last_basepri_change_time = mach_absolute_time();
}
thread_recompute_sched_pri(thread, SETPRI_DEFAULT);
}
thread_unlock(thread);
splx(s);
}
kern_return_t
thread_depress_abort(thread_t thread)
{
kern_return_t result = KERN_NOT_DEPRESSED;
spl_t s = splsched();
thread_lock(thread);
assert((thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != TH_SFLAG_DEPRESSED_MASK);
if ((thread->sched_flags & TH_SFLAG_POLLDEPRESS) == 0) {
result = thread_depress_abort_locked(thread);
}
thread_unlock(thread);
splx(s);
return result;
}
kern_return_t
thread_depress_abort_locked(thread_t thread)
{
if ((thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) == 0) {
return KERN_NOT_DEPRESSED;
}
assert((thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != TH_SFLAG_DEPRESSED_MASK);
thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
if ((thread->state & TH_RUN) == TH_RUN) {
thread->last_basepri_change_time = mach_absolute_time();
}
thread_recompute_sched_pri(thread, SETPRI_LAZY);
if (timer_call_cancel(&thread->depress_timer)) {
thread->depress_timer_active--;
}
return KERN_SUCCESS;
}
void
thread_poll_yield(thread_t self)
{
assert(self == current_thread());
assert((self->sched_flags & TH_SFLAG_DEPRESS) == 0);
if (self->sched_mode != TH_MODE_FIXED) {
return;
}
spl_t s = splsched();
uint64_t abstime = mach_absolute_time();
uint64_t total_computation = abstime -
self->computation_epoch + self->computation_metered;
if (total_computation >= max_poll_computation) {
thread_lock(self);
self->computation_epoch = abstime;
self->computation_metered = 0;
uint64_t yield_expiration = abstime +
(total_computation >> sched_poll_yield_shift);
if (!timer_call_enter(&self->depress_timer, yield_expiration,
TIMER_CALL_USER_CRITICAL)) {
self->depress_timer_active++;
}
self->sched_flags |= TH_SFLAG_POLLDEPRESS;
thread_recompute_sched_pri(self, SETPRI_DEFAULT);
thread_unlock(self);
}
splx(s);
}
void
thread_yield_internal(mach_msg_timeout_t ms)
{
thread_t self = current_thread();
assert((self->sched_flags & TH_SFLAG_DEPRESSED_MASK) != TH_SFLAG_DEPRESSED_MASK);
processor_t myprocessor;
disable_preemption();
myprocessor = current_processor();
if (!SCHED(thread_should_yield)(myprocessor, self)) {
mp_enable_preemption();
return;
}
enable_preemption();
thread_depress_ms(ms);
thread_block_reason(THREAD_CONTINUE_NULL, NULL, AST_YIELD);
thread_depress_abort(self);
}
void
thread_yield_to_preemption()
{
ast_t *pending_ast = ast_pending();
ast_t ast = AST_NONE;
processor_t p;
if (*pending_ast & AST_PREEMPT) {
thread_t self = current_thread();
spl_t s = splsched();
p = current_processor();
thread_lock(self);
ast = csw_check(self, p, AST_YIELD);
ast_on(ast);
thread_unlock(self);
if (ast != AST_NONE) {
(void)thread_block_reason(THREAD_CONTINUE_NULL, NULL, ast);
}
splx(s);
}
}