#include <mach/boolean.h>
#include <mach/thread_switch.h>
#include <ipc/ipc_port.h>
#include <ipc/ipc_space.h>
#include <kern/counters.h>
#include <kern/ipc_kobject.h>
#include <kern/processor.h>
#include <kern/sched.h>
#include <kern/sched_prim.h>
#include <kern/spl.h>
#include <kern/task.h>
#include <kern/thread.h>
#include <mach/policy.h>
#include <kern/syscall_subr.h>
#include <mach/mach_host_server.h>
#include <mach/mach_syscalls.h>
#ifdef MACH_BSD
extern void workqueue_thread_yielded(void);
extern sched_call_t workqueue_get_sched_callback(void);
#endif
kern_return_t
pfz_exit(
__unused struct pfz_exit_args *args)
{
return (KERN_SUCCESS);
}
static void
swtch_continue(void)
{
register processor_t myprocessor;
boolean_t result;
disable_preemption();
myprocessor = current_processor();
result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;
enable_preemption();
thread_syscall_return(result);
}
boolean_t
swtch(
__unused struct swtch_args *args)
{
register processor_t myprocessor;
boolean_t result;
disable_preemption();
myprocessor = current_processor();
if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) {
mp_enable_preemption();
return (FALSE);
}
enable_preemption();
counter(c_swtch_block++);
thread_block_reason((thread_continue_t)swtch_continue, NULL, AST_YIELD);
disable_preemption();
myprocessor = current_processor();
result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;
enable_preemption();
return (result);
}
static void
swtch_pri_continue(void)
{
register processor_t myprocessor;
boolean_t result;
thread_depress_abort_internal(current_thread());
disable_preemption();
myprocessor = current_processor();
result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;
mp_enable_preemption();
thread_syscall_return(result);
}
boolean_t
swtch_pri(
__unused struct swtch_pri_args *args)
{
register processor_t myprocessor;
boolean_t result;
disable_preemption();
myprocessor = current_processor();
if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) {
mp_enable_preemption();
return (FALSE);
}
enable_preemption();
counter(c_swtch_pri_block++);
thread_depress_abstime(thread_depress_time);
thread_block_reason((thread_continue_t)swtch_pri_continue, NULL, AST_YIELD);
thread_depress_abort_internal(current_thread());
disable_preemption();
myprocessor = current_processor();
result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0;
enable_preemption();
return (result);
}
static int
thread_switch_disable_workqueue_sched_callback(void)
{
sched_call_t callback = workqueue_get_sched_callback();
thread_t self = current_thread();
if (!callback || self->sched_call != callback) {
return FALSE;
}
spl_t s = splsched();
thread_lock(self);
thread_sched_call(self, NULL);
thread_unlock(self);
splx(s);
return TRUE;
}
static void
thread_switch_enable_workqueue_sched_callback(void)
{
sched_call_t callback = workqueue_get_sched_callback();
thread_t self = current_thread();
spl_t s = splsched();
thread_lock(self);
thread_sched_call(self, callback);
thread_unlock(self);
splx(s);
}
static void
thread_switch_continue(void)
{
register thread_t self = current_thread();
int option = self->saved.swtch.option;
boolean_t reenable_workq_callback = self->saved.swtch.reenable_workq_callback;
if (option == SWITCH_OPTION_DEPRESS || option == SWITCH_OPTION_OSLOCK_DEPRESS)
thread_depress_abort_internal(self);
if (reenable_workq_callback)
thread_switch_enable_workqueue_sched_callback();
thread_syscall_return(KERN_SUCCESS);
}
kern_return_t
thread_switch(
struct thread_switch_args *args)
{
register thread_t thread, self = current_thread();
mach_port_name_t thread_name = args->thread_name;
int option = args->option;
mach_msg_timeout_t option_time = args->option_time;
uint32_t scale_factor = NSEC_PER_MSEC;
boolean_t reenable_workq_callback = FALSE;
boolean_t depress_option = FALSE;
boolean_t wait_option = FALSE;
switch (option) {
case SWITCH_OPTION_NONE:
workqueue_thread_yielded();
break;
case SWITCH_OPTION_WAIT:
wait_option = TRUE;
workqueue_thread_yielded();
break;
case SWITCH_OPTION_DEPRESS:
depress_option = TRUE;
workqueue_thread_yielded();
break;
case SWITCH_OPTION_DISPATCH_CONTENTION:
scale_factor = NSEC_PER_USEC;
wait_option = TRUE;
if (thread_switch_disable_workqueue_sched_callback())
reenable_workq_callback = TRUE;
break;
case SWITCH_OPTION_OSLOCK_DEPRESS:
depress_option = TRUE;
if (thread_switch_disable_workqueue_sched_callback())
reenable_workq_callback = TRUE;
break;
case SWITCH_OPTION_OSLOCK_WAIT:
wait_option = TRUE;
if (thread_switch_disable_workqueue_sched_callback())
reenable_workq_callback = TRUE;
break;
default:
return (KERN_INVALID_ARGUMENT);
}
if (thread_name != MACH_PORT_NULL) {
ipc_port_t port;
if (ipc_port_translate_send(self->task->itk_space,
thread_name, &port) == KERN_SUCCESS) {
ip_reference(port);
ip_unlock(port);
thread = convert_port_to_thread(port);
ip_release(port);
if (thread == self) {
(void)thread_deallocate_internal(thread);
thread = THREAD_NULL;
}
}
else
thread = THREAD_NULL;
}
else
thread = THREAD_NULL;
if (option == SWITCH_OPTION_OSLOCK_DEPRESS || option == SWITCH_OPTION_OSLOCK_WAIT) {
if (thread != THREAD_NULL) {
if (thread->task != self->task) {
(void)thread_deallocate_internal(thread);
thread = THREAD_NULL;
} else {
int new_policy = proc_get_effective_thread_policy(self, TASK_POLICY_IO);
set_thread_iotier_override(thread, new_policy);
}
}
}
if (thread != THREAD_NULL) {
processor_t processor;
spl_t s;
s = splsched();
thread_lock(thread);
processor = current_processor();
if (processor->current_pri < BASEPRI_RTQUEUES &&
thread->sched_pri < BASEPRI_RTQUEUES &&
(thread->bound_processor == PROCESSOR_NULL ||
thread->bound_processor == processor) &&
thread_run_queue_remove(thread) ) {
thread_unlock(thread);
(void)thread_deallocate_internal(thread);
if (wait_option)
assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE,
option_time, scale_factor);
else
if (depress_option)
thread_depress_ms(option_time);
self->saved.swtch.option = option;
self->saved.swtch.reenable_workq_callback = reenable_workq_callback;
thread_run(self, (thread_continue_t)thread_switch_continue, NULL, thread);
}
thread_unlock(thread);
splx(s);
thread_deallocate(thread);
}
if (wait_option)
assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE, option_time, scale_factor);
else
if (depress_option)
thread_depress_ms(option_time);
self->saved.swtch.option = option;
self->saved.swtch.reenable_workq_callback = reenable_workq_callback;
thread_block_reason((thread_continue_t)thread_switch_continue, NULL, AST_YIELD);
if (depress_option)
thread_depress_abort_internal(self);
if (reenable_workq_callback)
thread_switch_enable_workqueue_sched_callback();
return (KERN_SUCCESS);
}
void
thread_depress_abstime(
uint64_t interval)
{
register thread_t self = current_thread();
uint64_t deadline;
spl_t s;
s = splsched();
thread_lock(self);
if (!(self->sched_flags & TH_SFLAG_DEPRESSED_MASK)) {
processor_t myprocessor = self->last_processor;
self->sched_pri = DEPRESSPRI;
myprocessor->current_pri = self->sched_pri;
self->sched_flags |= TH_SFLAG_DEPRESS;
if (interval != 0) {
clock_absolutetime_interval_to_deadline(interval, &deadline);
if (!timer_call_enter(&self->depress_timer, deadline, TIMER_CALL_USER_CRITICAL))
self->depress_timer_active++;
}
}
thread_unlock(self);
splx(s);
}
void
thread_depress_ms(
mach_msg_timeout_t interval)
{
uint64_t abstime;
clock_interval_to_absolutetime_interval(
interval, NSEC_PER_MSEC, &abstime);
thread_depress_abstime(abstime);
}
void
thread_depress_expire(
void *p0,
__unused void *p1)
{
thread_t thread = p0;
spl_t s;
s = splsched();
thread_lock(thread);
if (--thread->depress_timer_active == 0) {
thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
SCHED(compute_priority)(thread, FALSE);
}
thread_unlock(thread);
splx(s);
}
kern_return_t
thread_depress_abort_internal(
thread_t thread)
{
kern_return_t result = KERN_NOT_DEPRESSED;
spl_t s;
s = splsched();
thread_lock(thread);
if (!(thread->sched_flags & TH_SFLAG_POLLDEPRESS)) {
if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) {
thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
SCHED(compute_priority)(thread, FALSE);
result = KERN_SUCCESS;
}
if (timer_call_cancel(&thread->depress_timer))
thread->depress_timer_active--;
}
thread_unlock(thread);
splx(s);
return (result);
}
void
thread_poll_yield(
thread_t self)
{
spl_t s;
assert(self == current_thread());
s = splsched();
if (self->sched_mode == TH_MODE_FIXED) {
uint64_t total_computation, abstime;
abstime = mach_absolute_time();
total_computation = abstime - self->computation_epoch;
total_computation += self->computation_metered;
if (total_computation >= max_poll_computation) {
processor_t myprocessor = current_processor();
ast_t preempt;
thread_lock(self);
if (!(self->sched_flags & TH_SFLAG_DEPRESSED_MASK)) {
self->sched_pri = DEPRESSPRI;
myprocessor->current_pri = self->sched_pri;
}
self->computation_epoch = abstime;
self->computation_metered = 0;
self->sched_flags |= TH_SFLAG_POLLDEPRESS;
abstime += (total_computation >> sched_poll_yield_shift);
if (!timer_call_enter(&self->depress_timer, abstime, TIMER_CALL_USER_CRITICAL))
self->depress_timer_active++;
thread_unlock(self);
if ((preempt = csw_check(myprocessor)) != AST_NONE)
ast_on(preempt);
}
}
splx(s);
}
void
thread_yield_internal(
mach_msg_timeout_t ms)
{
processor_t myprocessor;
disable_preemption();
myprocessor = current_processor();
if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) {
mp_enable_preemption();
return;
}
enable_preemption();
thread_depress_ms(ms);
thread_block_reason(THREAD_CONTINUE_NULL, NULL, AST_YIELD);
thread_depress_abort_internal(current_thread());
}