#include <mach/mach_types.h>
#include <kern/sched_prim.h>
#include <kern/clock.h>
#include <kern/task.h>
#include <kern/thread.h>
#include <kern/thread_call.h>
#include <kern/call_entry.h>
#include <kern/timer_call.h>
#define internal_call_num 768
#define thread_call_thread_min 4
static
thread_call_data_t
internal_call_storage[internal_call_num];
decl_simple_lock_data(static,thread_call_lock)
static
timer_call_data_t
thread_call_delayed_timer;
static
queue_head_t
internal_call_free_queue,
pending_call_queue, delayed_call_queue;
static
struct wait_queue
call_thread_idle_queue;
static
thread_t
activate_thread;
static
boolean_t
activate_thread_awake;
static struct {
int pending_num,
pending_hiwat;
int active_num,
active_hiwat,
active_lowat;
int delayed_num,
delayed_hiwat;
int idle_thread_num;
int thread_num,
thread_hiwat,
thread_lowat;
} thread_calls;
static boolean_t
thread_call_initialized = FALSE;
static __inline__ thread_call_t
_internal_call_allocate(void);
static __inline__ void
_internal_call_release(
thread_call_t call
);
static __inline__ void
_pending_call_enqueue(
thread_call_t call
),
_pending_call_dequeue(
thread_call_t call
),
_delayed_call_enqueue(
thread_call_t call
),
_delayed_call_dequeue(
thread_call_t call
);
static void __inline__
_set_delayed_call_timer(
thread_call_t call
);
static boolean_t
_remove_from_pending_queue(
thread_call_func_t func,
thread_call_param_t param0,
boolean_t remove_all
),
_remove_from_delayed_queue(
thread_call_func_t func,
thread_call_param_t param0,
boolean_t remove_all
);
static __inline__ void
_call_thread_wake(void);
static void
_call_thread(void),
_activate_thread(void);
static void
_delayed_call_timer(
timer_call_param_t p0,
timer_call_param_t p1
);
#define qe(x) ((queue_entry_t)(x))
#define TC(x) ((thread_call_t)(x))
void
thread_call_initialize(void)
{
thread_call_t call;
spl_t s;
if (thread_call_initialized)
panic("thread_call_initialize");
simple_lock_init(&thread_call_lock, ETAP_MISC_TIMER);
s = splsched();
simple_lock(&thread_call_lock);
queue_init(&pending_call_queue);
queue_init(&delayed_call_queue);
queue_init(&internal_call_free_queue);
for (
call = internal_call_storage;
call < &internal_call_storage[internal_call_num];
call++) {
enqueue_tail(&internal_call_free_queue, qe(call));
}
timer_call_setup(&thread_call_delayed_timer, _delayed_call_timer, NULL);
wait_queue_init(&call_thread_idle_queue, SYNC_POLICY_FIFO);
thread_calls.thread_lowat = thread_call_thread_min;
activate_thread_awake = TRUE;
thread_call_initialized = TRUE;
simple_unlock(&thread_call_lock);
splx(s);
activate_thread = kernel_thread_with_priority(
kernel_task, MAXPRI_KERNEL - 2,
_activate_thread, TRUE, TRUE);
}
void
thread_call_setup(
thread_call_t call,
thread_call_func_t func,
thread_call_param_t param0
)
{
call_entry_setup(call, func, param0);
}
static __inline__ thread_call_t
_internal_call_allocate(void)
{
thread_call_t call;
if (queue_empty(&internal_call_free_queue))
panic("_internal_call_allocate");
call = TC(dequeue_head(&internal_call_free_queue));
return (call);
}
static __inline__
void
_internal_call_release(
thread_call_t call
)
{
if ( call >= internal_call_storage &&
call < &internal_call_storage[internal_call_num] )
enqueue_tail(&internal_call_free_queue, qe(call));
}
static __inline__
void
_pending_call_enqueue(
thread_call_t call
)
{
enqueue_tail(&pending_call_queue, qe(call));
if (++thread_calls.pending_num > thread_calls.pending_hiwat)
thread_calls.pending_hiwat = thread_calls.pending_num;
call->state = PENDING;
}
static __inline__
void
_pending_call_dequeue(
thread_call_t call
)
{
(void)remque(qe(call));
thread_calls.pending_num--;
call->state = IDLE;
}
static __inline__
void
_delayed_call_enqueue(
thread_call_t call
)
{
thread_call_t current;
current = TC(queue_first(&delayed_call_queue));
while (TRUE) {
if ( queue_end(&delayed_call_queue, qe(current)) ||
call->deadline < current->deadline ) {
current = TC(queue_prev(qe(current)));
break;
}
current = TC(queue_next(qe(current)));
}
insque(qe(call), qe(current));
if (++thread_calls.delayed_num > thread_calls.delayed_hiwat)
thread_calls.delayed_hiwat = thread_calls.delayed_num;
call->state = DELAYED;
}
static __inline__
void
_delayed_call_dequeue(
thread_call_t call
)
{
(void)remque(qe(call));
thread_calls.delayed_num--;
call->state = IDLE;
}
static __inline__ void
_set_delayed_call_timer(
thread_call_t call
)
{
timer_call_enter(&thread_call_delayed_timer, call->deadline);
}
static
boolean_t
_remove_from_pending_queue(
thread_call_func_t func,
thread_call_param_t param0,
boolean_t remove_all
)
{
boolean_t call_removed = FALSE;
thread_call_t call;
call = TC(queue_first(&pending_call_queue));
while (!queue_end(&pending_call_queue, qe(call))) {
if ( call->func == func &&
call->param0 == param0 ) {
thread_call_t next = TC(queue_next(qe(call)));
_pending_call_dequeue(call);
_internal_call_release(call);
call_removed = TRUE;
if (!remove_all)
break;
call = next;
}
else
call = TC(queue_next(qe(call)));
}
return (call_removed);
}
static
boolean_t
_remove_from_delayed_queue(
thread_call_func_t func,
thread_call_param_t param0,
boolean_t remove_all
)
{
boolean_t call_removed = FALSE;
thread_call_t call;
call = TC(queue_first(&delayed_call_queue));
while (!queue_end(&delayed_call_queue, qe(call))) {
if ( call->func == func &&
call->param0 == param0 ) {
thread_call_t next = TC(queue_next(qe(call)));
_delayed_call_dequeue(call);
_internal_call_release(call);
call_removed = TRUE;
if (!remove_all)
break;
call = next;
}
else
call = TC(queue_next(qe(call)));
}
return (call_removed);
}
void
thread_call_func(
thread_call_func_t func,
thread_call_param_t param,
boolean_t unique_call
)
{
thread_call_t call;
int s;
if (!thread_call_initialized)
panic("thread_call_func");
s = splsched();
simple_lock(&thread_call_lock);
call = TC(queue_first(&pending_call_queue));
while (unique_call && !queue_end(&pending_call_queue, qe(call))) {
if ( call->func == func &&
call->param0 == param ) {
break;
}
call = TC(queue_next(qe(call)));
}
if (!unique_call || queue_end(&pending_call_queue, qe(call))) {
call = _internal_call_allocate();
call->func = func;
call->param0 = param;
call->param1 = 0;
_pending_call_enqueue(call);
if (thread_calls.active_num <= 0)
_call_thread_wake();
}
simple_unlock(&thread_call_lock);
splx(s);
}
void
thread_call_func_delayed(
thread_call_func_t func,
thread_call_param_t param,
uint64_t deadline
)
{
thread_call_t call;
int s;
if (!thread_call_initialized)
panic("thread_call_func_delayed");
s = splsched();
simple_lock(&thread_call_lock);
call = _internal_call_allocate();
call->func = func;
call->param0 = param;
call->param1 = 0;
call->deadline = deadline;
_delayed_call_enqueue(call);
if (queue_first(&delayed_call_queue) == qe(call))
_set_delayed_call_timer(call);
simple_unlock(&thread_call_lock);
splx(s);
}
boolean_t
thread_call_func_cancel(
thread_call_func_t func,
thread_call_param_t param,
boolean_t cancel_all
)
{
boolean_t result;
int s;
s = splsched();
simple_lock(&thread_call_lock);
if (cancel_all)
result = _remove_from_pending_queue(func, param, cancel_all) |
_remove_from_delayed_queue(func, param, cancel_all);
else
result = _remove_from_pending_queue(func, param, cancel_all) ||
_remove_from_delayed_queue(func, param, cancel_all);
simple_unlock(&thread_call_lock);
splx(s);
return (result);
}
thread_call_t
thread_call_allocate(
thread_call_func_t func,
thread_call_param_t param0
)
{
thread_call_t call = (void *)kalloc(sizeof (thread_call_data_t));
call->func = func;
call->param0 = param0;
call->state = IDLE;
return (call);
}
boolean_t
thread_call_free(
thread_call_t call
)
{
int s;
s = splsched();
simple_lock(&thread_call_lock);
if (call->state != IDLE) {
simple_unlock(&thread_call_lock);
splx(s);
return (FALSE);
}
simple_unlock(&thread_call_lock);
splx(s);
kfree((vm_offset_t)call, sizeof (thread_call_data_t));
return (TRUE);
}
boolean_t
thread_call_enter(
thread_call_t call
)
{
boolean_t result = TRUE;
int s;
s = splsched();
simple_lock(&thread_call_lock);
if (call->state != PENDING) {
if (call->state == DELAYED)
_delayed_call_dequeue(call);
else if (call->state == IDLE)
result = FALSE;
_pending_call_enqueue(call);
if (thread_calls.active_num <= 0)
_call_thread_wake();
}
call->param1 = 0;
simple_unlock(&thread_call_lock);
splx(s);
return (result);
}
boolean_t
thread_call_enter1(
thread_call_t call,
thread_call_param_t param1
)
{
boolean_t result = TRUE;
int s;
s = splsched();
simple_lock(&thread_call_lock);
if (call->state != PENDING) {
if (call->state == DELAYED)
_delayed_call_dequeue(call);
else if (call->state == IDLE)
result = FALSE;
_pending_call_enqueue(call);
if (thread_calls.active_num <= 0)
_call_thread_wake();
}
call->param1 = param1;
simple_unlock(&thread_call_lock);
splx(s);
return (result);
}
boolean_t
thread_call_enter_delayed(
thread_call_t call,
uint64_t deadline
)
{
boolean_t result = TRUE;
int s;
s = splsched();
simple_lock(&thread_call_lock);
if (call->state == PENDING)
_pending_call_dequeue(call);
else if (call->state == DELAYED)
_delayed_call_dequeue(call);
else if (call->state == IDLE)
result = FALSE;
call->param1 = 0;
call->deadline = deadline;
_delayed_call_enqueue(call);
if (queue_first(&delayed_call_queue) == qe(call))
_set_delayed_call_timer(call);
simple_unlock(&thread_call_lock);
splx(s);
return (result);
}
boolean_t
thread_call_enter1_delayed(
thread_call_t call,
thread_call_param_t param1,
uint64_t deadline
)
{
boolean_t result = TRUE;
int s;
s = splsched();
simple_lock(&thread_call_lock);
if (call->state == PENDING)
_pending_call_dequeue(call);
else if (call->state == DELAYED)
_delayed_call_dequeue(call);
else if (call->state == IDLE)
result = FALSE;
call->param1 = param1;
call->deadline = deadline;
_delayed_call_enqueue(call);
if (queue_first(&delayed_call_queue) == qe(call))
_set_delayed_call_timer(call);
simple_unlock(&thread_call_lock);
splx(s);
return (result);
}
boolean_t
thread_call_cancel(
thread_call_t call
)
{
boolean_t result = TRUE;
int s;
s = splsched();
simple_lock(&thread_call_lock);
if (call->state == PENDING)
_pending_call_dequeue(call);
else if (call->state == DELAYED)
_delayed_call_dequeue(call);
else
result = FALSE;
simple_unlock(&thread_call_lock);
splx(s);
return (result);
}
boolean_t
thread_call_is_delayed(
thread_call_t call,
uint64_t *deadline)
{
boolean_t result = FALSE;
int s;
s = splsched();
simple_lock(&thread_call_lock);
if (call->state == DELAYED) {
if (deadline != NULL)
*deadline = call->deadline;
result = TRUE;
}
simple_unlock(&thread_call_lock);
splx(s);
return (result);
}
static __inline__
void
_call_thread_wake(void)
{
if (wait_queue_wakeup_one(
&call_thread_idle_queue, &call_thread_idle_queue,
THREAD_AWAKENED) == KERN_SUCCESS) {
thread_calls.idle_thread_num--;
if (++thread_calls.active_num > thread_calls.active_hiwat)
thread_calls.active_hiwat = thread_calls.active_num;
}
else
if (!activate_thread_awake) {
clear_wait(activate_thread, THREAD_AWAKENED);
activate_thread_awake = TRUE;
}
}
void
call_thread_block(void)
{
simple_lock(&thread_call_lock);
if (--thread_calls.active_num < thread_calls.active_lowat)
thread_calls.active_lowat = thread_calls.active_num;
if ( thread_calls.active_num <= 0 &&
thread_calls.pending_num > 0 )
_call_thread_wake();
simple_unlock(&thread_call_lock);
}
void
call_thread_unblock(void)
{
simple_lock(&thread_call_lock);
if (++thread_calls.active_num > thread_calls.active_hiwat)
thread_calls.active_hiwat = thread_calls.active_num;
simple_unlock(&thread_call_lock);
}
static
void
_call_thread_continue(void)
{
thread_t self = current_thread();
(void) splsched();
simple_lock(&thread_call_lock);
self->active_callout = TRUE;
while (thread_calls.pending_num > 0) {
thread_call_t call;
thread_call_func_t func;
thread_call_param_t param0, param1;
call = TC(dequeue_head(&pending_call_queue));
thread_calls.pending_num--;
func = call->func;
param0 = call->param0;
param1 = call->param1;
call->state = IDLE;
_internal_call_release(call);
simple_unlock(&thread_call_lock);
(void) spllo();
(*func)(param0, param1);
(void)thread_funnel_set(self->funnel_lock, FALSE);
(void) splsched();
simple_lock(&thread_call_lock);
}
self->active_callout = FALSE;
if (--thread_calls.active_num < thread_calls.active_lowat)
thread_calls.active_lowat = thread_calls.active_num;
if (thread_calls.idle_thread_num < thread_calls.thread_lowat) {
thread_calls.idle_thread_num++;
wait_queue_assert_wait(
&call_thread_idle_queue, &call_thread_idle_queue,
THREAD_INTERRUPTIBLE);
simple_unlock(&thread_call_lock);
(void) spllo();
thread_block(_call_thread_continue);
}
thread_calls.thread_num--;
simple_unlock(&thread_call_lock);
(void) spllo();
(void) thread_terminate(self->top_act);
}
static
void
_call_thread(void)
{
thread_t self = current_thread();
stack_privilege(self);
_call_thread_continue();
}
static
void
_activate_thread_continue(void)
{
(void) splsched();
simple_lock(&thread_call_lock);
while ( thread_calls.active_num <= 0 &&
thread_calls.pending_num > 0 ) {
if (++thread_calls.active_num > thread_calls.active_hiwat)
thread_calls.active_hiwat = thread_calls.active_num;
if (++thread_calls.thread_num > thread_calls.thread_hiwat)
thread_calls.thread_hiwat = thread_calls.thread_num;
simple_unlock(&thread_call_lock);
(void) spllo();
(void) kernel_thread_with_priority(
kernel_task, MAXPRI_KERNEL - 1,
_call_thread, TRUE, TRUE);
(void) splsched();
simple_lock(&thread_call_lock);
}
assert_wait(&activate_thread_awake, THREAD_INTERRUPTIBLE);
activate_thread_awake = FALSE;
simple_unlock(&thread_call_lock);
(void) spllo();
thread_block(_activate_thread_continue);
}
static
void
_activate_thread(void)
{
thread_t self = current_thread();
self->vm_privilege = TRUE;
vm_page_free_reserve(2);
stack_privilege(self);
_activate_thread_continue();
}
static
void
_delayed_call_timer(
timer_call_param_t p0,
timer_call_param_t p1
)
{
uint64_t timestamp;
thread_call_t call;
boolean_t new_pending = FALSE;
int s;
s = splsched();
simple_lock(&thread_call_lock);
clock_get_uptime(×tamp);
call = TC(queue_first(&delayed_call_queue));
while (!queue_end(&delayed_call_queue, qe(call))) {
if (call->deadline <= timestamp) {
_delayed_call_dequeue(call);
_pending_call_enqueue(call);
new_pending = TRUE;
}
else
break;
call = TC(queue_first(&delayed_call_queue));
}
if (!queue_end(&delayed_call_queue, qe(call)))
_set_delayed_call_timer(call);
if (new_pending && thread_calls.active_num <= 0)
_call_thread_wake();
simple_unlock(&thread_call_lock);
splx(s);
}