#include <mach_assert.h>
#include <task_swapper.h>
#include <kern/spl.h>
#include <kern/lock.h>
#include <kern/queue.h>
#include <kern/host.h>
#include <kern/task.h>
#include <kern/task_swap.h>
#include <kern/thread.h>
#include <kern/thread_swap.h>
#include <kern/host_statistics.h>
#include <kern/misc_protos.h>
#include <kern/assert.h>
#include <mach/policy.h>
#include <ipc/ipc_port.h>
#if TASK_SWAPPER
#define TASK_SW_DEBUG 1
#define TASK_SW_STATS 1
int task_swap_debug = 0;
int task_swap_stats = 0;
int task_swap_enable = 1;
int task_swap_on = 1;
queue_head_t swapped_tasks;
queue_head_t swapout_thread_q;
mutex_t task_swapper_lock;
#define task_swapper_lock() mutex_lock(&task_swapper_lock)
#define task_swapper_unlock() mutex_unlock(&task_swapper_lock)
#define task_swapper_wakeup() thread_wakeup((event_t)&swapout_thread_q)
#define task_swapper_sleep() thread_sleep_mutex((event_t)&swapout_thread_q, \
&task_swapper_lock, \
THREAD_UNINT)
queue_head_t eligible_tasks;
mutex_t task_swapout_list_lock;
#define task_swapout_lock() mutex_lock(&task_swapout_list_lock)
#define task_swapout_unlock() mutex_unlock(&task_swapout_list_lock)
#define SHORT_AVG_INTERVAL 5
#define LONG_AVG_INTERVAL 30
#define AVE_SCALE 1024
unsigned int short_avg_interval = SHORT_AVG_INTERVAL;
unsigned int long_avg_interval = LONG_AVG_INTERVAL;
#ifndef MIN_SWAP_PAGEOUT_RATE
#define MIN_SWAP_PAGEOUT_RATE 10
#endif
unsigned int vm_grab_rate_avg;
unsigned int vm_pageout_rate_avg = MIN_SWAP_PAGEOUT_RATE * AVE_SCALE;
unsigned int vm_pageout_rate_longavg = MIN_SWAP_PAGEOUT_RATE * AVE_SCALE;
unsigned int vm_pageout_rate_peakavg = MIN_SWAP_PAGEOUT_RATE * AVE_SCALE;
unsigned int vm_page_free_avg;
unsigned int vm_page_free_longavg;
#ifndef SWAP_PAGEOUT_HIGH_WATER_MARK
#define SWAP_PAGEOUT_HIGH_WATER_MARK 30
#endif
#ifndef SWAP_PAGEOUT_LOW_WATER_MARK
#define SWAP_PAGEOUT_LOW_WATER_MARK 10
#endif
#ifndef MAX_GRAB_RATE
#define MAX_GRAB_RATE ((unsigned int) -1)
#endif
unsigned int swap_pageout_high_water_mark = SWAP_PAGEOUT_HIGH_WATER_MARK;
unsigned int swap_pageout_low_water_mark = SWAP_PAGEOUT_LOW_WATER_MARK;
unsigned int swap_start_pageout_rate = MIN_SWAP_PAGEOUT_RATE * AVE_SCALE *
SWAP_PAGEOUT_HIGH_WATER_MARK / 100;
unsigned int swap_stop_pageout_rate = MIN_SWAP_PAGEOUT_RATE * AVE_SCALE *
SWAP_PAGEOUT_LOW_WATER_MARK / 100;
#if TASK_SW_DEBUG
unsigned int fixed_swap_start_pageout_rate = 0;
unsigned int fixed_swap_stop_pageout_rate = 0;
#endif
unsigned int max_grab_rate = MAX_GRAB_RATE;
#ifndef MIN_SWAP_TIME
#define MIN_SWAP_TIME 1
#endif
int min_swap_time = MIN_SWAP_TIME;
#ifndef MIN_RES_TIME
#define MIN_RES_TIME 6
#endif
int min_res_time = MIN_RES_TIME;
#ifndef MIN_ACTIVE_TASKS
#define MIN_ACTIVE_TASKS 4
#endif
int min_active_tasks = MIN_ACTIVE_TASKS;
#ifndef TASK_SWAP_CYCLE_TIME
#define TASK_SWAP_CYCLE_TIME 2
#endif
int task_swap_cycle_time = TASK_SWAP_CYCLE_TIME;
int last_task_swap_cycle = 0;
int task_swapouts = 0;
int task_swapins = 0;
int task_swaprss_out = 0;
int task_swaprss_in = 0;
int task_swap_total_time = 0;
int tasks_swapped_out = 0;
#ifdef TASK_SW_STATS
#define TASK_STATS_INCR(cnt) (cnt)++
#else
#define TASK_STATS_INCR(cnt)
#endif
#if TASK_SW_DEBUG
boolean_t on_swapped_list(task_t task);
boolean_t
on_swapped_list(task_t task)
{
task_t ltask;
if (queue_empty(&swapped_tasks)) {
return(FALSE);
}
ltask = (task_t)queue_first(&swapped_tasks);
while (!queue_end(&swapped_tasks, (queue_entry_t)ltask)) {
if (ltask->swap_state == TASK_SW_IN) {
printf("on_swapped_list and in: 0x%X\n",ltask);
Debugger("");
}
if (ltask == task)
return(TRUE);
ltask = (task_t)queue_next(<ask->swapped_tasks);
}
return(FALSE);
}
#endif
void
task_swapper_init()
{
queue_init(&swapped_tasks);
queue_init(&eligible_tasks);
queue_init(&swapout_thread_q);
mutex_init(&task_swapper_lock, ETAP_THREAD_TASK_SWAP);
mutex_init(&task_swapout_list_lock, ETAP_THREAD_TASK_SWAPOUT);
vm_page_free_avg = vm_page_free_count * AVE_SCALE;
vm_page_free_longavg = vm_page_free_count * AVE_SCALE;
}
#endif
kern_return_t
task_swappable(
host_priv_t host_priv,
task_t task,
boolean_t make_swappable)
{
if (host_priv == HOST_PRIV_NULL)
return(KERN_INVALID_ARGUMENT);
if (task == TASK_NULL)
return(KERN_INVALID_ARGUMENT);
#if !TASK_SWAPPER
return(KERN_SUCCESS);
#else
task_lock(task);
if (make_swappable) {
if (task->swap_state == TASK_SW_UNSWAPPABLE) {
task->swap_state = TASK_SW_IN;
task_unlock(task);
task_swapout_eligible(task);
}
} else {
switch (task->swap_state) {
case TASK_SW_IN:
task->swap_state = TASK_SW_UNSWAPPABLE;
task_unlock(task);
task_swapout_ineligible(task);
break;
case TASK_SW_UNSWAPPABLE:
task_unlock(task);
break;
default:
task_unlock(task);
task_swapin(task, TRUE);
break;
}
}
return(KERN_SUCCESS);
#endif
}
#if TASK_SWAPPER
kern_return_t
task_swapout(task_t task)
{
thread_act_t thr_act;
thread_t thread;
queue_head_t *list;
int s;
task_swapout_lock();
task_lock(task);
if ((task->swap_state != TASK_SW_IN) || (!task->active)) {
task_unlock(task);
task_swapout_unlock();
return(KERN_FAILURE);
}
if (task->swap_flags & TASK_SW_ELIGIBLE) {
queue_remove(&eligible_tasks, task, task_t, swapped_tasks);
task->swap_flags &= ~TASK_SW_ELIGIBLE;
}
task_swapout_unlock();
task->swap_state = TASK_SW_GOING_OUT;
task->swap_rss = pmap_resident_count(task->map->pmap);
task_swaprss_out += task->swap_rss;
task->swap_ast_waiting = task->thr_act_count;
list = &task->thr_acts;
thr_act = (thread_act_t) queue_first(list);
while (!queue_end(list, (queue_entry_t) thr_act)) {
boolean_t swappable;
thread_act_t ract;
thread = act_lock_thread(thr_act);
s = splsched();
if (!thread)
swappable = (thr_act->swap_state != TH_SW_UNSWAPPABLE);
else {
thread_lock(thread);
swappable = TRUE;
for (ract = thread->top_act; ract; ract = ract->lower)
if (ract->swap_state == TH_SW_UNSWAPPABLE) {
swappable = FALSE;
break;
}
}
if (swappable)
thread_ast_set(thr_act, AST_SWAPOUT);
if (thread)
thread_unlock(thread);
splx(s);
assert((thr_act->ast & AST_TERMINATE) == 0);
act_unlock_thread(thr_act);
thr_act = (thread_act_t) queue_next(&thr_act->thr_acts);
}
task->swap_stamp = sched_tick;
task->swap_nswap++;
assert((task->swap_flags&TASK_SW_WANT_IN) == 0);
task_swapper_lock();
#if TASK_SW_DEBUG
if (task_swap_debug && on_swapped_list(task)) {
printf("task 0x%X already on list\n", task);
Debugger("");
}
#endif
queue_enter(&swapped_tasks, task, task_t, swapped_tasks);
tasks_swapped_out++;
task_swapouts++;
task_swapper_unlock();
task_unlock(task);
return(KERN_SUCCESS);
}
#ifdef TASK_SW_STATS
int task_sw_race_in = 0;
int task_sw_race_coming_in = 0;
int task_sw_race_going_out = 0;
int task_sw_before_ast = 0;
int task_sw_before_swap = 0;
int task_sw_after_swap = 0;
int task_sw_race_in_won = 0;
int task_sw_unswappable = 0;
int task_sw_act_inactive = 0;
#endif
void
thread_swapout_enqueue(thread_act_t thr_act)
{
task_t task = thr_act->task;
task_lock(task);
if (task->swap_state != TASK_SW_GOING_OUT) {
task_unlock(task);
return;
}
if (--task->swap_ast_waiting == 0) {
vm_map_t map = task->map;
task->swap_state = TASK_SW_OUT;
task_unlock(task);
mutex_lock(&map->s_lock);
vm_map_res_deallocate(map);
mutex_unlock(&map->s_lock);
} else
task_unlock(task);
task_swapper_lock();
act_lock(thr_act);
if (! (thr_act->swap_state & TH_SW_TASK_SWAPPING)) {
} else {
queue_enter(&swapout_thread_q, thr_act,
thread_act_t, swap_queue);
task_swapper_wakeup();
}
act_unlock(thr_act);
task_swapper_unlock();
}
void
task_swap_swapout_thread(void)
{
thread_act_t thr_act;
thread_t thread, nthread;
task_t task;
int s;
thread_swappable(current_act(), FALSE);
stack_privilege(current_thread());
spllo();
task_swapper_lock();
while (TRUE) {
while (! queue_empty(&swapout_thread_q)) {
queue_remove_first(&swapout_thread_q, thr_act,
thread_act_t, swap_queue);
act_lock(thr_act);
queue_init((queue_t) &thr_act->swap_queue);
act_unlock(thr_act);
task_swapper_unlock();
thread = act_lock_thread(thr_act);
if (thread == THREAD_NULL)
act_unlock_thread(thr_act);
else {
boolean_t r;
thread_reference(thread);
thread_hold(thr_act);
act_unlock_thread(thr_act);
r = thread_stop_wait(thread);
nthread = act_lock_thread(thr_act);
thread_release(thr_act);
thread_deallocate(thread);
act_unlock_thread(thr_act);
if (!r || nthread != thread) {
task_swapper_lock();
continue;
}
}
task = thr_act->task;
task_lock(task);
if ((task->swap_state != TASK_SW_OUT) &&
(task->swap_state != TASK_SW_GOING_OUT)) {
task_unlock(task);
task_swapper_lock();
TASK_STATS_INCR(task_sw_race_in_won);
if (thread != THREAD_NULL)
thread_unstop(thread);
continue;
}
nthread = act_lock_thread(thr_act);
if (nthread != thread || thr_act->active == FALSE) {
act_unlock_thread(thr_act);
task_unlock(task);
task_swapper_lock();
TASK_STATS_INCR(task_sw_act_inactive);
if (thread != THREAD_NULL)
thread_unstop(thread);
continue;
}
s = splsched();
if (thread != THREAD_NULL)
thread_lock(thread);
assert(thr_act->swap_state & TH_SW_TASK_SWAPPING);
assert(thread == THREAD_NULL ||
!(thread->state & (TH_SWAPPED_OUT|TH_RUN)));
assert((thr_act->swap_state & TH_SW_STATE) == TH_SW_IN);
thr_act->swap_state = TH_SW_GOING_OUT;
if (thread != THREAD_NULL) {
if (thread->top_act == thr_act) {
thread->state |= TH_SWAPPED_OUT;
}
thread->ref_count++;
thread_unlock(thread);
thread_unstop(thread);
}
splx(s);
act_locked_act_reference(thr_act);
act_unlock_thread(thr_act);
task_unlock(task);
thread_swapout(thr_act);
if (thread != THREAD_NULL)
thread_deallocate(thread);
act_deallocate(thr_act);
task_swapper_lock();
}
task_swapper_sleep();
}
}
kern_return_t
task_swapin(task_t task, boolean_t make_unswappable)
{
register queue_head_t *list;
register thread_act_t thr_act, next;
thread_t thread;
int s;
boolean_t swappable = TRUE;
task_lock(task);
switch (task->swap_state) {
case TASK_SW_OUT:
{
vm_map_t map = task->map;
task->swap_state = TASK_SW_COMING_IN;
assert(task->swap_ast_waiting == 0);
assert(map->res_count >= 0);
task_unlock(task);
mutex_lock(&map->s_lock);
vm_map_res_reference(map);
mutex_unlock(&map->s_lock);
task_lock(task);
assert(task->swap_state == TASK_SW_COMING_IN);
}
break;
case TASK_SW_GOING_OUT:
task->swap_state = TASK_SW_COMING_IN;
assert(task->swap_ast_waiting > 0 ||
(task->swap_ast_waiting == 0 &&
task->thr_act_count == 0));
assert(task->map->res_count > 0);
TASK_STATS_INCR(task_sw_race_going_out);
break;
case TASK_SW_IN:
assert(task->map->res_count > 0);
#if TASK_SW_DEBUG
task_swapper_lock();
if (task_swap_debug && on_swapped_list(task)) {
printf("task 0x%X on list, state is SW_IN\n",
task);
Debugger("");
}
task_swapper_unlock();
#endif
TASK_STATS_INCR(task_sw_race_in);
if (make_unswappable) {
task->swap_state = TASK_SW_UNSWAPPABLE;
task_unlock(task);
task_swapout_ineligible(task);
} else
task_unlock(task);
return(KERN_SUCCESS);
case TASK_SW_COMING_IN:
assert(task->map->res_count >= 0);
if (make_unswappable)
task->swap_flags |= TASK_SW_MAKE_UNSWAPPABLE;
task->swap_flags |= TASK_SW_WANT_IN;
assert_wait((event_t)&task->swap_state, THREAD_UNINT);
task_unlock(task);
thread_block(THREAD_CONTINUE_NULL);
TASK_STATS_INCR(task_sw_race_coming_in);
return(KERN_SUCCESS);
case TASK_SW_UNSWAPPABLE:
task_unlock(task);
return(KERN_SUCCESS);
default:
panic("task_swapin bad state");
break;
}
if (make_unswappable)
task->swap_flags |= TASK_SW_MAKE_UNSWAPPABLE;
assert(task->swap_state == TASK_SW_COMING_IN);
task_swapper_lock();
#if TASK_SW_DEBUG
if (task_swap_debug && !on_swapped_list(task)) {
printf("task 0x%X not on list\n", task);
Debugger("");
}
#endif
queue_remove(&swapped_tasks, task, task_t, swapped_tasks);
tasks_swapped_out--;
task_swapins++;
task_swapper_unlock();
list = &task->thr_acts;
thr_act = (thread_act_t) queue_first(list);
while (!queue_end(list, (queue_entry_t) thr_act)) {
boolean_t need_to_release;
next = (thread_act_t) queue_next(&thr_act->thr_acts);
task_swapper_lock();
thread = act_lock_thread(thr_act);
s = splsched();
if (thr_act->ast & AST_SWAPOUT) {
thread_ast_clear(thr_act, AST_SWAPOUT);
need_to_release = FALSE;
TASK_STATS_INCR(task_sw_before_ast);
splx(s);
act_unlock_thread(thr_act);
} else {
need_to_release = TRUE;
if (thread)
thread_lock(thread);
if (thr_act->swap_state & TH_SW_TASK_SWAPPING) {
thr_act->swap_state &= ~TH_SW_TASK_SWAPPING;
assert(thr_act->thread == THREAD_NULL ||
!(thr_act->thread->state &
TH_SWAPPED_OUT));
queue_remove(&swapout_thread_q, thr_act,
thread_act_t, swap_queue);
TASK_STATS_INCR(task_sw_before_swap);
} else {
TASK_STATS_INCR(task_sw_after_swap);
if (thr_act->swap_state == TH_SW_UNSWAPPABLE) {
need_to_release = FALSE;
TASK_STATS_INCR(task_sw_unswappable);
}
}
if (thread)
thread_unlock(thread);
splx(s);
act_unlock_thread(thr_act);
}
task_swapper_unlock();
if (need_to_release) {
act_lock_thread(thr_act);
thread_release(thr_act);
act_unlock_thread(thr_act);
}
thr_act = next;
}
if (task->swap_flags & TASK_SW_MAKE_UNSWAPPABLE) {
task->swap_flags &= ~TASK_SW_MAKE_UNSWAPPABLE;
task->swap_state = TASK_SW_UNSWAPPABLE;
swappable = FALSE;
} else {
task->swap_state = TASK_SW_IN;
}
task_swaprss_in += pmap_resident_count(task->map->pmap);
task_swap_total_time += sched_tick - task->swap_stamp;
task->swap_stamp = sched_tick;
if (task->swap_flags & TASK_SW_WANT_IN) {
task->swap_flags &= ~TASK_SW_WANT_IN;
thread_wakeup((event_t)&task->swap_state);
}
assert((task->swap_flags & TASK_SW_ELIGIBLE) == 0);
task_unlock(task);
#if TASK_SW_DEBUG
task_swapper_lock();
if (task_swap_debug && on_swapped_list(task)) {
printf("task 0x%X on list at end of swap in\n", task);
Debugger("");
}
task_swapper_unlock();
#endif
if (swappable)
task_swapout_eligible(task);
return(KERN_SUCCESS);
}
void wake_task_swapper(boolean_t now);
void
wake_task_swapper(boolean_t now)
{
if (now ||
(sched_tick > (last_task_swap_cycle + task_swap_cycle_time))) {
last_task_swap_cycle = sched_tick;
if (task_swap_debug)
printf("wake_task_swapper: waking swapper\n");
thread_wakeup((event_t)&swapped_tasks);
}
}
task_t pick_intask(void);
task_t
pick_intask(void)
{
register task_t task = TASK_NULL;
task_swapper_lock();
if (!queue_empty(&swapped_tasks)) {
task = (task_t) queue_first(&swapped_tasks);
assert(task != TASK_NULL);
if ((sched_tick - task->swap_stamp) < min_swap_time)
task = TASK_NULL;
}
task_swapper_unlock();
return(task);
#if 0
task = (task_t)queue_first(&swapped_tasks);
while (!queue_end(&swapped_tasks, (queue_entry_t)task)) {
task_lock(task);
tmp_time = sched_tick - task->swap_stamp;
if (tmp_time > min_swap_time && tmp_time > time_swapped) {
target_task = task;
time_swapped = tmp_time;
}
task_unlock(task);
task = (task_t)queue_next(&task->swapped_tasks);
}
task_swapper_unlock();
return(target_task);
#endif
}
task_t pick_outtask(void);
#ifdef TASK_SW_STATS
int inactive_task_count = 0;
int empty_task_count = 0;
#endif
task_t
pick_outtask(void)
{
register task_t task;
register task_t target_task = TASK_NULL;
unsigned long task_rss;
unsigned long target_rss = 0;
boolean_t wired;
boolean_t active;
int nactive = 0;
task_swapout_lock();
if (queue_empty(&eligible_tasks)) {
task_swapout_unlock();
return(TASK_NULL);
}
task = (task_t)queue_first(&eligible_tasks);
while (!queue_end(&eligible_tasks, (queue_entry_t)task)) {
int s;
register thread_act_t thr_act;
thread_t th;
task_lock(task);
if (task->policy & POLICYCLASS_FIXEDPRI) {
goto tryagain;
}
if (!task->active) {
TASK_STATS_INCR(inactive_task_count);
goto tryagain;
}
if (task->res_act_count == 0) {
TASK_STATS_INCR(empty_task_count);
goto tryagain;
}
assert(!queue_empty(&task->thr_acts));
thr_act = (thread_act_t)queue_first(&task->thr_acts);
active = FALSE;
th = act_lock_thread(thr_act);
s = splsched();
if (th != THREAD_NULL)
thread_lock(th);
if ((th == THREAD_NULL) ||
(th->state == TH_RUN) ||
(th->state & TH_WAIT)) {
nactive++;
active = TRUE;
}
if (th != THREAD_NULL)
thread_unlock(th);
splx(s);
act_unlock_thread(thr_act);
if (active &&
(task->swap_state == TASK_SW_IN) &&
((sched_tick - task->swap_stamp) > min_res_time)) {
long rescount = pmap_resident_count(task->map->pmap);
#if 0
if (rescount > 1) {
task->ref_count++;
target_task = task;
task_unlock(task);
task_swapout_unlock();
return(target_task);
}
#else
if (rescount > target_rss) {
task->ref_count++;
target_rss = rescount;
assert(target_task != task);
if (target_task != TASK_NULL)
task_deallocate(target_task);
target_task = task;
}
#endif
}
tryagain:
task_unlock(task);
task = (task_t)queue_next(&task->swapped_tasks);
}
task_swapout_unlock();
if (nactive < min_active_tasks) {
if (target_task != TASK_NULL) {
task_deallocate(target_task);
target_task = TASK_NULL;
}
}
return(target_task);
}
#if TASK_SW_DEBUG
void print_pid(task_t task, unsigned long n1, unsigned long n2,
const char *comp, const char *inout);
void
print_pid(
task_t task,
unsigned long n1,
unsigned long n2,
const char *comp,
const char *inout)
{
long rescount;
task_lock(task);
rescount = pmap_resident_count(task->map->pmap);
task_unlock(task);
printf("task_swapper: swapped %s task %x; %d %s %d; res=%d\n",
inout, task, n1, comp, n2, rescount);
}
#endif
#define MAX_LOOP 3
void
task_swapper(void)
{
task_t outtask, intask;
int timeout;
int loopcnt = 0;
boolean_t start_swapping;
boolean_t stop_swapping;
int local_page_free_avg;
extern int hz;
thread_swappable(current_act(), FALSE);
stack_privilege(current_thread());
spllo();
for (;;) {
local_page_free_avg = vm_page_free_avg;
while (TRUE) {
#if 0
if (task_swap_debug)
printf("task_swapper: top of loop; cnt = %d\n",loopcnt);
#endif
intask = pick_intask();
start_swapping = ((vm_pageout_rate_avg > swap_start_pageout_rate) ||
(vm_grab_rate_avg > max_grab_rate));
stop_swapping = (vm_pageout_rate_avg < swap_stop_pageout_rate);
outtask = TASK_NULL;
if (start_swapping ||
(!stop_swapping && intask &&
((local_page_free_avg / AVE_SCALE) < vm_page_free_target))
) {
if (task_swap_enable &&
(outtask = pick_outtask()) &&
(task_swapout(outtask) == KERN_SUCCESS)) {
unsigned long rss;
#if TASK_SW_DEBUG
if (task_swap_debug)
print_pid(outtask, local_page_free_avg / AVE_SCALE,
vm_page_free_target, "<",
"out");
#endif
rss = outtask->swap_rss;
if (outtask->swap_nswap == 1)
rss /= 2;
local_page_free_avg += (rss/short_avg_interval) * AVE_SCALE;
}
if (outtask != TASK_NULL)
task_deallocate(outtask);
}
if (intask && ((local_page_free_avg / AVE_SCALE) >=
vm_page_free_target ||
stop_swapping || !task_swap_enable)) {
if (task_swapin(intask, FALSE) == KERN_SUCCESS) {
unsigned long rss;
#if TASK_SW_DEBUG
if (task_swap_debug)
print_pid(intask, local_page_free_avg / AVE_SCALE,
vm_page_free_target, ">=",
"in");
#endif
rss = intask->swap_rss;
if (intask->swap_nswap == 1)
rss /= 2;
local_page_free_avg -= (rss/short_avg_interval) * AVE_SCALE;
}
}
if (++loopcnt < MAX_LOOP)
continue;
timeout = 0;
if (start_swapping)
timeout = task_swap_cycle_time;
else {
task_swapper_lock();
if (!queue_empty(&swapped_tasks))
timeout = min_swap_time;
task_swapper_unlock();
}
assert_wait((event_t)&swapped_tasks, THREAD_UNINT);
if (timeout) {
if (task_swap_debug)
printf("task_swapper: set timeout of %d\n",
timeout);
thread_set_timeout(timeout, NSEC_PER_SEC);
}
if (task_swap_debug)
printf("task_swapper: blocking\n");
thread_block(THREAD_CONTINUE_NULL);
if (timeout) {
thread_cancel_timeout(current_thread());
}
loopcnt = 0;
local_page_free_avg = vm_page_free_avg;
}
}
}
#define ave(smooth, cnt, time) \
smooth = ((time - 1) * (smooth) + ((cnt) * AVE_SCALE)) / (time)
long last_grab_count = 0;
long last_pageout_count = 0;
void
compute_vm_averages(void)
{
extern unsigned long vm_page_grab_count;
long grab_count, pageout_count;
int i;
ave(vm_page_free_avg, vm_page_free_count, short_avg_interval);
ave(vm_page_free_longavg, vm_page_free_count, long_avg_interval);
grab_count = vm_page_grab_count;
pageout_count = 0;
for (i = 0; i < NCPUS; i++) {
pageout_count += vm_stat[i].pageouts;
}
ave(vm_pageout_rate_avg, pageout_count - last_pageout_count,
short_avg_interval);
ave(vm_pageout_rate_longavg, pageout_count - last_pageout_count,
long_avg_interval);
ave(vm_grab_rate_avg, grab_count - last_grab_count,
short_avg_interval);
last_grab_count = grab_count;
last_pageout_count = pageout_count;
if (vm_pageout_rate_avg > vm_pageout_rate_peakavg) {
unsigned int desired_max;
vm_pageout_rate_peakavg = vm_pageout_rate_avg;
swap_start_pageout_rate =
vm_pageout_rate_peakavg * swap_pageout_high_water_mark / 100;
swap_stop_pageout_rate =
vm_pageout_rate_peakavg * swap_pageout_low_water_mark / 100;
}
#if TASK_SW_DEBUG
if (fixed_swap_start_pageout_rate)
swap_start_pageout_rate = fixed_swap_start_pageout_rate;
if (fixed_swap_stop_pageout_rate)
swap_stop_pageout_rate = fixed_swap_stop_pageout_rate;
#endif
#if TASK_SW_DEBUG
if (task_swap_stats)
printf("vm_avgs: pageout_rate: %d %d (on/off: %d/%d); page_free: %d %d (tgt: %d)\n",
vm_pageout_rate_avg / AVE_SCALE,
vm_pageout_rate_longavg / AVE_SCALE,
swap_start_pageout_rate / AVE_SCALE,
swap_stop_pageout_rate / AVE_SCALE,
vm_page_free_avg / AVE_SCALE,
vm_page_free_longavg / AVE_SCALE,
vm_page_free_target);
#endif
if (vm_page_free_avg / AVE_SCALE <= vm_page_free_target) {
if (task_swap_on) {
if ((vm_pageout_rate_longavg > swap_stop_pageout_rate) &&
(vm_pageout_rate_avg > swap_start_pageout_rate) ||
(vm_pageout_rate_avg > vm_pageout_rate_peakavg) ||
(vm_grab_rate_avg > max_grab_rate))
wake_task_swapper(FALSE);
}
} else {
if (tasks_swapped_out != 0)
wake_task_swapper(TRUE);
}
}
void
task_swapout_eligible(task_t task)
{
#if TASK_SW_DEBUG
task_swapper_lock();
if (task_swap_debug && on_swapped_list(task)) {
printf("swapout_eligible: task 0x%X on swapped list\n", task);
Debugger("");
}
task_swapper_unlock();
#endif
task_swapout_lock();
task_lock(task);
#if TASK_SW_DEBUG
if (task->swap_flags & TASK_SW_ELIGIBLE) {
printf("swapout_eligible: task 0x%X already eligible\n", task);
}
#endif
if ((task->swap_state == TASK_SW_IN) &&
((task->swap_flags & TASK_SW_ELIGIBLE) == 0)) {
queue_enter(&eligible_tasks,task,task_t,swapped_tasks);
task->swap_flags |= TASK_SW_ELIGIBLE;
}
task_unlock(task);
task_swapout_unlock();
}
void
task_swapout_ineligible(task_t task)
{
#if TASK_SW_DEBUG
task_swapper_lock();
if (task_swap_debug && on_swapped_list(task)) {
printf("swapout_ineligible: task 0x%X on swapped list\n", task);
Debugger("");
}
task_swapper_unlock();
#endif
task_swapout_lock();
task_lock(task);
#if TASK_SW_DEBUG
if (!(task->swap_flags & TASK_SW_ELIGIBLE))
printf("swapout_ineligible: task 0x%X already inel.\n", task);
#endif
if ((task->swap_state != TASK_SW_IN) &&
(task->swap_flags & TASK_SW_ELIGIBLE)) {
queue_remove(&eligible_tasks, task, task_t, swapped_tasks);
task->swap_flags &= ~TASK_SW_ELIGIBLE;
}
task_unlock(task);
task_swapout_unlock();
}
int task_swap_ast_aborted = 0;
void
swapout_ast()
{
spl_t s;
thread_act_t act;
thread_t thread;
act = current_act();
thread = act_lock_thread(act);
s = splsched();
if (thread)
thread_lock(thread);
if ((act->ast & AST_SWAPOUT) == 0) {
task_swap_ast_aborted++;
if (thread)
thread_unlock(thread);
splx(s);
act_unlock_thread(act);
} else if (act->swap_state == TH_SW_IN) {
act->swap_state |= TH_SW_TASK_SWAPPING;
if (thread)
thread_unlock(thread);
if (act->suspend_count++ == 0)
install_special_handler(act);
thread_ast_clear(act, AST_SWAPOUT);
queue_init((queue_t) &act->swap_queue);
splx(s);
act_unlock_thread(act);
thread_swapout_enqueue(act);
} else {
assert(act->swap_state == TH_SW_UNSWAPPABLE);
if (thread)
thread_unlock(thread);
thread_ast_clear(act, AST_SWAPOUT);
splx(s);
act_unlock_thread(act);
}
}
#endif