#include <mach_pagemap.h>
#include <mach_cluster_stats.h>
#include <mach_kdb.h>
#include <advisory_pageout.h>
#include <mach/mach_types.h>
#include <mach/memory_object.h>
#include <mach/memory_object_default.h>
#include <mach/memory_object_control_server.h>
#include <mach/mach_host_server.h>
#include <mach/vm_param.h>
#include <mach/vm_statistics.h>
#include <kern/host_statistics.h>
#include <kern/counters.h>
#include <kern/thread.h>
#include <kern/xpr.h>
#include <vm/pmap.h>
#include <vm/vm_fault.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_pageout.h>
#include <machine/vm_tuning.h>
#include <kern/misc_protos.h>
extern ipc_port_t memory_manager_default;
#ifndef VM_PAGE_LAUNDRY_MAX
#define VM_PAGE_LAUNDRY_MAX 6
#endif
#ifndef VM_PAGEOUT_BURST_MAX
#define VM_PAGEOUT_BURST_MAX 32
#endif
#ifndef VM_PAGEOUT_DISCARD_MAX
#define VM_PAGEOUT_DISCARD_MAX 68
#endif
#ifndef VM_PAGEOUT_BURST_WAIT
#define VM_PAGEOUT_BURST_WAIT 30
#endif
#ifndef VM_PAGEOUT_EMPTY_WAIT
#define VM_PAGEOUT_EMPTY_WAIT 200
#endif
#ifndef VM_PAGE_INACTIVE_TARGET
#define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 1 / 3)
#endif
#ifndef VM_PAGE_FREE_TARGET
#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80)
#endif
#ifndef VM_PAGE_FREE_MIN
#define VM_PAGE_FREE_MIN(free) (10 + (free) / 100)
#endif
#ifndef VM_PAGE_FREE_RESERVED
#define VM_PAGE_FREE_RESERVED \
((16 * VM_PAGE_LAUNDRY_MAX) + NCPUS)
#endif
unsigned int vm_pageout_scan_event_counter = 0;
extern void vm_pageout_continue(void);
extern void vm_pageout_scan(void);
extern void vm_pageout_throttle(vm_page_t m);
extern vm_page_t vm_pageout_cluster_page(
vm_object_t object,
vm_object_offset_t offset,
boolean_t precious_clean);
unsigned int vm_pageout_reserved_internal = 0;
unsigned int vm_pageout_reserved_really = 0;
unsigned int vm_page_laundry_max = 0;
unsigned int vm_page_laundry_min = 0;
unsigned int vm_pageout_burst_max = 0;
unsigned int vm_pageout_burst_wait = 0;
unsigned int vm_pageout_empty_wait = 0;
unsigned int vm_pageout_burst_min = 0;
unsigned int vm_pageout_pause_count = 0;
unsigned int vm_pageout_pause_max = 0;
unsigned int vm_free_page_pause = 100;
unsigned int vm_accellerate_zf_pageout_trigger = 400;
unsigned int vm_zf_iterator;
unsigned int vm_zf_iterator_count = 40;
unsigned int last_page_zf;
unsigned int vm_zf_count = 0;
unsigned int vm_pageout_active = 0;
unsigned int vm_pageout_inactive = 0;
unsigned int vm_pageout_inactive_throttled = 0;
unsigned int vm_pageout_inactive_forced = 0;
unsigned int vm_pageout_inactive_nolock = 0;
unsigned int vm_pageout_inactive_avoid = 0;
unsigned int vm_pageout_inactive_busy = 0;
unsigned int vm_pageout_inactive_absent = 0;
unsigned int vm_pageout_inactive_used = 0;
unsigned int vm_pageout_inactive_clean = 0;
unsigned int vm_pageout_inactive_dirty = 0;
unsigned int vm_pageout_dirty_no_pager = 0;
unsigned int vm_stat_discard = 0;
unsigned int vm_stat_discard_sent = 0;
unsigned int vm_stat_discard_failure = 0;
unsigned int vm_stat_discard_throttle = 0;
unsigned int vm_pageout_scan_active_emm_throttle = 0;
unsigned int vm_pageout_scan_active_emm_throttle_success = 0;
unsigned int vm_pageout_scan_active_emm_throttle_failure = 0;
unsigned int vm_pageout_scan_inactive_emm_throttle = 0;
unsigned int vm_pageout_scan_inactive_emm_throttle_success = 0;
unsigned int vm_pageout_scan_inactive_emm_throttle_failure = 0;
unsigned int vm_pageout_out_of_line = 0;
unsigned int vm_pageout_in_place = 0;
vm_object_t
vm_pageout_object_allocate(
vm_page_t m,
vm_size_t size,
vm_object_offset_t offset)
{
vm_object_t object = m->object;
vm_object_t new_object;
assert(object->pager_ready);
if (object->pager_trusted || object->internal)
vm_pageout_throttle(m);
new_object = vm_object_allocate(size);
if (object->pager_trusted) {
assert (offset < object->size);
vm_object_lock(new_object);
new_object->pageout = TRUE;
new_object->shadow = object;
new_object->can_persist = FALSE;
new_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
new_object->shadow_offset = offset;
vm_object_unlock(new_object);
vm_object_lock(object);
vm_object_paging_begin(object);
vm_object_unlock(object);
vm_pageout_in_place++;
} else
vm_pageout_out_of_line++;
return(new_object);
}
#if MACH_CLUSTER_STATS
unsigned long vm_pageout_cluster_dirtied = 0;
unsigned long vm_pageout_cluster_cleaned = 0;
unsigned long vm_pageout_cluster_collisions = 0;
unsigned long vm_pageout_cluster_clusters = 0;
unsigned long vm_pageout_cluster_conversions = 0;
unsigned long vm_pageout_target_collisions = 0;
unsigned long vm_pageout_target_page_dirtied = 0;
unsigned long vm_pageout_target_page_freed = 0;
#define CLUSTER_STAT(clause) clause
#else
#define CLUSTER_STAT(clause)
#endif
void
vm_pageout_object_terminate(
vm_object_t object)
{
vm_object_t shadow_object;
assert(object->pageout);
shadow_object = object->shadow;
vm_object_lock(shadow_object);
while (!queue_empty(&object->memq)) {
vm_page_t p, m;
vm_object_offset_t offset;
p = (vm_page_t) queue_first(&object->memq);
assert(p->private);
assert(p->pageout);
p->pageout = FALSE;
assert(!p->cleaning);
offset = p->offset;
VM_PAGE_FREE(p);
p = VM_PAGE_NULL;
m = vm_page_lookup(shadow_object,
offset + object->shadow_offset);
if(m == VM_PAGE_NULL)
continue;
assert(m->cleaning);
m->dump_cleaning = FALSE;
vm_object_paging_end(shadow_object);
assert((m->dirty) || (m->precious) ||
(m->busy && m->cleaning));
vm_page_lock_queues();
if (m->laundry) {
vm_page_laundry_count--;
m->laundry = FALSE;
if (vm_page_laundry_count < vm_page_laundry_min) {
vm_page_laundry_min = 0;
thread_wakeup((event_t) &vm_page_laundry_count);
}
}
if (m->pageout) {
assert(m->busy);
assert(m->wire_count == 1);
m->cleaning = FALSE;
m->pageout = FALSE;
#if MACH_CLUSTER_STATS
if (m->wanted) vm_pageout_target_collisions++;
#endif
pmap_page_protect(m->phys_page, VM_PROT_NONE);
m->dirty = pmap_is_modified(m->phys_page);
if (m->dirty) {
CLUSTER_STAT(vm_pageout_target_page_dirtied++;)
vm_page_unwire(m);
VM_STAT(reactivations++);
PAGE_WAKEUP_DONE(m);
} else {
CLUSTER_STAT(vm_pageout_target_page_freed++;)
vm_page_free(m);
}
vm_page_unlock_queues();
continue;
}
if (!m->active && !m->inactive && !m->private) {
if (m->reference)
vm_page_activate(m);
else
vm_page_deactivate(m);
}
if((m->busy) && (m->cleaning)) {
m->busy = FALSE;
pmap_clear_modify(m->phys_page);
if(m->absent) {
m->absent = FALSE;
if(shadow_object->absent_count == 1)
vm_object_absent_release(shadow_object);
else
shadow_object->absent_count--;
}
m->overwriting = FALSE;
} else if (m->overwriting) {
assert(m->wire_count != 0);
vm_page_unwire(m);
m->overwriting = FALSE;
} else {
#if MACH_CLUSTER_STATS
m->dirty = pmap_is_modified(m->phys_page);
if (m->dirty) vm_pageout_cluster_dirtied++;
else vm_pageout_cluster_cleaned++;
if (m->wanted) vm_pageout_cluster_collisions++;
#else
m->dirty = 0;
#endif
}
m->cleaning = FALSE;
PAGE_WAKEUP(m);
vm_page_unlock_queues();
}
vm_object_paging_end(shadow_object);
vm_object_unlock(shadow_object);
assert(object->ref_count == 0);
assert(object->paging_in_progress == 0);
assert(object->resident_page_count == 0);
return;
}
vm_page_t
vm_pageout_setup(
register vm_page_t m,
register vm_object_t new_object,
vm_object_offset_t new_offset)
{
register vm_object_t old_object = m->object;
vm_object_offset_t paging_offset;
vm_object_offset_t offset;
register vm_page_t holding_page;
register vm_page_t new_m;
register vm_page_t new_page;
boolean_t need_to_wire = FALSE;
XPR(XPR_VM_PAGEOUT,
"vm_pageout_setup, obj 0x%X off 0x%X page 0x%X new obj 0x%X offset 0x%X\n",
(integer_t)m->object, (integer_t)m->offset,
(integer_t)m, (integer_t)new_object,
(integer_t)new_offset);
assert(m && m->busy && !m->absent && !m->fictitious && !m->error &&
!m->restart);
assert(m->dirty || m->precious);
VM_PAGE_GRAB_FICTITIOUS(holding_page);
vm_object_lock(old_object);
offset = m->offset;
paging_offset = offset + old_object->paging_offset;
if (old_object->pager_trusted) {
new_m = holding_page;
holding_page = VM_PAGE_NULL;
new_m->phys_page = m->phys_page;
new_m->fictitious = FALSE;
new_m->pageout = TRUE;
pmap_clear_modify(m->phys_page);
vm_page_lock_queues();
new_m->private = TRUE;
vm_page_wire(new_m);
m->cleaning = TRUE;
m->pageout = TRUE;
vm_page_wire(m);
assert(m->wire_count == 1);
vm_page_unlock_queues();
m->dirty = TRUE;
m->precious = FALSE;
m->page_lock = VM_PROT_NONE;
m->unusual = FALSE;
m->unlock_request = VM_PROT_NONE;
} else {
vm_page_lock_queues();
VM_PAGE_QUEUES_REMOVE(m);
vm_page_remove(m);
vm_page_insert(holding_page, old_object, offset);
vm_page_unlock_queues();
m->dirty = TRUE;
m->precious = FALSE;
new_m = m;
new_m->page_lock = VM_PROT_NONE;
new_m->unlock_request = VM_PROT_NONE;
if (old_object->internal)
need_to_wire = TRUE;
}
#if MACH_PAGEMAP
vm_external_state_set(old_object->existence_map, offset);
#endif
vm_object_unlock(old_object);
vm_object_lock(new_object);
vm_page_lock_queues();
vm_page_insert(new_m, new_object, new_offset);
if (need_to_wire)
vm_page_wire(new_m);
else
vm_page_activate(new_m);
PAGE_WAKEUP_DONE(new_m);
vm_page_unlock_queues();
vm_object_unlock(new_object);
return (holding_page);
}
void
vm_pageclean_setup(
vm_page_t m,
vm_page_t new_m,
vm_object_t new_object,
vm_object_offset_t new_offset)
{
vm_object_t old_object = m->object;
assert(!m->busy);
assert(!m->cleaning);
XPR(XPR_VM_PAGEOUT,
"vm_pageclean_setup, obj 0x%X off 0x%X page 0x%X new 0x%X new_off 0x%X\n",
(integer_t)old_object, m->offset, (integer_t)m,
(integer_t)new_m, new_offset);
pmap_clear_modify(m->phys_page);
vm_object_paging_begin(old_object);
#if MACH_PAGEMAP
vm_external_state_set(old_object->existence_map, m->offset);
#endif
m->cleaning = TRUE;
m->dirty = TRUE;
m->precious = FALSE;
assert(new_m->fictitious);
new_m->fictitious = FALSE;
new_m->private = TRUE;
new_m->pageout = TRUE;
new_m->phys_page = m->phys_page;
vm_page_wire(new_m);
vm_page_insert(new_m, new_object, new_offset);
assert(!new_m->wanted);
new_m->busy = FALSE;
}
void
vm_pageclean_copy(
vm_page_t m,
vm_page_t new_m,
vm_object_t new_object,
vm_object_offset_t new_offset)
{
XPR(XPR_VM_PAGEOUT,
"vm_pageclean_copy, page 0x%X new_m 0x%X new_obj 0x%X offset 0x%X\n",
m, new_m, new_object, new_offset, 0);
assert((!m->busy) && (!m->cleaning));
assert(!new_m->private && !new_m->fictitious);
pmap_clear_modify(m->phys_page);
m->busy = TRUE;
vm_object_paging_begin(m->object);
vm_page_unlock_queues();
vm_object_unlock(m->object);
vm_page_copy(m, new_m);
vm_object_lock(m->object);
m->dirty = FALSE;
vm_object_paging_end(m->object);
vm_page_lock_queues();
if (!m->active && !m->inactive)
vm_page_activate(m);
PAGE_WAKEUP_DONE(m);
vm_page_insert(new_m, new_object, new_offset);
vm_page_activate(new_m);
new_m->busy = FALSE;
}
void
vm_pageout_initialize_page(
vm_page_t m)
{
vm_map_copy_t copy;
vm_object_t new_object;
vm_object_t object;
vm_object_offset_t paging_offset;
vm_page_t holding_page;
XPR(XPR_VM_PAGEOUT,
"vm_pageout_initialize_page, page 0x%X\n",
(integer_t)m, 0, 0, 0, 0);
assert(m->busy);
assert(!m->absent);
assert(!m->error);
assert(m->dirty);
object = m->object;
paging_offset = m->offset + object->paging_offset;
vm_object_paging_begin(object);
vm_object_unlock(object);
if (m->absent || m->error || m->restart ||
(!m->dirty && !m->precious)) {
VM_PAGE_FREE(m);
panic("reservation without pageout?");
return;
}
holding_page = NULL;
vm_object_lock(m->object);
vm_page_lock_queues();
pmap_clear_modify(m->phys_page);
m->dirty = TRUE;
m->busy = TRUE;
m->list_req_pending = TRUE;
m->cleaning = TRUE;
m->pageout = TRUE;
vm_page_wire(m);
vm_page_unlock_queues();
vm_object_unlock(m->object);
vm_pageout_throttle(m);
memory_object_data_initialize(object->pager,
paging_offset,
PAGE_SIZE);
vm_object_lock(object);
}
#if MACH_CLUSTER_STATS
#define MAXCLUSTERPAGES 16
struct {
unsigned long pages_in_cluster;
unsigned long pages_at_higher_offsets;
unsigned long pages_at_lower_offsets;
} cluster_stats[MAXCLUSTERPAGES];
#endif
boolean_t allow_clustered_pageouts = FALSE;
void
vm_pageout_cluster(
vm_page_t m)
{
vm_object_t object = m->object;
vm_object_offset_t offset = m->offset;
vm_object_offset_t paging_offset = m->offset + object->paging_offset;
vm_object_t new_object;
vm_object_offset_t new_offset;
vm_size_t cluster_size;
vm_object_offset_t cluster_offset;
vm_object_offset_t cluster_lower_bound;
vm_object_offset_t cluster_upper_bound;
vm_object_offset_t cluster_start, cluster_end;
vm_object_offset_t offset_within_cluster;
vm_size_t length_of_data;
vm_page_t friend, holding_page;
kern_return_t rc;
boolean_t precious_clean = TRUE;
int pages_in_cluster;
CLUSTER_STAT(int pages_at_higher_offsets = 0;)
CLUSTER_STAT(int pages_at_lower_offsets = 0;)
XPR(XPR_VM_PAGEOUT,
"vm_pageout_cluster, object 0x%X offset 0x%X page 0x%X\n",
(integer_t)object, offset, (integer_t)m, 0, 0);
CLUSTER_STAT(vm_pageout_cluster_clusters++;)
assert(m->busy && (m->dirty || m->precious) && (m->wire_count == 0));
assert(!m->cleaning && !m->pageout && !m->inactive && !m->active);
vm_object_lock(object);
cluster_size = object->cluster_size;
assert(cluster_size >= PAGE_SIZE);
if (cluster_size < PAGE_SIZE) cluster_size = PAGE_SIZE;
assert(object->pager_created && object->pager_initialized);
assert(object->internal || object->pager_ready);
if (m->precious && !m->dirty)
precious_clean = TRUE;
if (!object->pager_trusted || !allow_clustered_pageouts)
cluster_size = PAGE_SIZE;
vm_object_unlock(object);
cluster_offset = paging_offset & (vm_object_offset_t)(cluster_size - 1);
if (offset > cluster_offset) {
cluster_lower_bound = offset - cluster_offset;
} else {
cluster_lower_bound = 0;
}
cluster_upper_bound = (offset - cluster_offset) +
(vm_object_offset_t)cluster_size;
holding_page = NULL;
vm_object_lock(m->object);
vm_page_lock_queues();
m->busy = TRUE;
m->list_req_pending = TRUE;
m->cleaning = TRUE;
m->pageout = TRUE;
vm_page_wire(m);
vm_page_unlock_queues();
vm_object_unlock(m->object);
vm_pageout_throttle(m);
cluster_start = offset;
if (offset) {
for (cluster_start = offset - PAGE_SIZE_64;
cluster_start >= cluster_lower_bound;
cluster_start -= PAGE_SIZE_64) {
assert(cluster_size > PAGE_SIZE);
vm_object_lock(object);
vm_page_lock_queues();
if ((friend = vm_pageout_cluster_page(object, cluster_start,
precious_clean)) == VM_PAGE_NULL) {
vm_page_unlock_queues();
vm_object_unlock(object);
break;
}
new_offset = (cluster_start + object->paging_offset)
& (cluster_size - 1);
assert(new_offset < cluster_offset);
m->list_req_pending = TRUE;
m->cleaning = TRUE;
if (vm_page_free_count <= vm_page_free_reserved) {
m->busy = TRUE;
m->pageout = TRUE;
vm_page_wire(m);
}
vm_page_unlock_queues();
vm_object_unlock(object);
if(m->dirty || m->object->internal) {
CLUSTER_STAT(pages_at_lower_offsets++;)
}
}
cluster_start += PAGE_SIZE_64;
}
assert(cluster_start >= cluster_lower_bound);
assert(cluster_start <= offset);
for (cluster_end = offset + PAGE_SIZE_64;
cluster_end < cluster_upper_bound;
cluster_end += PAGE_SIZE_64) {
assert(cluster_size > PAGE_SIZE);
vm_object_lock(object);
vm_page_lock_queues();
if ((friend = vm_pageout_cluster_page(object, cluster_end,
precious_clean)) == VM_PAGE_NULL) {
vm_page_unlock_queues();
vm_object_unlock(object);
break;
}
new_offset = (cluster_end + object->paging_offset)
& (cluster_size - 1);
assert(new_offset < cluster_size);
m->list_req_pending = TRUE;
m->cleaning = TRUE;
if (vm_page_free_count <= vm_page_free_reserved) {
m->busy = TRUE;
m->pageout = TRUE;
vm_page_wire(m);
}
vm_page_unlock_queues();
vm_object_unlock(object);
if(m->dirty || m->object->internal) {
CLUSTER_STAT(pages_at_higher_offsets++;)
}
}
assert(cluster_end <= cluster_upper_bound);
assert(cluster_end >= offset + PAGE_SIZE);
offset_within_cluster = cluster_start - (offset - cluster_offset);
length_of_data = cluster_end - cluster_start;
assert(offset_within_cluster < cluster_size);
assert((offset_within_cluster + length_of_data) <= cluster_size);
rc = KERN_SUCCESS;
assert(rc == KERN_SUCCESS);
pages_in_cluster = length_of_data/PAGE_SIZE;
#if MACH_CLUSTER_STATS
(cluster_stats[pages_at_lower_offsets].pages_at_lower_offsets)++;
(cluster_stats[pages_at_higher_offsets].pages_at_higher_offsets)++;
(cluster_stats[pages_in_cluster].pages_in_cluster)++;
#endif
paging_offset = cluster_start + object->paging_offset;
rc = memory_object_data_return(object->pager,
paging_offset,
length_of_data,
!precious_clean,
FALSE);
vm_object_lock(object);
vm_object_paging_end(object);
if (holding_page) {
assert(!object->pager_trusted);
VM_PAGE_FREE(holding_page);
vm_object_paging_end(object);
}
vm_object_unlock(object);
}
void
vm_pageout_throttle(
register vm_page_t m)
{
vm_page_lock_queues();
assert(!m->laundry);
m->laundry = TRUE;
while (vm_page_laundry_count >= vm_page_laundry_max) {
vm_page_laundry_min = vm_page_laundry_max/2;
assert_wait((event_t) &vm_page_laundry_count, THREAD_UNINT);
vm_page_unlock_queues();
thread_block((void (*)(void)) 0);
vm_page_lock_queues();
}
vm_page_laundry_count++;
vm_page_unlock_queues();
}
int vm_pageout_clean_active_pages = 1;
vm_page_t
vm_pageout_cluster_page(
vm_object_t object,
vm_object_offset_t offset,
boolean_t precious_clean)
{
vm_page_t m;
XPR(XPR_VM_PAGEOUT,
"vm_pageout_cluster_page, object 0x%X offset 0x%X\n",
(integer_t)object, offset, 0, 0, 0);
if ((m = vm_page_lookup(object, offset)) == VM_PAGE_NULL)
return(VM_PAGE_NULL);
if (m->busy || m->absent || m->cleaning ||
(m->wire_count != 0) || m->error)
return(VM_PAGE_NULL);
if (vm_pageout_clean_active_pages) {
if (!m->active && !m->inactive) return(VM_PAGE_NULL);
} else {
if (!m->inactive) return(VM_PAGE_NULL);
}
assert(!m->private);
assert(!m->fictitious);
if (!m->dirty) m->dirty = pmap_is_modified(m->phys_page);
if (precious_clean) {
if (!m->precious || !m->dirty)
return(VM_PAGE_NULL);
} else {
if (!m->dirty)
return(VM_PAGE_NULL);
}
return(m);
}
extern void vm_pageout_scan_continue(void);
void
vm_pageout_scan(void)
{
unsigned int burst_count;
boolean_t now = FALSE;
unsigned int laundry_pages;
boolean_t need_more_inactive_pages;
unsigned int loop_detect;
XPR(XPR_VM_PAGEOUT, "vm_pageout_scan\n", 0, 0, 0, 0, 0);
Restart:
#if THREAD_SWAPPER
mutex_lock(&vm_page_queue_free_lock);
now = (vm_page_free_count < vm_page_free_min);
mutex_unlock(&vm_page_queue_free_lock);
swapout_threads(now);
#endif
stack_collect();
consider_task_collect();
consider_thread_collect();
consider_zone_gc();
consider_machine_collect();
loop_detect = vm_page_active_count + vm_page_inactive_count;
#if 0
if (vm_page_free_count <= vm_page_free_reserved) {
need_more_inactive_pages = TRUE;
} else {
need_more_inactive_pages = FALSE;
}
#else
need_more_inactive_pages = FALSE;
#endif
for (burst_count = 0;;) {
register vm_page_t m;
register vm_object_t object;
vm_page_lock_queues();
vm_page_inactive_target =
VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
vm_page_inactive_count);
while ((vm_page_inactive_count < vm_page_inactive_target ||
need_more_inactive_pages) &&
!queue_empty(&vm_page_queue_active)) {
register vm_object_t object;
vm_pageout_active++;
m = (vm_page_t) queue_first(&vm_page_queue_active);
if (need_more_inactive_pages &&
(IP_VALID(memory_manager_default))) {
vm_pageout_scan_active_emm_throttle++;
do {
assert(m->active && !m->inactive);
object = m->object;
if (vm_object_lock_try(object)) {
#if 0
if (object->pager_trusted ||
object->internal) {
vm_pageout_scan_active_emm_throttle_success++;
goto object_locked_active;
}
#else
vm_pageout_scan_active_emm_throttle_success++;
goto object_locked_active;
#endif
vm_object_unlock(object);
}
m = (vm_page_t) queue_next(&m->pageq);
} while (!queue_end(&vm_page_queue_active,
(queue_entry_t) m));
if (queue_end(&vm_page_queue_active,
(queue_entry_t) m)) {
vm_pageout_scan_active_emm_throttle_failure++;
m = (vm_page_t)
queue_first(&vm_page_queue_active);
}
}
assert(m->active && !m->inactive);
object = m->object;
if (!vm_object_lock_try(object)) {
queue_remove(&vm_page_queue_active, m,
vm_page_t, pageq);
queue_enter(&vm_page_queue_active, m,
vm_page_t, pageq);
vm_page_unlock_queues();
mutex_pause();
vm_page_lock_queues();
continue;
}
object_locked_active:
if (m->busy) {
vm_object_unlock(object);
queue_remove(&vm_page_queue_active, m,
vm_page_t, pageq);
m->active = FALSE;
if (!m->fictitious)
vm_page_active_count--;
continue;
}
vm_page_deactivate(m);
vm_object_unlock(object);
}
if (vm_page_free_count >= vm_page_free_target) {
mutex_lock(&vm_page_queue_free_lock);
if ((vm_page_free_count >= vm_page_free_target) &&
(vm_page_free_wanted == 0)) {
vm_page_unlock_queues();
break;
}
mutex_unlock(&vm_page_queue_free_lock);
}
if ((queue_empty(&vm_page_queue_inactive) &&
(queue_empty(&vm_page_queue_zf))) ||
((--loop_detect) == 0) ||
(burst_count >= vm_pageout_burst_max)) {
unsigned int pages, msecs;
int wait_result;
consider_machine_adjust();
pages = burst_count;
if (loop_detect == 0) {
printf("Warning: No physical memory suitable for pageout or reclaim, pageout thread temporarily going to sleep\n");
msecs = vm_free_page_pause;
}
else {
msecs = burst_count * vm_pageout_burst_wait;
}
if (queue_empty(&vm_page_queue_inactive) &&
queue_empty(&vm_page_queue_zf) &&
(msecs < vm_pageout_empty_wait))
msecs = vm_pageout_empty_wait;
vm_page_unlock_queues();
assert_wait_timeout(msecs, THREAD_INTERRUPTIBLE);
counter(c_vm_pageout_scan_block++);
wait_result = thread_block((void (*)(void)) 0);
if (wait_result != THREAD_TIMED_OUT)
thread_cancel_timer();
vm_pageout_scan_continue();
goto Restart;
}
vm_pageout_inactive++;
if (vm_zf_count < vm_accellerate_zf_pageout_trigger) {
vm_zf_iterator = 0;
} else {
last_page_zf = 0;
if((vm_zf_iterator+=1) >= vm_zf_iterator_count) {
vm_zf_iterator = 0;
}
}
if(queue_empty(&vm_page_queue_zf) ||
(((last_page_zf) || (vm_zf_iterator == 0)) &&
!queue_empty(&vm_page_queue_inactive))) {
m = (vm_page_t) queue_first(&vm_page_queue_inactive);
last_page_zf = 0;
} else {
m = (vm_page_t) queue_first(&vm_page_queue_zf);
last_page_zf = 1;
}
if ((vm_page_free_count <= vm_page_free_reserved) &&
(IP_VALID(memory_manager_default))) {
vm_pageout_scan_inactive_emm_throttle++;
do {
assert(!m->active && m->inactive);
object = m->object;
if (vm_object_lock_try(object)) {
#if 0
if (object->pager_trusted ||
object->internal) {
vm_pageout_scan_inactive_emm_throttle_success++;
goto object_locked_inactive;
}
#else
vm_pageout_scan_inactive_emm_throttle_success++;
goto object_locked_inactive;
#endif
vm_object_unlock(object);
}
m = (vm_page_t) queue_next(&m->pageq);
} while ((!queue_end(&vm_page_queue_zf,
(queue_entry_t) m))
&& (!queue_end(&vm_page_queue_inactive,
(queue_entry_t) m)));
if ((queue_end(&vm_page_queue_zf,
(queue_entry_t) m))
|| (queue_end(&vm_page_queue_inactive,
(queue_entry_t) m))) {
vm_pageout_scan_inactive_emm_throttle_failure++;
need_more_inactive_pages = TRUE;
if(last_page_zf == 0) {
last_page_zf = 1;
vm_zf_iterator = vm_zf_iterator_count - 1;
} else {
last_page_zf = 0;
vm_zf_iterator = vm_zf_iterator_count - 2;
}
vm_page_unlock_queues();
goto Restart;
}
}
assert(!m->active && m->inactive);
object = m->object;
if (!vm_object_lock_try(object)) {
if(m->zero_fill) {
queue_remove(&vm_page_queue_zf, m,
vm_page_t, pageq);
queue_enter(&vm_page_queue_zf, m,
vm_page_t, pageq);
} else {
queue_remove(&vm_page_queue_inactive, m,
vm_page_t, pageq);
queue_enter(&vm_page_queue_inactive, m,
vm_page_t, pageq);
}
vm_page_unlock_queues();
mutex_pause();
vm_pageout_inactive_nolock++;
continue;
}
object_locked_inactive:
if (!object->pager_initialized && object->pager_created) {
if(m->zero_fill) {
queue_remove(&vm_page_queue_zf, m,
vm_page_t, pageq);
queue_enter(&vm_page_queue_zf, m,
vm_page_t, pageq);
last_page_zf = 1;
vm_zf_iterator = vm_zf_iterator_count - 1;
} else {
queue_remove(&vm_page_queue_inactive, m,
vm_page_t, pageq);
queue_enter(&vm_page_queue_inactive, m,
vm_page_t, pageq);
last_page_zf = 0;
vm_zf_iterator = 1;
}
vm_page_unlock_queues();
vm_object_unlock(object);
vm_pageout_inactive_avoid++;
continue;
}
if(m->zero_fill) {
queue_remove(&vm_page_queue_zf, m, vm_page_t, pageq);
} else {
queue_remove(&vm_page_queue_inactive, m, vm_page_t, pageq);
}
m->inactive = FALSE;
if (!m->fictitious)
vm_page_inactive_count--;
if (m->busy || !object->alive) {
vm_page_unlock_queues();
vm_object_unlock(object);
vm_pageout_inactive_busy++;
continue;
}
if (m->absent || m->error) {
vm_pageout_inactive_absent++;
reclaim_page:
vm_page_free(m);
vm_page_unlock_queues();
vm_object_unlock(object);
continue;
}
assert(!m->private);
assert(!m->fictitious);
if (m->cleaning) {
#if MACH_CLUSTER_STATS
vm_pageout_cluster_conversions++;
#endif
m->busy = TRUE;
m->pageout = TRUE;
m->dump_cleaning = TRUE;
vm_page_wire(m);
vm_object_unlock(object);
vm_page_unlock_queues();
continue;
}
if (m->reference || pmap_is_referenced(m->phys_page)) {
vm_pageout_inactive_used++;
reactivate_page:
#if ADVISORY_PAGEOUT
if (m->discard_request) {
m->discard_request = FALSE;
}
#endif
last_page_zf = 0;
vm_object_unlock(object);
vm_page_activate(m);
VM_STAT(reactivations++);
vm_page_unlock_queues();
continue;
}
#if ADVISORY_PAGEOUT
if (object->advisory_pageout) {
boolean_t do_throttle;
memory_object_t pager;
vm_object_offset_t discard_offset;
if (m->discard_request) {
vm_stat_discard_failure++;
goto mandatory_pageout;
}
assert(object->pager_initialized);
m->discard_request = TRUE;
pager = object->pager;
do_throttle = (vm_page_free_count <=
vm_page_free_reserved);
#if 0
if (!do_throttle) {
ip_lock(port);
do_throttle= imq_full(&port->ip_messages);
ip_unlock(port);
}
#endif
if (do_throttle) {
vm_stat_discard_throttle++;
#if 0
vm_page_unlock_queues();
vm_object_unlock(object);
continue;
#else
goto mandatory_pageout;
#endif
}
vm_page_activate(m);
vm_stat_discard++;
VM_STAT(reactivations++);
discard_offset = m->offset + object->paging_offset;
vm_stat_discard_sent++;
vm_page_unlock_queues();
vm_object_unlock(object);
continue;
}
mandatory_pageout:
#endif
XPR(XPR_VM_PAGEOUT,
"vm_pageout_scan, replace object 0x%X offset 0x%X page 0x%X\n",
(integer_t)object, (integer_t)m->offset, (integer_t)m, 0,0);
m->busy = TRUE;
pmap_page_protect(m->phys_page, VM_PROT_NONE);
if (!m->dirty)
m->dirty = pmap_is_modified(m->phys_page);
if (!m->dirty && !m->precious) {
vm_pageout_inactive_clean++;
goto reclaim_page;
}
vm_page_unlock_queues();
if (!object->pager_initialized)
vm_object_collapse(object);
if (!object->pager_initialized)
vm_object_pager_create(object);
if (!object->pager_initialized) {
vm_page_lock_queues();
vm_page_activate(m);
vm_page_unlock_queues();
PAGE_WAKEUP_DONE(m);
vm_object_unlock(object);
vm_pageout_dirty_no_pager++;
continue;
}
if ((object->pager_initialized) &&
(object->pager == MEMORY_OBJECT_NULL)) {
VM_PAGE_FREE(m);
vm_object_unlock(object);
continue;
}
vm_pageout_inactive_dirty++;
vm_object_paging_begin(object);
vm_object_unlock(object);
vm_pageout_cluster(m);
}
consider_machine_adjust();
}
counter(unsigned int c_vm_pageout_scan_continue = 0;)
void
vm_pageout_scan_continue(void)
{
vm_page_lock_queues();
counter(++c_vm_pageout_scan_continue);
if (vm_page_laundry_count > vm_pageout_burst_min) {
vm_pageout_burst_wait++;
vm_pageout_pause_count = 0;
} else if (++vm_pageout_pause_count > vm_pageout_pause_max) {
vm_pageout_burst_wait = (vm_pageout_burst_wait * 3) / 4;
if (vm_pageout_burst_wait < 1)
vm_pageout_burst_wait = 1;
vm_pageout_pause_count = 0;
}
vm_page_unlock_queues();
}
void vm_page_free_reserve(int pages);
int vm_page_free_count_init;
void
vm_page_free_reserve(
int pages)
{
int free_after_reserve;
vm_page_free_reserved += pages;
free_after_reserve = vm_page_free_count_init - vm_page_free_reserved;
vm_page_free_min = vm_page_free_reserved +
VM_PAGE_FREE_MIN(free_after_reserve);
vm_page_free_target = vm_page_free_reserved +
VM_PAGE_FREE_TARGET(free_after_reserve);
if (vm_page_free_target < vm_page_free_min + 5)
vm_page_free_target = vm_page_free_min + 5;
}
void
vm_pageout(void)
{
thread_t self = current_thread();
spl_t s;
self->vm_privilege = TRUE;
stack_privilege(self);
s = splsched();
thread_lock(self);
self->priority = BASEPRI_PREEMPT - 1;
set_sched_pri(self, self->priority);
thread_unlock(self);
splx(s);
if (vm_page_laundry_max == 0)
vm_page_laundry_max = VM_PAGE_LAUNDRY_MAX;
if (vm_pageout_burst_max == 0)
vm_pageout_burst_max = VM_PAGEOUT_BURST_MAX;
if (vm_pageout_burst_wait == 0)
vm_pageout_burst_wait = VM_PAGEOUT_BURST_WAIT;
if (vm_pageout_empty_wait == 0)
vm_pageout_empty_wait = VM_PAGEOUT_EMPTY_WAIT;
vm_page_free_count_init = vm_page_free_count;
vm_zf_iterator = 0;
if (vm_page_free_reserved < VM_PAGE_FREE_RESERVED)
vm_page_free_reserve(VM_PAGE_FREE_RESERVED - vm_page_free_reserved);
else
vm_page_free_reserve(0);
for (;;) {
vm_pageout_scan_event_counter++;
vm_pageout_scan();
assert(vm_page_free_wanted == 0);
assert_wait((event_t) &vm_page_free_wanted, THREAD_UNINT);
mutex_unlock(&vm_page_queue_free_lock);
counter(c_vm_pageout_block++);
thread_block((void (*)(void)) 0);
}
}
kern_return_t
vm_pageout_emergency_availability_request()
{
vm_page_t m;
vm_object_t object;
vm_page_lock_queues();
m = (vm_page_t) queue_first(&vm_page_queue_inactive);
while (!queue_end(&vm_page_queue_inactive, (queue_entry_t) m)) {
if(m->fictitious) {
m = (vm_page_t) queue_next(&m->pageq);
continue;
}
if (!m->dirty)
m->dirty = pmap_is_modified(m->phys_page);
if(m->dirty || m->busy || m->wire_count || m->absent
|| m->precious || m->cleaning
|| m->dump_cleaning || m->error
|| m->pageout || m->laundry
|| m->list_req_pending
|| m->overwriting) {
m = (vm_page_t) queue_next(&m->pageq);
continue;
}
object = m->object;
if (vm_object_lock_try(object)) {
if((!object->alive) ||
(object->pageout)) {
vm_object_unlock(object);
m = (vm_page_t) queue_next(&m->pageq);
continue;
}
m->busy = TRUE;
pmap_page_protect(m->phys_page, VM_PROT_NONE);
vm_page_free(m);
vm_object_unlock(object);
vm_page_unlock_queues();
return KERN_SUCCESS;
}
m = (vm_page_t) queue_next(&m->pageq);
}
m = (vm_page_t) queue_first(&vm_page_queue_active);
while (!queue_end(&vm_page_queue_active, (queue_entry_t) m)) {
if(m->fictitious) {
m = (vm_page_t) queue_next(&m->pageq);
continue;
}
if (!m->dirty)
m->dirty = pmap_is_modified(m->phys_page);
if(m->dirty || m->busy || m->wire_count || m->absent
|| m->precious || m->cleaning
|| m->dump_cleaning || m->error
|| m->pageout || m->laundry
|| m->list_req_pending
|| m->overwriting) {
m = (vm_page_t) queue_next(&m->pageq);
continue;
}
object = m->object;
if (vm_object_lock_try(object)) {
if((!object->alive) ||
(object->pageout)) {
vm_object_unlock(object);
m = (vm_page_t) queue_next(&m->pageq);
continue;
}
m->busy = TRUE;
pmap_page_protect(m->phys_page, VM_PROT_NONE);
vm_page_free(m);
vm_object_unlock(object);
vm_page_unlock_queues();
return KERN_SUCCESS;
}
m = (vm_page_t) queue_next(&m->pageq);
}
vm_page_unlock_queues();
return KERN_FAILURE;
}
static upl_t
upl_create(
int flags,
vm_size_t size)
{
upl_t upl;
int page_field_size;
page_field_size = 0;
if (flags & UPL_CREATE_LITE) {
page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
page_field_size = (page_field_size + 3) & 0xFFFFFFFC;
}
if(flags & UPL_CREATE_INTERNAL) {
upl = (upl_t)kalloc(sizeof(struct upl)
+ (sizeof(struct upl_page_info)*(size/PAGE_SIZE))
+ page_field_size);
} else {
upl = (upl_t)kalloc(sizeof(struct upl) + page_field_size);
}
upl->flags = 0;
upl->src_object = NULL;
upl->kaddr = (vm_offset_t)0;
upl->size = 0;
upl->map_object = NULL;
upl->ref_count = 1;
upl_lock_init(upl);
#ifdef UBC_DEBUG
upl->ubc_alias1 = 0;
upl->ubc_alias2 = 0;
#endif
return(upl);
}
static void
upl_destroy(
upl_t upl)
{
int page_field_size;
#ifdef UBC_DEBUG
{
upl_t upl_ele;
vm_object_t object;
if (upl->map_object->pageout) {
object = upl->map_object->shadow;
} else {
object = upl->map_object;
}
vm_object_lock(object);
queue_iterate(&object->uplq, upl_ele, upl_t, uplq) {
if(upl_ele == upl) {
queue_remove(&object->uplq,
upl_ele, upl_t, uplq);
break;
}
}
vm_object_unlock(object);
}
#endif
if(upl->map_object->pageout)
vm_object_deallocate(upl->map_object);
page_field_size = 0;
if (upl->flags & UPL_LITE) {
page_field_size = ((upl->size/PAGE_SIZE) + 7) >> 3;
page_field_size = (page_field_size + 3) & 0xFFFFFFFC;
}
if(upl->flags & UPL_INTERNAL) {
kfree((vm_offset_t)upl,
sizeof(struct upl) +
(sizeof(struct upl_page_info) * (upl->size/PAGE_SIZE))
+ page_field_size);
} else {
kfree((vm_offset_t)upl, sizeof(struct upl) + page_field_size);
}
}
__private_extern__ void
uc_upl_dealloc(
upl_t upl)
{
upl->ref_count -= 1;
if(upl->ref_count == 0) {
upl_destroy(upl);
}
}
void
upl_deallocate(
upl_t upl)
{
upl->ref_count -= 1;
if(upl->ref_count == 0) {
upl_destroy(upl);
}
}
__private_extern__ kern_return_t
vm_object_upl_request(
vm_object_t object,
vm_object_offset_t offset,
vm_size_t size,
upl_t *upl_ptr,
upl_page_info_array_t user_page_list,
unsigned int *page_list_count,
int cntrl_flags)
{
vm_page_t dst_page;
vm_object_offset_t dst_offset = offset;
vm_size_t xfer_size = size;
boolean_t do_m_lock = FALSE;
boolean_t dirty;
upl_t upl = NULL;
int entry;
boolean_t encountered_lrp = FALSE;
vm_page_t alias_page = NULL;
int page_ticket;
wpl_array_t lite_list;
page_ticket = (cntrl_flags & UPL_PAGE_TICKET_MASK)
>> UPL_PAGE_TICKET_SHIFT;
if(((size/PAGE_SIZE) > MAX_UPL_TRANSFER) && !object->phys_contiguous) {
size = MAX_UPL_TRANSFER * PAGE_SIZE;
}
if(cntrl_flags & UPL_SET_INTERNAL)
if(page_list_count != NULL)
*page_list_count = MAX_UPL_TRANSFER;
if(((cntrl_flags & UPL_SET_INTERNAL) && !(object->phys_contiguous)) &&
((page_list_count != NULL) && (*page_list_count != 0)
&& *page_list_count < (size/page_size)))
return KERN_INVALID_ARGUMENT;
if((!object->internal) && (object->paging_offset != 0))
panic("vm_object_upl_request: vnode object with non-zero paging offset\n");
if((cntrl_flags & UPL_COPYOUT_FROM) && (upl_ptr == NULL)) {
return KERN_SUCCESS;
}
if(upl_ptr) {
if(cntrl_flags & UPL_SET_INTERNAL) {
if(cntrl_flags & UPL_SET_LITE) {
vm_offset_t page_field_size;
upl = upl_create(
UPL_CREATE_INTERNAL | UPL_CREATE_LITE,
size);
user_page_list = (upl_page_info_t *)
(((vm_offset_t)upl) + sizeof(struct upl));
lite_list = (wpl_array_t)
(((vm_offset_t)user_page_list) +
((size/PAGE_SIZE) *
sizeof(upl_page_info_t)));
page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
page_field_size =
(page_field_size + 3) & 0xFFFFFFFC;
bzero((char *)lite_list, page_field_size);
upl->flags =
UPL_LITE | UPL_INTERNAL;
} else {
upl = upl_create(UPL_CREATE_INTERNAL, size);
user_page_list = (upl_page_info_t *)
(((vm_offset_t)upl)
+ sizeof(struct upl));
upl->flags = UPL_INTERNAL;
}
} else {
if(cntrl_flags & UPL_SET_LITE) {
vm_offset_t page_field_size;
upl = upl_create(UPL_CREATE_LITE, size);
lite_list = (wpl_array_t)
(((vm_offset_t)upl) + sizeof(struct upl));
page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
page_field_size =
(page_field_size + 3) & 0xFFFFFFFC;
bzero((char *)lite_list, page_field_size);
upl->flags = UPL_LITE;
} else {
upl = upl_create(UPL_CREATE_EXTERNAL, size);
upl->flags = 0;
}
}
if(object->phys_contiguous) {
upl->size = size;
upl->offset = offset + object->paging_offset;
*upl_ptr = upl;
if(user_page_list) {
user_page_list[0].phys_addr =
(offset + object->shadow_offset)>>12;
user_page_list[0].device = TRUE;
}
upl->map_object = object;
upl->flags |= UPL_DEVICE_MEMORY;
vm_object_lock(object);
vm_object_paging_begin(object);
vm_object_unlock(object);
if(page_list_count != NULL) {
if (upl->flags & UPL_INTERNAL) {
*page_list_count = 0;
} else {
*page_list_count = 1;
}
}
return KERN_SUCCESS;
}
if(cntrl_flags & UPL_SET_LITE) {
upl->map_object = object;
} else {
upl->map_object = vm_object_allocate(size);
vm_object_lock(upl->map_object);
upl->map_object->shadow = object;
upl->map_object->pageout = TRUE;
upl->map_object->can_persist = FALSE;
upl->map_object->copy_strategy =
MEMORY_OBJECT_COPY_NONE;
upl->map_object->shadow_offset = offset;
upl->map_object->wimg_bits = object->wimg_bits;
vm_object_unlock(upl->map_object);
}
upl->size = size;
upl->offset = offset + object->paging_offset;
*upl_ptr = upl;
}
if (!(cntrl_flags & UPL_SET_LITE)) {
VM_PAGE_GRAB_FICTITIOUS(alias_page);
}
vm_object_lock(object);
#ifdef UBC_DEBUG
if(upl_ptr)
queue_enter(&object->uplq, upl, upl_t, uplq);
#endif
vm_object_paging_begin(object);
entry = 0;
if(cntrl_flags & UPL_COPYOUT_FROM) {
upl->flags |= UPL_PAGE_SYNC_DONE;
while (xfer_size) {
if((alias_page == NULL) &&
!(cntrl_flags & UPL_SET_LITE)) {
vm_object_unlock(object);
VM_PAGE_GRAB_FICTITIOUS(alias_page);
vm_object_lock(object);
}
if(((dst_page = vm_page_lookup(object,
dst_offset)) == VM_PAGE_NULL) ||
dst_page->fictitious ||
dst_page->absent ||
dst_page->error ||
(dst_page->wire_count != 0 &&
!dst_page->pageout) ||
((!(dst_page->dirty || dst_page->precious ||
pmap_is_modified(dst_page->phys_page)))
&& (cntrl_flags & UPL_RET_ONLY_DIRTY)) ||
((!(dst_page->inactive))
&& (dst_page->page_ticket != page_ticket)
&& ((dst_page->page_ticket+1) != page_ticket)
&& (cntrl_flags & UPL_FOR_PAGEOUT)) ||
((!dst_page->list_req_pending) &&
(cntrl_flags & UPL_RET_ONLY_DIRTY) &&
pmap_is_referenced(dst_page->phys_page))) {
if(user_page_list) {
user_page_list[entry].phys_addr = 0;
user_page_list[entry].device = FALSE;
}
} else {
if(dst_page->busy &&
(!(dst_page->list_req_pending &&
dst_page->pageout))) {
if(cntrl_flags & UPL_NOBLOCK) {
if(user_page_list) {
user_page_list[entry].phys_addr = 0;
user_page_list[entry].device = FALSE;
}
entry++;
dst_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
continue;
}
PAGE_SLEEP(object, dst_page, THREAD_UNINT);
continue;
}
if((dst_page->cleaning || dst_page->absent ||
dst_page->wire_count != 0) &&
!dst_page->list_req_pending) {
if(user_page_list) {
user_page_list[entry].phys_addr = 0;
user_page_list[entry].device = FALSE;
}
entry++;
dst_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
continue;
}
vm_page_lock_queues();
if( !(cntrl_flags & UPL_FILE_IO)) {
pmap_page_protect(dst_page->phys_page, VM_PROT_NONE);
}
if(dst_page->list_req_pending)
encountered_lrp = TRUE;
if((dst_page->dirty ||
(dst_page->object->internal &&
dst_page->precious)) &&
(dst_page->list_req_pending
== FALSE)) {
if(encountered_lrp) {
CLUSTER_STAT
(pages_at_higher_offsets++;)
} else {
CLUSTER_STAT
(pages_at_lower_offsets++;)
}
}
dst_page->list_req_pending = FALSE;
dst_page->busy = FALSE;
dst_page->cleaning = FALSE;
dirty = pmap_is_modified(dst_page->phys_page);
dirty = dirty ? TRUE : dst_page->dirty;
if(cntrl_flags & UPL_SET_LITE) {
int pg_num;
pg_num = (dst_offset-offset)/PAGE_SIZE;
lite_list[pg_num>>5] |=
1 << (pg_num & 31);
pmap_clear_modify(dst_page->phys_page);
#if MACH_PAGEMAP
vm_external_state_set(
object->existence_map,
dst_page->offset);
#endif
dst_page->cleaning = TRUE;
dst_page->dirty = TRUE;
dst_page->precious = FALSE;
} else {
vm_pageclean_setup(dst_page,
alias_page, upl->map_object,
size - xfer_size);
alias_page->absent = FALSE;
alias_page = NULL;
}
if(!dirty) {
dst_page->dirty = FALSE;
dst_page->precious = TRUE;
}
if(dst_page->pageout)
dst_page->busy = TRUE;
if((!(cntrl_flags & UPL_CLEAN_IN_PLACE))
|| (cntrl_flags & UPL_FOR_PAGEOUT)) {
if((!dst_page->pageout) &&
(dst_page->wire_count == 0)) {
dst_page->busy = TRUE;
dst_page->pageout = TRUE;
vm_page_wire(dst_page);
}
}
if(user_page_list) {
user_page_list[entry].phys_addr
= dst_page->phys_page;
user_page_list[entry].dirty =
dst_page->dirty;
user_page_list[entry].pageout =
dst_page->pageout;
user_page_list[entry].absent =
dst_page->absent;
user_page_list[entry].precious =
dst_page->precious;
user_page_list[entry].device =
FALSE;
}
vm_page_unlock_queues();
}
entry++;
dst_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
}
} else {
while (xfer_size) {
if((alias_page == NULL) &&
!(cntrl_flags & UPL_SET_LITE)) {
vm_object_unlock(object);
VM_PAGE_GRAB_FICTITIOUS(alias_page);
vm_object_lock(object);
}
dst_page = vm_page_lookup(object, dst_offset);
if(dst_page != VM_PAGE_NULL) {
if((cntrl_flags & UPL_RET_ONLY_ABSENT) &&
!((dst_page->list_req_pending)
&& (dst_page->absent))) {
if(user_page_list) {
user_page_list[entry].phys_addr = 0;
user_page_list[entry].device = FALSE;
}
entry++;
dst_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
continue;
}
if((dst_page->cleaning) &&
!(dst_page->list_req_pending)) {
PAGE_SLEEP(object,dst_page,THREAD_UNINT);
continue;
}
if ((dst_page->fictitious &&
dst_page->list_req_pending)) {
dst_page->list_req_pending = FALSE;
dst_page->clustered = FALSE;
vm_page_lock_queues();
vm_page_free(dst_page);
vm_page_unlock_queues();
} else if ((dst_page->absent &&
dst_page->list_req_pending)) {
dst_page->list_req_pending = FALSE;
dst_page->busy = FALSE;
dst_page->clustered = FALSE;
}
}
if((dst_page = vm_page_lookup(object, dst_offset)) ==
VM_PAGE_NULL) {
if(object->private) {
if(user_page_list) {
user_page_list[entry].phys_addr = 0;
user_page_list[entry].device = FALSE;
}
entry++;
dst_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
continue;
}
dst_page = vm_page_alloc(object, dst_offset);
if (dst_page == VM_PAGE_NULL) {
vm_object_unlock(object);
VM_PAGE_WAIT();
vm_object_lock(object);
continue;
}
dst_page->busy = FALSE;
#if 0
if(cntrl_flags & UPL_NO_SYNC) {
dst_page->page_lock = 0;
dst_page->unlock_request = 0;
}
#endif
dst_page->absent = TRUE;
object->absent_count++;
}
#if 1
if(cntrl_flags & UPL_NO_SYNC) {
dst_page->page_lock = 0;
dst_page->unlock_request = 0;
}
#endif
dst_page->overwriting = TRUE;
if(dst_page->fictitious) {
panic("need corner case for fictitious page");
}
if(dst_page->page_lock) {
do_m_lock = TRUE;
}
if(upl_ptr) {
if(dst_page->busy) {
PAGE_SLEEP(object, dst_page, THREAD_UNINT);
continue;
}
vm_page_lock_queues();
if( !(cntrl_flags & UPL_FILE_IO)) {
pmap_page_protect(dst_page->phys_page, VM_PROT_NONE);
}
dirty = pmap_is_modified(dst_page->phys_page);
dirty = dirty ? TRUE : dst_page->dirty;
if(cntrl_flags & UPL_SET_LITE) {
int pg_num;
pg_num = (dst_offset-offset)/PAGE_SIZE;
lite_list[pg_num>>5] |=
1 << (pg_num & 31);
pmap_clear_modify(dst_page->phys_page);
#if MACH_PAGEMAP
vm_external_state_set(
object->existence_map,
dst_page->offset);
#endif
dst_page->cleaning = TRUE;
dst_page->dirty = TRUE;
dst_page->precious = FALSE;
} else {
vm_pageclean_setup(dst_page,
alias_page, upl->map_object,
size - xfer_size);
alias_page->absent = FALSE;
alias_page = NULL;
}
if(cntrl_flags & UPL_CLEAN_IN_PLACE) {
upl->flags |= UPL_CLEAR_DIRTY;
}
if(!dirty) {
dst_page->dirty = FALSE;
dst_page->precious = TRUE;
}
if (dst_page->wire_count == 0) {
dst_page->busy = TRUE;
} else {
vm_page_wire(dst_page);
}
dst_page->reference = TRUE;
dst_page->precious =
(cntrl_flags & UPL_PRECIOUS)
? TRUE : FALSE;
if(user_page_list) {
user_page_list[entry].phys_addr
= dst_page->phys_page;
user_page_list[entry].dirty =
dst_page->dirty;
user_page_list[entry].pageout =
dst_page->pageout;
user_page_list[entry].absent =
dst_page->absent;
user_page_list[entry].precious =
dst_page->precious;
user_page_list[entry].device =
FALSE;
}
vm_page_unlock_queues();
}
entry++;
dst_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
}
}
if (upl->flags & UPL_INTERNAL) {
if(page_list_count != NULL)
*page_list_count = 0;
} else if (*page_list_count > entry) {
if(page_list_count != NULL)
*page_list_count = entry;
}
if(alias_page != NULL) {
vm_page_lock_queues();
vm_page_free(alias_page);
vm_page_unlock_queues();
}
if(do_m_lock) {
vm_prot_t access_required;
access_required = (cntrl_flags & UPL_COPYOUT_FROM)
? VM_PROT_READ : VM_PROT_WRITE;
while (TRUE) {
kern_return_t rc;
if(!object->pager_ready) {
wait_result_t wait_result;
wait_result = vm_object_sleep(object,
VM_OBJECT_EVENT_PAGER_READY,
THREAD_UNINT);
if (wait_result != THREAD_AWAKENED) {
vm_object_unlock(object);
return(KERN_FAILURE);
}
continue;
}
vm_object_unlock(object);
if (rc = memory_object_data_unlock(
object->pager,
dst_offset + object->paging_offset,
size,
access_required)) {
if (rc == MACH_SEND_INTERRUPTED)
continue;
else
return KERN_FAILURE;
}
break;
}
if(dst_page != VM_PAGE_NULL &&
(access_required & dst_page->page_lock) != access_required) {
PAGE_ASSERT_WAIT(dst_page, THREAD_UNINT);
thread_block((void (*)(void))0);
vm_object_lock(object);
}
}
vm_object_unlock(object);
return KERN_SUCCESS;
}
kern_return_t
vm_fault_list_request(
memory_object_control_t control,
vm_object_offset_t offset,
vm_size_t size,
upl_t *upl_ptr,
upl_page_info_t **user_page_list_ptr,
int page_list_count,
int cntrl_flags)
{
int local_list_count;
upl_page_info_t *user_page_list;
kern_return_t kr;
if (user_page_list_ptr != NULL) {
local_list_count = page_list_count;
user_page_list = *user_page_list_ptr;
} else {
local_list_count = 0;
user_page_list = NULL;
}
kr = memory_object_upl_request(control,
offset,
size,
upl_ptr,
user_page_list,
&local_list_count,
cntrl_flags);
if(kr != KERN_SUCCESS)
return kr;
if ((user_page_list_ptr != NULL) && (cntrl_flags & UPL_INTERNAL)) {
*user_page_list_ptr = UPL_GET_INTERNAL_PAGE_LIST(*upl_ptr);
}
return KERN_SUCCESS;
}
__private_extern__ kern_return_t
vm_object_super_upl_request(
vm_object_t object,
vm_object_offset_t offset,
vm_size_t size,
vm_size_t super_cluster,
upl_t *upl,
upl_page_info_t *user_page_list,
unsigned int *page_list_count,
int cntrl_flags)
{
vm_page_t target_page;
int ticket;
if(object->paging_offset > offset)
return KERN_FAILURE;
offset = offset - object->paging_offset;
if(cntrl_flags & UPL_FOR_PAGEOUT) {
if((target_page = vm_page_lookup(object, offset))
!= VM_PAGE_NULL) {
ticket = target_page->page_ticket;
cntrl_flags = cntrl_flags & ~(int)UPL_PAGE_TICKET_MASK;
cntrl_flags = cntrl_flags |
((ticket << UPL_PAGE_TICKET_SHIFT)
& UPL_PAGE_TICKET_MASK);
}
}
if ((super_cluster > size) &&
(vm_page_free_count > vm_page_free_reserved)) {
vm_object_offset_t base_offset;
vm_size_t super_size;
base_offset = (offset &
~((vm_object_offset_t) super_cluster - 1));
super_size = (offset+size) > (base_offset + super_cluster) ?
super_cluster<<1 : super_cluster;
super_size = ((base_offset + super_size) > object->size) ?
(object->size - base_offset) : super_size;
if(offset > (base_offset + super_size))
panic("vm_object_super_upl_request: Missed target pageout 0x%x,0x%x, 0x%x, 0x%x, 0x%x, 0x%x\n", offset, base_offset, super_size, super_cluster, size, object->paging_offset);
if((offset + size) > (base_offset + super_size))
super_size = (offset + size) - base_offset;
offset = base_offset;
size = super_size;
}
vm_object_upl_request(object, offset, size,
upl, user_page_list, page_list_count,
cntrl_flags);
}
kern_return_t
vm_upl_map(
vm_map_t map,
upl_t upl,
vm_offset_t *dst_addr)
{
vm_size_t size;
vm_object_offset_t offset;
vm_offset_t addr;
vm_page_t m;
kern_return_t kr;
if (upl == UPL_NULL)
return KERN_INVALID_ARGUMENT;
upl_lock(upl);
if(UPL_PAGE_LIST_MAPPED & upl->flags) {
upl_unlock(upl);
return KERN_FAILURE;
}
if((!(upl->map_object->pageout)) &&
!((upl->flags & (UPL_DEVICE_MEMORY | UPL_IO_WIRE)) ||
(upl->map_object->phys_contiguous))) {
vm_object_t object;
vm_page_t alias_page;
vm_object_offset_t new_offset;
int pg_num;
wpl_array_t lite_list;
if(upl->flags & UPL_INTERNAL) {
lite_list = (wpl_array_t)
((((vm_offset_t)upl) + sizeof(struct upl))
+ ((upl->size/PAGE_SIZE)
* sizeof(upl_page_info_t)));
} else {
lite_list = (wpl_array_t)
(((vm_offset_t)upl) + sizeof(struct upl));
}
object = upl->map_object;
upl->map_object = vm_object_allocate(upl->size);
vm_object_lock(upl->map_object);
upl->map_object->shadow = object;
upl->map_object->pageout = TRUE;
upl->map_object->can_persist = FALSE;
upl->map_object->copy_strategy =
MEMORY_OBJECT_COPY_NONE;
upl->map_object->shadow_offset =
upl->offset - object->paging_offset;
upl->map_object->wimg_bits = object->wimg_bits;
vm_object_unlock(upl->map_object);
offset = upl->map_object->shadow_offset;
new_offset = 0;
size = upl->size;
vm_object_lock(object);
while(size) {
pg_num = (new_offset)/PAGE_SIZE;
if(lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
vm_object_unlock(object);
VM_PAGE_GRAB_FICTITIOUS(alias_page);
vm_object_lock(object);
m = vm_page_lookup(object, offset);
if (m == VM_PAGE_NULL) {
panic("vm_upl_map: page missing\n");
}
vm_object_paging_begin(object);
assert(alias_page->fictitious);
alias_page->fictitious = FALSE;
alias_page->private = TRUE;
alias_page->pageout = TRUE;
alias_page->phys_page = m->phys_page;
vm_page_wire(alias_page);
vm_page_insert(alias_page,
upl->map_object, new_offset);
assert(!alias_page->wanted);
alias_page->busy = FALSE;
alias_page->absent = FALSE;
}
size -= PAGE_SIZE;
offset += PAGE_SIZE_64;
new_offset += PAGE_SIZE_64;
}
vm_object_unlock(object);
}
offset = 0;
size = upl->size;
vm_object_lock(upl->map_object);
upl->map_object->ref_count++;
vm_object_res_reference(upl->map_object);
vm_object_unlock(upl->map_object);
*dst_addr = 0;
kr = vm_map_enter(map, dst_addr, size, (vm_offset_t) 0, TRUE,
upl->map_object, offset, FALSE,
VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
if (kr != KERN_SUCCESS) {
upl_unlock(upl);
return(kr);
}
for(addr=*dst_addr; size > 0; size-=PAGE_SIZE,addr+=PAGE_SIZE) {
m = vm_page_lookup(upl->map_object, offset);
if(m) {
unsigned int cache_attr;
cache_attr = ((unsigned int)m->object->wimg_bits) & VM_WIMG_MASK;
PMAP_ENTER(map->pmap, addr,
m, VM_PROT_ALL,
cache_attr, TRUE);
}
offset+=PAGE_SIZE_64;
}
upl->ref_count++;
upl->flags |= UPL_PAGE_LIST_MAPPED;
upl->kaddr = *dst_addr;
upl_unlock(upl);
return KERN_SUCCESS;
}
kern_return_t
vm_upl_unmap(
vm_map_t map,
upl_t upl)
{
vm_address_t addr;
vm_size_t size;
if (upl == UPL_NULL)
return KERN_INVALID_ARGUMENT;
upl_lock(upl);
if(upl->flags & UPL_PAGE_LIST_MAPPED) {
addr = upl->kaddr;
size = upl->size;
assert(upl->ref_count > 1);
upl->ref_count--;
upl->flags &= ~UPL_PAGE_LIST_MAPPED;
upl->kaddr = (vm_offset_t) 0;
upl_unlock(upl);
vm_deallocate(map, addr, size);
return KERN_SUCCESS;
}
upl_unlock(upl);
return KERN_FAILURE;
}
kern_return_t
upl_commit_range(
upl_t upl,
vm_offset_t offset,
vm_size_t size,
int flags,
upl_page_info_t *page_list,
mach_msg_type_number_t count,
boolean_t *empty)
{
vm_size_t xfer_size = size;
vm_object_t shadow_object;
vm_object_t object = upl->map_object;
vm_object_offset_t target_offset;
int entry;
wpl_array_t lite_list;
int occupied;
*empty = FALSE;
if (upl == UPL_NULL)
return KERN_INVALID_ARGUMENT;
if (count == 0)
page_list = NULL;
if(object->pageout) {
shadow_object = object->shadow;
} else {
shadow_object = object;
}
upl_lock(upl);
if(upl->flags & UPL_DEVICE_MEMORY) {
xfer_size = 0;
} else if ((offset + size) > upl->size) {
upl_unlock(upl);
return KERN_FAILURE;
}
if(upl->flags & UPL_INTERNAL) {
lite_list = (wpl_array_t)
((((vm_offset_t)upl) + sizeof(struct upl))
+ ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t)));
} else {
lite_list = (wpl_array_t)
(((vm_offset_t)upl) + sizeof(struct upl));
}
vm_object_lock(shadow_object);
entry = offset/PAGE_SIZE;
target_offset = (vm_object_offset_t)offset;
while(xfer_size) {
vm_page_t t,m;
upl_page_info_t *p;
m = VM_PAGE_NULL;
if(upl->flags & UPL_LITE) {
int pg_num;
pg_num = target_offset/PAGE_SIZE;
if(lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
lite_list[pg_num>>5] &= ~(1 << (pg_num & 31));
m = vm_page_lookup(shadow_object,
target_offset + (upl->offset -
shadow_object->paging_offset));
}
}
if(object->pageout) {
if ((t = vm_page_lookup(object, target_offset))
!= NULL) {
t->pageout = FALSE;
VM_PAGE_FREE(t);
if(m == NULL) {
m = vm_page_lookup(
shadow_object,
target_offset +
object->shadow_offset);
}
if(m != VM_PAGE_NULL)
vm_object_paging_end(m->object);
}
}
if(m != VM_PAGE_NULL) {
if(upl->flags & UPL_IO_WIRE) {
vm_page_lock_queues();
vm_page_unwire(m);
vm_page_unlock_queues();
if(page_list) {
page_list[entry].phys_addr = 0;
}
if (flags & UPL_COMMIT_SET_DIRTY) {
m->dirty = TRUE;
} else if ((upl->flags & UPL_CLEAR_DIRTY) ||
(flags & UPL_COMMIT_CLEAR_DIRTY)) {
pmap_clear_modify(m->phys_page);
m->dirty = FALSE;
}
if (flags & UPL_COMMIT_INACTIVATE) {
vm_page_deactivate(m);
m->reference = FALSE;
pmap_clear_reference(m->phys_page);
}
target_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
entry++;
continue;
}
vm_page_lock_queues();
if ((upl->flags & UPL_CLEAR_DIRTY) ||
(flags & UPL_COMMIT_CLEAR_DIRTY)) {
pmap_clear_modify(m->phys_page);
m->dirty = FALSE;
}
if(page_list) {
p = &(page_list[entry]);
if(p->phys_addr && p->pageout && !m->pageout) {
m->busy = TRUE;
m->pageout = TRUE;
vm_page_wire(m);
} else if (page_list[entry].phys_addr &&
!p->pageout && m->pageout &&
!m->dump_cleaning) {
m->pageout = FALSE;
m->absent = FALSE;
m->overwriting = FALSE;
vm_page_unwire(m);
PAGE_WAKEUP_DONE(m);
}
page_list[entry].phys_addr = 0;
}
m->dump_cleaning = FALSE;
if(m->laundry) {
vm_page_laundry_count--;
m->laundry = FALSE;
if (vm_page_laundry_count < vm_page_laundry_min) {
vm_page_laundry_min = 0;
thread_wakeup((event_t)
&vm_page_laundry_count);
}
}
if(m->pageout) {
m->cleaning = FALSE;
m->pageout = FALSE;
#if MACH_CLUSTER_STATS
if (m->wanted) vm_pageout_target_collisions++;
#endif
pmap_page_protect(m->phys_page, VM_PROT_NONE);
m->dirty = pmap_is_modified(m->phys_page);
if(m->dirty) {
CLUSTER_STAT(
vm_pageout_target_page_dirtied++;)
vm_page_unwire(m);
VM_STAT(reactivations++);
PAGE_WAKEUP_DONE(m);
} else {
CLUSTER_STAT(
vm_pageout_target_page_freed++;)
vm_page_free(m);
VM_STAT(pageouts++);
}
vm_page_unlock_queues();
target_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
entry++;
continue;
}
if (flags & UPL_COMMIT_INACTIVATE) {
vm_page_deactivate(m);
m->reference = FALSE;
pmap_clear_reference(m->phys_page);
} else if (!m->active && !m->inactive) {
if (m->reference)
vm_page_activate(m);
else
vm_page_deactivate(m);
}
#if MACH_CLUSTER_STATS
m->dirty = pmap_is_modified(m->phys_page);
if (m->dirty) vm_pageout_cluster_dirtied++;
else vm_pageout_cluster_cleaned++;
if (m->wanted) vm_pageout_cluster_collisions++;
#else
m->dirty = 0;
#endif
if((m->busy) && (m->cleaning)) {
if(m->absent) {
m->absent = FALSE;
if(shadow_object->absent_count == 1)
vm_object_absent_release(shadow_object);
else
shadow_object->absent_count--;
}
m->overwriting = FALSE;
m->busy = FALSE;
m->dirty = FALSE;
} else if (m->overwriting) {
assert(m->wire_count != 0);
vm_page_unwire(m);
m->overwriting = FALSE;
}
m->cleaning = FALSE;
if (upl->flags & UPL_PAGE_SYNC_DONE)
m->precious = FALSE;
if (flags & UPL_COMMIT_SET_DIRTY) {
m->dirty = TRUE;
}
PAGE_WAKEUP(m);
vm_page_unlock_queues();
}
target_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
entry++;
}
occupied = 1;
if (upl->flags & UPL_DEVICE_MEMORY) {
occupied = 0;
} else if (upl->flags & UPL_LITE) {
int pg_num;
int i;
pg_num = upl->size/PAGE_SIZE;
pg_num = (pg_num + 31) >> 5;
occupied = 0;
for(i= 0; i<pg_num; i++) {
if(lite_list[i] != 0) {
occupied = 1;
break;
}
}
} else {
if(queue_empty(&upl->map_object->memq)) {
occupied = 0;
}
}
if(occupied == 0) {
if(upl->flags & UPL_COMMIT_NOTIFY_EMPTY) {
*empty = TRUE;
}
if(object == shadow_object)
vm_object_paging_end(shadow_object);
}
vm_object_unlock(shadow_object);
upl_unlock(upl);
return KERN_SUCCESS;
}
kern_return_t
upl_abort_range(
upl_t upl,
vm_offset_t offset,
vm_size_t size,
int error,
boolean_t *empty)
{
vm_size_t xfer_size = size;
vm_object_t shadow_object;
vm_object_t object = upl->map_object;
vm_object_offset_t target_offset;
vm_object_offset_t page_offset;
int entry;
wpl_array_t lite_list;
int occupied;
*empty = FALSE;
if (upl == UPL_NULL)
return KERN_INVALID_ARGUMENT;
if (upl->flags & UPL_IO_WIRE) {
return upl_commit_range(upl,
offset, size, 0,
NULL, 0, empty);
}
if(object->pageout) {
shadow_object = object->shadow;
} else {
shadow_object = object;
}
upl_lock(upl);
if(upl->flags & UPL_DEVICE_MEMORY) {
xfer_size = 0;
} else if ((offset + size) > upl->size) {
upl_unlock(upl);
return KERN_FAILURE;
}
vm_object_lock(shadow_object);
if(upl->flags & UPL_INTERNAL) {
lite_list = (wpl_array_t)
((((vm_offset_t)upl) + sizeof(struct upl))
+ ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t)));
} else {
lite_list = (wpl_array_t)
(((vm_offset_t)upl) + sizeof(struct upl));
}
entry = offset/PAGE_SIZE;
target_offset = (vm_object_offset_t)offset;
while(xfer_size) {
vm_page_t t,m;
upl_page_info_t *p;
m = VM_PAGE_NULL;
if(upl->flags & UPL_LITE) {
int pg_num;
pg_num = target_offset/PAGE_SIZE;
if(lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
lite_list[pg_num>>5] &= ~(1 << (pg_num & 31));
m = vm_page_lookup(shadow_object,
target_offset + (upl->offset -
shadow_object->paging_offset));
}
}
if(object->pageout) {
if ((t = vm_page_lookup(object, target_offset))
!= NULL) {
t->pageout = FALSE;
VM_PAGE_FREE(t);
if(m == NULL) {
m = vm_page_lookup(
shadow_object,
target_offset +
object->shadow_offset);
}
if(m != VM_PAGE_NULL)
vm_object_paging_end(m->object);
}
}
if(m != VM_PAGE_NULL) {
vm_page_lock_queues();
if(m->absent) {
if(error & UPL_ABORT_RESTART) {
m->restart = TRUE;
m->absent = FALSE;
vm_object_absent_release(m->object);
m->page_error = KERN_MEMORY_ERROR;
m->error = TRUE;
} else if(error & UPL_ABORT_UNAVAILABLE) {
m->restart = FALSE;
m->unusual = TRUE;
m->clustered = FALSE;
} else if(error & UPL_ABORT_ERROR) {
m->restart = FALSE;
m->absent = FALSE;
vm_object_absent_release(m->object);
m->page_error = KERN_MEMORY_ERROR;
m->error = TRUE;
} else if(error & UPL_ABORT_DUMP_PAGES) {
m->clustered = TRUE;
} else {
m->clustered = TRUE;
}
m->cleaning = FALSE;
m->overwriting = FALSE;
PAGE_WAKEUP_DONE(m);
if(m->clustered) {
vm_page_free(m);
} else {
vm_page_activate(m);
}
vm_page_unlock_queues();
target_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
entry++;
continue;
}
if (m->laundry) {
vm_page_laundry_count--;
m->laundry = FALSE;
if (vm_page_laundry_count
< vm_page_laundry_min) {
vm_page_laundry_min = 0;
thread_wakeup((event_t)
&vm_page_laundry_count);
}
}
if(m->pageout) {
assert(m->busy);
assert(m->wire_count == 1);
m->pageout = FALSE;
vm_page_unwire(m);
}
m->dump_cleaning = FALSE;
m->cleaning = FALSE;
m->busy = FALSE;
m->overwriting = FALSE;
#if MACH_PAGEMAP
vm_external_state_clr(
m->object->existence_map, m->offset);
#endif
if(error & UPL_ABORT_DUMP_PAGES) {
vm_page_free(m);
pmap_page_protect(m->phys_page, VM_PROT_NONE);
} else {
PAGE_WAKEUP(m);
}
vm_page_unlock_queues();
}
target_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
entry++;
}
occupied = 1;
if (upl->flags & UPL_DEVICE_MEMORY) {
occupied = 0;
} else if (upl->flags & UPL_LITE) {
int pg_num;
int i;
pg_num = upl->size/PAGE_SIZE;
pg_num = (pg_num + 31) >> 5;
occupied = 0;
for(i= 0; i<pg_num; i++) {
if(lite_list[i] != 0) {
occupied = 1;
break;
}
}
} else {
if(queue_empty(&upl->map_object->memq)) {
occupied = 0;
}
}
if(occupied == 0) {
if(upl->flags & UPL_COMMIT_NOTIFY_EMPTY) {
*empty = TRUE;
}
if(object == shadow_object)
vm_object_paging_end(shadow_object);
}
vm_object_unlock(shadow_object);
upl_unlock(upl);
return KERN_SUCCESS;
}
kern_return_t
upl_abort(
upl_t upl,
int error)
{
vm_object_t object = NULL;
vm_object_t shadow_object = NULL;
vm_object_offset_t offset;
vm_object_offset_t shadow_offset;
vm_object_offset_t target_offset;
int i;
wpl_array_t lite_list;
vm_page_t t,m;
int occupied;
if (upl == UPL_NULL)
return KERN_INVALID_ARGUMENT;
if (upl->flags & UPL_IO_WIRE) {
boolean_t empty;
return upl_commit_range(upl,
0, upl->size, 0,
NULL, 0, &empty);
}
upl_lock(upl);
if(upl->flags & UPL_DEVICE_MEMORY) {
upl_unlock(upl);
return KERN_SUCCESS;
}
object = upl->map_object;
if (object == NULL) {
panic("upl_abort: upl object is not backed by an object");
upl_unlock(upl);
return KERN_INVALID_ARGUMENT;
}
if(object->pageout) {
shadow_object = object->shadow;
shadow_offset = object->shadow_offset;
} else {
shadow_object = object;
shadow_offset = upl->offset - object->paging_offset;
}
if(upl->flags & UPL_INTERNAL) {
lite_list = (wpl_array_t)
((((vm_offset_t)upl) + sizeof(struct upl))
+ ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t)));
} else {
lite_list = (wpl_array_t)
(((vm_offset_t)upl) + sizeof(struct upl));
}
offset = 0;
vm_object_lock(shadow_object);
for(i = 0; i<(upl->size); i+=PAGE_SIZE, offset += PAGE_SIZE_64) {
m = VM_PAGE_NULL;
target_offset = offset + shadow_offset;
if(upl->flags & UPL_LITE) {
int pg_num;
pg_num = offset/PAGE_SIZE;
if(lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
lite_list[pg_num>>5] &= ~(1 << (pg_num & 31));
m = vm_page_lookup(
shadow_object, target_offset);
}
}
if(object->pageout) {
if ((t = vm_page_lookup(object, offset)) != NULL) {
t->pageout = FALSE;
VM_PAGE_FREE(t);
if(m == NULL) {
m = vm_page_lookup(
shadow_object, target_offset);
}
if(m != VM_PAGE_NULL)
vm_object_paging_end(m->object);
}
}
if(m != VM_PAGE_NULL) {
vm_page_lock_queues();
if(m->absent) {
if(error & UPL_ABORT_RESTART) {
m->restart = TRUE;
m->absent = FALSE;
vm_object_absent_release(m->object);
m->page_error = KERN_MEMORY_ERROR;
m->error = TRUE;
} else if(error & UPL_ABORT_UNAVAILABLE) {
m->restart = FALSE;
m->unusual = TRUE;
m->clustered = FALSE;
} else if(error & UPL_ABORT_ERROR) {
m->restart = FALSE;
m->absent = FALSE;
vm_object_absent_release(m->object);
m->page_error = KERN_MEMORY_ERROR;
m->error = TRUE;
} else if(error & UPL_ABORT_DUMP_PAGES) {
m->clustered = TRUE;
} else {
m->clustered = TRUE;
}
m->cleaning = FALSE;
m->overwriting = FALSE;
PAGE_WAKEUP_DONE(m);
if(m->clustered) {
vm_page_free(m);
} else {
vm_page_activate(m);
}
vm_page_unlock_queues();
continue;
}
if (m->laundry) {
vm_page_laundry_count--;
m->laundry = FALSE;
if (vm_page_laundry_count
< vm_page_laundry_min) {
vm_page_laundry_min = 0;
thread_wakeup((event_t)
&vm_page_laundry_count);
}
}
if(m->pageout) {
assert(m->busy);
assert(m->wire_count == 1);
m->pageout = FALSE;
vm_page_unwire(m);
}
m->dump_cleaning = FALSE;
m->cleaning = FALSE;
m->busy = FALSE;
m->overwriting = FALSE;
#if MACH_PAGEMAP
vm_external_state_clr(
m->object->existence_map, m->offset);
#endif
if(error & UPL_ABORT_DUMP_PAGES) {
vm_page_free(m);
pmap_page_protect(m->phys_page, VM_PROT_NONE);
} else {
PAGE_WAKEUP(m);
}
vm_page_unlock_queues();
}
}
occupied = 1;
if (upl->flags & UPL_DEVICE_MEMORY) {
occupied = 0;
} else if (upl->flags & UPL_LITE) {
int pg_num;
int i;
pg_num = upl->size/PAGE_SIZE;
pg_num = (pg_num + 31) >> 5;
occupied = 0;
for(i= 0; i<pg_num; i++) {
if(lite_list[i] != 0) {
occupied = 1;
break;
}
}
} else {
if(queue_empty(&upl->map_object->memq)) {
occupied = 0;
}
}
if(occupied == 0) {
if(object == shadow_object)
vm_object_paging_end(shadow_object);
}
vm_object_unlock(shadow_object);
upl_unlock(upl);
return KERN_SUCCESS;
}
kern_return_t
upl_commit(
upl_t upl,
upl_page_info_t *page_list,
mach_msg_type_number_t count)
{
if (upl == UPL_NULL)
return KERN_INVALID_ARGUMENT;
if(upl->flags & (UPL_LITE | UPL_IO_WIRE)) {
boolean_t empty;
return upl_commit_range(upl, 0, upl->size, 0,
page_list, count, &empty);
}
if (count == 0)
page_list = NULL;
upl_lock(upl);
if (upl->flags & UPL_DEVICE_MEMORY)
page_list = NULL;
if ((upl->flags & UPL_CLEAR_DIRTY) ||
(upl->flags & UPL_PAGE_SYNC_DONE) || page_list) {
vm_object_t shadow_object = upl->map_object->shadow;
vm_object_t object = upl->map_object;
vm_object_offset_t target_offset;
vm_size_t xfer_end;
int entry;
vm_page_t t, m;
upl_page_info_t *p;
vm_object_lock(shadow_object);
entry = 0;
target_offset = object->shadow_offset;
xfer_end = upl->size + object->shadow_offset;
while(target_offset < xfer_end) {
if ((t = vm_page_lookup(object,
target_offset - object->shadow_offset))
== NULL) {
target_offset += PAGE_SIZE_64;
entry++;
continue;
}
m = vm_page_lookup(shadow_object, target_offset);
if(m != VM_PAGE_NULL) {
if (upl->flags & UPL_CLEAR_DIRTY) {
pmap_clear_modify(m->phys_page);
m->dirty = FALSE;
}
if (upl->flags & UPL_PAGE_SYNC_DONE)
m->precious = FALSE;
if(page_list) {
p = &(page_list[entry]);
if(page_list[entry].phys_addr &&
p->pageout && !m->pageout) {
vm_page_lock_queues();
m->busy = TRUE;
m->pageout = TRUE;
vm_page_wire(m);
vm_page_unlock_queues();
} else if (page_list[entry].phys_addr &&
!p->pageout && m->pageout &&
!m->dump_cleaning) {
vm_page_lock_queues();
m->pageout = FALSE;
m->absent = FALSE;
m->overwriting = FALSE;
vm_page_unwire(m);
PAGE_WAKEUP_DONE(m);
vm_page_unlock_queues();
}
page_list[entry].phys_addr = 0;
}
}
target_offset += PAGE_SIZE_64;
entry++;
}
vm_object_unlock(shadow_object);
}
if (upl->flags & UPL_DEVICE_MEMORY) {
vm_object_lock(upl->map_object->shadow);
if(upl->map_object == upl->map_object->shadow)
vm_object_paging_end(upl->map_object->shadow);
vm_object_unlock(upl->map_object->shadow);
}
upl_unlock(upl);
return KERN_SUCCESS;
}
kern_return_t
vm_object_iopl_request(
vm_object_t object,
vm_object_offset_t offset,
vm_size_t size,
upl_t *upl_ptr,
upl_page_info_array_t user_page_list,
unsigned int *page_list_count,
int cntrl_flags)
{
vm_page_t dst_page;
vm_object_offset_t dst_offset = offset;
vm_size_t xfer_size = size;
upl_t upl = NULL;
int entry;
wpl_array_t lite_list;
int page_field_size;
vm_page_t alias_page = NULL;
kern_return_t ret;
vm_prot_t prot;
if(cntrl_flags & UPL_COPYOUT_FROM) {
prot = VM_PROT_READ;
} else {
prot = VM_PROT_READ | VM_PROT_WRITE;
}
if(((size/page_size) > MAX_UPL_TRANSFER) && !object->phys_contiguous) {
size = MAX_UPL_TRANSFER * page_size;
}
if(cntrl_flags & UPL_SET_INTERNAL)
if(page_list_count != NULL)
*page_list_count = MAX_UPL_TRANSFER;
if(((cntrl_flags & UPL_SET_INTERNAL) && !(object->phys_contiguous)) &&
((page_list_count != NULL) && (*page_list_count != 0)
&& *page_list_count < (size/page_size)))
return KERN_INVALID_ARGUMENT;
if((!object->internal) && (object->paging_offset != 0))
panic("vm_object_upl_request: vnode object with non-zero paging offset\n");
if(object->phys_contiguous) {
cntrl_flags |= UPL_SET_LITE;
}
if(upl_ptr) {
if(cntrl_flags & UPL_SET_INTERNAL) {
if(cntrl_flags & UPL_SET_LITE) {
upl = upl_create(
UPL_CREATE_INTERNAL | UPL_CREATE_LITE,
size);
user_page_list = (upl_page_info_t *)
(((vm_offset_t)upl) + sizeof(struct upl));
lite_list = (wpl_array_t)
(((vm_offset_t)user_page_list) +
((size/PAGE_SIZE) *
sizeof(upl_page_info_t)));
page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
page_field_size =
(page_field_size + 3) & 0xFFFFFFFC;
bzero((char *)lite_list, page_field_size);
upl->flags =
UPL_LITE | UPL_INTERNAL | UPL_IO_WIRE;
} else {
upl = upl_create(UPL_CREATE_INTERNAL, size);
user_page_list = (upl_page_info_t *)
(((vm_offset_t)upl)
+ sizeof(struct upl));
upl->flags = UPL_INTERNAL | UPL_IO_WIRE;
}
} else {
if(cntrl_flags & UPL_SET_LITE) {
upl = upl_create(UPL_CREATE_LITE, size);
lite_list = (wpl_array_t)
(((vm_offset_t)upl) + sizeof(struct upl));
page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
page_field_size =
(page_field_size + 3) & 0xFFFFFFFC;
bzero((char *)lite_list, page_field_size);
upl->flags = UPL_LITE | UPL_IO_WIRE;
} else {
upl = upl_create(UPL_CREATE_EXTERNAL, size);
upl->flags = UPL_IO_WIRE;
}
}
if(object->phys_contiguous) {
upl->size = size;
upl->offset = offset + object->paging_offset;
*upl_ptr = upl;
if(user_page_list) {
user_page_list[0].phys_addr =
(offset + object->shadow_offset)>>12;
user_page_list[0].device = TRUE;
}
upl->map_object = object;
upl->flags |= UPL_DEVICE_MEMORY;
vm_object_lock(object);
vm_object_paging_begin(object);
vm_object_unlock(object);
if(page_list_count != NULL) {
if (upl->flags & UPL_INTERNAL) {
*page_list_count = 0;
} else {
*page_list_count = 1;
}
}
return KERN_SUCCESS;
}
if(cntrl_flags & UPL_SET_LITE) {
upl->map_object = object;
} else {
upl->map_object = vm_object_allocate(size);
vm_object_lock(upl->map_object);
upl->map_object->shadow = object;
upl->map_object->pageout = TRUE;
upl->map_object->can_persist = FALSE;
upl->map_object->copy_strategy =
MEMORY_OBJECT_COPY_NONE;
upl->map_object->shadow_offset = offset;
upl->map_object->wimg_bits = object->wimg_bits;
vm_object_unlock(upl->map_object);
}
upl->size = size;
upl->offset = offset + object->paging_offset;
*upl_ptr = upl;
}
vm_object_lock(object);
if (!object->phys_contiguous) {
object->true_share = TRUE;
if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
}
#ifdef UBC_DEBUG
if(upl_ptr)
queue_enter(&object->uplq, upl, upl_t, uplq);
#endif
vm_object_paging_begin(object);
entry = 0;
while (xfer_size) {
if((alias_page == NULL) && !(cntrl_flags & UPL_SET_LITE)) {
vm_object_unlock(object);
VM_PAGE_GRAB_FICTITIOUS(alias_page);
vm_object_lock(object);
}
dst_page = vm_page_lookup(object, dst_offset);
if ((dst_page == VM_PAGE_NULL) || (dst_page->busy) ||
(dst_page->unusual && (dst_page->error ||
dst_page->restart || dst_page->absent ||
dst_page->fictitious ||
prot & dst_page->page_lock))) {
vm_fault_return_t result;
do {
vm_page_t top_page;
kern_return_t error_code;
int interruptible;
vm_object_offset_t lo_offset = offset;
vm_object_offset_t hi_offset = offset + size;
if(cntrl_flags & UPL_SET_INTERRUPTIBLE) {
interruptible = THREAD_ABORTSAFE;
} else {
interruptible = THREAD_UNINT;
}
result = vm_fault_page(object, dst_offset,
prot | VM_PROT_WRITE, FALSE,
interruptible,
lo_offset, hi_offset,
VM_BEHAVIOR_SEQUENTIAL,
&prot, &dst_page, &top_page,
(int *)0,
&error_code, FALSE, FALSE, NULL, 0);
switch(result) {
case VM_FAULT_SUCCESS:
PAGE_WAKEUP_DONE(dst_page);
if(top_page != VM_PAGE_NULL) {
vm_object_t local_object;
local_object =
top_page->object;
if(top_page->object
!= dst_page->object) {
vm_object_lock(
local_object);
VM_PAGE_FREE(top_page);
vm_object_paging_end(
local_object);
vm_object_unlock(
local_object);
} else {
VM_PAGE_FREE(top_page);
vm_object_paging_end(
local_object);
}
}
break;
case VM_FAULT_RETRY:
vm_object_lock(object);
vm_object_paging_begin(object);
break;
case VM_FAULT_FICTITIOUS_SHORTAGE:
vm_page_more_fictitious();
vm_object_lock(object);
vm_object_paging_begin(object);
break;
case VM_FAULT_MEMORY_SHORTAGE:
if (vm_page_wait(interruptible)) {
vm_object_lock(object);
vm_object_paging_begin(object);
break;
}
case VM_FAULT_INTERRUPTED:
error_code = MACH_SEND_INTERRUPTED;
case VM_FAULT_MEMORY_ERROR:
ret = (error_code ? error_code:
KERN_MEMORY_ERROR);
vm_object_lock(object);
for(; offset < dst_offset;
offset += PAGE_SIZE) {
dst_page = vm_page_lookup(
object, offset);
if(dst_page == VM_PAGE_NULL)
panic("vm_object_iopl_request: Wired pages missing. \n");
vm_page_lock_queues();
vm_page_unwire(dst_page);
vm_page_unlock_queues();
VM_STAT(reactivations++);
}
vm_object_unlock(object);
upl_destroy(upl);
return ret;
}
} while ((result != VM_FAULT_SUCCESS)
|| (result == VM_FAULT_INTERRUPTED));
}
vm_page_lock_queues();
vm_page_wire(dst_page);
vm_page_unlock_queues();
if(upl_ptr) {
vm_page_lock_queues();
if(cntrl_flags & UPL_SET_LITE) {
int pg_num;
pg_num = (dst_offset-offset)/PAGE_SIZE;
lite_list[pg_num>>5] |= 1 << (pg_num & 31);
} else {
assert(alias_page->fictitious);
alias_page->fictitious = FALSE;
alias_page->private = TRUE;
alias_page->pageout = TRUE;
alias_page->phys_page = dst_page->phys_page;
vm_page_wire(alias_page);
vm_page_insert(alias_page,
upl->map_object, size - xfer_size);
assert(!alias_page->wanted);
alias_page->busy = FALSE;
alias_page->absent = FALSE;
}
dst_page->reference = TRUE;
if (!(cntrl_flags & UPL_COPYOUT_FROM))
dst_page->dirty = TRUE;
alias_page = NULL;
if(user_page_list) {
user_page_list[entry].phys_addr
= dst_page->phys_page;
user_page_list[entry].dirty =
dst_page->dirty;
user_page_list[entry].pageout =
dst_page->pageout;
user_page_list[entry].absent =
dst_page->absent;
user_page_list[entry].precious =
dst_page->precious;
}
vm_page_unlock_queues();
}
entry++;
dst_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
}
if (upl->flags & UPL_INTERNAL) {
if(page_list_count != NULL)
*page_list_count = 0;
} else if (*page_list_count > entry) {
if(page_list_count != NULL)
*page_list_count = entry;
}
if(alias_page != NULL) {
vm_page_lock_queues();
vm_page_free(alias_page);
vm_page_unlock_queues();
}
vm_object_unlock(object);
return KERN_SUCCESS;
}
vm_size_t
upl_get_internal_pagelist_offset()
{
return sizeof(struct upl);
}
void
upl_set_dirty(
upl_t upl)
{
upl->flags |= UPL_CLEAR_DIRTY;
}
void
upl_clear_dirty(
upl_t upl)
{
upl->flags &= ~UPL_CLEAR_DIRTY;
}
#ifdef MACH_BSD
boolean_t upl_page_present(upl_page_info_t *upl, int index)
{
return(UPL_PAGE_PRESENT(upl, index));
}
boolean_t upl_dirty_page(upl_page_info_t *upl, int index)
{
return(UPL_DIRTY_PAGE(upl, index));
}
boolean_t upl_valid_page(upl_page_info_t *upl, int index)
{
return(UPL_VALID_PAGE(upl, index));
}
vm_offset_t upl_phys_page(upl_page_info_t *upl, int index)
{
return((vm_offset_t)UPL_PHYS_PAGE(upl, index));
}
void
vm_countdirtypages(void)
{
vm_page_t m;
int dpages;
int pgopages;
int precpages;
dpages=0;
pgopages=0;
precpages=0;
vm_page_lock_queues();
m = (vm_page_t) queue_first(&vm_page_queue_inactive);
do {
if (m ==(vm_page_t )0) break;
if(m->dirty) dpages++;
if(m->pageout) pgopages++;
if(m->precious) precpages++;
m = (vm_page_t) queue_next(&m->pageq);
if (m ==(vm_page_t )0) break;
} while (!queue_end(&vm_page_queue_inactive,(queue_entry_t) m));
vm_page_unlock_queues();
vm_page_lock_queues();
m = (vm_page_t) queue_first(&vm_page_queue_zf);
do {
if (m ==(vm_page_t )0) break;
if(m->dirty) dpages++;
if(m->pageout) pgopages++;
if(m->precious) precpages++;
m = (vm_page_t) queue_next(&m->pageq);
if (m ==(vm_page_t )0) break;
} while (!queue_end(&vm_page_queue_zf,(queue_entry_t) m));
vm_page_unlock_queues();
printf("IN Q: %d : %d : %d\n", dpages, pgopages, precpages);
dpages=0;
pgopages=0;
precpages=0;
vm_page_lock_queues();
m = (vm_page_t) queue_first(&vm_page_queue_active);
do {
if(m == (vm_page_t )0) break;
if(m->dirty) dpages++;
if(m->pageout) pgopages++;
if(m->precious) precpages++;
m = (vm_page_t) queue_next(&m->pageq);
if(m == (vm_page_t )0) break;
} while (!queue_end(&vm_page_queue_active,(queue_entry_t) m));
vm_page_unlock_queues();
printf("AC Q: %d : %d : %d\n", dpages, pgopages, precpages);
}
#endif
#ifdef UBC_DEBUG
kern_return_t upl_ubc_alias_set(upl_t upl, unsigned int alias1, unsigned int alias2)
{
upl->ubc_alias1 = alias1;
upl->ubc_alias2 = alias2;
return KERN_SUCCESS;
}
int upl_ubc_alias_get(upl_t upl, unsigned int * al, unsigned int * al2)
{
if(al)
*al = upl->ubc_alias1;
if(al2)
*al2 = upl->ubc_alias2;
return KERN_SUCCESS;
}
#endif
#if MACH_KDB
#include <ddb/db_output.h>
#include <ddb/db_print.h>
#include <vm/vm_print.h>
#define printf kdbprintf
extern int db_indent;
void db_pageout(void);
void
db_vm(void)
{
extern int vm_page_gobble_count;
iprintf("VM Statistics:\n");
db_indent += 2;
iprintf("pages:\n");
db_indent += 2;
iprintf("activ %5d inact %5d free %5d",
vm_page_active_count, vm_page_inactive_count,
vm_page_free_count);
printf(" wire %5d gobbl %5d\n",
vm_page_wire_count, vm_page_gobble_count);
iprintf("laund %5d\n",
vm_page_laundry_count);
db_indent -= 2;
iprintf("target:\n");
db_indent += 2;
iprintf("min %5d inact %5d free %5d",
vm_page_free_min, vm_page_inactive_target,
vm_page_free_target);
printf(" resrv %5d\n", vm_page_free_reserved);
db_indent -= 2;
iprintf("burst:\n");
db_indent += 2;
iprintf("max %5d min %5d wait %5d empty %5d\n",
vm_pageout_burst_max, vm_pageout_burst_min,
vm_pageout_burst_wait, vm_pageout_empty_wait);
db_indent -= 2;
iprintf("pause:\n");
db_indent += 2;
iprintf("count %5d max %5d\n",
vm_pageout_pause_count, vm_pageout_pause_max);
#if MACH_COUNTERS
iprintf("scan_continue called %8d\n", c_vm_pageout_scan_continue);
#endif
db_indent -= 2;
db_pageout();
db_indent -= 2;
}
void
db_pageout(void)
{
#if MACH_COUNTERS
extern int c_laundry_pages_freed;
#endif
iprintf("Pageout Statistics:\n");
db_indent += 2;
iprintf("active %5d inactv %5d\n",
vm_pageout_active, vm_pageout_inactive);
iprintf("nolock %5d avoid %5d busy %5d absent %5d\n",
vm_pageout_inactive_nolock, vm_pageout_inactive_avoid,
vm_pageout_inactive_busy, vm_pageout_inactive_absent);
iprintf("used %5d clean %5d dirty %5d\n",
vm_pageout_inactive_used, vm_pageout_inactive_clean,
vm_pageout_inactive_dirty);
#if MACH_COUNTERS
iprintf("laundry_pages_freed %d\n", c_laundry_pages_freed);
#endif
#if MACH_CLUSTER_STATS
iprintf("Cluster Statistics:\n");
db_indent += 2;
iprintf("dirtied %5d cleaned %5d collisions %5d\n",
vm_pageout_cluster_dirtied, vm_pageout_cluster_cleaned,
vm_pageout_cluster_collisions);
iprintf("clusters %5d conversions %5d\n",
vm_pageout_cluster_clusters, vm_pageout_cluster_conversions);
db_indent -= 2;
iprintf("Target Statistics:\n");
db_indent += 2;
iprintf("collisions %5d page_dirtied %5d page_freed %5d\n",
vm_pageout_target_collisions, vm_pageout_target_page_dirtied,
vm_pageout_target_page_freed);
db_indent -= 2;
#endif
db_indent -= 2;
}
#if MACH_CLUSTER_STATS
unsigned long vm_pageout_cluster_dirtied = 0;
unsigned long vm_pageout_cluster_cleaned = 0;
unsigned long vm_pageout_cluster_collisions = 0;
unsigned long vm_pageout_cluster_clusters = 0;
unsigned long vm_pageout_cluster_conversions = 0;
unsigned long vm_pageout_target_collisions = 0;
unsigned long vm_pageout_target_page_dirtied = 0;
unsigned long vm_pageout_target_page_freed = 0;
#define CLUSTER_STAT(clause) clause
#else
#define CLUSTER_STAT(clause)
#endif
#endif