#include <debug.h>
#include <mach_pagemap.h>
#include <task_swapper.h>
#include <mach/mach_types.h>
#include <mach/memory_object.h>
#include <mach/memory_object_default.h>
#include <mach/memory_object_control_server.h>
#include <mach/vm_param.h>
#include <ipc/ipc_types.h>
#include <ipc/ipc_port.h>
#include <kern/kern_types.h>
#include <kern/assert.h>
#include <kern/lock.h>
#include <kern/queue.h>
#include <kern/xpr.h>
#include <kern/zalloc.h>
#include <kern/host.h>
#include <kern/host_statistics.h>
#include <kern/processor.h>
#include <kern/misc_protos.h>
#include <vm/memory_object.h>
#include <vm/vm_fault.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_pageout.h>
#include <vm/vm_protos.h>
#include <vm/vm_purgeable_internal.h>
static kern_return_t vm_object_terminate(
vm_object_t object);
extern void vm_object_remove(
vm_object_t object);
static vm_object_t vm_object_cache_trim(
boolean_t called_from_vm_object_deallocate);
static void vm_object_deactivate_all_pages(
vm_object_t object);
static kern_return_t vm_object_copy_call(
vm_object_t src_object,
vm_object_offset_t src_offset,
vm_object_size_t size,
vm_object_t *_result_object);
static void vm_object_do_collapse(
vm_object_t object,
vm_object_t backing_object);
static void vm_object_do_bypass(
vm_object_t object,
vm_object_t backing_object);
static void vm_object_release_pager(
memory_object_t pager);
static zone_t vm_object_zone;
static struct vm_object kernel_object_store;
vm_object_t kernel_object;
static struct vm_object vm_submap_object_store;
static struct vm_object vm_object_template;
static queue_head_t vm_object_cached_list;
static int vm_object_cached_count=0;
static int vm_object_cached_high;
static int vm_object_cached_max = 512;
static decl_mutex_data(,vm_object_cached_lock_data)
#define vm_object_cache_lock() \
mutex_lock(&vm_object_cached_lock_data)
#define vm_object_cache_lock_try() \
mutex_try(&vm_object_cached_lock_data)
#define vm_object_cache_unlock() \
mutex_unlock(&vm_object_cached_lock_data)
#define VM_OBJECT_HASH_COUNT 1024
static queue_head_t vm_object_hashtable[VM_OBJECT_HASH_COUNT];
static struct zone *vm_object_hash_zone;
struct vm_object_hash_entry {
queue_chain_t hash_link;
memory_object_t pager;
vm_object_t object;
boolean_t waiting;
};
typedef struct vm_object_hash_entry *vm_object_hash_entry_t;
#define VM_OBJECT_HASH_ENTRY_NULL ((vm_object_hash_entry_t) 0)
#define VM_OBJECT_HASH_SHIFT 8
#define vm_object_hash(pager) \
((((unsigned)pager) >> VM_OBJECT_HASH_SHIFT) % VM_OBJECT_HASH_COUNT)
void vm_object_hash_entry_free(
vm_object_hash_entry_t entry);
static void vm_object_reap(vm_object_t object);
static void vm_object_reap_async(vm_object_t object);
static void vm_object_reaper_thread(void);
static queue_head_t vm_object_reaper_queue;
unsigned int vm_object_reap_count = 0;
unsigned int vm_object_reap_count_async = 0;
static vm_object_hash_entry_t
vm_object_hash_lookup(
memory_object_t pager,
boolean_t remove_entry)
{
register queue_t bucket;
register vm_object_hash_entry_t entry;
bucket = &vm_object_hashtable[vm_object_hash(pager)];
entry = (vm_object_hash_entry_t)queue_first(bucket);
while (!queue_end(bucket, (queue_entry_t)entry)) {
if (entry->pager == pager && !remove_entry)
return(entry);
else if (entry->pager == pager) {
queue_remove(bucket, entry,
vm_object_hash_entry_t, hash_link);
return(entry);
}
entry = (vm_object_hash_entry_t)queue_next(&entry->hash_link);
}
return(VM_OBJECT_HASH_ENTRY_NULL);
}
static void
vm_object_hash_insert(
vm_object_hash_entry_t entry)
{
register queue_t bucket;
bucket = &vm_object_hashtable[vm_object_hash(entry->pager)];
queue_enter(bucket, entry, vm_object_hash_entry_t, hash_link);
}
static vm_object_hash_entry_t
vm_object_hash_entry_alloc(
memory_object_t pager)
{
vm_object_hash_entry_t entry;
entry = (vm_object_hash_entry_t)zalloc(vm_object_hash_zone);
entry->pager = pager;
entry->object = VM_OBJECT_NULL;
entry->waiting = FALSE;
return(entry);
}
void
vm_object_hash_entry_free(
vm_object_hash_entry_t entry)
{
zfree(vm_object_hash_zone, entry);
}
__private_extern__ void
_vm_object_allocate(
vm_object_size_t size,
vm_object_t object)
{
XPR(XPR_VM_OBJECT,
"vm_object_allocate, object 0x%X size 0x%X\n",
(integer_t)object, size, 0,0,0);
*object = vm_object_template;
queue_init(&object->memq);
queue_init(&object->msr_q);
#ifdef UPL_DEBUG
queue_init(&object->uplq);
#endif
vm_object_lock_init(object);
object->size = size;
}
__private_extern__ vm_object_t
vm_object_allocate(
vm_object_size_t size)
{
register vm_object_t object;
object = (vm_object_t) zalloc(vm_object_zone);
if (object != VM_OBJECT_NULL)
_vm_object_allocate(size, object);
return object;
}
lck_grp_t vm_object_lck_grp;
lck_grp_attr_t vm_object_lck_grp_attr;
lck_attr_t vm_object_lck_attr;
lck_attr_t kernel_object_lck_attr;
__private_extern__ void
vm_object_bootstrap(void)
{
register int i;
vm_object_zone = zinit((vm_size_t) sizeof(struct vm_object),
round_page_32(512*1024),
round_page_32(12*1024),
"vm objects");
queue_init(&vm_object_reaper_queue);
queue_init(&vm_object_cached_list);
mutex_init(&vm_object_cached_lock_data, 0);
vm_object_hash_zone =
zinit((vm_size_t) sizeof (struct vm_object_hash_entry),
round_page_32(512*1024),
round_page_32(12*1024),
"vm object hash entries");
for (i = 0; i < VM_OBJECT_HASH_COUNT; i++)
queue_init(&vm_object_hashtable[i]);
vm_object_init_lck_grp();
vm_object_template.memq.prev = NULL;
vm_object_template.memq.next = NULL;
#if 0
vm_object_lock_init(&vm_object_template);
#endif
vm_object_template.size = 0;
vm_object_template.memq_hint = VM_PAGE_NULL;
vm_object_template.ref_count = 1;
#if TASK_SWAPPER
vm_object_template.res_count = 1;
#endif
vm_object_template.resident_page_count = 0;
vm_object_template.copy = VM_OBJECT_NULL;
vm_object_template.shadow = VM_OBJECT_NULL;
vm_object_template.shadow_offset = (vm_object_offset_t) 0;
vm_object_template.pager = MEMORY_OBJECT_NULL;
vm_object_template.paging_offset = 0;
vm_object_template.pager_control = MEMORY_OBJECT_CONTROL_NULL;
vm_object_template.copy_strategy = MEMORY_OBJECT_COPY_SYMMETRIC;
vm_object_template.paging_in_progress = 0;
vm_object_template.all_wanted = 0;
vm_object_template.pager_created = FALSE;
vm_object_template.pager_initialized = FALSE;
vm_object_template.pager_ready = FALSE;
vm_object_template.pager_trusted = FALSE;
vm_object_template.can_persist = FALSE;
vm_object_template.internal = TRUE;
vm_object_template.temporary = TRUE;
vm_object_template.private = FALSE;
vm_object_template.pageout = FALSE;
vm_object_template.alive = TRUE;
vm_object_template.purgable = VM_PURGABLE_DENY;
vm_object_template.shadowed = FALSE;
vm_object_template.silent_overwrite = FALSE;
vm_object_template.advisory_pageout = FALSE;
vm_object_template.true_share = FALSE;
vm_object_template.terminating = FALSE;
vm_object_template.named = FALSE;
vm_object_template.shadow_severed = FALSE;
vm_object_template.phys_contiguous = FALSE;
vm_object_template.nophyscache = FALSE;
vm_object_template.cached_list.prev = NULL;
vm_object_template.cached_list.next = NULL;
vm_object_template.msr_q.prev = NULL;
vm_object_template.msr_q.next = NULL;
vm_object_template.last_alloc = (vm_object_offset_t) 0;
vm_object_template.sequential = (vm_object_offset_t) 0;
vm_object_template.pages_created = 0;
vm_object_template.pages_used = 0;
#if MACH_PAGEMAP
vm_object_template.existence_map = VM_EXTERNAL_NULL;
#endif
vm_object_template.cow_hint = ~(vm_offset_t)0;
#if MACH_ASSERT
vm_object_template.paging_object = VM_OBJECT_NULL;
#endif
vm_object_template.wimg_bits = VM_WIMG_DEFAULT;
vm_object_template.code_signed = FALSE;
vm_object_template.not_in_use = 0;
#ifdef UPL_DEBUG
vm_object_template.uplq.prev = NULL;
vm_object_template.uplq.next = NULL;
#endif
#ifdef VM_PIP_DEBUG
bzero(&vm_object_template.pip_holders,
sizeof (vm_object_template.pip_holders));
#endif
vm_object_template.objq.next=NULL;
vm_object_template.objq.prev=NULL;
kernel_object = &kernel_object_store;
#ifdef ppc
_vm_object_allocate((vm_last_addr - VM_MIN_KERNEL_ADDRESS) + 1,
kernel_object);
#else
_vm_object_allocate((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) + 1,
kernel_object);
#endif
kernel_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
vm_submap_object = &vm_submap_object_store;
#ifdef ppc
_vm_object_allocate((vm_last_addr - VM_MIN_KERNEL_ADDRESS) + 1,
vm_submap_object);
#else
_vm_object_allocate((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) + 1,
vm_submap_object);
#endif
vm_submap_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
vm_object_reference(vm_submap_object);
#if MACH_PAGEMAP
vm_external_module_initialize();
#endif
}
void
vm_object_reaper_init(void)
{
kern_return_t kr;
thread_t thread;
kr = kernel_thread_start_priority(
(thread_continue_t) vm_object_reaper_thread,
NULL,
BASEPRI_PREEMPT - 1,
&thread);
if (kr != KERN_SUCCESS) {
panic("failed to launch vm_object_reaper_thread kr=0x%x", kr);
}
thread_deallocate(thread);
}
__private_extern__ void
vm_object_init(void)
{
}
__private_extern__ void
vm_object_init_lck_grp(void)
{
lck_grp_attr_setdefault(&vm_object_lck_grp_attr);
lck_grp_init(&vm_object_lck_grp, "vm_object", &vm_object_lck_grp_attr);
lck_attr_setdefault(&vm_object_lck_attr);
lck_attr_setdefault(&kernel_object_lck_attr);
lck_attr_cleardebug(&kernel_object_lck_attr);
}
#define MIGHT_NOT_CACHE_SHADOWS 1
#if MIGHT_NOT_CACHE_SHADOWS
static int cache_shadows = TRUE;
#endif
unsigned long vm_object_deallocate_shared_successes = 0;
unsigned long vm_object_deallocate_shared_failures = 0;
unsigned long vm_object_deallocate_shared_swap_failures = 0;
__private_extern__ void
vm_object_deallocate(
register vm_object_t object)
{
boolean_t retry_cache_trim = FALSE;
vm_object_t shadow = VM_OBJECT_NULL;
uint32_t try_failed_count = 0;
if (object == VM_OBJECT_NULL)
return;
if (object == kernel_object) {
vm_object_lock(kernel_object);
kernel_object->ref_count--;
if (kernel_object->ref_count == 0) {
panic("vm_object_deallocate: losing kernel_object\n");
}
vm_object_unlock(kernel_object);
return;
}
if (object->ref_count > 2 ||
(!object->named && object->ref_count > 1)) {
UInt32 original_ref_count;
volatile UInt32 *ref_count_p;
Boolean atomic_swap;
vm_object_lock_shared(object);
ref_count_p = (volatile UInt32 *) &object->ref_count;
original_ref_count = object->ref_count;
if (original_ref_count > 2 ||
(!object->named && original_ref_count > 1)) {
atomic_swap = OSCompareAndSwap(
original_ref_count,
original_ref_count - 1,
(UInt32 *) &object->ref_count);
if (atomic_swap == FALSE) {
vm_object_deallocate_shared_swap_failures++;
}
} else {
atomic_swap = FALSE;
}
vm_object_unlock(object);
if (atomic_swap) {
vm_object_deallocate_shared_successes++;
return;
}
vm_object_deallocate_shared_failures++;
}
while (object != VM_OBJECT_NULL) {
for (;;) {
vm_object_cache_lock();
if (vm_object_lock_try(object))
break;
vm_object_cache_unlock();
try_failed_count++;
mutex_pause(try_failed_count);
}
assert(object->ref_count > 0);
if ((object->ref_count == 2) && (object->named)) {
memory_object_t pager = object->pager;
if (pager != MEMORY_OBJECT_NULL) {
vm_object_unlock(object);
vm_object_cache_unlock();
memory_object_unmap(pager);
try_failed_count = 0;
for (;;) {
vm_object_cache_lock();
if (vm_object_lock_try(object))
break;
vm_object_cache_unlock();
try_failed_count++;
mutex_pause(try_failed_count);
}
assert(object->ref_count > 0);
}
}
if ((object->ref_count > 1) || object->terminating) {
vm_object_lock_assert_exclusive(object);
object->ref_count--;
vm_object_res_deallocate(object);
vm_object_cache_unlock();
if (object->ref_count == 1 &&
object->shadow != VM_OBJECT_NULL) {
vm_object_collapse(object, 0, FALSE);
}
vm_object_unlock(object);
if (retry_cache_trim &&
((object = vm_object_cache_trim(TRUE)) !=
VM_OBJECT_NULL)) {
continue;
}
return;
}
if (object->pager_created && ! object->pager_initialized) {
assert(! object->can_persist);
vm_object_assert_wait(object,
VM_OBJECT_EVENT_INITIALIZED,
THREAD_UNINT);
vm_object_unlock(object);
vm_object_cache_unlock();
thread_block(THREAD_CONTINUE_NULL);
continue;
}
if ((object->can_persist) && (object->alive)) {
vm_object_lock_assert_exclusive(object);
if (--object->ref_count > 0) {
vm_object_res_deallocate(object);
vm_object_unlock(object);
vm_object_cache_unlock();
if (retry_cache_trim &&
((object = vm_object_cache_trim(TRUE)) !=
VM_OBJECT_NULL)) {
continue;
}
return;
}
#if MIGHT_NOT_CACHE_SHADOWS
if (! cache_shadows) {
shadow = object->shadow;
object->shadow = VM_OBJECT_NULL;
}
#endif
assert(object->shadow == VM_OBJECT_NULL);
VM_OBJ_RES_DECR(object);
XPR(XPR_VM_OBJECT,
"vm_o_deallocate: adding %x to cache, queue = (%x, %x)\n",
(integer_t)object,
(integer_t)vm_object_cached_list.next,
(integer_t)vm_object_cached_list.prev,0,0);
vm_object_cached_count++;
if (vm_object_cached_count > vm_object_cached_high)
vm_object_cached_high = vm_object_cached_count;
queue_enter(&vm_object_cached_list, object,
vm_object_t, cached_list);
vm_object_cache_unlock();
vm_object_deactivate_all_pages(object);
vm_object_unlock(object);
#if MIGHT_NOT_CACHE_SHADOWS
if (! cache_shadows && shadow != VM_OBJECT_NULL) {
object = shadow;
retry_cache_trim = TRUE;
continue;
}
#endif
object = vm_object_cache_trim(TRUE);
if (object == VM_OBJECT_NULL) {
return;
}
retry_cache_trim = TRUE;
} else {
XPR(XPR_VM_OBJECT,
"vm_o_deallocate: !cacheable 0x%X res %d paging_ops %d thread 0x%p ref %d\n",
(integer_t)object, object->resident_page_count,
object->paging_in_progress,
(void *)current_thread(),object->ref_count);
VM_OBJ_RES_DECR(object);
shadow = object->pageout?VM_OBJECT_NULL:object->shadow;
if(vm_object_terminate(object) != KERN_SUCCESS) {
return;
}
if (shadow != VM_OBJECT_NULL) {
object = shadow;
continue;
}
if (retry_cache_trim &&
((object = vm_object_cache_trim(TRUE)) !=
VM_OBJECT_NULL)) {
continue;
}
return;
}
}
assert(! retry_cache_trim);
}
vm_object_t
vm_object_cache_trim(
boolean_t called_from_vm_object_deallocate)
{
register vm_object_t object = VM_OBJECT_NULL;
vm_object_t shadow;
for (;;) {
vm_object_cache_lock();
if (vm_object_cached_count <= vm_object_cached_max) {
vm_object_cache_unlock();
return VM_OBJECT_NULL;
}
XPR(XPR_VM_OBJECT,
"vm_object_cache_trim: removing from front of cache (%x, %x)\n",
(integer_t)vm_object_cached_list.next,
(integer_t)vm_object_cached_list.prev, 0, 0, 0);
object = (vm_object_t) queue_first(&vm_object_cached_list);
if(object == (vm_object_t) &vm_object_cached_list) {
if(vm_object_cached_max < 0)
vm_object_cached_max = 0;
vm_object_cached_count = 0;
vm_object_cache_unlock();
return VM_OBJECT_NULL;
}
vm_object_lock(object);
queue_remove(&vm_object_cached_list, object, vm_object_t,
cached_list);
vm_object_cached_count--;
assert(object->pager_initialized);
assert(object->ref_count == 0);
vm_object_lock_assert_exclusive(object);
object->ref_count++;
shadow = object->pageout?VM_OBJECT_NULL:object->shadow;
if(vm_object_terminate(object) != KERN_SUCCESS)
continue;
if (shadow != VM_OBJECT_NULL) {
if (called_from_vm_object_deallocate) {
return shadow;
} else {
vm_object_deallocate(shadow);
}
}
}
}
#define VM_OBJ_TERM_STATS DEBUG
#if VM_OBJ_TERM_STATS
uint32_t vm_object_terminate_pages_freed = 0;
uint32_t vm_object_terminate_pages_removed = 0;
uint32_t vm_object_terminate_batches = 0;
uint32_t vm_object_terminate_biggest_batch = 0;
#endif
#define V_O_T_MAX_BATCH 256
static kern_return_t
vm_object_terminate(
register vm_object_t object)
{
register vm_page_t p;
vm_object_t shadow_object;
vm_page_t local_free_q;
int loop_count;
#if VM_OBJ_TERM_STATS
uint32_t local_free_count;
uint32_t pages_removed;
#endif
#if VM_OBJ_TERM_STATS
#define VM_OBJ_TERM_FREELIST_DEBUG(_pages_removed, _local_free_count) \
MACRO_BEGIN \
if (_pages_removed) { \
hw_atomic_add(&vm_object_terminate_batches, 1); \
hw_atomic_add(&vm_object_terminate_pages_removed, \
_pages_removed); \
hw_atomic_add(&vm_object_terminate_pages_freed, \
_local_free_count); \
if (_local_free_count > \
vm_object_terminate_biggest_batch) { \
vm_object_terminate_biggest_batch = \
_local_free_count; \
} \
_local_free_count = 0; \
} \
MACRO_END
#else
#define VM_OBJ_TERM_FREELIST_DEBUG(_pages_removed, _local_free_count)
#endif
#define VM_OBJ_TERM_FREELIST(_pages_removed, _local_free_count, _local_free_q) \
MACRO_BEGIN \
VM_OBJ_TERM_FREELIST_DEBUG(_pages_removed, _local_free_count); \
if (_local_free_q) { \
vm_page_free_list(_local_free_q); \
_local_free_q = VM_PAGE_NULL; \
} \
MACRO_END
XPR(XPR_VM_OBJECT, "vm_object_terminate, object 0x%X ref %d\n",
(integer_t)object, object->ref_count, 0, 0, 0);
local_free_q = VM_PAGE_NULL;
#if VM_OBJ_TERM_STATS
local_free_count = 0;
pages_removed = 0;
#endif
if (!object->pageout && (!object->temporary || object->can_persist)
&& (object->pager != NULL || object->shadow_severed)) {
vm_object_cache_unlock();
loop_count = V_O_T_MAX_BATCH;
vm_page_lock_queues();
while (!queue_empty(&object->memq)) {
if (--loop_count == 0) {
VM_OBJ_TERM_FREELIST(pages_removed,
local_free_count,
local_free_q);
mutex_yield(&vm_page_queue_lock);
loop_count = V_O_T_MAX_BATCH;
}
object->pager_trusted = FALSE;
p = (vm_page_t) queue_first(&object->memq);
VM_PAGE_CHECK(p);
if (p->busy || p->cleaning) {
if(p->cleaning || p->absent) {
VM_OBJ_TERM_FREELIST(pages_removed,
local_free_count,
local_free_q);
vm_page_unlock_queues();
vm_object_paging_wait(object, THREAD_UNINT);
vm_page_lock_queues();
continue;
} else {
panic("vm_object_terminate.3 %p %p", object, p);
}
}
p->busy = TRUE;
VM_PAGE_QUEUES_REMOVE(p);
#if VM_OBJ_TERM_STATS
pages_removed++;
#endif
if (p->absent || p->private) {
goto free_page;
}
if (p->fictitious) {
if (p->phys_page == vm_page_guard_addr) {
goto free_page;
}
panic("vm_object_terminate.4 %p %p", object, p);
}
if (!p->dirty && p->pmapped)
p->dirty = pmap_is_modified(p->phys_page);
if ((p->dirty || p->precious) && !p->error && object->alive) {
VM_OBJ_TERM_FREELIST(pages_removed,
local_free_count,
local_free_q);
vm_page_unlock_queues();
vm_pageout_cluster(p);
vm_object_paging_wait(object, THREAD_UNINT);
XPR(XPR_VM_OBJECT,
"vm_object_terminate restart, object 0x%X ref %d\n",
(integer_t)object, object->ref_count, 0, 0, 0);
vm_page_lock_queues();
} else {
free_page:
vm_page_free_prepare(p);
p->pageq.next = (queue_entry_t) local_free_q;
local_free_q = p;
#if VM_OBJ_TERM_STATS
local_free_count++;
#endif
}
}
VM_OBJ_TERM_FREELIST(pages_removed,
local_free_count,
local_free_q);
vm_page_unlock_queues();
vm_object_unlock(object);
vm_object_cache_lock();
vm_object_lock(object);
}
if(object->terminating) {
vm_object_lock_assert_exclusive(object);
object->ref_count--;
assert(object->ref_count > 0);
vm_object_cache_unlock();
vm_object_unlock(object);
return KERN_FAILURE;
}
if(object->ref_count != 1) {
vm_object_lock_assert_exclusive(object);
object->ref_count--;
assert(object->ref_count > 0);
vm_object_res_deallocate(object);
vm_object_cache_unlock();
vm_object_unlock(object);
return KERN_FAILURE;
}
object->terminating = TRUE;
object->alive = FALSE;
vm_object_remove(object);
if (((shadow_object = object->shadow) != VM_OBJECT_NULL) &&
!(object->pageout)) {
vm_object_lock(shadow_object);
if (shadow_object->copy == object)
shadow_object->copy = VM_OBJECT_NULL;
vm_object_unlock(shadow_object);
}
if (object->paging_in_progress != 0) {
vm_object_reap_async(object);
vm_object_cache_unlock();
vm_object_unlock(object);
return KERN_FAILURE;
}
vm_object_reap(object);
object = VM_OBJECT_NULL;
return KERN_SUCCESS;
}
void
vm_object_reap(
vm_object_t object)
{
memory_object_t pager;
vm_page_t p;
vm_page_t local_free_q;
int loop_count;
#if VM_OBJ_TERM_STATS
uint32_t local_free_count;
#endif
#if DEBUG
mutex_assert(&vm_object_cached_lock_data, MA_OWNED);
#endif
vm_object_lock_assert_exclusive(object);
assert(object->paging_in_progress == 0);
vm_object_reap_count++;
local_free_q = VM_PAGE_NULL;
#if VM_OBJ_TERM_STATS
local_free_count = 0;
#endif
pager = object->pager;
object->pager = MEMORY_OBJECT_NULL;
if (pager != MEMORY_OBJECT_NULL)
memory_object_control_disable(object->pager_control);
vm_object_cache_unlock();
vm_object_lock_assert_exclusive(object);
object->ref_count--;
#if TASK_SWAPPER
assert(object->res_count == 0);
#endif
assert (object->ref_count == 0);
if (object->objq.next || object->objq.prev) {
purgeable_q_t queue = vm_purgeable_object_remove(object);
assert(queue);
vm_page_lock_queues();
vm_purgeable_token_delete_first(queue);
assert(queue->debug_count_objects>=0);
vm_page_unlock_queues();
}
if (object->pageout) {
assert(object->shadow != VM_OBJECT_NULL);
vm_pageout_object_terminate(object);
} else if ((object->temporary && !object->can_persist) ||
(pager == MEMORY_OBJECT_NULL)) {
loop_count = V_O_T_MAX_BATCH;
vm_page_lock_queues();
while (!queue_empty(&object->memq)) {
if (--loop_count == 0) {
VM_OBJ_TERM_FREELIST(local_free_count,
local_free_count,
local_free_q);
mutex_yield(&vm_page_queue_lock);
loop_count = V_O_T_MAX_BATCH;
}
p = (vm_page_t) queue_first(&object->memq);
vm_page_free_prepare(p);
assert(p->pageq.next == NULL && p->pageq.prev == NULL);
p->pageq.next = (queue_entry_t) local_free_q;
local_free_q = p;
#if VM_OBJ_TERM_STATS
local_free_count++;
#endif
}
VM_OBJ_TERM_FREELIST(local_free_count,
local_free_count,
local_free_q);
vm_page_unlock_queues();
} else if (!queue_empty(&object->memq)) {
panic("vm_object_reap: queue just emptied isn't");
}
assert(object->paging_in_progress == 0);
assert(object->ref_count == 0);
if (pager != MEMORY_OBJECT_NULL) {
vm_object_unlock(object);
vm_object_release_pager(pager);
vm_object_lock(object);
}
object->terminating = FALSE;
vm_object_paging_begin(object);
vm_object_paging_end(object);
vm_object_unlock(object);
#if MACH_PAGEMAP
vm_external_destroy(object->existence_map, object->size);
#endif
object->shadow = VM_OBJECT_NULL;
vm_object_lock_destroy(object);
zfree(vm_object_zone, object);
object = VM_OBJECT_NULL;
}
void
vm_object_reap_async(
vm_object_t object)
{
#if DEBUG
mutex_assert(&vm_object_cached_lock_data, MA_OWNED);
#endif
vm_object_lock_assert_exclusive(object);
vm_object_reap_count_async++;
queue_enter(&vm_object_reaper_queue, object,
vm_object_t, cached_list);
thread_wakeup((event_t) &vm_object_reaper_queue);
}
void
vm_object_reaper_thread(void)
{
vm_object_t object, shadow_object;
vm_object_cache_lock();
while (!queue_empty(&vm_object_reaper_queue)) {
queue_remove_first(&vm_object_reaper_queue,
object,
vm_object_t,
cached_list);
vm_object_lock(object);
assert(object->terminating);
assert(!object->alive);
while (object->paging_in_progress != 0) {
vm_object_cache_unlock();
vm_object_wait(object,
VM_OBJECT_EVENT_PAGING_IN_PROGRESS,
THREAD_UNINT);
vm_object_cache_lock();
vm_object_lock(object);
}
shadow_object =
object->pageout ? VM_OBJECT_NULL : object->shadow;
vm_object_reap(object);
object = VM_OBJECT_NULL;
if (shadow_object != VM_OBJECT_NULL) {
vm_object_deallocate(shadow_object);
shadow_object = VM_OBJECT_NULL;
}
vm_object_cache_lock();
}
assert_wait((event_t) &vm_object_reaper_queue, THREAD_UNINT);
vm_object_cache_unlock();
thread_block((thread_continue_t) vm_object_reaper_thread);
}
static void
vm_object_pager_wakeup(
memory_object_t pager)
{
vm_object_hash_entry_t entry;
boolean_t waiting = FALSE;
vm_object_cache_lock();
entry = vm_object_hash_lookup(pager, TRUE);
if (entry != VM_OBJECT_HASH_ENTRY_NULL)
waiting = entry->waiting;
vm_object_cache_unlock();
if (entry != VM_OBJECT_HASH_ENTRY_NULL) {
if (waiting)
thread_wakeup((event_t) pager);
vm_object_hash_entry_free(entry);
}
}
static void
vm_object_release_pager(
memory_object_t pager)
{
(void) memory_object_terminate(pager);
vm_object_pager_wakeup(pager);
memory_object_deallocate(pager);
}
kern_return_t
vm_object_destroy(
vm_object_t object,
__unused kern_return_t reason)
{
memory_object_t old_pager;
if (object == VM_OBJECT_NULL)
return(KERN_SUCCESS);
vm_object_cache_lock();
vm_object_lock(object);
object->can_persist = FALSE;
object->named = FALSE;
object->alive = FALSE;
vm_object_remove(object);
old_pager = object->pager;
object->pager = MEMORY_OBJECT_NULL;
if (old_pager != MEMORY_OBJECT_NULL)
memory_object_control_disable(object->pager_control);
vm_object_cache_unlock();
vm_object_paging_wait(object, THREAD_UNINT);
vm_object_unlock(object);
if (old_pager != MEMORY_OBJECT_NULL) {
vm_object_release_pager(old_pager);
vm_object_deallocate(object);
}
return(KERN_SUCCESS);
}
#define VM_OBJ_DEACT_ALL_STATS DEBUG
#if VM_OBJ_DEACT_ALL_STATS
uint32_t vm_object_deactivate_all_pages_batches = 0;
uint32_t vm_object_deactivate_all_pages_pages = 0;
#endif
static void
vm_object_deactivate_all_pages(
register vm_object_t object)
{
register vm_page_t p;
int loop_count;
#if VM_OBJ_DEACT_ALL_STATS
int pages_count;
#endif
#define V_O_D_A_P_MAX_BATCH 256
loop_count = V_O_D_A_P_MAX_BATCH;
#if VM_OBJ_DEACT_ALL_STATS
pages_count = 0;
#endif
vm_page_lock_queues();
queue_iterate(&object->memq, p, vm_page_t, listq) {
if (--loop_count == 0) {
#if VM_OBJ_DEACT_ALL_STATS
hw_atomic_add(&vm_object_deactivate_all_pages_batches,
1);
hw_atomic_add(&vm_object_deactivate_all_pages_pages,
pages_count);
pages_count = 0;
#endif
mutex_yield(&vm_page_queue_lock);
loop_count = V_O_D_A_P_MAX_BATCH;
}
if (!p->busy && !p->throttled) {
#if VM_OBJ_DEACT_ALL_STATS
pages_count++;
#endif
vm_page_deactivate(p);
}
}
#if VM_OBJ_DEACT_ALL_STATS
if (pages_count) {
hw_atomic_add(&vm_object_deactivate_all_pages_batches, 1);
hw_atomic_add(&vm_object_deactivate_all_pages_pages,
pages_count);
pages_count = 0;
}
#endif
vm_page_unlock_queues();
}
__private_extern__ void
vm_object_deactivate_pages(
vm_object_t object,
vm_object_offset_t offset,
vm_object_size_t size,
boolean_t kill_page)
{
vm_object_t orig_object;
int pages_moved = 0;
int pages_found = 0;
orig_object = object;
for (;;) {
register vm_page_t m;
vm_object_offset_t toffset;
vm_object_size_t tsize;
vm_object_paging_begin(object);
vm_page_lock_queues();
for (tsize = size, toffset = offset; tsize; tsize -= PAGE_SIZE, toffset += PAGE_SIZE) {
if ((m = vm_page_lookup(object, toffset)) != VM_PAGE_NULL) {
pages_found++;
if ((m->wire_count == 0) && (!m->private) && (!m->gobbled) && (!m->busy)) {
assert(!m->laundry);
m->reference = FALSE;
pmap_clear_reference(m->phys_page);
if ((kill_page) && (object->internal)) {
m->precious = FALSE;
m->dirty = FALSE;
pmap_clear_modify(m->phys_page);
#if MACH_PAGEMAP
vm_external_state_clr(object->existence_map, offset);
#endif
}
if (!m->throttled) {
VM_PAGE_QUEUES_REMOVE(m);
assert(!m->laundry);
assert(m->object != kernel_object);
assert(m->pageq.next == NULL &&
m->pageq.prev == NULL);
if(m->zero_fill) {
queue_enter_first(
&vm_page_queue_zf,
m, vm_page_t, pageq);
vm_zf_queue_count++;
} else {
queue_enter_first(
&vm_page_queue_inactive,
m, vm_page_t, pageq);
}
m->inactive = TRUE;
if (!m->fictitious) {
vm_page_inactive_count++;
token_new_pagecount++;
} else {
assert(m->phys_page == vm_page_fictitious_addr);
}
pages_moved++;
}
}
}
}
vm_page_unlock_queues();
vm_object_paging_end(object);
if (object->shadow) {
vm_object_t tmp_object;
kill_page = 0;
offset += object->shadow_offset;
tmp_object = object->shadow;
vm_object_lock(tmp_object);
if (object != orig_object)
vm_object_unlock(object);
object = tmp_object;
} else
break;
}
if (object != orig_object)
vm_object_unlock(object);
}
__private_extern__ void
vm_object_pmap_protect(
register vm_object_t object,
register vm_object_offset_t offset,
vm_object_size_t size,
pmap_t pmap,
vm_map_offset_t pmap_start,
vm_prot_t prot)
{
if (object == VM_OBJECT_NULL)
return;
size = vm_object_round_page(size);
offset = vm_object_trunc_page(offset);
vm_object_lock(object);
if (object->phys_contiguous) {
if (pmap != NULL) {
vm_object_unlock(object);
pmap_protect(pmap, pmap_start, pmap_start + size, prot);
} else {
vm_object_offset_t phys_start, phys_end, phys_addr;
phys_start = object->shadow_offset + offset;
phys_end = phys_start + size;
assert(phys_start <= phys_end);
assert(phys_end <= object->shadow_offset + object->size);
vm_object_unlock(object);
for (phys_addr = phys_start;
phys_addr < phys_end;
phys_addr += PAGE_SIZE_64) {
pmap_page_protect(phys_addr >> 12, prot);
}
}
return;
}
assert(object->internal);
while (TRUE) {
if (ptoa_64(object->resident_page_count) > size/2 && pmap != PMAP_NULL) {
vm_object_unlock(object);
pmap_protect(pmap, pmap_start, pmap_start + size, prot);
return;
}
if (ptoa_64(object->resident_page_count / 4) < size) {
vm_page_t p;
vm_object_offset_t end;
end = offset + size;
if (pmap != PMAP_NULL) {
queue_iterate(&object->memq, p, vm_page_t, listq) {
if (!p->fictitious &&
(offset <= p->offset) && (p->offset < end)) {
vm_map_offset_t start;
start = pmap_start + p->offset - offset;
pmap_protect(pmap, start, start + PAGE_SIZE_64, prot);
}
}
} else {
queue_iterate(&object->memq, p, vm_page_t, listq) {
if (!p->fictitious &&
(offset <= p->offset) && (p->offset < end)) {
pmap_page_protect(p->phys_page, prot);
}
}
}
} else {
vm_page_t p;
vm_object_offset_t end;
vm_object_offset_t target_off;
end = offset + size;
if (pmap != PMAP_NULL) {
for(target_off = offset;
target_off < end;
target_off += PAGE_SIZE) {
p = vm_page_lookup(object, target_off);
if (p != VM_PAGE_NULL) {
vm_offset_t start;
start = pmap_start +
(vm_offset_t)(p->offset - offset);
pmap_protect(pmap, start,
start + PAGE_SIZE, prot);
}
}
} else {
for(target_off = offset;
target_off < end; target_off += PAGE_SIZE) {
p = vm_page_lookup(object, target_off);
if (p != VM_PAGE_NULL) {
pmap_page_protect(p->phys_page, prot);
}
}
}
}
if (prot == VM_PROT_NONE) {
register vm_object_t next_object;
next_object = object->shadow;
if (next_object != VM_OBJECT_NULL) {
offset += object->shadow_offset;
vm_object_lock(next_object);
vm_object_unlock(object);
object = next_object;
}
else {
break;
}
}
else {
break;
}
}
vm_object_unlock(object);
}
__private_extern__ kern_return_t
vm_object_copy_slowly(
register vm_object_t src_object,
vm_object_offset_t src_offset,
vm_object_size_t size,
boolean_t interruptible,
vm_object_t *_result_object)
{
vm_object_t new_object;
vm_object_offset_t new_offset;
struct vm_object_fault_info fault_info;
XPR(XPR_VM_OBJECT, "v_o_c_slowly obj 0x%x off 0x%x size 0x%x\n",
src_object, src_offset, size, 0, 0);
if (size == 0) {
vm_object_unlock(src_object);
*_result_object = VM_OBJECT_NULL;
return(KERN_INVALID_ARGUMENT);
}
vm_object_reference_locked(src_object);
vm_object_unlock(src_object);
new_object = vm_object_allocate(size);
new_offset = 0;
assert(size == trunc_page_64(size));
fault_info.interruptible = interruptible;
fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL;
fault_info.user_tag = 0;
fault_info.lo_offset = src_offset;
fault_info.hi_offset = src_offset + size;
fault_info.no_cache = FALSE;
for ( ;
size != 0 ;
src_offset += PAGE_SIZE_64,
new_offset += PAGE_SIZE_64, size -= PAGE_SIZE_64
) {
vm_page_t new_page;
vm_fault_return_t result;
vm_object_lock(new_object);
while ((new_page = vm_page_alloc(new_object, new_offset))
== VM_PAGE_NULL) {
vm_object_unlock(new_object);
if (!vm_page_wait(interruptible)) {
vm_object_deallocate(new_object);
vm_object_deallocate(src_object);
*_result_object = VM_OBJECT_NULL;
return(MACH_SEND_INTERRUPTED);
}
vm_object_lock(new_object);
}
vm_object_unlock(new_object);
do {
vm_prot_t prot = VM_PROT_READ;
vm_page_t _result_page;
vm_page_t top_page;
register
vm_page_t result_page;
kern_return_t error_code;
vm_object_lock(src_object);
vm_object_paging_begin(src_object);
fault_info.cluster_size = size;
XPR(XPR_VM_FAULT,"vm_object_copy_slowly -> vm_fault_page",0,0,0,0,0);
result = vm_fault_page(src_object, src_offset,
VM_PROT_READ, FALSE,
&prot, &_result_page, &top_page,
(int *)0,
&error_code, FALSE, FALSE, &fault_info);
switch(result) {
case VM_FAULT_SUCCESS:
result_page = _result_page;
vm_object_unlock(result_page->object);
vm_page_copy(result_page, new_page);
vm_object_lock(new_object);
new_page->dirty = TRUE;
PAGE_WAKEUP_DONE(new_page);
vm_object_unlock(new_object);
vm_object_lock(result_page->object);
PAGE_WAKEUP_DONE(result_page);
vm_page_lockspin_queues();
if (!result_page->active &&
!result_page->inactive &&
!result_page->throttled)
vm_page_activate(result_page);
vm_page_activate(new_page);
vm_page_unlock_queues();
vm_fault_cleanup(result_page->object,
top_page);
break;
case VM_FAULT_RETRY:
break;
case VM_FAULT_FICTITIOUS_SHORTAGE:
vm_page_more_fictitious();
break;
case VM_FAULT_MEMORY_SHORTAGE:
if (vm_page_wait(interruptible))
break;
case VM_FAULT_INTERRUPTED:
vm_page_free(new_page);
vm_object_deallocate(new_object);
vm_object_deallocate(src_object);
*_result_object = VM_OBJECT_NULL;
return(MACH_SEND_INTERRUPTED);
case VM_FAULT_MEMORY_ERROR:
vm_page_lock_queues();
vm_page_free(new_page);
vm_page_unlock_queues();
vm_object_deallocate(new_object);
vm_object_deallocate(src_object);
*_result_object = VM_OBJECT_NULL;
return(error_code ? error_code:
KERN_MEMORY_ERROR);
}
} while (result != VM_FAULT_SUCCESS);
}
vm_object_deallocate(src_object);
*_result_object = new_object;
return(KERN_SUCCESS);
}
__private_extern__ boolean_t
vm_object_copy_quickly(
vm_object_t *_object,
__unused vm_object_offset_t offset,
__unused vm_object_size_t size,
boolean_t *_src_needs_copy,
boolean_t *_dst_needs_copy)
{
vm_object_t object = *_object;
memory_object_copy_strategy_t copy_strategy;
XPR(XPR_VM_OBJECT, "v_o_c_quickly obj 0x%x off 0x%x size 0x%x\n",
*_object, offset, size, 0, 0);
if (object == VM_OBJECT_NULL) {
*_src_needs_copy = FALSE;
*_dst_needs_copy = FALSE;
return(TRUE);
}
vm_object_lock(object);
copy_strategy = object->copy_strategy;
switch (copy_strategy) {
case MEMORY_OBJECT_COPY_SYMMETRIC:
vm_object_reference_locked(object);
object->shadowed = TRUE;
vm_object_unlock(object);
*_src_needs_copy = TRUE;
*_dst_needs_copy = TRUE;
break;
case MEMORY_OBJECT_COPY_DELAY:
vm_object_unlock(object);
return(FALSE);
default:
vm_object_unlock(object);
return(FALSE);
}
return(TRUE);
}
static int copy_call_count = 0;
static int copy_call_sleep_count = 0;
static int copy_call_restart_count = 0;
static kern_return_t
vm_object_copy_call(
vm_object_t src_object,
vm_object_offset_t src_offset,
vm_object_size_t size,
vm_object_t *_result_object)
{
kern_return_t kr;
vm_object_t copy;
boolean_t check_ready = FALSE;
uint32_t try_failed_count = 0;
copy_call_count++;
while (vm_object_wanted(src_object, VM_OBJECT_EVENT_COPY_CALL)) {
vm_object_sleep(src_object, VM_OBJECT_EVENT_COPY_CALL,
THREAD_UNINT);
copy_call_restart_count++;
}
vm_object_set_wanted(src_object, VM_OBJECT_EVENT_COPY_CALL);
vm_object_unlock(src_object);
kr = KERN_FAILURE;
if (kr != KERN_SUCCESS) {
return kr;
}
vm_object_lock(src_object);
while (vm_object_wanted(src_object, VM_OBJECT_EVENT_COPY_CALL)) {
vm_object_sleep(src_object, VM_OBJECT_EVENT_COPY_CALL,
THREAD_UNINT);
copy_call_sleep_count++;
}
Retry:
assert(src_object->copy != VM_OBJECT_NULL);
copy = src_object->copy;
if (!vm_object_lock_try(copy)) {
vm_object_unlock(src_object);
try_failed_count++;
mutex_pause(try_failed_count);
vm_object_lock(src_object);
goto Retry;
}
if (copy->size < src_offset+size)
copy->size = src_offset+size;
if (!copy->pager_ready)
check_ready = TRUE;
*_result_object = copy;
vm_object_unlock(copy);
vm_object_unlock(src_object);
if (check_ready == TRUE) {
vm_object_lock(copy);
while (!copy->pager_ready) {
vm_object_sleep(copy, VM_OBJECT_EVENT_PAGER_READY, THREAD_UNINT);
}
vm_object_unlock(copy);
}
return KERN_SUCCESS;
}
static int copy_delayed_lock_collisions = 0;
static int copy_delayed_max_collisions = 0;
static int copy_delayed_lock_contention = 0;
static int copy_delayed_protect_iterate = 0;
__private_extern__ vm_object_t
vm_object_copy_delayed(
vm_object_t src_object,
vm_object_offset_t src_offset,
vm_object_size_t size,
boolean_t src_object_shared)
{
vm_object_t new_copy = VM_OBJECT_NULL;
vm_object_t old_copy;
vm_page_t p;
vm_object_size_t copy_size = src_offset + size;
int collisions = 0;
Retry:
if (!src_object->true_share && src_object->paging_in_progress) {
if (src_object_shared == TRUE) {
vm_object_unlock(src_object);
vm_object_lock(src_object);
src_object_shared = FALSE;
}
vm_object_paging_wait(src_object, THREAD_UNINT);
}
old_copy = src_object->copy;
if (old_copy != VM_OBJECT_NULL) {
int lock_granted;
if (src_object_shared == TRUE)
lock_granted = vm_object_lock_try_shared(old_copy);
else
lock_granted = vm_object_lock_try(old_copy);
if (!lock_granted) {
vm_object_unlock(src_object);
if (collisions++ == 0)
copy_delayed_lock_contention++;
mutex_pause(collisions);
copy_delayed_lock_collisions++;
if (collisions > copy_delayed_max_collisions)
copy_delayed_max_collisions = collisions;
if (src_object_shared == TRUE)
vm_object_lock_shared(src_object);
else
vm_object_lock(src_object);
goto Retry;
}
if (old_copy->resident_page_count == 0 &&
!old_copy->pager_created) {
if (old_copy->size < copy_size) {
if (src_object_shared == TRUE) {
vm_object_unlock(old_copy);
vm_object_unlock(src_object);
vm_object_lock(src_object);
src_object_shared = FALSE;
goto Retry;
}
copy_delayed_protect_iterate++;
queue_iterate(&src_object->memq, p, vm_page_t, listq) {
if (!p->fictitious &&
p->offset >= old_copy->size &&
p->offset < copy_size) {
if (p->wire_count > 0) {
vm_object_unlock(old_copy);
vm_object_unlock(src_object);
if (new_copy != VM_OBJECT_NULL) {
vm_object_unlock(new_copy);
vm_object_deallocate(new_copy);
}
return VM_OBJECT_NULL;
} else {
pmap_page_protect(p->phys_page,
(VM_PROT_ALL & ~VM_PROT_WRITE));
}
}
}
old_copy->size = copy_size;
}
if (src_object_shared == TRUE)
vm_object_reference_shared(old_copy);
else
vm_object_reference_locked(old_copy);
vm_object_unlock(old_copy);
vm_object_unlock(src_object);
if (new_copy != VM_OBJECT_NULL) {
vm_object_unlock(new_copy);
vm_object_deallocate(new_copy);
}
return(old_copy);
}
if (old_copy->size > copy_size)
copy_size = old_copy->size;
if (new_copy == VM_OBJECT_NULL) {
vm_object_unlock(old_copy);
vm_object_unlock(src_object);
new_copy = vm_object_allocate(copy_size);
vm_object_lock(src_object);
vm_object_lock(new_copy);
src_object_shared = FALSE;
goto Retry;
}
new_copy->size = copy_size;
assert((old_copy->shadow == src_object) &&
(old_copy->shadow_offset == (vm_object_offset_t) 0));
} else if (new_copy == VM_OBJECT_NULL) {
vm_object_unlock(src_object);
new_copy = vm_object_allocate(copy_size);
vm_object_lock(src_object);
vm_object_lock(new_copy);
src_object_shared = FALSE;
goto Retry;
}
copy_delayed_protect_iterate++;
queue_iterate(&src_object->memq, p, vm_page_t, listq) {
if (!p->fictitious && p->offset < copy_size) {
if (p->wire_count > 0) {
if (old_copy)
vm_object_unlock(old_copy);
vm_object_unlock(src_object);
vm_object_unlock(new_copy);
vm_object_deallocate(new_copy);
return VM_OBJECT_NULL;
} else {
pmap_page_protect(p->phys_page,
(VM_PROT_ALL & ~VM_PROT_WRITE));
}
}
}
if (old_copy != VM_OBJECT_NULL) {
vm_object_lock_assert_exclusive(src_object);
src_object->ref_count--;
assert(src_object->ref_count > 0);
vm_object_lock_assert_exclusive(old_copy);
old_copy->shadow = new_copy;
vm_object_lock_assert_exclusive(new_copy);
assert(new_copy->ref_count > 0);
new_copy->ref_count++;
#if TASK_SWAPPER
if (old_copy->res_count) {
VM_OBJ_RES_INCR(new_copy);
VM_OBJ_RES_DECR(src_object);
}
#endif
vm_object_unlock(old_copy);
}
vm_object_lock_assert_exclusive(new_copy);
new_copy->shadow = src_object;
new_copy->shadow_offset = 0;
new_copy->shadowed = TRUE;
vm_object_lock_assert_exclusive(src_object);
vm_object_reference_locked(src_object);
src_object->copy = new_copy;
vm_object_unlock(src_object);
vm_object_unlock(new_copy);
XPR(XPR_VM_OBJECT,
"vm_object_copy_delayed: used copy object %X for source %X\n",
(integer_t)new_copy, (integer_t)src_object, 0, 0, 0);
return new_copy;
}
__private_extern__ kern_return_t
vm_object_copy_strategically(
register vm_object_t src_object,
vm_object_offset_t src_offset,
vm_object_size_t size,
vm_object_t *dst_object,
vm_object_offset_t *dst_offset,
boolean_t *dst_needs_copy)
{
boolean_t result;
boolean_t interruptible = THREAD_ABORTSAFE;
boolean_t object_lock_shared = FALSE;
memory_object_copy_strategy_t copy_strategy;
assert(src_object != VM_OBJECT_NULL);
copy_strategy = src_object->copy_strategy;
if (copy_strategy == MEMORY_OBJECT_COPY_DELAY) {
vm_object_lock_shared(src_object);
object_lock_shared = TRUE;
} else
vm_object_lock(src_object);
while (!src_object->internal && !src_object->pager_ready) {
wait_result_t wait_result;
if (object_lock_shared == TRUE) {
vm_object_unlock(src_object);
vm_object_lock(src_object);
object_lock_shared = FALSE;
continue;
}
wait_result = vm_object_sleep( src_object,
VM_OBJECT_EVENT_PAGER_READY,
interruptible);
if (wait_result != THREAD_AWAKENED) {
vm_object_unlock(src_object);
*dst_object = VM_OBJECT_NULL;
*dst_offset = 0;
*dst_needs_copy = FALSE;
return(MACH_SEND_INTERRUPTED);
}
}
switch (copy_strategy) {
case MEMORY_OBJECT_COPY_DELAY:
*dst_object = vm_object_copy_delayed(src_object,
src_offset, size, object_lock_shared);
if (*dst_object != VM_OBJECT_NULL) {
*dst_offset = src_offset;
*dst_needs_copy = TRUE;
result = KERN_SUCCESS;
break;
}
vm_object_lock(src_object);
case MEMORY_OBJECT_COPY_NONE:
result = vm_object_copy_slowly(src_object, src_offset, size,
interruptible, dst_object);
if (result == KERN_SUCCESS) {
*dst_offset = 0;
*dst_needs_copy = FALSE;
}
break;
case MEMORY_OBJECT_COPY_CALL:
result = vm_object_copy_call(src_object, src_offset, size,
dst_object);
if (result == KERN_SUCCESS) {
*dst_offset = src_offset;
*dst_needs_copy = TRUE;
}
break;
case MEMORY_OBJECT_COPY_SYMMETRIC:
XPR(XPR_VM_OBJECT, "v_o_c_strategically obj 0x%x off 0x%x size 0x%x\n",(natural_t)src_object, src_offset, size, 0, 0);
vm_object_unlock(src_object);
result = KERN_MEMORY_RESTART_COPY;
break;
default:
panic("copy_strategically: bad strategy");
result = KERN_INVALID_ARGUMENT;
}
return(result);
}
boolean_t vm_object_shadow_check = FALSE;
__private_extern__ boolean_t
vm_object_shadow(
vm_object_t *object,
vm_object_offset_t *offset,
vm_object_size_t length)
{
register vm_object_t source;
register vm_object_t result;
source = *object;
#if 0
assert(source->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC);
#endif
if (vm_object_shadow_check && source->ref_count == 1 &&
(source->shadow == VM_OBJECT_NULL ||
source->shadow->copy == VM_OBJECT_NULL))
{
source->shadowed = FALSE;
return FALSE;
}
if ((result = vm_object_allocate(length)) == VM_OBJECT_NULL)
panic("vm_object_shadow: no object for shadowing");
result->shadow = source;
result->shadow_offset = *offset;
*offset = 0;
*object = result;
return TRUE;
}
vm_object_t
vm_object_enter(
memory_object_t pager,
vm_object_size_t size,
boolean_t internal,
boolean_t init,
boolean_t named)
{
register vm_object_t object;
vm_object_t new_object;
boolean_t must_init;
vm_object_hash_entry_t entry, new_entry;
uint32_t try_failed_count = 0;
if (pager == MEMORY_OBJECT_NULL)
return(vm_object_allocate(size));
new_object = VM_OBJECT_NULL;
new_entry = VM_OBJECT_HASH_ENTRY_NULL;
must_init = init;
Retry:
vm_object_cache_lock();
do {
entry = vm_object_hash_lookup(pager, FALSE);
if (entry == VM_OBJECT_HASH_ENTRY_NULL) {
if (new_object == VM_OBJECT_NULL) {
vm_object_cache_unlock();
assert(new_entry == VM_OBJECT_HASH_ENTRY_NULL);
new_entry = vm_object_hash_entry_alloc(pager);
new_object = vm_object_allocate(size);
vm_object_cache_lock();
} else {
vm_object_hash_insert(new_entry);
entry = new_entry;
entry->object = new_object;
new_entry = VM_OBJECT_HASH_ENTRY_NULL;
new_object = VM_OBJECT_NULL;
must_init = TRUE;
}
} else if (entry->object == VM_OBJECT_NULL) {
entry->waiting = TRUE;
entry = VM_OBJECT_HASH_ENTRY_NULL;
assert_wait((event_t) pager, THREAD_UNINT);
vm_object_cache_unlock();
thread_block(THREAD_CONTINUE_NULL);
vm_object_cache_lock();
}
} while (entry == VM_OBJECT_HASH_ENTRY_NULL);
object = entry->object;
assert(object != VM_OBJECT_NULL);
if (!must_init) {
if (!vm_object_lock_try(object)) {
vm_object_cache_unlock();
try_failed_count++;
mutex_pause(try_failed_count);
goto Retry;
}
assert(!internal || object->internal);
if (named) {
assert(!object->named);
object->named = TRUE;
}
if (object->ref_count == 0) {
XPR(XPR_VM_OBJECT_CACHE,
"vm_object_enter: removing %x from cache, head (%x, %x)\n",
(integer_t)object,
(integer_t)vm_object_cached_list.next,
(integer_t)vm_object_cached_list.prev, 0,0);
queue_remove(&vm_object_cached_list, object,
vm_object_t, cached_list);
vm_object_cached_count--;
}
vm_object_lock_assert_exclusive(object);
object->ref_count++;
vm_object_res_reference(object);
vm_object_unlock(object);
VM_STAT_INCR(hits);
}
assert(object->ref_count > 0);
VM_STAT_INCR(lookups);
vm_object_cache_unlock();
XPR(XPR_VM_OBJECT,
"vm_o_enter: pager 0x%x obj 0x%x must_init %d\n",
(integer_t)pager, (integer_t)object, must_init, 0, 0);
if (new_object != VM_OBJECT_NULL)
vm_object_deallocate(new_object);
if (new_entry != VM_OBJECT_HASH_ENTRY_NULL)
vm_object_hash_entry_free(new_entry);
if (must_init) {
memory_object_control_t control;
control = memory_object_control_allocate(object);
assert (control != MEMORY_OBJECT_CONTROL_NULL);
vm_object_lock(object);
assert(object != kernel_object);
memory_object_reference(pager);
object->pager_created = TRUE;
object->pager = pager;
object->internal = internal;
object->pager_trusted = internal;
if (!internal) {
object->copy_strategy = MEMORY_OBJECT_COPY_INVALID;
}
object->pager_control = control;
object->pager_ready = FALSE;
vm_object_unlock(object);
(void) memory_object_init(pager,
object->pager_control,
PAGE_SIZE);
vm_object_lock(object);
if (named)
object->named = TRUE;
if (internal) {
object->pager_ready = TRUE;
vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
}
object->pager_initialized = TRUE;
vm_object_wakeup(object, VM_OBJECT_EVENT_INITIALIZED);
} else {
vm_object_lock(object);
}
while (!object->pager_initialized) {
vm_object_sleep(object,
VM_OBJECT_EVENT_INITIALIZED,
THREAD_UNINT);
}
vm_object_unlock(object);
XPR(XPR_VM_OBJECT,
"vm_object_enter: vm_object %x, memory_object %x, internal %d\n",
(integer_t)object, (integer_t)object->pager, internal, 0,0);
return(object);
}
void
vm_object_pager_create(
register vm_object_t object)
{
memory_object_t pager;
vm_object_hash_entry_t entry;
#if MACH_PAGEMAP
vm_object_size_t size;
vm_external_map_t map;
#endif
XPR(XPR_VM_OBJECT, "vm_object_pager_create, object 0x%X\n",
(integer_t)object, 0,0,0,0);
assert(object != kernel_object);
if (memory_manager_default_check() != KERN_SUCCESS)
return;
vm_object_paging_begin(object);
if (object->pager_created) {
while (!object->pager_initialized) {
vm_object_sleep(object,
VM_OBJECT_EVENT_INITIALIZED,
THREAD_UNINT);
}
vm_object_paging_end(object);
return;
}
object->pager_created = TRUE;
object->paging_offset = 0;
#if MACH_PAGEMAP
size = object->size;
#endif
vm_object_unlock(object);
#if MACH_PAGEMAP
map = vm_external_create(size);
vm_object_lock(object);
assert(object->size == size);
object->existence_map = map;
vm_object_unlock(object);
#endif
{
memory_object_default_t dmm;
dmm = memory_manager_default_reference();
assert(object->temporary);
(void) memory_object_create(dmm, object->size, &pager);
memory_object_default_deallocate(dmm);
}
entry = vm_object_hash_entry_alloc(pager);
vm_object_cache_lock();
vm_object_hash_insert(entry);
entry->object = object;
vm_object_cache_unlock();
if (vm_object_enter(pager, object->size, TRUE, TRUE, FALSE) != object)
panic("vm_object_pager_create: mismatch");
memory_object_deallocate(pager);
vm_object_lock(object);
vm_object_paging_end(object);
}
__private_extern__ void
vm_object_remove(
vm_object_t object)
{
memory_object_t pager;
if ((pager = object->pager) != MEMORY_OBJECT_NULL) {
vm_object_hash_entry_t entry;
entry = vm_object_hash_lookup(pager, FALSE);
if (entry != VM_OBJECT_HASH_ENTRY_NULL)
entry->object = VM_OBJECT_NULL;
}
}
static long object_collapses = 0;
static long object_bypasses = 0;
static boolean_t vm_object_collapse_allowed = TRUE;
static boolean_t vm_object_bypass_allowed = TRUE;
#if MACH_PAGEMAP
static int vm_external_discarded;
static int vm_external_collapsed;
#endif
unsigned long vm_object_collapse_encrypted = 0;
static void
vm_object_do_collapse(
vm_object_t object,
vm_object_t backing_object)
{
vm_page_t p, pp;
vm_object_offset_t new_offset, backing_offset;
vm_object_size_t size;
backing_offset = object->shadow_offset;
size = object->size;
while (!queue_empty(&backing_object->memq)) {
p = (vm_page_t) queue_first(&backing_object->memq);
new_offset = (p->offset - backing_offset);
assert(!p->busy || p->absent);
if (p->offset < backing_offset || new_offset >= size) {
VM_PAGE_FREE(p);
} else {
if (p->encrypted) {
vm_object_collapse_encrypted++;
}
pp = vm_page_lookup(object, new_offset);
if (pp == VM_PAGE_NULL) {
vm_page_rename(p, object, new_offset, TRUE);
#if MACH_PAGEMAP
} else if (pp->absent) {
VM_PAGE_FREE(pp);
vm_page_rename(p, object, new_offset, TRUE);
#endif
} else {
assert(! pp->absent);
VM_PAGE_FREE(p);
}
}
}
#if !MACH_PAGEMAP
assert((!object->pager_created && (object->pager == MEMORY_OBJECT_NULL))
|| (!backing_object->pager_created
&& (backing_object->pager == MEMORY_OBJECT_NULL)));
#else
assert(!object->pager_created && object->pager == MEMORY_OBJECT_NULL);
#endif
if (backing_object->pager != MEMORY_OBJECT_NULL) {
vm_object_hash_entry_t entry;
assert(!object->paging_in_progress);
object->pager = backing_object->pager;
entry = vm_object_hash_lookup(object->pager, FALSE);
assert(entry != VM_OBJECT_HASH_ENTRY_NULL);
entry->object = object;
object->pager_created = backing_object->pager_created;
object->pager_control = backing_object->pager_control;
object->pager_ready = backing_object->pager_ready;
object->pager_initialized = backing_object->pager_initialized;
object->paging_offset =
backing_object->paging_offset + backing_offset;
if (object->pager_control != MEMORY_OBJECT_CONTROL_NULL) {
memory_object_control_collapse(object->pager_control,
object);
}
}
vm_object_cache_unlock();
#if MACH_PAGEMAP
assert(object->existence_map == VM_EXTERNAL_NULL);
if (backing_offset || (size != backing_object->size)) {
vm_external_discarded++;
vm_external_destroy(backing_object->existence_map,
backing_object->size);
}
else {
vm_external_collapsed++;
object->existence_map = backing_object->existence_map;
}
backing_object->existence_map = VM_EXTERNAL_NULL;
#endif
assert(!object->phys_contiguous);
assert(!backing_object->phys_contiguous);
object->shadow = backing_object->shadow;
if (object->shadow) {
object->shadow_offset += backing_object->shadow_offset;
} else {
object->shadow_offset = 0;
}
assert((object->shadow == VM_OBJECT_NULL) ||
(object->shadow->copy != backing_object));
assert((backing_object->ref_count == 1) &&
(backing_object->resident_page_count == 0) &&
(backing_object->paging_in_progress == 0));
backing_object->alive = FALSE;
vm_object_unlock(backing_object);
XPR(XPR_VM_OBJECT, "vm_object_collapse, collapsed 0x%X\n",
(integer_t)backing_object, 0,0,0,0);
vm_object_lock_destroy(backing_object);
zfree(vm_object_zone, backing_object);
object_collapses++;
}
static void
vm_object_do_bypass(
vm_object_t object,
vm_object_t backing_object)
{
vm_object_lock_assert_exclusive(backing_object);
#if TASK_SWAPPER
if (backing_object->shadow != VM_OBJECT_NULL) {
vm_object_lock(backing_object->shadow);
vm_object_lock_assert_exclusive(backing_object->shadow);
backing_object->shadow->ref_count++;
if (object->res_count != 0)
vm_object_res_reference(backing_object->shadow);
vm_object_unlock(backing_object->shadow);
}
#else
vm_object_reference(backing_object->shadow);
#endif
assert(!object->phys_contiguous);
assert(!backing_object->phys_contiguous);
object->shadow = backing_object->shadow;
if (object->shadow) {
object->shadow_offset += backing_object->shadow_offset;
} else {
object->shadow_offset = 0;
}
if (backing_object->copy == object) {
backing_object->copy = VM_OBJECT_NULL;
}
if (backing_object->ref_count > 1) {
vm_object_lock_assert_exclusive(backing_object);
backing_object->ref_count--;
#if TASK_SWAPPER
if (object->res_count != 0)
vm_object_res_deallocate(backing_object);
assert(backing_object->ref_count > 0);
#endif
vm_object_unlock(backing_object);
} else {
#if TASK_SWAPPER
if (object->res_count == 0) {
vm_object_res_reference(backing_object);
}
#endif
vm_object_unlock(object);
vm_object_unlock(backing_object);
vm_object_deallocate(backing_object);
vm_object_lock(object);
}
object_bypasses++;
}
static unsigned long vm_object_collapse_calls = 0;
static unsigned long vm_object_collapse_objects = 0;
static unsigned long vm_object_collapse_do_collapse = 0;
static unsigned long vm_object_collapse_do_bypass = 0;
static unsigned long vm_object_collapse_delays = 0;
__private_extern__ void
vm_object_collapse(
register vm_object_t object,
register vm_object_offset_t hint_offset,
boolean_t can_bypass)
{
register vm_object_t backing_object;
register unsigned int rcount;
register unsigned int size;
vm_object_t original_object;
vm_object_collapse_calls++;
if (! vm_object_collapse_allowed &&
! (can_bypass && vm_object_bypass_allowed)) {
return;
}
XPR(XPR_VM_OBJECT, "vm_object_collapse, obj 0x%X\n",
(integer_t)object, 0,0,0,0);
if (object == VM_OBJECT_NULL)
return;
original_object = object;
while (TRUE) {
vm_object_collapse_objects++;
backing_object = object->shadow;
if (backing_object == VM_OBJECT_NULL) {
if (object != original_object) {
vm_object_unlock(object);
}
return;
}
if (object->paging_in_progress != 0) {
vm_object_lock(backing_object);
if (object != original_object) {
vm_object_unlock(object);
}
object = backing_object;
continue;
}
vm_object_lock(backing_object);
if (!backing_object->internal ||
backing_object->paging_in_progress != 0) {
if (object != original_object) {
vm_object_unlock(object);
}
object = backing_object;
continue;
}
if (backing_object->shadow != VM_OBJECT_NULL &&
backing_object->shadow->copy == backing_object) {
if (object != original_object) {
vm_object_unlock(object);
}
object = backing_object;
continue;
}
if (backing_object->ref_count == 1 &&
(!object->pager_created
#if !MACH_PAGEMAP
|| !backing_object->pager_created
#endif
) && vm_object_collapse_allowed) {
XPR(XPR_VM_OBJECT,
"vm_object_collapse: %x to %x, pager %x, pager_control %x\n",
(integer_t)backing_object, (integer_t)object,
(integer_t)backing_object->pager,
(integer_t)backing_object->pager_control, 0);
if (! vm_object_cache_lock_try()) {
if (object != original_object) {
vm_object_unlock(object);
}
vm_object_unlock(backing_object);
return;
}
vm_object_do_collapse(object, backing_object);
vm_object_collapse_do_collapse++;
continue;
}
if (! (can_bypass && vm_object_bypass_allowed)) {
if (object != original_object) {
vm_object_unlock(object);
}
object = backing_object;
continue;
}
size = atop(object->size);
rcount = object->resident_page_count;
if (rcount != size) {
vm_object_offset_t offset;
vm_object_offset_t backing_offset;
unsigned int backing_rcount;
unsigned int lookups = 0;
if (backing_object->pager_created
#if MACH_PAGEMAP
&& (backing_object->existence_map == VM_EXTERNAL_NULL)
#endif
) {
if (object != original_object) {
vm_object_unlock(object);
}
object = backing_object;
continue;
}
if (object->pager_created
#if MACH_PAGEMAP
&& (object->existence_map == VM_EXTERNAL_NULL)
#endif
) {
if (object != original_object) {
vm_object_unlock(object);
}
object = backing_object;
continue;
}
backing_offset = object->shadow_offset;
backing_rcount = backing_object->resident_page_count;
#if MACH_PAGEMAP
#define EXISTS_IN_OBJECT(obj, off, rc) \
(vm_external_state_get((obj)->existence_map, \
(vm_offset_t)(off)) == VM_EXTERNAL_STATE_EXISTS || \
((rc) && ++lookups && vm_page_lookup((obj), (off)) != VM_PAGE_NULL && (rc)--))
#else
#define EXISTS_IN_OBJECT(obj, off, rc) \
(((rc) && ++lookups && vm_page_lookup((obj), (off)) != VM_PAGE_NULL && (rc)--))
#endif
if (object->cow_hint != ~(vm_offset_t)0)
hint_offset = (vm_object_offset_t)object->cow_hint;
else
hint_offset = (hint_offset > 8 * PAGE_SIZE_64) ?
(hint_offset - 8 * PAGE_SIZE_64) : 0;
if (EXISTS_IN_OBJECT(backing_object, hint_offset +
backing_offset, backing_rcount) &&
!EXISTS_IN_OBJECT(object, hint_offset, rcount)) {
object->cow_hint = (vm_offset_t)hint_offset;
if (object != original_object) {
vm_object_unlock(object);
}
object = backing_object;
continue;
}
if (backing_rcount &&
#if MACH_PAGEMAP
size > ((backing_object->existence_map) ?
backing_rcount : (backing_rcount >> 1))
#else
size > (backing_rcount >> 1)
#endif
) {
unsigned int rc = rcount;
vm_page_t p;
backing_rcount = backing_object->resident_page_count;
p = (vm_page_t)queue_first(&backing_object->memq);
do {
if (lookups > 256) {
vm_object_collapse_delays++;
lookups = 0;
mutex_pause(0);
}
offset = (p->offset - backing_offset);
if (offset < object->size &&
offset != hint_offset &&
!EXISTS_IN_OBJECT(object, offset, rc)) {
object->cow_hint = (vm_offset_t)offset;
break;
}
p = (vm_page_t) queue_next(&p->listq);
} while (--backing_rcount);
if (backing_rcount != 0 ) {
if (object != original_object) {
vm_object_unlock(object);
}
object = backing_object;
continue;
}
}
#if MACH_PAGEMAP
if (backing_rcount || backing_object->existence_map) {
#else
if (backing_rcount) {
#endif
offset = hint_offset;
while((offset =
(offset + PAGE_SIZE_64 < object->size) ?
(offset + PAGE_SIZE_64) : 0) != hint_offset) {
if (lookups > 256) {
vm_object_collapse_delays++;
lookups = 0;
mutex_pause(0);
}
if (EXISTS_IN_OBJECT(backing_object, offset +
backing_offset, backing_rcount) &&
!EXISTS_IN_OBJECT(object, offset, rcount)) {
object->cow_hint = (vm_offset_t)offset;
break;
}
}
if (offset != hint_offset) {
if (object != original_object) {
vm_object_unlock(object);
}
object = backing_object;
continue;
}
}
}
object->cow_hint = (vm_offset_t)0;
vm_object_do_bypass(object, backing_object);
vm_object_collapse_do_bypass++;
continue;
}
if (object != original_object) {
vm_object_unlock(object);
}
}
unsigned int vm_object_page_remove_lookup = 0;
unsigned int vm_object_page_remove_iterate = 0;
__private_extern__ void
vm_object_page_remove(
register vm_object_t object,
register vm_object_offset_t start,
register vm_object_offset_t end)
{
register vm_page_t p, next;
if (atop_64(end - start) < (unsigned)object->resident_page_count/16) {
vm_object_page_remove_lookup++;
for (; start < end; start += PAGE_SIZE_64) {
p = vm_page_lookup(object, start);
if (p != VM_PAGE_NULL) {
assert(!p->cleaning && !p->pageout);
if (!p->fictitious && p->pmapped)
pmap_disconnect(p->phys_page);
VM_PAGE_FREE(p);
}
}
} else {
vm_object_page_remove_iterate++;
p = (vm_page_t) queue_first(&object->memq);
while (!queue_end(&object->memq, (queue_entry_t) p)) {
next = (vm_page_t) queue_next(&p->listq);
if ((start <= p->offset) && (p->offset < end)) {
assert(!p->cleaning && !p->pageout);
if (!p->fictitious && p->pmapped)
pmap_disconnect(p->phys_page);
VM_PAGE_FREE(p);
}
p = next;
}
}
}
static int vm_object_coalesce_count = 0;
__private_extern__ boolean_t
vm_object_coalesce(
register vm_object_t prev_object,
vm_object_t next_object,
vm_object_offset_t prev_offset,
__unused vm_object_offset_t next_offset,
vm_object_size_t prev_size,
vm_object_size_t next_size)
{
vm_object_size_t newsize;
#ifdef lint
next_offset++;
#endif
if (next_object != VM_OBJECT_NULL) {
return(FALSE);
}
if (prev_object == VM_OBJECT_NULL) {
return(TRUE);
}
XPR(XPR_VM_OBJECT,
"vm_object_coalesce: 0x%X prev_off 0x%X prev_size 0x%X next_size 0x%X\n",
(integer_t)prev_object, prev_offset, prev_size, next_size, 0);
vm_object_lock(prev_object);
vm_object_collapse(prev_object, prev_offset, TRUE);
if ((prev_object->ref_count > 1) ||
prev_object->pager_created ||
(prev_object->shadow != VM_OBJECT_NULL) ||
(prev_object->copy != VM_OBJECT_NULL) ||
(prev_object->true_share != FALSE) ||
(prev_object->purgable != VM_PURGABLE_DENY) ||
(prev_object->paging_in_progress != 0)) {
vm_object_unlock(prev_object);
return(FALSE);
}
vm_object_coalesce_count++;
vm_object_page_remove(prev_object,
prev_offset + prev_size,
prev_offset + prev_size + next_size);
newsize = prev_offset + prev_size + next_size;
if (newsize > prev_object->size) {
#if MACH_PAGEMAP
assert(prev_object->existence_map == VM_EXTERNAL_NULL);
#endif
prev_object->size = newsize;
}
vm_object_unlock(prev_object);
return(TRUE);
}
void
vm_object_page_map(
vm_object_t object,
vm_object_offset_t offset,
vm_object_size_t size,
vm_object_offset_t (*map_fn)(void *map_fn_data,
vm_object_offset_t offset),
void *map_fn_data)
{
int num_pages;
int i;
vm_page_t m;
vm_page_t old_page;
vm_object_offset_t addr;
num_pages = atop_64(size);
for (i = 0; i < num_pages; i++, offset += PAGE_SIZE_64) {
addr = (*map_fn)(map_fn_data, offset);
while ((m = vm_page_grab_fictitious()) == VM_PAGE_NULL)
vm_page_more_fictitious();
vm_object_lock(object);
if ((old_page = vm_page_lookup(object, offset))
!= VM_PAGE_NULL)
{
vm_page_lock_queues();
vm_page_free(old_page);
vm_page_unlock_queues();
}
vm_page_init(m, addr);
m->private = TRUE;
m->wire_count = 1;
vm_page_insert(m, object, offset);
PAGE_WAKEUP_DONE(m);
vm_object_unlock(object);
}
}
#include <mach_kdb.h>
#if MACH_KDB
#include <ddb/db_output.h>
#include <vm/vm_print.h>
#define printf kdbprintf
extern boolean_t vm_object_cached(
vm_object_t object);
extern void print_bitstring(
char byte);
boolean_t vm_object_print_pages = FALSE;
void
print_bitstring(
char byte)
{
printf("%c%c%c%c%c%c%c%c",
((byte & (1 << 0)) ? '1' : '0'),
((byte & (1 << 1)) ? '1' : '0'),
((byte & (1 << 2)) ? '1' : '0'),
((byte & (1 << 3)) ? '1' : '0'),
((byte & (1 << 4)) ? '1' : '0'),
((byte & (1 << 5)) ? '1' : '0'),
((byte & (1 << 6)) ? '1' : '0'),
((byte & (1 << 7)) ? '1' : '0'));
}
boolean_t
vm_object_cached(
register vm_object_t object)
{
register vm_object_t o;
queue_iterate(&vm_object_cached_list, o, vm_object_t, cached_list) {
if (object == o) {
return TRUE;
}
}
return FALSE;
}
#if MACH_PAGEMAP
void
vm_external_print(
vm_external_map_t emap,
vm_size_t size)
{
if (emap == VM_EXTERNAL_NULL) {
printf("0 ");
} else {
vm_size_t existence_size = stob(size);
printf("{ size=%d, map=[", existence_size);
if (existence_size > 0) {
print_bitstring(emap[0]);
}
if (existence_size > 1) {
print_bitstring(emap[1]);
}
if (existence_size > 2) {
printf("...");
print_bitstring(emap[existence_size-1]);
}
printf("] }\n");
}
return;
}
#endif
int
vm_follow_object(
vm_object_t object)
{
int count = 0;
int orig_db_indent = db_indent;
while (TRUE) {
if (object == VM_OBJECT_NULL) {
db_indent = orig_db_indent;
return count;
}
count += 1;
iprintf("object 0x%x", object);
printf(", shadow=0x%x", object->shadow);
printf(", copy=0x%x", object->copy);
printf(", pager=0x%x", object->pager);
printf(", ref=%d\n", object->ref_count);
db_indent += 2;
object = object->shadow;
}
}
void
vm_object_print(db_expr_t db_addr, __unused boolean_t have_addr,
__unused db_expr_t arg_count, __unused char *modif)
{
vm_object_t object;
register vm_page_t p;
const char *s;
register int count;
object = (vm_object_t) (long) db_addr;
if (object == VM_OBJECT_NULL)
return;
iprintf("object 0x%x\n", object);
db_indent += 2;
iprintf("size=0x%x", object->size);
printf(", memq_hint=%p", object->memq_hint);
printf(", ref_count=%d\n", object->ref_count);
iprintf("");
#if TASK_SWAPPER
printf("res_count=%d, ", object->res_count);
#endif
printf("resident_page_count=%d\n", object->resident_page_count);
iprintf("shadow=0x%x", object->shadow);
if (object->shadow) {
register int i = 0;
vm_object_t shadow = object;
while((shadow = shadow->shadow))
i++;
printf(" (depth %d)", i);
}
printf(", copy=0x%x", object->copy);
printf(", shadow_offset=0x%x", object->shadow_offset);
printf(", last_alloc=0x%x\n", object->last_alloc);
iprintf("pager=0x%x", object->pager);
printf(", paging_offset=0x%x", object->paging_offset);
printf(", pager_control=0x%x\n", object->pager_control);
iprintf("copy_strategy=%d[", object->copy_strategy);
switch (object->copy_strategy) {
case MEMORY_OBJECT_COPY_NONE:
printf("copy_none");
break;
case MEMORY_OBJECT_COPY_CALL:
printf("copy_call");
break;
case MEMORY_OBJECT_COPY_DELAY:
printf("copy_delay");
break;
case MEMORY_OBJECT_COPY_SYMMETRIC:
printf("copy_symmetric");
break;
case MEMORY_OBJECT_COPY_INVALID:
printf("copy_invalid");
break;
default:
printf("?");
}
printf("]");
iprintf("all_wanted=0x%x<", object->all_wanted);
s = "";
if (vm_object_wanted(object, VM_OBJECT_EVENT_INITIALIZED)) {
printf("%sinit", s);
s = ",";
}
if (vm_object_wanted(object, VM_OBJECT_EVENT_PAGER_READY)) {
printf("%sready", s);
s = ",";
}
if (vm_object_wanted(object, VM_OBJECT_EVENT_PAGING_IN_PROGRESS)) {
printf("%spaging", s);
s = ",";
}
if (vm_object_wanted(object, VM_OBJECT_EVENT_LOCK_IN_PROGRESS)) {
printf("%slock", s);
s = ",";
}
if (vm_object_wanted(object, VM_OBJECT_EVENT_UNCACHING)) {
printf("%suncaching", s);
s = ",";
}
if (vm_object_wanted(object, VM_OBJECT_EVENT_COPY_CALL)) {
printf("%scopy_call", s);
s = ",";
}
if (vm_object_wanted(object, VM_OBJECT_EVENT_CACHING)) {
printf("%scaching", s);
s = ",";
}
printf(">");
printf(", paging_in_progress=%d\n", object->paging_in_progress);
iprintf("%screated, %sinit, %sready, %spersist, %strusted, %spageout, %s, %s\n",
(object->pager_created ? "" : "!"),
(object->pager_initialized ? "" : "!"),
(object->pager_ready ? "" : "!"),
(object->can_persist ? "" : "!"),
(object->pager_trusted ? "" : "!"),
(object->pageout ? "" : "!"),
(object->internal ? "internal" : "external"),
(object->temporary ? "temporary" : "permanent"));
iprintf("%salive, %spurgeable, %spurgeable_volatile, %spurgeable_empty, %sshadowed, %scached, %sprivate\n",
(object->alive ? "" : "!"),
((object->purgable != VM_PURGABLE_DENY) ? "" : "!"),
((object->purgable == VM_PURGABLE_VOLATILE) ? "" : "!"),
((object->purgable == VM_PURGABLE_EMPTY) ? "" : "!"),
(object->shadowed ? "" : "!"),
(vm_object_cached(object) ? "" : "!"),
(object->private ? "" : "!"));
iprintf("%sadvisory_pageout, %ssilent_overwrite\n",
(object->advisory_pageout ? "" : "!"),
(object->silent_overwrite ? "" : "!"));
#if MACH_PAGEMAP
iprintf("existence_map=");
vm_external_print(object->existence_map, object->size);
#endif
#if MACH_ASSERT
iprintf("paging_object=0x%x\n", object->paging_object);
#endif
if (vm_object_print_pages) {
count = 0;
p = (vm_page_t) queue_first(&object->memq);
while (!queue_end(&object->memq, (queue_entry_t) p)) {
if (count == 0) {
iprintf("memory:=");
} else if (count == 2) {
printf("\n");
iprintf(" ...");
count = 0;
} else {
printf(",");
}
count++;
printf("(off=0x%llX,page=%p)", p->offset, p);
p = (vm_page_t) queue_next(&p->listq);
}
if (count != 0) {
printf("\n");
}
}
db_indent -= 2;
}
boolean_t vm_object_find(vm_object_t object);
boolean_t vm_object_print_verbose = FALSE;
boolean_t
vm_object_find(
vm_object_t object)
{
task_t task;
vm_map_t map;
vm_map_entry_t entry;
boolean_t found = FALSE;
queue_iterate(&tasks, task, task_t, tasks) {
map = task->map;
for (entry = vm_map_first_entry(map);
entry && entry != vm_map_to_entry(map);
entry = entry->vme_next) {
vm_object_t obj;
if (entry->is_sub_map)
continue;
if (entry)
obj = entry->object.vm_object;
else
continue;
while (obj != VM_OBJECT_NULL) {
if (obj == object) {
if (!found) {
printf("TASK\t\tMAP\t\tENTRY\n");
found = TRUE;
}
printf("0x%x\t0x%x\t0x%x\n",
task, map, entry);
}
obj = obj->shadow;
}
}
}
return(found);
}
#endif
kern_return_t
vm_object_populate_with_private(
vm_object_t object,
vm_object_offset_t offset,
ppnum_t phys_page,
vm_size_t size)
{
ppnum_t base_page;
vm_object_offset_t base_offset;
if(!object->private)
return KERN_FAILURE;
base_page = phys_page;
vm_object_lock(object);
if(!object->phys_contiguous) {
vm_page_t m;
if((base_offset = trunc_page_64(offset)) != offset) {
vm_object_unlock(object);
return KERN_FAILURE;
}
base_offset += object->paging_offset;
while(size) {
m = vm_page_lookup(object, base_offset);
if(m != VM_PAGE_NULL) {
if(m->fictitious) {
if (m->phys_page !=
vm_page_guard_addr) {
vm_page_lockspin_queues();
m->fictitious = FALSE;
m->private = TRUE;
m->phys_page = base_page;
if(!m->busy) {
m->busy = TRUE;
}
if(!m->absent) {
m->absent = TRUE;
}
m->list_req_pending = TRUE;
vm_page_unlock_queues();
}
} else if (m->phys_page != base_page) {
if (m->pmapped) {
pmap_disconnect(m->phys_page);
}
m->phys_page = base_page;
}
m->encrypted = FALSE;
} else {
while ((m = vm_page_grab_fictitious())
== VM_PAGE_NULL)
vm_page_more_fictitious();
vm_page_lockspin_queues();
m->fictitious = FALSE;
m->private = TRUE;
m->phys_page = base_page;
m->list_req_pending = TRUE;
m->absent = TRUE;
m->unusual = TRUE;
vm_page_unlock_queues();
vm_page_insert(m, object, base_offset);
}
base_page++;
base_offset += PAGE_SIZE;
size -= PAGE_SIZE;
}
} else {
object->shadow_offset = (vm_object_offset_t)(phys_page << 12);
object->size = size;
}
vm_object_unlock(object);
return KERN_SUCCESS;
}
__private_extern__ kern_return_t
memory_object_free_from_cache(
__unused host_t host,
memory_object_pager_ops_t pager_ops,
int *count)
{
int object_released = 0;
register vm_object_t object = VM_OBJECT_NULL;
vm_object_t shadow;
try_again:
vm_object_cache_lock();
queue_iterate(&vm_object_cached_list, object,
vm_object_t, cached_list) {
if (object->pager &&
(pager_ops == object->pager->mo_pager_ops)) {
vm_object_lock(object);
queue_remove(&vm_object_cached_list, object,
vm_object_t, cached_list);
vm_object_cached_count--;
assert(object->pager_initialized);
assert(object->ref_count == 0);
vm_object_lock_assert_exclusive(object);
object->ref_count++;
shadow = object->pageout?VM_OBJECT_NULL:object->shadow;
if ((vm_object_terminate(object) == KERN_SUCCESS)
&& (shadow != VM_OBJECT_NULL)) {
vm_object_deallocate(shadow);
}
if(object_released++ == *count)
return KERN_SUCCESS;
goto try_again;
}
}
vm_object_cache_unlock();
*count = object_released;
return KERN_SUCCESS;
}
kern_return_t
memory_object_create_named(
memory_object_t pager,
memory_object_offset_t size,
memory_object_control_t *control)
{
vm_object_t object;
vm_object_hash_entry_t entry;
*control = MEMORY_OBJECT_CONTROL_NULL;
if (pager == MEMORY_OBJECT_NULL)
return KERN_INVALID_ARGUMENT;
vm_object_cache_lock();
entry = vm_object_hash_lookup(pager, FALSE);
if ((entry != VM_OBJECT_HASH_ENTRY_NULL) &&
(entry->object != VM_OBJECT_NULL)) {
if (entry->object->named == TRUE)
panic("memory_object_create_named: caller already holds the right"); }
vm_object_cache_unlock();
if ((object = vm_object_enter(pager, size, FALSE, FALSE, TRUE))
== VM_OBJECT_NULL) {
return(KERN_INVALID_OBJECT);
}
if (object != VM_OBJECT_NULL) {
vm_object_lock(object);
object->named = TRUE;
while (!object->pager_ready) {
vm_object_sleep(object,
VM_OBJECT_EVENT_PAGER_READY,
THREAD_UNINT);
}
*control = object->pager_control;
vm_object_unlock(object);
}
return (KERN_SUCCESS);
}
kern_return_t
memory_object_recover_named(
memory_object_control_t control,
boolean_t wait_on_terminating)
{
vm_object_t object;
vm_object_cache_lock();
object = memory_object_control_to_vm_object(control);
if (object == VM_OBJECT_NULL) {
vm_object_cache_unlock();
return (KERN_INVALID_ARGUMENT);
}
restart:
vm_object_lock(object);
if (object->terminating && wait_on_terminating) {
vm_object_cache_unlock();
vm_object_wait(object,
VM_OBJECT_EVENT_PAGING_IN_PROGRESS,
THREAD_UNINT);
vm_object_cache_lock();
goto restart;
}
if (!object->alive) {
vm_object_cache_unlock();
vm_object_unlock(object);
return KERN_FAILURE;
}
if (object->named == TRUE) {
vm_object_cache_unlock();
vm_object_unlock(object);
return KERN_SUCCESS;
}
if((object->ref_count == 0) && (!object->terminating)){
queue_remove(&vm_object_cached_list, object,
vm_object_t, cached_list);
vm_object_cached_count--;
XPR(XPR_VM_OBJECT_CACHE,
"memory_object_recover_named: removing %X, head (%X, %X)\n",
(integer_t)object,
(integer_t)vm_object_cached_list.next,
(integer_t)vm_object_cached_list.prev, 0,0);
}
vm_object_cache_unlock();
object->named = TRUE;
vm_object_lock_assert_exclusive(object);
object->ref_count++;
vm_object_res_reference(object);
while (!object->pager_ready) {
vm_object_sleep(object,
VM_OBJECT_EVENT_PAGER_READY,
THREAD_UNINT);
}
vm_object_unlock(object);
return (KERN_SUCCESS);
}
__private_extern__ kern_return_t
vm_object_release_name(
vm_object_t object,
int flags)
{
vm_object_t shadow;
boolean_t original_object = TRUE;
while (object != VM_OBJECT_NULL) {
vm_object_cache_lock();
vm_object_lock(object);
assert(object->alive);
if(original_object)
assert(object->named);
assert(object->ref_count > 0);
if (object->pager_created && !object->pager_initialized) {
assert(!object->can_persist);
vm_object_assert_wait(object,
VM_OBJECT_EVENT_INITIALIZED,
THREAD_UNINT);
vm_object_unlock(object);
vm_object_cache_unlock();
thread_block(THREAD_CONTINUE_NULL);
continue;
}
if (((object->ref_count > 1)
&& (flags & MEMORY_OBJECT_TERMINATE_IDLE))
|| (object->terminating)) {
vm_object_unlock(object);
vm_object_cache_unlock();
return KERN_FAILURE;
} else {
if (flags & MEMORY_OBJECT_RELEASE_NO_OP) {
vm_object_unlock(object);
vm_object_cache_unlock();
return KERN_SUCCESS;
}
}
if ((flags & MEMORY_OBJECT_RESPECT_CACHE) &&
(object->ref_count == 1)) {
if(original_object)
object->named = FALSE;
vm_object_unlock(object);
vm_object_cache_unlock();
vm_object_deallocate(object);
return KERN_SUCCESS;
}
VM_OBJ_RES_DECR(object);
shadow = object->pageout?VM_OBJECT_NULL:object->shadow;
if(object->ref_count == 1) {
if(vm_object_terminate(object) != KERN_SUCCESS) {
if(original_object) {
return KERN_FAILURE;
} else {
return KERN_SUCCESS;
}
}
if (shadow != VM_OBJECT_NULL) {
original_object = FALSE;
object = shadow;
continue;
}
return KERN_SUCCESS;
} else {
vm_object_lock_assert_exclusive(object);
object->ref_count--;
assert(object->ref_count > 0);
if(original_object)
object->named = FALSE;
vm_object_unlock(object);
vm_object_cache_unlock();
return KERN_SUCCESS;
}
}
assert(0);
return KERN_FAILURE;
}
__private_extern__ kern_return_t
vm_object_lock_request(
vm_object_t object,
vm_object_offset_t offset,
vm_object_size_t size,
memory_object_return_t should_return,
int flags,
vm_prot_t prot)
{
__unused boolean_t should_flush;
should_flush = flags & MEMORY_OBJECT_DATA_FLUSH;
XPR(XPR_MEMORY_OBJECT,
"vm_o_lock_request, obj 0x%X off 0x%X size 0x%X flags %X prot %X\n",
(integer_t)object, offset, size,
(((should_return&1)<<1)|should_flush), prot);
if (object == VM_OBJECT_NULL)
return (KERN_INVALID_ARGUMENT);
if ((prot & ~VM_PROT_ALL) != 0 && prot != VM_PROT_NO_CHANGE)
return (KERN_INVALID_ARGUMENT);
size = round_page_64(size);
vm_object_lock(object);
vm_object_paging_begin(object);
(void)vm_object_update(object,
offset, size, NULL, NULL, should_return, flags, prot);
vm_object_paging_end(object);
vm_object_unlock(object);
return (KERN_SUCCESS);
}
unsigned int
vm_object_purge(vm_object_t object)
{
vm_page_t p, next;
unsigned int num_purged_pages;
vm_page_t local_freeq;
unsigned long local_freed;
int purge_loop_quota;
#define PURGE_BATCH_FREE_LIMIT 50
#define PURGE_LOOP_QUOTA 100
num_purged_pages = 0;
if (object->purgable == VM_PURGABLE_DENY)
return num_purged_pages;
assert(object->purgable != VM_PURGABLE_NONVOLATILE);
object->purgable = VM_PURGABLE_EMPTY;
assert(object->copy == VM_OBJECT_NULL);
assert(object->copy_strategy == MEMORY_OBJECT_COPY_NONE);
purge_loop_quota = PURGE_LOOP_QUOTA;
local_freeq = VM_PAGE_NULL;
local_freed = 0;
next = (vm_page_t)queue_first(&object->memq);
while (!queue_end(&object->memq, (queue_entry_t)next)) {
p = next;
next = (vm_page_t)queue_next(&next->listq);
if (purge_loop_quota-- == 0) {
if (local_freeq != VM_PAGE_NULL) {
vm_page_free_list(local_freeq);
local_freeq = VM_PAGE_NULL;
local_freed = 0;
}
mutex_yield(&vm_page_queue_lock);
purge_loop_quota = PURGE_LOOP_QUOTA;
}
if (p->busy || p->cleaning || p->laundry ||
p->list_req_pending) {
continue;
}
if (p->wire_count) {
continue;
}
assert(!p->laundry);
assert(p->object != kernel_object);
p->busy = TRUE;
if (p->pmapped == TRUE) {
int refmod_state;
refmod_state = pmap_disconnect(p->phys_page);
if (refmod_state & VM_MEM_MODIFIED) {
p->dirty = TRUE;
}
}
if (p->dirty || p->precious) {
num_purged_pages++;
vm_page_purged_count++;
}
vm_page_free_prepare(p);
assert(p->pageq.next == NULL &&
p->pageq.prev == NULL);
p->pageq.next = (queue_entry_t) local_freeq;
local_freeq = p;
if (++local_freed >= PURGE_BATCH_FREE_LIMIT) {
vm_page_free_list(local_freeq);
local_freeq = VM_PAGE_NULL;
local_freed = 0;
}
}
if (local_freeq != VM_PAGE_NULL) {
vm_page_free_list(local_freeq);
local_freeq = VM_PAGE_NULL;
local_freed = 0;
}
return num_purged_pages;
}
kern_return_t
vm_object_purgable_control(
vm_object_t object,
vm_purgable_t control,
int *state)
{
int old_state;
int new_state;
if (object == VM_OBJECT_NULL) {
return KERN_INVALID_ARGUMENT;
}
old_state = object->purgable;
if (old_state == VM_PURGABLE_DENY)
return KERN_INVALID_ARGUMENT;
assert(object->copy == VM_OBJECT_NULL);
assert(object->copy_strategy == MEMORY_OBJECT_COPY_NONE);
if (control == VM_PURGABLE_GET_STATE) {
*state = old_state;
return KERN_SUCCESS;
}
new_state = *state & VM_PURGABLE_STATE_MASK;
switch (new_state) {
case VM_PURGABLE_DENY:
case VM_PURGABLE_NONVOLATILE:
object->purgable = new_state;
if (old_state != VM_PURGABLE_NONVOLATILE) {
vm_page_lock_queues();
assert(vm_page_purgeable_count >=
object->resident_page_count);
vm_page_purgeable_count -= object->resident_page_count;
if (old_state==VM_PURGABLE_VOLATILE) {
assert(object->objq.next != NULL && object->objq.prev != NULL);
purgeable_q_t queue = vm_purgeable_object_remove(object);
assert(queue);
vm_purgeable_token_delete_first(queue);
assert(queue->debug_count_objects>=0);
};
vm_page_unlock_queues();
}
break;
case VM_PURGABLE_VOLATILE:
if ((old_state != VM_PURGABLE_NONVOLATILE) && (old_state != VM_PURGABLE_VOLATILE))
break;
purgeable_q_t queue;
if ((*state&VM_PURGABLE_ORDERING_MASK) == VM_PURGABLE_ORDERING_OBSOLETE)
queue = &purgeable_queues[PURGEABLE_Q_TYPE_FIFO];
else {
if ((*state&VM_PURGABLE_BEHAVIOR_MASK) == VM_PURGABLE_BEHAVIOR_FIFO)
queue = &purgeable_queues[PURGEABLE_Q_TYPE_FIFO];
else
queue = &purgeable_queues[PURGEABLE_Q_TYPE_LIFO];
}
if (old_state == VM_PURGABLE_NONVOLATILE) {
vm_page_lock_queues();
kern_return_t result = vm_purgeable_token_add(queue);
if (result != KERN_SUCCESS) {
vm_page_unlock_queues();
return result;
}
vm_page_purgeable_count += object->resident_page_count;
vm_page_unlock_queues();
object->purgable = new_state;
assert(object->objq.next == NULL && object->objq.prev == NULL);
}
else if (old_state == VM_PURGABLE_VOLATILE) {
assert(object->objq.next != NULL && object->objq.prev != NULL);
purgeable_q_t old_queue=vm_purgeable_object_remove(object);
assert(old_queue);
if (old_queue != queue) {
kern_return_t result;
vm_page_lock_queues();
vm_purgeable_token_delete_first(old_queue);
result = vm_purgeable_token_add(queue);
vm_page_unlock_queues();
assert(result==KERN_SUCCESS);
}
};
vm_purgeable_object_add(object, queue, (*state&VM_VOLATILE_GROUP_MASK)>>VM_VOLATILE_GROUP_SHIFT );
assert(queue->debug_count_objects>=0);
break;
case VM_PURGABLE_EMPTY:
if (old_state != new_state)
{
assert(old_state==VM_PURGABLE_NONVOLATILE || old_state==VM_PURGABLE_VOLATILE);
if(old_state==VM_PURGABLE_VOLATILE) {
assert(object->objq.next != NULL && object->objq.prev != NULL);
purgeable_q_t old_queue=vm_purgeable_object_remove(object);
assert(old_queue);
vm_page_lock_queues();
vm_purgeable_token_delete_first(old_queue);
}
if (old_state==VM_PURGABLE_NONVOLATILE) {
vm_page_purgeable_count += object->resident_page_count;
vm_page_lock_queues();
}
(void) vm_object_purge(object);
vm_page_unlock_queues();
}
break;
}
*state = old_state;
return KERN_SUCCESS;
}
#if TASK_SWAPPER
__private_extern__ void
vm_object_res_deallocate(
vm_object_t object)
{
vm_object_t orig_object = object;
assert(object->res_count > 0);
while (--object->res_count == 0) {
assert(object->ref_count >= object->res_count);
vm_object_deactivate_all_pages(object);
if (object->shadow != VM_OBJECT_NULL) {
vm_object_t tmp_object = object->shadow;
vm_object_lock(tmp_object);
if (object != orig_object)
vm_object_unlock(object);
object = tmp_object;
assert(object->res_count > 0);
} else
break;
}
if (object != orig_object)
vm_object_unlock(object);
}
__private_extern__ void
vm_object_res_reference(
vm_object_t object)
{
vm_object_t orig_object = object;
while ((++object->res_count == 1) &&
(object->shadow != VM_OBJECT_NULL)) {
vm_object_t tmp_object = object->shadow;
assert(object->ref_count >= object->res_count);
vm_object_lock(tmp_object);
if (object != orig_object)
vm_object_unlock(object);
object = tmp_object;
}
if (object != orig_object)
vm_object_unlock(object);
assert(orig_object->ref_count >= orig_object->res_count);
}
#endif
#ifdef vm_object_reference
#undef vm_object_reference
#endif
__private_extern__ void
vm_object_reference(
register vm_object_t object)
{
if (object == VM_OBJECT_NULL)
return;
vm_object_lock(object);
assert(object->ref_count > 0);
vm_object_reference_locked(object);
vm_object_unlock(object);
}
#ifdef MACH_BSD
kern_return_t
adjust_vm_object_cache(
__unused vm_size_t oval,
vm_size_t nval)
{
vm_object_cached_max = nval;
vm_object_cache_trim(FALSE);
return (KERN_SUCCESS);
}
#endif
kern_return_t
vm_object_transpose(
vm_object_t object1,
vm_object_t object2,
vm_object_size_t transpose_size)
{
vm_object_t tmp_object;
kern_return_t retval;
boolean_t object1_locked, object2_locked;
boolean_t object1_paging, object2_paging;
vm_page_t page;
vm_object_offset_t page_offset;
tmp_object = VM_OBJECT_NULL;
object1_locked = FALSE; object2_locked = FALSE;
object1_paging = FALSE; object2_paging = FALSE;
if (object1 == object2 ||
object1 == VM_OBJECT_NULL ||
object2 == VM_OBJECT_NULL) {
retval = KERN_INVALID_VALUE;
goto done;
}
vm_object_lock(object1);
object1_locked = TRUE;
if (!object1->alive || object1->terminating ||
object1->copy || object1->shadow || object1->shadowed ||
object1->purgable != VM_PURGABLE_DENY) {
retval = KERN_INVALID_VALUE;
goto done;
}
vm_object_paging_begin(object1);
object1_paging = TRUE;
vm_object_unlock(object1);
object1_locked = FALSE;
vm_object_lock(object2);
object2_locked = TRUE;
if (! object2->alive || object2->terminating ||
object2->copy || object2->shadow || object2->shadowed ||
object2->purgable != VM_PURGABLE_DENY) {
retval = KERN_INVALID_VALUE;
goto done;
}
vm_object_paging_begin(object2);
object2_paging = TRUE;
vm_object_unlock(object2);
object2_locked = FALSE;
tmp_object = vm_object_allocate(transpose_size);
vm_object_lock(tmp_object);
vm_object_paging_begin(tmp_object);
tmp_object->can_persist = FALSE;
if (object1 < object2) {
vm_object_lock(object1);
vm_object_lock(object2);
} else {
vm_object_lock(object2);
vm_object_lock(object1);
}
object1_locked = TRUE;
object2_locked = TRUE;
if (object1->size != object2->size ||
object1->size != transpose_size) {
retval = KERN_INVALID_VALUE;
goto done;
}
if (object1->phys_contiguous || queue_empty(&object1->memq)) {
while (!queue_empty(&object2->memq)) {
page = (vm_page_t) queue_first(&object2->memq);
vm_page_rename(page, object1, page->offset, FALSE);
}
assert(queue_empty(&object2->memq));
} else if (object2->phys_contiguous || queue_empty(&object2->memq)) {
while (!queue_empty(&object1->memq)) {
page = (vm_page_t) queue_first(&object1->memq);
vm_page_rename(page, object2, page->offset, FALSE);
}
assert(queue_empty(&object1->memq));
} else {
vm_page_lock_queues();
while (!queue_empty(&object1->memq)) {
page = (vm_page_t) queue_first(&object1->memq);
page_offset = page->offset;
vm_page_remove(page);
page->offset = page_offset;
queue_enter(&tmp_object->memq, page, vm_page_t, listq);
}
vm_page_unlock_queues();
assert(queue_empty(&object1->memq));
while (!queue_empty(&object2->memq)) {
page = (vm_page_t) queue_first(&object2->memq);
vm_page_rename(page, object1, page->offset, FALSE);
}
assert(queue_empty(&object2->memq));
while (!queue_empty(&tmp_object->memq)) {
page = (vm_page_t) queue_first(&tmp_object->memq);
queue_remove(&tmp_object->memq, page,
vm_page_t, listq);
vm_page_insert(page, object2, page->offset);
}
assert(queue_empty(&tmp_object->memq));
}
#define __TRANSPOSE_FIELD(field) \
MACRO_BEGIN \
tmp_object->field = object1->field; \
object1->field = object2->field; \
object2->field = tmp_object->field; \
MACRO_END
assert(object1->size == object2->size);
#if TASK_SWAPPER
#endif
assert(!object1->copy);
assert(!object2->copy);
assert(!object1->shadow);
assert(!object2->shadow);
__TRANSPOSE_FIELD(shadow_offset);
__TRANSPOSE_FIELD(pager);
__TRANSPOSE_FIELD(paging_offset);
__TRANSPOSE_FIELD(pager_control);
if (object1->pager_control != MEMORY_OBJECT_CONTROL_NULL) {
memory_object_control_collapse(object1->pager_control,
object1);
}
if (object2->pager_control != MEMORY_OBJECT_CONTROL_NULL) {
memory_object_control_collapse(object2->pager_control,
object2);
}
__TRANSPOSE_FIELD(copy_strategy);
assert(object1->paging_in_progress);
assert(object2->paging_in_progress);
__TRANSPOSE_FIELD(pager_created);
__TRANSPOSE_FIELD(pager_initialized);
__TRANSPOSE_FIELD(pager_ready);
__TRANSPOSE_FIELD(pager_trusted);
__TRANSPOSE_FIELD(can_persist);
__TRANSPOSE_FIELD(internal);
__TRANSPOSE_FIELD(temporary);
__TRANSPOSE_FIELD(private);
__TRANSPOSE_FIELD(pageout);
assert(object1->alive);
assert(object2->alive);
assert(object1->purgable == VM_PURGABLE_DENY);
assert(object2->purgable == VM_PURGABLE_DENY);
__TRANSPOSE_FIELD(silent_overwrite);
__TRANSPOSE_FIELD(advisory_pageout);
__TRANSPOSE_FIELD(true_share);
assert(!object1->terminating);
assert(!object2->terminating);
__TRANSPOSE_FIELD(named);
__TRANSPOSE_FIELD(phys_contiguous);
__TRANSPOSE_FIELD(nophyscache);
assert(object1->cached_list.prev == NULL);
assert(object1->cached_list.next == NULL);
assert(object2->cached_list.prev == NULL);
assert(object2->cached_list.next == NULL);
assert(queue_empty(&object1->msr_q));
assert(queue_empty(&object2->msr_q));
__TRANSPOSE_FIELD(last_alloc);
__TRANSPOSE_FIELD(sequential);
__TRANSPOSE_FIELD(pages_created);
__TRANSPOSE_FIELD(pages_used);
#if MACH_PAGEMAP
__TRANSPOSE_FIELD(existence_map);
#endif
__TRANSPOSE_FIELD(cow_hint);
#if MACH_ASSERT
__TRANSPOSE_FIELD(paging_object);
#endif
__TRANSPOSE_FIELD(wimg_bits);
__TRANSPOSE_FIELD(code_signed);
__TRANSPOSE_FIELD(not_in_use);
#ifdef UPL_DEBUG
#endif
#undef __TRANSPOSE_FIELD
retval = KERN_SUCCESS;
done:
if (tmp_object != VM_OBJECT_NULL) {
vm_object_paging_end(tmp_object);
vm_object_unlock(tmp_object);
_vm_object_allocate(transpose_size, tmp_object);
vm_object_deallocate(tmp_object);
tmp_object = VM_OBJECT_NULL;
}
if (object1_locked) {
vm_object_unlock(object1);
object1_locked = FALSE;
}
if (object2_locked) {
vm_object_unlock(object2);
object2_locked = FALSE;
}
if (object1_paging) {
vm_object_lock(object1);
vm_object_paging_end(object1);
vm_object_unlock(object1);
object1_paging = FALSE;
}
if (object2_paging) {
vm_object_lock(object2);
vm_object_paging_end(object2);
vm_object_unlock(object2);
object2_paging = FALSE;
}
return retval;
}
extern int speculative_reads_disabled;
uint32_t pre_heat_scaling[MAX_UPL_TRANSFER];
uint32_t pre_heat_cluster[MAX_UPL_TRANSFER];
#define PRE_HEAT_MULTIPLIER 4
__private_extern__ void
vm_object_cluster_size(vm_object_t object, vm_object_offset_t *start,
vm_size_t *length, vm_object_fault_info_t fault_info)
{
vm_size_t pre_heat_size;
vm_size_t tail_size;
vm_size_t head_size;
vm_size_t max_length;
vm_size_t cluster_size;
vm_object_offset_t object_size;
vm_object_offset_t orig_start;
vm_object_offset_t target_start;
vm_object_offset_t offset;
vm_behavior_t behavior;
boolean_t look_behind = TRUE;
boolean_t look_ahead = TRUE;
int sequential_run;
int sequential_behavior = VM_BEHAVIOR_SEQUENTIAL;
assert( !(*length & PAGE_MASK));
assert( !(*start & PAGE_MASK_64));
if ( (max_length = *length) > (MAX_UPL_TRANSFER * PAGE_SIZE) )
max_length = (MAX_UPL_TRANSFER * PAGE_SIZE);
*length = PAGE_SIZE;
if (speculative_reads_disabled || fault_info == NULL || max_length == 0) {
return;
}
orig_start = *start;
target_start = orig_start;
cluster_size = round_page_32(fault_info->cluster_size);
behavior = fault_info->behavior;
vm_object_lock(object);
if (object->internal)
object_size = object->size;
else if (object->pager != MEMORY_OBJECT_NULL)
vnode_pager_get_object_size(object->pager, &object_size);
else
goto out;
object_size = round_page_64(object_size);
if (orig_start >= object_size) {
goto out;
}
if (object->pages_used > object->pages_created) {
object->pages_used = object->pages_created = 0;
}
if ((sequential_run = object->sequential)) {
if (sequential_run < 0) {
sequential_behavior = VM_BEHAVIOR_RSEQNTL;
sequential_run = 0 - sequential_run;
} else {
sequential_behavior = VM_BEHAVIOR_SEQUENTIAL;
}
}
switch(behavior) {
default:
behavior = VM_BEHAVIOR_DEFAULT;
case VM_BEHAVIOR_DEFAULT:
if (object->internal && fault_info->user_tag == VM_MEMORY_STACK)
goto out;
if (sequential_run >= (3 * PAGE_SIZE)) {
pre_heat_size = sequential_run + PAGE_SIZE;
if ((behavior = sequential_behavior) == VM_BEHAVIOR_SEQUENTIAL)
look_behind = FALSE;
else
look_ahead = FALSE;
} else {
uint32_t pages_unused;
if (object->pages_created < 32 * PRE_HEAT_MULTIPLIER) {
pre_heat_size = PAGE_SIZE * 8 * PRE_HEAT_MULTIPLIER;
break;
}
pages_unused = object->pages_created - object->pages_used;
if (pages_unused < (object->pages_created / 8)) {
pre_heat_size = PAGE_SIZE * 32 * PRE_HEAT_MULTIPLIER;
} else if (pages_unused < (object->pages_created / 4)) {
pre_heat_size = PAGE_SIZE * 16 * PRE_HEAT_MULTIPLIER;
} else if (pages_unused < (object->pages_created / 2)) {
pre_heat_size = PAGE_SIZE * 8 * PRE_HEAT_MULTIPLIER;
} else {
pre_heat_size = PAGE_SIZE * 4 * PRE_HEAT_MULTIPLIER;
}
}
break;
case VM_BEHAVIOR_RANDOM:
if ((pre_heat_size = cluster_size) <= PAGE_SIZE)
goto out;
break;
case VM_BEHAVIOR_SEQUENTIAL:
if ((pre_heat_size = cluster_size) == 0)
pre_heat_size = sequential_run + PAGE_SIZE;
look_behind = FALSE;
break;
case VM_BEHAVIOR_RSEQNTL:
if ((pre_heat_size = cluster_size) == 0)
pre_heat_size = sequential_run + PAGE_SIZE;
look_ahead = FALSE;
break;
}
if (pre_heat_size > max_length)
pre_heat_size = max_length;
if (behavior == VM_BEHAVIOR_DEFAULT && vm_page_free_count < vm_page_free_target)
pre_heat_size /= 2;
if (look_ahead == TRUE) {
if (look_behind == TRUE)
target_start &= ~(pre_heat_size - 1);
if ((target_start + pre_heat_size) > object_size)
pre_heat_size = (vm_size_t)(trunc_page_64(object_size - target_start));
tail_size = pre_heat_size - (orig_start - target_start) - PAGE_SIZE;
} else {
if (pre_heat_size > target_start)
pre_heat_size = target_start;
tail_size = 0;
}
pre_heat_scaling[pre_heat_size / PAGE_SIZE]++;
if (pre_heat_size <= PAGE_SIZE)
goto out;
if (look_behind == TRUE) {
head_size = pre_heat_size - tail_size - PAGE_SIZE;
for (offset = orig_start - PAGE_SIZE_64; head_size; offset -= PAGE_SIZE_64, head_size -= PAGE_SIZE) {
if (offset < fault_info->lo_offset)
break;
#if MACH_PAGEMAP
if (vm_external_state_get(object->existence_map, offset) == VM_EXTERNAL_STATE_ABSENT) {
break;
}
#endif
if (vm_page_lookup(object, offset) != VM_PAGE_NULL) {
break;
}
*start = offset;
*length += PAGE_SIZE;
}
}
if (look_ahead == TRUE) {
for (offset = orig_start + PAGE_SIZE_64; tail_size; offset += PAGE_SIZE_64, tail_size -= PAGE_SIZE) {
if (offset >= fault_info->hi_offset)
break;
#if MACH_PAGEMAP
if (vm_external_state_get(object->existence_map, offset) == VM_EXTERNAL_STATE_ABSENT) {
break;
}
#endif
if (vm_page_lookup(object, offset) != VM_PAGE_NULL) {
break;
}
*length += PAGE_SIZE;
}
}
out:
pre_heat_cluster[*length / PAGE_SIZE]++;
vm_object_unlock(object);
}
kern_return_t
vm_object_page_op(
vm_object_t object,
vm_object_offset_t offset,
int ops,
ppnum_t *phys_entry,
int *flags)
{
vm_page_t dst_page;
vm_object_lock(object);
if(ops & UPL_POP_PHYSICAL) {
if(object->phys_contiguous) {
if (phys_entry) {
*phys_entry = (ppnum_t)
(object->shadow_offset >> 12);
}
vm_object_unlock(object);
return KERN_SUCCESS;
} else {
vm_object_unlock(object);
return KERN_INVALID_OBJECT;
}
}
if(object->phys_contiguous) {
vm_object_unlock(object);
return KERN_INVALID_OBJECT;
}
while(TRUE) {
if((dst_page = vm_page_lookup(object,offset)) == VM_PAGE_NULL) {
vm_object_unlock(object);
return KERN_FAILURE;
}
if((dst_page->busy || dst_page->cleaning) &&
(((ops & UPL_POP_SET) &&
(ops & UPL_POP_BUSY)) || (ops & UPL_POP_DUMP))) {
PAGE_SLEEP(object, dst_page, THREAD_UNINT);
continue;
}
if (ops & UPL_POP_DUMP) {
if (dst_page->pmapped == TRUE)
pmap_disconnect(dst_page->phys_page);
vm_page_lock_queues();
vm_page_free(dst_page);
vm_page_unlock_queues();
break;
}
if (flags) {
*flags = 0;
if(dst_page->dirty) *flags |= UPL_POP_DIRTY;
if(dst_page->pageout) *flags |= UPL_POP_PAGEOUT;
if(dst_page->precious) *flags |= UPL_POP_PRECIOUS;
if(dst_page->absent) *flags |= UPL_POP_ABSENT;
if(dst_page->busy) *flags |= UPL_POP_BUSY;
}
if(ops & UPL_POP_SET) {
assert(dst_page->busy || (ops & UPL_POP_BUSY));
if (ops & UPL_POP_DIRTY) dst_page->dirty = TRUE;
if (ops & UPL_POP_PAGEOUT) dst_page->pageout = TRUE;
if (ops & UPL_POP_PRECIOUS) dst_page->precious = TRUE;
if (ops & UPL_POP_ABSENT) dst_page->absent = TRUE;
if (ops & UPL_POP_BUSY) dst_page->busy = TRUE;
}
if(ops & UPL_POP_CLR) {
assert(dst_page->busy);
if (ops & UPL_POP_DIRTY) dst_page->dirty = FALSE;
if (ops & UPL_POP_PAGEOUT) dst_page->pageout = FALSE;
if (ops & UPL_POP_PRECIOUS) dst_page->precious = FALSE;
if (ops & UPL_POP_ABSENT) dst_page->absent = FALSE;
if (ops & UPL_POP_BUSY) {
dst_page->busy = FALSE;
PAGE_WAKEUP(dst_page);
}
}
if (dst_page->encrypted) {
if ((ops & UPL_POP_SET) && (ops & UPL_POP_BUSY) &&
dst_page->busy) {
vm_page_decrypt(dst_page, 0);
} else {
assert(!phys_entry);
}
}
if (phys_entry) {
assert(dst_page->busy);
assert(!dst_page->encrypted);
*phys_entry = dst_page->phys_page;
}
break;
}
vm_object_unlock(object);
return KERN_SUCCESS;
}
kern_return_t
vm_object_range_op(
vm_object_t object,
vm_object_offset_t offset_beg,
vm_object_offset_t offset_end,
int ops,
int *range)
{
vm_object_offset_t offset;
vm_page_t dst_page;
if (object->resident_page_count == 0) {
if (range) {
if (ops & UPL_ROP_PRESENT)
*range = 0;
else
*range = offset_end - offset_beg;
}
return KERN_SUCCESS;
}
vm_object_lock(object);
if (object->phys_contiguous) {
vm_object_unlock(object);
return KERN_INVALID_OBJECT;
}
offset = offset_beg & ~PAGE_MASK_64;
while (offset < offset_end) {
dst_page = vm_page_lookup(object, offset);
if (dst_page != VM_PAGE_NULL) {
if (ops & UPL_ROP_DUMP) {
if (dst_page->busy || dst_page->cleaning) {
PAGE_SLEEP(object, dst_page, THREAD_UNINT);
continue;
}
if (dst_page->pmapped == TRUE)
pmap_disconnect(dst_page->phys_page);
vm_page_lock_queues();
vm_page_free(dst_page);
vm_page_unlock_queues();
} else if (ops & UPL_ROP_ABSENT)
break;
} else if (ops & UPL_ROP_PRESENT)
break;
offset += PAGE_SIZE;
}
vm_object_unlock(object);
if (range) {
if (offset > offset_end)
offset = offset_end;
*range = offset - offset_beg;
}
return KERN_SUCCESS;
}
uint32_t scan_object_collision = 0;
void
vm_object_lock(vm_object_t object)
{
if (object == vm_pageout_scan_wants_object) {
scan_object_collision++;
mutex_pause(2);
}
lck_rw_lock_exclusive(&object->Lock);
}
boolean_t
vm_object_lock_try(vm_object_t object)
{
if (object == vm_pageout_scan_wants_object) {
scan_object_collision++;
mutex_pause(2);
}
return (lck_rw_try_lock_exclusive(&object->Lock));
}
void
vm_object_lock_shared(vm_object_t object)
{
if (object == vm_pageout_scan_wants_object) {
scan_object_collision++;
mutex_pause(2);
}
lck_rw_lock_shared(&object->Lock);
}
boolean_t
vm_object_lock_try_shared(vm_object_t object)
{
if (object == vm_pageout_scan_wants_object) {
scan_object_collision++;
mutex_pause(2);
}
return (lck_rw_try_lock_shared(&object->Lock));
}