dp_memory_object.c [plain text]
#include "default_pager_internal.h"
#include <default_pager/default_pager_object_server.h>
#include <mach/memory_object_default_server.h>
#include <mach/memory_object_control.h>
#include <mach/memory_object_types.h>
#include <mach/memory_object_server.h>
#include <mach/upl.h>
#include <mach/vm_map.h>
#include <vm/memory_object.h>
#include <vm/vm_pageout.h>
#include <vm/vm_map.h>
#include <vm/vm_protos.h>
vstruct_t vs_object_create(dp_size_t size);
struct vstruct_list_head vstruct_list;
__private_extern__ void
vstruct_list_insert(
vstruct_t vs)
{
VSL_LOCK();
queue_enter(&vstruct_list.vsl_queue, vs, vstruct_t, vs_links);
vstruct_list.vsl_count++;
VSL_UNLOCK();
}
__private_extern__ void
vstruct_list_delete(
vstruct_t vs)
{
queue_remove(&vstruct_list.vsl_queue, vs, vstruct_t, vs_links);
vstruct_list.vsl_count--;
}
static unsigned int default_pager_total = 0;
static unsigned int default_pager_wait_seqno = 0;
static unsigned int default_pager_wait_read = 0;
static unsigned int default_pager_wait_write = 0;
__private_extern__ void
vs_async_wait(
vstruct_t vs)
{
ASSERT(vs->vs_async_pending >= 0);
while (vs->vs_async_pending > 0) {
vs->vs_waiting_async = TRUE;
assert_wait(&vs->vs_async_pending, THREAD_UNINT);
VS_UNLOCK(vs);
thread_block(THREAD_CONTINUE_NULL);
VS_LOCK(vs);
}
ASSERT(vs->vs_async_pending == 0);
}
#if PARALLEL
__private_extern__ void
vs_lock(
vstruct_t vs)
{
mach_port_seqno_t seqno;
default_pager_total++;
VS_LOCK(vs);
seqno = vs->vs_next_seqno++;
while (vs->vs_seqno != seqno) {
default_pager_wait_seqno++;
vs->vs_waiting_seqno = TRUE;
assert_wait(&vs->vs_seqno, THREAD_UNINT);
VS_UNLOCK(vs);
thread_block(THREAD_CONTINUE_NULL);
VS_LOCK(vs);
}
}
__private_extern__ void
vs_unlock(vstruct_t vs)
{
vs->vs_seqno++;
if (vs->vs_waiting_seqno) {
vs->vs_waiting_seqno = FALSE;
VS_UNLOCK(vs);
thread_wakeup(&vs->vs_seqno);
return;
}
VS_UNLOCK(vs);
}
__private_extern__ void
vs_start_read(
vstruct_t vs)
{
vs->vs_readers++;
}
__private_extern__ void
vs_wait_for_readers(
vstruct_t vs)
{
while (vs->vs_readers != 0) {
default_pager_wait_read++;
vs->vs_waiting_read = TRUE;
assert_wait(&vs->vs_readers, THREAD_UNINT);
VS_UNLOCK(vs);
thread_block(THREAD_CONTINUE_NULL);
VS_LOCK(vs);
}
}
__private_extern__ void
vs_finish_read(
vstruct_t vs)
{
VS_LOCK(vs);
if (--vs->vs_readers == 0 && vs->vs_waiting_read) {
vs->vs_waiting_read = FALSE;
VS_UNLOCK(vs);
thread_wakeup(&vs->vs_readers);
return;
}
VS_UNLOCK(vs);
}
__private_extern__ void
vs_start_write(
vstruct_t vs)
{
vs->vs_writers++;
}
__private_extern__ void
vs_wait_for_writers(
vstruct_t vs)
{
while (vs->vs_writers != 0) {
default_pager_wait_write++;
vs->vs_waiting_write = TRUE;
assert_wait(&vs->vs_writers, THREAD_UNINT);
VS_UNLOCK(vs);
thread_block(THREAD_CONTINUE_NULL);
VS_LOCK(vs);
}
vs_async_wait(vs);
}
__private_extern__ void
vs_wait_for_sync_writers(
vstruct_t vs)
{
while (vs->vs_writers != 0) {
default_pager_wait_write++;
vs->vs_waiting_write = TRUE;
assert_wait(&vs->vs_writers, THREAD_UNINT);
VS_UNLOCK(vs);
thread_block(THREAD_CONTINUE_NULL);
VS_LOCK(vs);
}
}
__private_extern__ void
vs_finish_write(
vstruct_t vs)
{
VS_LOCK(vs);
if (--vs->vs_writers == 0 && vs->vs_waiting_write) {
vs->vs_waiting_write = FALSE;
VS_UNLOCK(vs);
thread_wakeup(&vs->vs_writers);
return;
}
VS_UNLOCK(vs);
}
#endif
vstruct_t
vs_object_create(
dp_size_t size)
{
vstruct_t vs;
vs = ps_vstruct_create(size);
if (vs == VSTRUCT_NULL) {
dprintf(("vs_object_create: unable to allocate %s\n",
"-- either run swapon command or reboot"));
return VSTRUCT_NULL;
}
return vs;
}
#if 0
void default_pager_add(vstruct_t, boolean_t);
void
default_pager_add(
vstruct_t vs,
boolean_t internal)
{
memory_object_t mem_obj = vs->vs_mem_obj;
mach_port_t pset;
mach_port_mscount_t sync;
mach_port_t previous;
kern_return_t kr;
static char here[] = "default_pager_add";
if (internal) {
sync = 0;
pset = default_pager_internal_set;
} else {
sync = 1;
pset = default_pager_external_set;
}
ip_lock(mem_obj);
ipc_port_make_sonce_locked(mem_obj);
ipc_port_nsrequest(mem_obj, sync, mem_obj, &previous);
}
#endif
const struct memory_object_pager_ops default_pager_ops = {
dp_memory_object_reference,
dp_memory_object_deallocate,
dp_memory_object_init,
dp_memory_object_terminate,
dp_memory_object_data_request,
dp_memory_object_data_return,
dp_memory_object_data_initialize,
dp_memory_object_data_unlock,
dp_memory_object_synchronize,
dp_memory_object_map,
dp_memory_object_last_unmap,
dp_memory_object_data_reclaim,
"default pager"
};
kern_return_t
dp_memory_object_init(
memory_object_t mem_obj,
memory_object_control_t control,
__unused memory_object_cluster_size_t pager_page_size)
{
vstruct_t vs;
assert(pager_page_size == vm_page_size);
memory_object_control_reference(control);
vs_lookup(mem_obj, vs);
vs_lock(vs);
if (vs->vs_control != MEMORY_OBJECT_CONTROL_NULL)
Panic("bad request");
vs->vs_control = control;
vs_unlock(vs);
return KERN_SUCCESS;
}
kern_return_t
dp_memory_object_synchronize(
memory_object_t mem_obj,
memory_object_offset_t offset,
memory_object_size_t length,
__unused vm_sync_t flags)
{
vstruct_t vs;
vs_lookup(mem_obj, vs);
vs_lock(vs);
vs_unlock(vs);
memory_object_synchronize_completed(vs->vs_control, offset, length);
return KERN_SUCCESS;
}
kern_return_t
dp_memory_object_map(
__unused memory_object_t mem_obj,
__unused vm_prot_t prot)
{
panic("dp_memory_object_map");
return KERN_FAILURE;
}
kern_return_t
dp_memory_object_last_unmap(
__unused memory_object_t mem_obj)
{
panic("dp_memory_object_last_unmap");
return KERN_FAILURE;
}
kern_return_t
dp_memory_object_data_reclaim(
memory_object_t mem_obj,
boolean_t reclaim_backing_store)
{
vstruct_t vs;
kern_return_t retval;
vs_lookup(mem_obj, vs);
for (;;) {
vs_lock(vs);
vs_async_wait(vs);
if (!vs->vs_xfer_pending) {
break;
}
}
vs->vs_xfer_pending = TRUE;
vs_unlock(vs);
retval = ps_vstruct_reclaim(vs, TRUE, reclaim_backing_store);
vs_lock(vs);
vs->vs_xfer_pending = FALSE;
vs_unlock(vs);
return retval;
}
kern_return_t
dp_memory_object_terminate(
memory_object_t mem_obj)
{
memory_object_control_t control;
vstruct_t vs;
vs_lookup(mem_obj, vs);
vs_lock(vs);
vs_wait_for_readers(vs);
vs_wait_for_writers(vs);
control = vs->vs_control;
vs->vs_control = MEMORY_OBJECT_CONTROL_NULL;
thread_wakeup(&vs->vs_writers);
thread_wakeup(&vs->vs_async_pending);
vs_unlock(vs);
memory_object_control_deallocate(control);
return KERN_SUCCESS;
}
void
dp_memory_object_reference(
memory_object_t mem_obj)
{
vstruct_t vs;
vs_lookup_safe(mem_obj, vs);
if (vs == VSTRUCT_NULL)
return;
VS_LOCK(vs);
assert(vs->vs_references > 0);
vs->vs_references++;
VS_UNLOCK(vs);
}
void
dp_memory_object_deallocate(
memory_object_t mem_obj)
{
vstruct_t vs;
mach_port_seqno_t seqno;
vs_lookup_safe(mem_obj, vs);
if (vs == VSTRUCT_NULL)
return;
VS_LOCK(vs);
if (--vs->vs_references > 0) {
VS_UNLOCK(vs);
return;
}
seqno = vs->vs_next_seqno++;
while (vs->vs_seqno != seqno) {
default_pager_wait_seqno++;
vs->vs_waiting_seqno = TRUE;
assert_wait(&vs->vs_seqno, THREAD_UNINT);
VS_UNLOCK(vs);
thread_block(THREAD_CONTINUE_NULL);
VS_LOCK(vs);
}
vs_async_wait(vs);
while(!VSL_LOCK_TRY()) {
VS_UNLOCK(vs);
VSL_LOCK();
VSL_UNLOCK();
VS_LOCK(vs);
vs_async_wait(vs);
}
if (vs->vs_control != MEMORY_OBJECT_CONTROL_NULL)
Panic("bad request");
VS_UNLOCK(vs);
backing_store_release_trigger_disable += 1;
vstruct_list_delete(vs);
VSL_UNLOCK();
ps_vstruct_dealloc(vs);
VSL_LOCK();
backing_store_release_trigger_disable -= 1;
if(backing_store_release_trigger_disable == 0) {
thread_wakeup((event_t)&backing_store_release_trigger_disable);
}
VSL_UNLOCK();
}
kern_return_t
dp_memory_object_data_request(
memory_object_t mem_obj,
memory_object_offset_t offset,
memory_object_cluster_size_t length,
__unused vm_prot_t protection_required,
memory_object_fault_info_t fault_info)
{
vstruct_t vs;
kern_return_t kr = KERN_SUCCESS;
GSTAT(global_stats.gs_pagein_calls++);
vs_lookup(mem_obj, vs);
vs_lock(vs);
if (vs->vs_writers != 0) {
vs_unlock(vs);
VS_LOCK(vs);
while (vs->vs_writers != 0) {
default_pager_wait_write++;
vs->vs_waiting_write = TRUE;
assert_wait(&vs->vs_writers, THREAD_UNINT);
VS_UNLOCK(vs);
thread_block(THREAD_CONTINUE_NULL);
VS_LOCK(vs);
vs_async_wait(vs);
}
if(vs->vs_control == MEMORY_OBJECT_CONTROL_NULL) {
VS_UNLOCK(vs);
return KERN_FAILURE;
}
vs_start_read(vs);
VS_UNLOCK(vs);
} else {
vs_start_read(vs);
vs_unlock(vs);
}
if ((offset & vm_page_mask) != 0 || (length & vm_page_mask) != 0)
Panic("bad alignment");
assert((dp_offset_t) offset == offset);
kr = pvs_cluster_read(vs, (dp_offset_t) offset, length, fault_info);
if(length) {
kr = KERN_SUCCESS;
}
vs_finish_read(vs);
return kr;
}
kern_return_t
dp_memory_object_data_initialize(
memory_object_t mem_obj,
memory_object_offset_t offset,
memory_object_cluster_size_t size)
{
vstruct_t vs;
DP_DEBUG(DEBUG_MO_EXTERNAL,
("mem_obj=0x%x,offset=0x%x,cnt=0x%x\n",
(int)mem_obj, (int)offset, (int)size));
GSTAT(global_stats.gs_pages_init += atop_32(size));
vs_lookup(mem_obj, vs);
vs_lock(vs);
vs_start_write(vs);
vs_unlock(vs);
assert((upl_offset_t) offset == offset);
vs_cluster_write(vs, 0, (upl_offset_t)offset, size, FALSE, 0);
vs_finish_write(vs);
return KERN_SUCCESS;
}
kern_return_t
dp_memory_object_data_unlock(
__unused memory_object_t mem_obj,
__unused memory_object_offset_t offset,
__unused memory_object_size_t size,
__unused vm_prot_t desired_access)
{
Panic("dp_memory_object_data_unlock: illegal");
return KERN_FAILURE;
}
kern_return_t
dp_memory_object_data_return(
memory_object_t mem_obj,
memory_object_offset_t offset,
memory_object_cluster_size_t size,
__unused memory_object_offset_t *resid_offset,
__unused int *io_error,
__unused boolean_t dirty,
__unused boolean_t kernel_copy,
__unused int upl_flags)
{
vstruct_t vs;
DP_DEBUG(DEBUG_MO_EXTERNAL,
("mem_obj=0x%x,offset=0x%x,size=0x%x\n",
(int)mem_obj, (int)offset, (int)size));
GSTAT(global_stats.gs_pageout_calls++);
vs_lookup(mem_obj, vs);
default_pager_total++;
__unreachable_ok_push
if(!VS_TRY_LOCK(vs)) {
upl_t upl;
unsigned int page_list_count = 0;
memory_object_super_upl_request(vs->vs_control,
(memory_object_offset_t)offset,
size, size,
&upl, NULL, &page_list_count,
UPL_NOBLOCK | UPL_CLEAN_IN_PLACE
| UPL_NO_SYNC | UPL_COPYOUT_FROM);
upl_abort(upl,0);
upl_deallocate(upl);
return KERN_SUCCESS;
}
__unreachable_ok_pop
if ((vs->vs_seqno != vs->vs_next_seqno++)
|| (vs->vs_readers)
|| (vs->vs_xfer_pending)) {
upl_t upl;
unsigned int page_list_count = 0;
vs->vs_next_seqno--;
VS_UNLOCK(vs);
memory_object_super_upl_request(vs->vs_control,
(memory_object_offset_t)offset,
size, size,
&upl, NULL, &page_list_count,
UPL_NOBLOCK | UPL_CLEAN_IN_PLACE
| UPL_NO_SYNC | UPL_COPYOUT_FROM);
upl_abort(upl,0);
upl_deallocate(upl);
return KERN_SUCCESS;
}
if ((size % vm_page_size) != 0)
Panic("bad alignment");
vs_start_write(vs);
vs->vs_async_pending += 1;
vs_unlock(vs);
assert((upl_offset_t) offset == offset);
vs_cluster_write(vs, 0, (upl_offset_t) offset, size, FALSE, 0);
vs_finish_write(vs);
VS_LOCK(vs);
vs->vs_async_pending -= 1;
if (vs->vs_async_pending == 0 && vs->vs_waiting_async) {
vs->vs_waiting_async = FALSE;
VS_UNLOCK(vs);
thread_wakeup(&vs->vs_async_pending);
} else {
VS_UNLOCK(vs);
}
return KERN_SUCCESS;
}
kern_return_t
default_pager_memory_object_create(
__unused memory_object_default_t dmm,
vm_size_t new_size,
memory_object_t *new_mem_obj)
{
vstruct_t vs;
assert(dmm == default_pager_object);
if ((dp_size_t) new_size != new_size) {
return KERN_INVALID_ARGUMENT;
}
vs = vs_object_create((dp_size_t) new_size);
if (vs == VSTRUCT_NULL)
return KERN_RESOURCE_SHORTAGE;
vs->vs_next_seqno = 0;
vs->vs_pager_ops = &default_pager_ops;
vs->vs_pager_header.io_bits = IKOT_MEMORY_OBJECT;
vstruct_list_insert(vs);
*new_mem_obj = vs_to_mem_obj(vs);
return KERN_SUCCESS;
}
kern_return_t
default_pager_object_create(
default_pager_t default_pager,
vm_size_t size,
memory_object_t *mem_objp)
{
vstruct_t vs;
if (default_pager != default_pager_object)
return KERN_INVALID_ARGUMENT;
if ((dp_size_t) size != size) {
return KERN_INVALID_ARGUMENT;
}
vs = vs_object_create((dp_size_t) size);
if (vs == VSTRUCT_NULL)
return KERN_RESOURCE_SHORTAGE;
vs->vs_pager_ops = &default_pager_ops;
vstruct_list_insert(vs);
*mem_objp = vs_to_mem_obj(vs);
return KERN_SUCCESS;
}
kern_return_t
default_pager_objects(
default_pager_t default_pager,
default_pager_object_array_t *objectsp,
mach_msg_type_number_t *ocountp,
mach_port_array_t *portsp,
mach_msg_type_number_t *pcountp)
{
vm_offset_t oaddr = 0;
vm_size_t osize = 0;
default_pager_object_t * objects;
unsigned int opotential = 0;
vm_map_copy_t pcopy = 0;
vm_size_t psize = 0;
memory_object_t * pagers;
unsigned int ppotential = 0;
unsigned int actual;
unsigned int num_objects;
kern_return_t kr;
vstruct_t entry;
if (default_pager != default_pager_object)
return KERN_INVALID_ARGUMENT;
actual = vstruct_list.vsl_count;
psize = vm_map_round_page(actual * sizeof (*pagers),
vm_map_page_mask(ipc_kernel_map));
ppotential = (unsigned int) (psize / sizeof (*pagers));
pagers = (memory_object_t *)kalloc(psize);
if (0 == pagers)
return KERN_RESOURCE_SHORTAGE;
osize = vm_map_round_page(actual * sizeof (*objects),
vm_map_page_mask(ipc_kernel_map));
opotential = (unsigned int) (osize / sizeof (*objects));
kr = kmem_alloc(ipc_kernel_map, &oaddr, osize, VM_KERN_MEMORY_IPC);
if (KERN_SUCCESS != kr) {
kfree(pagers, psize);
return KERN_RESOURCE_SHORTAGE;
}
objects = (default_pager_object_t *)oaddr;
VSL_LOCK();
num_objects = 0;
queue_iterate(&vstruct_list.vsl_queue, entry, vstruct_t, vs_links) {
memory_object_t pager;
vm_size_t size;
if ((num_objects >= opotential) ||
(num_objects >= ppotential)) {
break;
}
if (!VS_MAP_TRY_LOCK(entry))
goto not_this_one;
size = ps_vstruct_allocated_size(entry);
VS_MAP_UNLOCK(entry);
VS_LOCK(entry);
VS_LOCK(entry);
if (entry->vs_references == 0) {
VS_UNLOCK(entry);
goto not_this_one;
}
pager = vs_to_mem_obj(entry);
dp_memory_object_reference(pager);
VS_UNLOCK(entry);
objects[num_objects].dpo_object = (vm_offset_t) entry;
objects[num_objects].dpo_size = size;
pagers [num_objects++] = pager;
continue;
not_this_one:
objects[num_objects].dpo_object = (vm_offset_t) 0;
objects[num_objects].dpo_size = 0;
pagers[num_objects++] = MEMORY_OBJECT_NULL;
}
VSL_UNLOCK();
while (num_objects < opotential) {
objects[--opotential].dpo_object = (vm_offset_t) 0;
objects[opotential].dpo_size = 0;
}
while (num_objects < ppotential) {
pagers[--ppotential] = MEMORY_OBJECT_NULL;
}
kr = vm_map_unwire(ipc_kernel_map,
vm_map_trunc_page(oaddr,
vm_map_page_mask(ipc_kernel_map)),
vm_map_round_page(oaddr + osize,
vm_map_page_mask(ipc_kernel_map)),
FALSE);
assert(KERN_SUCCESS == kr);
kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)oaddr,
(vm_map_size_t)(num_objects * sizeof(*objects)), TRUE, &pcopy);
assert(KERN_SUCCESS == kr);
*objectsp = (default_pager_object_array_t)objects;
*ocountp = num_objects;
*portsp = (mach_port_array_t)pcopy;
*pcountp = num_objects;
return KERN_SUCCESS;
}
kern_return_t
default_pager_object_pages(
default_pager_t default_pager,
mach_port_t memory_object,
default_pager_page_array_t *pagesp,
mach_msg_type_number_t *countp)
{
vm_offset_t addr = 0;
vm_size_t size = 0;
vm_map_copy_t copy;
default_pager_page_t * pages = 0;
unsigned int potential;
unsigned int actual;
kern_return_t kr;
memory_object_t object;
if (default_pager != default_pager_object)
return KERN_INVALID_ARGUMENT;
object = (memory_object_t) memory_object;
potential = 0;
for (;;) {
vstruct_t entry;
VSL_LOCK();
queue_iterate(&vstruct_list.vsl_queue, entry, vstruct_t,
vs_links) {
VS_LOCK(entry);
if (vs_to_mem_obj(entry) == object) {
VSL_UNLOCK();
goto found_object;
}
VS_UNLOCK(entry);
}
VSL_UNLOCK();
if (0 != addr)
kmem_free(ipc_kernel_map, addr, size);
return KERN_INVALID_ARGUMENT;
found_object:
if (!VS_MAP_TRY_LOCK(entry)) {
int wresult;
VS_UNLOCK(entry);
assert_wait_timeout((event_t)assert_wait_timeout, THREAD_UNINT, 1, 1000*NSEC_PER_USEC);
wresult = thread_block(THREAD_CONTINUE_NULL);
assert(wresult == THREAD_TIMED_OUT);
continue;
}
actual = ps_vstruct_allocated_pages(entry, pages, potential);
VS_MAP_UNLOCK(entry);
VS_UNLOCK(entry);
if (actual <= potential)
break;
if (0 != addr)
kmem_free(ipc_kernel_map, addr, size);
size = vm_map_round_page(actual * sizeof (*pages),
vm_map_page_mask(ipc_kernel_map));
kr = kmem_alloc(ipc_kernel_map, &addr, size, VM_KERN_MEMORY_IPC);
if (KERN_SUCCESS != kr)
return KERN_RESOURCE_SHORTAGE;
pages = (default_pager_page_t *)addr;
potential = (unsigned int) (size / sizeof (*pages));
}
while (actual < potential)
pages[--potential].dpp_offset = 0;
kr = vm_map_unwire(ipc_kernel_map,
vm_map_trunc_page(addr,
vm_map_page_mask(ipc_kernel_map)),
vm_map_round_page(addr + size,
vm_map_page_mask(ipc_kernel_map)),
FALSE);
assert(KERN_SUCCESS == kr);
kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)addr,
(vm_map_size_t)(actual * sizeof(*pages)), TRUE, ©);
assert(KERN_SUCCESS == kr);
*pagesp = (default_pager_page_array_t)copy;
*countp = actual;
return KERN_SUCCESS;
}