#include <mach/clock_types.h>
#include <mach/vm_prot.h>
#include <mach/vm_statistics.h>
#include <kern/counters.h>
#include <kern/sched_prim.h>
#include <kern/task.h>
#include <kern/thread.h>
#include <kern/zalloc.h>
#include <kern/xpr.h>
#include <vm/pmap.h>
#include <vm/vm_init.h>
#include <vm/vm_map.h>
#include <vm/vm_page.h>
#include <vm/vm_pageout.h>
#include <vm/vm_kern.h>
#include <kern/misc_protos.h>
#include <zone_debug.h>
#include <vm/cpm.h>
#include <ppc/mappings.h>
#include <pexpert/pexpert.h>
int vm_page_ticket_roll = 0;
int vm_page_ticket = 0;
vm_offset_t virtual_space_start;
vm_offset_t virtual_space_end;
int vm_page_pages;
typedef struct {
vm_page_t pages;
#if MACH_PAGE_HASH_STATS
int cur_count;
int hi_count;
#endif
} vm_page_bucket_t;
vm_page_bucket_t *vm_page_buckets;
unsigned int vm_page_bucket_count = 0;
unsigned int vm_page_hash_mask;
unsigned int vm_page_hash_shift;
uint32_t vm_page_bucket_hash;
decl_simple_lock_data(,vm_page_bucket_lock)
#if MACH_PAGE_HASH_STATS
void
hash_debug(void)
{
int i;
int numbuckets = 0;
int highsum = 0;
int maxdepth = 0;
for (i = 0; i < vm_page_bucket_count; i++) {
if (vm_page_buckets[i].hi_count) {
numbuckets++;
highsum += vm_page_buckets[i].hi_count;
if (vm_page_buckets[i].hi_count > maxdepth)
maxdepth = vm_page_buckets[i].hi_count;
}
}
printf("Total number of buckets: %d\n", vm_page_bucket_count);
printf("Number used buckets: %d = %d%%\n",
numbuckets, 100*numbuckets/vm_page_bucket_count);
printf("Number unused buckets: %d = %d%%\n",
vm_page_bucket_count - numbuckets,
100*(vm_page_bucket_count-numbuckets)/vm_page_bucket_count);
printf("Sum of bucket max depth: %d\n", highsum);
printf("Average bucket depth: %d.%2d\n",
highsum/vm_page_bucket_count,
highsum%vm_page_bucket_count);
printf("Maximum bucket depth: %d\n", maxdepth);
}
#endif
#ifndef PAGE_SIZE_FIXED
vm_size_t page_size = 4096;
vm_size_t page_mask = 4095;
int page_shift = 12;
#else
vm_size_t page_size = PAGE_SIZE;
vm_size_t page_mask = PAGE_MASK;
int page_shift = PAGE_SHIFT;
#endif
struct vm_page vm_page_template;
vm_page_t vm_page_queue_free;
vm_page_t vm_page_queue_fictitious;
decl_mutex_data(,vm_page_queue_free_lock)
unsigned int vm_page_free_wanted;
int vm_page_free_count;
int vm_page_fictitious_count;
unsigned int vm_page_free_count_minimum;
zone_t vm_page_zone;
decl_mutex_data(,vm_page_alloc_lock)
unsigned int io_throttle_zero_fill;
decl_mutex_data(,vm_page_zero_fill_lock)
vm_offset_t vm_page_fictitious_addr = (vm_offset_t) -1;
queue_head_t vm_page_queue_active;
queue_head_t vm_page_queue_inactive;
queue_head_t vm_page_queue_zf;
decl_mutex_data(,vm_page_queue_lock)
int vm_page_active_count;
int vm_page_inactive_count;
int vm_page_wire_count;
int vm_page_gobble_count = 0;
int vm_page_wire_count_warning = 0;
int vm_page_gobble_count_warning = 0;
queue_head_t vm_page_queue_limbo;
int vm_page_limbo_count = 0;
int vm_page_limbo_real_count = 0;
int vm_page_pin_count = 0;
decl_simple_lock_data(,vm_page_preppin_lock)
int vm_page_free_target = 0;
int vm_page_free_min = 0;
int vm_page_inactive_target = 0;
int vm_page_free_reserved = 0;
int vm_page_laundry_count = 0;
int vm_page_burst_count = 0;
int vm_page_throttled_count = 0;
boolean_t vm_page_deactivate_hint = TRUE;
void
vm_set_page_size(void)
{
#ifndef PAGE_SIZE_FIXED
page_mask = page_size - 1;
if ((page_mask & page_size) != 0)
panic("vm_set_page_size: page size not a power of two");
for (page_shift = 0; ; page_shift++)
if ((1 << page_shift) == page_size)
break;
#endif
}
void
vm_page_bootstrap(
vm_offset_t *startp,
vm_offset_t *endp)
{
register vm_page_t m;
int i;
unsigned int log1;
unsigned int log2;
unsigned int size;
m = &vm_page_template;
m->object = VM_OBJECT_NULL;
m->offset = 0;
m->wire_count = 0;
m->inactive = FALSE;
m->active = FALSE;
m->laundry = FALSE;
m->free = FALSE;
m->no_isync = TRUE;
m->reference = FALSE;
m->pageout = FALSE;
m->dump_cleaning = FALSE;
m->list_req_pending = FALSE;
m->busy = TRUE;
m->wanted = FALSE;
m->tabled = FALSE;
m->fictitious = FALSE;
m->private = FALSE;
m->absent = FALSE;
m->error = FALSE;
m->dirty = FALSE;
m->cleaning = FALSE;
m->precious = FALSE;
m->clustered = FALSE;
m->lock_supplied = FALSE;
m->unusual = FALSE;
m->restart = FALSE;
m->zero_fill = FALSE;
m->phys_page = 0;
m->page_lock = VM_PROT_NONE;
m->unlock_request = VM_PROT_NONE;
m->page_error = KERN_SUCCESS;
mutex_init(&vm_page_queue_free_lock, ETAP_VM_PAGEQ_FREE);
mutex_init(&vm_page_queue_lock, ETAP_VM_PAGEQ);
simple_lock_init(&vm_page_preppin_lock, ETAP_VM_PREPPIN);
vm_page_queue_free = VM_PAGE_NULL;
vm_page_queue_fictitious = VM_PAGE_NULL;
queue_init(&vm_page_queue_active);
queue_init(&vm_page_queue_inactive);
queue_init(&vm_page_queue_zf);
queue_init(&vm_page_queue_limbo);
vm_page_free_wanted = 0;
vm_map_steal_memory();
zone_steal_memory();
simple_lock_init(&vm_page_bucket_lock, ETAP_VM_BUCKET);
if (vm_page_bucket_count == 0) {
unsigned int npages = pmap_free_pages();
vm_page_bucket_count = 1;
while (vm_page_bucket_count < npages)
vm_page_bucket_count <<= 1;
}
vm_page_hash_mask = vm_page_bucket_count - 1;
size = vm_page_bucket_count;
for (log1 = 0; size > 1; log1++)
size /= 2;
size = sizeof(struct vm_object);
for (log2 = 0; size > 1; log2++)
size /= 2;
vm_page_hash_shift = log1/2 - log2 + 1;
vm_page_bucket_hash = 1 << ((log1 + 1) >> 1);
vm_page_bucket_hash |= 1 << ((log1 + 1) >> 2);
vm_page_bucket_hash |= 1;
if (vm_page_hash_mask & vm_page_bucket_count)
printf("vm_page_bootstrap: WARNING -- strange page hash\n");
vm_page_buckets = (vm_page_bucket_t *)
pmap_steal_memory(vm_page_bucket_count *
sizeof(vm_page_bucket_t));
for (i = 0; i < vm_page_bucket_count; i++) {
register vm_page_bucket_t *bucket = &vm_page_buckets[i];
bucket->pages = VM_PAGE_NULL;
#if MACH_PAGE_HASH_STATS
bucket->cur_count = 0;
bucket->hi_count = 0;
#endif
}
pmap_startup(&virtual_space_start, &virtual_space_end);
virtual_space_start = round_page_32(virtual_space_start);
virtual_space_end = trunc_page_32(virtual_space_end);
*startp = virtual_space_start;
*endp = virtual_space_end;
vm_page_wire_count = atop_64(max_mem) - vm_page_free_count;
printf("vm_page_bootstrap: %d free pages\n", vm_page_free_count);
vm_page_free_count_minimum = vm_page_free_count;
}
#ifndef MACHINE_PAGES
vm_offset_t
pmap_steal_memory(
vm_size_t size)
{
vm_offset_t addr, vaddr;
ppnum_t phys_page;
size = (size + sizeof (void *) - 1) &~ (sizeof (void *) - 1);
if (virtual_space_start == virtual_space_end) {
pmap_virtual_space(&virtual_space_start, &virtual_space_end);
virtual_space_start = round_page_32(virtual_space_start);
virtual_space_end = trunc_page_32(virtual_space_end);
}
addr = virtual_space_start;
virtual_space_start += size;
kprintf("pmap_steal_memory: %08X - %08X; size=%08X\n", addr, virtual_space_start, size);
for (vaddr = round_page_32(addr);
vaddr < addr + size;
vaddr += PAGE_SIZE) {
if (!pmap_next_page(&phys_page))
panic("pmap_steal_memory");
pmap_enter(kernel_pmap, vaddr, phys_page,
VM_PROT_READ|VM_PROT_WRITE,
VM_WIMG_USE_DEFAULT, FALSE);
vm_page_wire_count++;
}
return addr;
}
void
pmap_startup(
vm_offset_t *startp,
vm_offset_t *endp)
{
unsigned int i, npages, pages_initialized, fill, fillval;
vm_page_t pages;
ppnum_t phys_page;
addr64_t tmpaddr;
tmpaddr = (addr64_t)pmap_free_pages() * (addr64_t)PAGE_SIZE;
tmpaddr = tmpaddr + (addr64_t)(round_page_32(virtual_space_start) - virtual_space_start);
npages = (unsigned int)(tmpaddr / (addr64_t)(PAGE_SIZE + sizeof(*pages)));
pages = (vm_page_t) pmap_steal_memory(npages * sizeof *pages);
for (i = 0, pages_initialized = 0; i < npages; i++) {
if (!pmap_next_page(&phys_page))
break;
vm_page_init(&pages[i], phys_page);
vm_page_pages++;
pages_initialized++;
}
fill = 0;
if (PE_parse_boot_arg("fill", &fillval)) fill = 1;
for (i = pages_initialized; i > 0; i--) {
extern void fillPage(ppnum_t phys_page, unsigned int fillval);
if(fill) fillPage(pages[i - 1].phys_page, fillval);
vm_page_release(&pages[i - 1]);
}
#if 0
{
vm_page_t xx, xxo, xxl;
int j, k, l;
j = 0;
xxl = 0;
for(xx = vm_page_queue_free; xx; xxl = xx, xx = xx->pageq.next) {
j++;
if(j > vm_page_free_count) {
panic("pmap_startup: too many pages, xx = %08X, xxl = %08X\n", xx, xxl);
}
l = vm_page_free_count - j;
k = 0;
if(((j - 1) & 0xFFFF) == 0) kprintf("checking number %d of %d\n", j, vm_page_free_count);
for(xxo = xx->pageq.next; xxo; xxo = xxo->pageq.next) {
k++;
if(k > l) panic("pmap_startup: too many in secondary check %d %d\n", k, l);
if((xx->phys_page & 0xFFFFFFFF) == (xxo->phys_page & 0xFFFFFFFF)) {
panic("pmap_startup: duplicate physaddr, xx = %08X, xxo = %08X\n", xx, xxo);
}
}
}
if(j != vm_page_free_count) {
panic("pmap_startup: vm_page_free_count does not match, calc = %d, vm_page_free_count = %08X\n", j, vm_page_free_count);
}
}
#endif
virtual_space_start = round_page_32(virtual_space_start);
*startp = virtual_space_start;
*endp = virtual_space_end;
}
#endif
void
vm_page_module_init(void)
{
vm_page_zone = zinit((vm_size_t) sizeof(struct vm_page),
0, PAGE_SIZE, "vm pages");
#if ZONE_DEBUG
zone_debug_disable(vm_page_zone);
#endif
zone_change(vm_page_zone, Z_EXPAND, FALSE);
zone_change(vm_page_zone, Z_EXHAUST, TRUE);
zone_change(vm_page_zone, Z_FOREIGN, TRUE);
vm_page_zone->count += vm_page_pages;
vm_page_zone->cur_size += vm_page_pages * vm_page_zone->elem_size;
mutex_init(&vm_page_alloc_lock, ETAP_VM_PAGE_ALLOC);
mutex_init(&vm_page_zero_fill_lock, ETAP_VM_PAGE_ALLOC);
}
void
vm_page_create(
ppnum_t start,
ppnum_t end)
{
ppnum_t phys_page;
vm_page_t m;
for (phys_page = start;
phys_page < end;
phys_page++) {
while ((m = (vm_page_t) vm_page_grab_fictitious())
== VM_PAGE_NULL)
vm_page_more_fictitious();
vm_page_init(m, phys_page);
vm_page_pages++;
vm_page_release(m);
}
}
#define vm_page_hash(object, offset) (\
( (natural_t)((uint32_t)object * vm_page_bucket_hash) + ((uint32_t)atop_64(offset) ^ vm_page_bucket_hash))\
& vm_page_hash_mask)
void
vm_page_insert(
register vm_page_t mem,
register vm_object_t object,
register vm_object_offset_t offset)
{
register vm_page_bucket_t *bucket;
XPR(XPR_VM_PAGE,
"vm_page_insert, object 0x%X offset 0x%X page 0x%X\n",
(integer_t)object, (integer_t)offset, (integer_t)mem, 0,0);
VM_PAGE_CHECK(mem);
if (mem->tabled)
panic("vm_page_insert");
assert(!object->internal || offset < object->size);
assert(object->pageout == mem->pageout);
mem->object = object;
mem->offset = offset;
bucket = &vm_page_buckets[vm_page_hash(object, offset)];
simple_lock(&vm_page_bucket_lock);
mem->next = bucket->pages;
bucket->pages = mem;
#if MACH_PAGE_HASH_STATS
if (++bucket->cur_count > bucket->hi_count)
bucket->hi_count = bucket->cur_count;
#endif
simple_unlock(&vm_page_bucket_lock);
queue_enter(&object->memq, mem, vm_page_t, listq);
mem->tabled = TRUE;
object->resident_page_count++;
}
void
vm_page_replace(
register vm_page_t mem,
register vm_object_t object,
register vm_object_offset_t offset)
{
register vm_page_bucket_t *bucket;
VM_PAGE_CHECK(mem);
if (mem->tabled)
panic("vm_page_replace");
mem->object = object;
mem->offset = offset;
bucket = &vm_page_buckets[vm_page_hash(object, offset)];
simple_lock(&vm_page_bucket_lock);
if (bucket->pages) {
vm_page_t *mp = &bucket->pages;
register vm_page_t m = *mp;
do {
if (m->object == object && m->offset == offset) {
*mp = m->next;
queue_remove(&object->memq, m, vm_page_t,
listq);
m->tabled = FALSE;
object->resident_page_count--;
vm_page_free(m);
break;
}
mp = &m->next;
} while (m = *mp);
mem->next = bucket->pages;
} else {
mem->next = VM_PAGE_NULL;
}
bucket->pages = mem;
simple_unlock(&vm_page_bucket_lock);
queue_enter(&object->memq, mem, vm_page_t, listq);
mem->tabled = TRUE;
object->resident_page_count++;
}
void
vm_page_remove(
register vm_page_t mem)
{
register vm_page_bucket_t *bucket;
register vm_page_t this;
XPR(XPR_VM_PAGE,
"vm_page_remove, object 0x%X offset 0x%X page 0x%X\n",
(integer_t)mem->object, (integer_t)mem->offset,
(integer_t)mem, 0,0);
assert(mem->tabled);
assert(!mem->cleaning);
VM_PAGE_CHECK(mem);
bucket = &vm_page_buckets[vm_page_hash(mem->object, mem->offset)];
simple_lock(&vm_page_bucket_lock);
if ((this = bucket->pages) == mem) {
bucket->pages = mem->next;
} else {
register vm_page_t *prev;
for (prev = &this->next;
(this = *prev) != mem;
prev = &this->next)
continue;
*prev = this->next;
}
#if MACH_PAGE_HASH_STATS
bucket->cur_count--;
#endif
simple_unlock(&vm_page_bucket_lock);
queue_remove(&mem->object->memq, mem, vm_page_t, listq);
mem->object->resident_page_count--;
mem->tabled = FALSE;
mem->object = VM_OBJECT_NULL;
mem->offset = 0;
}
vm_page_t
vm_page_lookup(
register vm_object_t object,
register vm_object_offset_t offset)
{
register vm_page_t mem;
register vm_page_bucket_t *bucket;
bucket = &vm_page_buckets[vm_page_hash(object, offset)];
simple_lock(&vm_page_bucket_lock);
for (mem = bucket->pages; mem != VM_PAGE_NULL; mem = mem->next) {
VM_PAGE_CHECK(mem);
if ((mem->object == object) && (mem->offset == offset))
break;
}
simple_unlock(&vm_page_bucket_lock);
return(mem);
}
void
vm_page_rename(
register vm_page_t mem,
register vm_object_t new_object,
vm_object_offset_t new_offset)
{
assert(mem->object != new_object);
XPR(XPR_VM_PAGE,
"vm_page_rename, new object 0x%X, offset 0x%X page 0x%X\n",
(integer_t)new_object, (integer_t)new_offset,
(integer_t)mem, 0,0);
vm_page_lock_queues();
vm_page_remove(mem);
vm_page_insert(mem, new_object, new_offset);
vm_page_unlock_queues();
}
void
vm_page_init(
vm_page_t mem,
ppnum_t phys_page)
{
*mem = vm_page_template;
mem->phys_page = phys_page;
}
int c_vm_page_grab_fictitious = 0;
int c_vm_page_release_fictitious = 0;
int c_vm_page_more_fictitious = 0;
vm_page_t
vm_page_grab_fictitious(void)
{
register vm_page_t m;
m = (vm_page_t)zget(vm_page_zone);
if (m) {
vm_page_init(m, vm_page_fictitious_addr);
m->fictitious = TRUE;
}
c_vm_page_grab_fictitious++;
return m;
}
void
vm_page_release_fictitious(
register vm_page_t m)
{
assert(!m->free);
assert(m->busy);
assert(m->fictitious);
assert(m->phys_page == vm_page_fictitious_addr);
c_vm_page_release_fictitious++;
if (m->free)
panic("vm_page_release_fictitious");
m->free = TRUE;
zfree(vm_page_zone, (vm_offset_t)m);
}
void vm_page_more_fictitious(void)
{
extern vm_map_t zone_map;
register vm_page_t m;
vm_offset_t addr;
kern_return_t retval;
int i;
c_vm_page_more_fictitious++;
mutex_lock(&vm_page_alloc_lock);
if (zone_free_count(vm_page_zone) > 5) {
mutex_unlock(&vm_page_alloc_lock);
return;
}
if ((retval = kernel_memory_allocate(zone_map,
&addr, PAGE_SIZE, VM_PROT_ALL,
KMA_KOBJECT|KMA_NOPAGEWAIT)) != KERN_SUCCESS) {
mutex_unlock(&vm_page_alloc_lock);
vm_page_wait(THREAD_UNINT);
return;
}
m = (vm_page_t)addr;
for (i = PAGE_SIZE/sizeof(struct vm_page); i > 0; i--) {
vm_page_init(m, vm_page_fictitious_addr);
m->fictitious = TRUE;
m++;
}
zcram(vm_page_zone, addr, PAGE_SIZE);
mutex_unlock(&vm_page_alloc_lock);
}
boolean_t
vm_page_convert(
register vm_page_t m)
{
register vm_page_t real_m;
assert(m->busy);
assert(m->fictitious);
assert(!m->dirty);
real_m = vm_page_grab();
if (real_m == VM_PAGE_NULL)
return FALSE;
m->phys_page = real_m->phys_page;
m->fictitious = FALSE;
m->no_isync = TRUE;
vm_page_lock_queues();
if (m->active)
vm_page_active_count++;
else if (m->inactive)
vm_page_inactive_count++;
vm_page_unlock_queues();
real_m->phys_page = vm_page_fictitious_addr;
real_m->fictitious = TRUE;
vm_page_release_fictitious(real_m);
return TRUE;
}
int
vm_pool_low(void)
{
return( vm_page_free_count < vm_page_free_reserved );
}
unsigned long vm_page_grab_count = 0;
vm_page_t
vm_page_grab(void)
{
register vm_page_t mem;
mutex_lock(&vm_page_queue_free_lock);
vm_page_grab_count++;
if (vm_page_wire_count_warning > 0
&& vm_page_wire_count >= vm_page_wire_count_warning) {
printf("mk: vm_page_grab(): high wired page count of %d\n",
vm_page_wire_count);
assert(vm_page_wire_count < vm_page_wire_count_warning);
}
if (vm_page_gobble_count_warning > 0
&& vm_page_gobble_count >= vm_page_gobble_count_warning) {
printf("mk: vm_page_grab(): high gobbled page count of %d\n",
vm_page_gobble_count);
assert(vm_page_gobble_count < vm_page_gobble_count_warning);
}
if ((vm_page_free_count < vm_page_free_reserved) &&
!current_thread()->vm_privilege) {
mutex_unlock(&vm_page_queue_free_lock);
mem = VM_PAGE_NULL;
goto wakeup_pageout;
}
while (vm_page_queue_free == VM_PAGE_NULL) {
printf("vm_page_grab: no free pages, trouble expected...\n");
mutex_unlock(&vm_page_queue_free_lock);
VM_PAGE_WAIT();
mutex_lock(&vm_page_queue_free_lock);
}
if (--vm_page_free_count < vm_page_free_count_minimum)
vm_page_free_count_minimum = vm_page_free_count;
mem = vm_page_queue_free;
vm_page_queue_free = (vm_page_t) mem->pageq.next;
mem->free = FALSE;
mem->no_isync = TRUE;
mutex_unlock(&vm_page_queue_free_lock);
wakeup_pageout:
if ((vm_page_free_count < vm_page_free_min) ||
((vm_page_free_count < vm_page_free_target) &&
(vm_page_inactive_count < vm_page_inactive_target)))
thread_wakeup((event_t) &vm_page_free_wanted);
return mem;
}
void
vm_page_release(
register vm_page_t mem)
{
#if 0
unsigned int pindex;
phys_entry *physent;
physent = mapping_phys_lookup(mem->phys_page, &pindex);
if(physent->ppLink & ppN) {
panic("vm_page_release: already released - %08X %08X\n", mem, mem->phys_page);
}
physent->ppLink = physent->ppLink | ppN;
#endif
assert(!mem->private && !mem->fictitious);
mutex_lock(&vm_page_queue_free_lock);
if (mem->free)
panic("vm_page_release");
mem->free = TRUE;
mem->pageq.next = (queue_entry_t) vm_page_queue_free;
vm_page_queue_free = mem;
vm_page_free_count++;
if ((vm_page_free_wanted > 0) &&
(vm_page_free_count >= vm_page_free_reserved)) {
vm_page_free_wanted--;
thread_wakeup_one((event_t) &vm_page_free_count);
}
mutex_unlock(&vm_page_queue_free_lock);
}
#define VM_PAGEOUT_DEADLOCK_TIMEOUT 3
boolean_t
vm_page_wait(
int interruptible )
{
uint64_t abstime;
kern_return_t wait_result;
kern_return_t kr;
int need_wakeup = 0;
mutex_lock(&vm_page_queue_free_lock);
if (vm_page_free_count < vm_page_free_target) {
if (vm_page_free_wanted++ == 0)
need_wakeup = 1;
wait_result = assert_wait((event_t)&vm_page_free_count,
interruptible);
mutex_unlock(&vm_page_queue_free_lock);
counter(c_vm_page_wait_block++);
if (need_wakeup)
thread_wakeup((event_t)&vm_page_free_wanted);
if (wait_result == THREAD_WAITING) {
clock_interval_to_absolutetime_interval(
VM_PAGEOUT_DEADLOCK_TIMEOUT,
NSEC_PER_SEC, &abstime);
clock_absolutetime_interval_to_deadline(
abstime, &abstime);
thread_set_timer_deadline(abstime);
wait_result = thread_block(THREAD_CONTINUE_NULL);
if(wait_result == THREAD_TIMED_OUT) {
kr = vm_pageout_emergency_availability_request();
return TRUE;
} else {
thread_cancel_timer();
}
}
return(wait_result == THREAD_AWAKENED);
} else {
mutex_unlock(&vm_page_queue_free_lock);
return TRUE;
}
}
vm_page_t
vm_page_alloc(
vm_object_t object,
vm_object_offset_t offset)
{
register vm_page_t mem;
mem = vm_page_grab();
if (mem == VM_PAGE_NULL)
return VM_PAGE_NULL;
vm_page_insert(mem, object, offset);
return(mem);
}
counter(unsigned int c_laundry_pages_freed = 0;)
int vm_pagein_cluster_unused = 0;
boolean_t vm_page_free_verify = FALSE;
void
vm_page_free(
register vm_page_t mem)
{
vm_object_t object = mem->object;
assert(!mem->free);
assert(!mem->cleaning);
assert(!mem->pageout);
assert(!vm_page_free_verify || pmap_verify_free(mem->phys_page));
if (mem->tabled)
vm_page_remove(mem);
VM_PAGE_QUEUES_REMOVE(mem);
if (mem->clustered) {
mem->clustered = FALSE;
vm_pagein_cluster_unused++;
}
if (mem->wire_count) {
if (!mem->private && !mem->fictitious)
vm_page_wire_count--;
mem->wire_count = 0;
assert(!mem->gobbled);
} else if (mem->gobbled) {
if (!mem->private && !mem->fictitious)
vm_page_wire_count--;
vm_page_gobble_count--;
}
mem->gobbled = FALSE;
if (mem->laundry) {
extern int vm_page_laundry_min;
if (!object->internal)
vm_page_burst_count--;
vm_page_laundry_count--;
mem->laundry = FALSE;
counter(++c_laundry_pages_freed);
if (vm_page_laundry_count < vm_page_laundry_min) {
vm_page_laundry_min = 0;
thread_wakeup((event_t) &vm_page_laundry_count);
}
}
mem->discard_request = FALSE;
PAGE_WAKEUP(mem);
if (mem->absent)
vm_object_absent_release(object);
mem->page_lock = 0;
mem->unlock_request = 0;
mem->busy = TRUE;
mem->absent = FALSE;
mem->error = FALSE;
mem->dirty = FALSE;
mem->precious = FALSE;
mem->reference = FALSE;
mem->page_error = KERN_SUCCESS;
if (mem->private) {
mem->private = FALSE;
mem->fictitious = TRUE;
mem->phys_page = vm_page_fictitious_addr;
}
if (mem->fictitious) {
vm_page_release_fictitious(mem);
} else {
if(mem->zero_fill) {
vm_zf_count-=1;
mem->zero_fill = FALSE;
}
vm_page_init(mem, mem->phys_page);
vm_page_release(mem);
}
}
void
vm_page_free_list(
register vm_page_t mem)
{
register vm_page_t nxt;
register vm_page_t first = NULL;
register vm_page_t last;
register int pg_count = 0;
while (mem) {
nxt = (vm_page_t)(mem->pageq.next);
if (mem->clustered)
vm_pagein_cluster_unused++;
if (mem->laundry) {
extern int vm_page_laundry_min;
if (!mem->object->internal)
vm_page_burst_count--;
vm_page_laundry_count--;
counter(++c_laundry_pages_freed);
if (vm_page_laundry_count < vm_page_laundry_min) {
vm_page_laundry_min = 0;
thread_wakeup((event_t) &vm_page_laundry_count);
}
}
mem->busy = TRUE;
PAGE_WAKEUP(mem);
if (mem->private)
mem->fictitious = TRUE;
if (!mem->fictitious) {
if (mem->zero_fill)
vm_zf_count -= 1;
vm_page_init(mem, mem->phys_page);
mem->free = TRUE;
if (first == NULL)
last = mem;
mem->pageq.next = (queue_t) first;
first = mem;
pg_count++;
} else {
mem->phys_page = vm_page_fictitious_addr;
vm_page_release_fictitious(mem);
}
mem = nxt;
}
if (first) {
mutex_lock(&vm_page_queue_free_lock);
last->pageq.next = (queue_entry_t) vm_page_queue_free;
vm_page_queue_free = first;
vm_page_free_count += pg_count;
if ((vm_page_free_wanted > 0) &&
(vm_page_free_count >= vm_page_free_reserved)) {
int available_pages;
available_pages = vm_page_free_count - vm_page_free_reserved;
if (available_pages >= vm_page_free_wanted) {
vm_page_free_wanted = 0;
thread_wakeup((event_t) &vm_page_free_count);
} else {
while (available_pages--) {
vm_page_free_wanted--;
thread_wakeup_one((event_t) &vm_page_free_count);
}
}
}
mutex_unlock(&vm_page_queue_free_lock);
}
}
void
vm_page_wire(
register vm_page_t mem)
{
VM_PAGE_CHECK(mem);
if (mem->wire_count == 0) {
VM_PAGE_QUEUES_REMOVE(mem);
if (!mem->private && !mem->fictitious && !mem->gobbled)
vm_page_wire_count++;
if (mem->gobbled)
vm_page_gobble_count--;
mem->gobbled = FALSE;
if(mem->zero_fill) {
vm_zf_count-=1;
mem->zero_fill = FALSE;
}
}
assert(!mem->gobbled);
mem->wire_count++;
}
void
vm_page_gobble(
register vm_page_t mem)
{
vm_page_lock_queues();
VM_PAGE_CHECK(mem);
assert(!mem->gobbled);
assert(mem->wire_count == 0);
if (!mem->gobbled && mem->wire_count == 0) {
if (!mem->private && !mem->fictitious)
vm_page_wire_count++;
}
vm_page_gobble_count++;
mem->gobbled = TRUE;
vm_page_unlock_queues();
}
void
vm_page_unwire(
register vm_page_t mem)
{
VM_PAGE_CHECK(mem);
assert(mem->wire_count > 0);
if (--mem->wire_count == 0) {
assert(!mem->private && !mem->fictitious);
vm_page_wire_count--;
queue_enter(&vm_page_queue_active, mem, vm_page_t, pageq);
vm_page_active_count++;
mem->active = TRUE;
mem->reference = TRUE;
}
}
void
vm_page_deactivate(
register vm_page_t m)
{
VM_PAGE_CHECK(m);
if (m->gobbled) {
assert(m->wire_count == 0);
if (!m->private && !m->fictitious)
vm_page_wire_count--;
vm_page_gobble_count--;
m->gobbled = FALSE;
}
if (m->private || (m->wire_count != 0))
return;
if (m->active || (m->inactive && m->reference)) {
if (!m->fictitious && !m->absent)
pmap_clear_reference(m->phys_page);
m->reference = FALSE;
VM_PAGE_QUEUES_REMOVE(m);
}
if (m->wire_count == 0 && !m->inactive) {
m->page_ticket = vm_page_ticket;
vm_page_ticket_roll++;
if(vm_page_ticket_roll == VM_PAGE_TICKETS_IN_ROLL) {
vm_page_ticket_roll = 0;
if(vm_page_ticket == VM_PAGE_TICKET_ROLL_IDS)
vm_page_ticket= 0;
else
vm_page_ticket++;
}
if(m->zero_fill) {
queue_enter(&vm_page_queue_zf, m, vm_page_t, pageq);
} else {
queue_enter(&vm_page_queue_inactive,
m, vm_page_t, pageq);
}
m->inactive = TRUE;
if (!m->fictitious)
vm_page_inactive_count++;
}
}
void
vm_page_activate(
register vm_page_t m)
{
VM_PAGE_CHECK(m);
if (m->gobbled) {
assert(m->wire_count == 0);
if (!m->private && !m->fictitious)
vm_page_wire_count--;
vm_page_gobble_count--;
m->gobbled = FALSE;
}
if (m->private)
return;
if (m->inactive) {
if (m->zero_fill) {
queue_remove(&vm_page_queue_zf, m, vm_page_t, pageq);
} else {
queue_remove(&vm_page_queue_inactive,
m, vm_page_t, pageq);
}
if (!m->fictitious)
vm_page_inactive_count--;
m->inactive = FALSE;
}
if (m->wire_count == 0) {
if (m->active)
panic("vm_page_activate: already active");
queue_enter(&vm_page_queue_active, m, vm_page_t, pageq);
m->active = TRUE;
m->reference = TRUE;
if (!m->fictitious)
vm_page_active_count++;
}
}
void
vm_page_part_zero_fill(
vm_page_t m,
vm_offset_t m_pa,
vm_size_t len)
{
vm_page_t tmp;
VM_PAGE_CHECK(m);
#ifdef PMAP_ZERO_PART_PAGE_IMPLEMENTED
pmap_zero_part_page(m->phys_page, m_pa, len);
#else
while (1) {
tmp = vm_page_grab();
if (tmp == VM_PAGE_NULL) {
vm_page_wait(THREAD_UNINT);
continue;
}
break;
}
vm_page_zero_fill(tmp);
if(m_pa != 0) {
vm_page_part_copy(m, 0, tmp, 0, m_pa);
}
if((m_pa + len) < PAGE_SIZE) {
vm_page_part_copy(m, m_pa + len, tmp,
m_pa + len, PAGE_SIZE - (m_pa + len));
}
vm_page_copy(tmp,m);
vm_page_lock_queues();
vm_page_free(tmp);
vm_page_unlock_queues();
#endif
}
void
vm_page_zero_fill(
vm_page_t m)
{
XPR(XPR_VM_PAGE,
"vm_page_zero_fill, object 0x%X offset 0x%X page 0x%X\n",
(integer_t)m->object, (integer_t)m->offset, (integer_t)m, 0,0);
VM_PAGE_CHECK(m);
pmap_zero_page(m->phys_page);
}
void
vm_page_part_copy(
vm_page_t src_m,
vm_offset_t src_pa,
vm_page_t dst_m,
vm_offset_t dst_pa,
vm_size_t len)
{
VM_PAGE_CHECK(src_m);
VM_PAGE_CHECK(dst_m);
pmap_copy_part_page(src_m->phys_page, src_pa,
dst_m->phys_page, dst_pa, len);
}
void
vm_page_copy(
vm_page_t src_m,
vm_page_t dest_m)
{
XPR(XPR_VM_PAGE,
"vm_page_copy, object 0x%X offset 0x%X to object 0x%X offset 0x%X\n",
(integer_t)src_m->object, src_m->offset,
(integer_t)dest_m->object, dest_m->offset,
0);
VM_PAGE_CHECK(src_m);
VM_PAGE_CHECK(dest_m);
pmap_copy_page(src_m->phys_page, dest_m->phys_page);
}
#define SET_NEXT_PAGE(m,n) ((m)->pageq.next = (struct queue_entry *) (n))
#if MACH_ASSERT
int vm_page_verify_contiguous(
vm_page_t pages,
unsigned int npages);
#endif
cpm_counter(unsigned int vpfls_pages_handled = 0;)
cpm_counter(unsigned int vpfls_head_insertions = 0;)
cpm_counter(unsigned int vpfls_tail_insertions = 0;)
cpm_counter(unsigned int vpfls_general_insertions = 0;)
cpm_counter(unsigned int vpfc_failed = 0;)
cpm_counter(unsigned int vpfc_satisfied = 0;)
#if MACH_ASSERT
int
vm_page_verify_contiguous(
vm_page_t pages,
unsigned int npages)
{
register vm_page_t m;
unsigned int page_count;
ppnum_t prev_addr;
prev_addr = pages->phys_page;
page_count = 1;
for (m = NEXT_PAGE(pages); m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
if (m->phys_page != prev_addr + 1) {
printf("m 0x%x prev_addr 0x%x, current addr 0x%x\n",
m, prev_addr, m->phys_page);
printf("pages 0x%x page_count %u\n", pages, page_count);
panic("vm_page_verify_contiguous: not contiguous!");
}
prev_addr = m->phys_page;
++page_count;
}
if (page_count != npages) {
printf("pages 0x%x actual count 0x%x but requested 0x%x\n",
pages, page_count, npages);
panic("vm_page_verify_contiguous: count error");
}
return 1;
}
#endif
static vm_page_t
vm_page_find_contiguous(
unsigned int contig_pages)
{
vm_page_t sort_list;
vm_page_t *contfirstprev, contlast;
vm_page_t m, m1;
ppnum_t prevcontaddr;
ppnum_t nextcontaddr;
unsigned int npages;
#if MACH_ASSERT
npages = 0;
for (m = vm_page_queue_free; m != VM_PAGE_NULL; m = NEXT_PAGE(m))
++npages;
if (npages != vm_page_free_count)
panic("vm_sort_free_list: prelim: npages %u free_count %d",
npages, vm_page_free_count);
#endif
if (contig_pages == 0 || vm_page_queue_free == VM_PAGE_NULL)
return VM_PAGE_NULL;
#define PPNUM_PREV(x) (((x) > 0) ? ((x) - 1) : 0)
#define PPNUM_NEXT(x) (((x) < PPNUM_MAX) ? ((x) + 1) : PPNUM_MAX)
npages = 1;
contfirstprev = &sort_list;
contlast = sort_list = vm_page_queue_free;
vm_page_queue_free = NEXT_PAGE(sort_list);
SET_NEXT_PAGE(sort_list, VM_PAGE_NULL);
prevcontaddr = PPNUM_PREV(sort_list->phys_page);
nextcontaddr = PPNUM_NEXT(sort_list->phys_page);
while (npages < contig_pages &&
(m = vm_page_queue_free) != VM_PAGE_NULL)
{
cpm_counter(++vpfls_pages_handled);
if (m->phys_page == prevcontaddr)
{
vm_page_queue_free = NEXT_PAGE(m);
cpm_counter(++vpfls_head_insertions);
prevcontaddr = PPNUM_PREV(prevcontaddr);
SET_NEXT_PAGE(m, *contfirstprev);
*contfirstprev = m;
npages++;
continue;
}
else if (m->phys_page == nextcontaddr)
{
vm_page_queue_free = NEXT_PAGE(m);
cpm_counter(++vpfls_tail_insertions);
nextcontaddr = PPNUM_NEXT(nextcontaddr);
SET_NEXT_PAGE(m, NEXT_PAGE(contlast));
SET_NEXT_PAGE(contlast, m);
contlast = m;
npages++;
}
else if (m->phys_page < sort_list->phys_page)
{
vm_page_queue_free = NEXT_PAGE(m);
cpm_counter(++vpfls_general_insertions);
prevcontaddr = PPNUM_PREV(m->phys_page);
nextcontaddr = PPNUM_NEXT(m->phys_page);
SET_NEXT_PAGE(m, sort_list);
contfirstprev = &sort_list;
contlast = sort_list = m;
npages = 1;
}
else
{
if (m->phys_page < nextcontaddr)
{
prevcontaddr = PPNUM_PREV(sort_list->phys_page);
nextcontaddr = PPNUM_NEXT(sort_list->phys_page);
contfirstprev = &sort_list;
contlast = sort_list;
npages = 1;
}
for (m1 = NEXT_PAGE(contlast);
npages < contig_pages &&
m1 != VM_PAGE_NULL && m1->phys_page < m->phys_page;
m1 = NEXT_PAGE(m1))
{
if (m1->phys_page != nextcontaddr) {
prevcontaddr = PPNUM_PREV(m1->phys_page);
contfirstprev = NEXT_PAGE_PTR(contlast);
npages = 1;
} else {
npages++;
}
nextcontaddr = PPNUM_NEXT(m1->phys_page);
contlast = m1;
}
if (npages == contig_pages)
break;
if (m->phys_page != nextcontaddr) {
contfirstprev = NEXT_PAGE_PTR(contlast);
prevcontaddr = PPNUM_PREV(m->phys_page);
nextcontaddr = PPNUM_NEXT(m->phys_page);
npages = 1;
} else {
nextcontaddr = PPNUM_NEXT(nextcontaddr);
npages++;
}
vm_page_queue_free = NEXT_PAGE(m);
cpm_counter(++vpfls_general_insertions);
SET_NEXT_PAGE(m, NEXT_PAGE(contlast));
SET_NEXT_PAGE(contlast, m);
contlast = m;
}
for (m1 = NEXT_PAGE(m);
npages < contig_pages &&
m1 != VM_PAGE_NULL && m1->phys_page == nextcontaddr;
m1 = NEXT_PAGE(m1))
{
nextcontaddr = PPNUM_NEXT(nextcontaddr);
contlast = m1;
npages++;
}
}
if (npages == contig_pages)
{
cpm_counter(++vpfc_satisfied);
m = *contfirstprev;
*contfirstprev = NEXT_PAGE(contlast);
SET_NEXT_PAGE(contlast, VM_PAGE_NULL);
assert(vm_page_verify_contiguous(m, npages));
for (m1 = m; m1 != VM_PAGE_NULL; m1 = NEXT_PAGE(m1)) {
assert(m1->free);
assert(!m1->wanted);
m1->free = FALSE;
m1->no_isync = TRUE;
m1->gobbled = TRUE;
}
vm_page_wire_count += npages;
vm_page_gobble_count += npages;
vm_page_free_count -= npages;
while ((m1 = *contfirstprev) != VM_PAGE_NULL)
contfirstprev = (vm_page_t *)&m1->pageq.next;
*contfirstprev = vm_page_queue_free;
}
vm_page_queue_free = sort_list;
return m;
}
kern_return_t
cpm_allocate(
vm_size_t size,
vm_page_t *list,
boolean_t wire)
{
register vm_page_t m;
vm_page_t *first_contig;
vm_page_t free_list, pages;
unsigned int npages, n1pages;
int vm_pages_available;
boolean_t wakeup;
if (size % page_size != 0)
return KERN_INVALID_ARGUMENT;
vm_page_lock_queues();
mutex_lock(&vm_page_queue_free_lock);
npages = size / page_size;
vm_pages_available = vm_page_free_count - vm_page_free_reserved;
if (npages > vm_pages_available) {
mutex_unlock(&vm_page_queue_free_lock);
vm_page_unlock_queues();
return KERN_RESOURCE_SHORTAGE;
}
pages = vm_page_find_contiguous(npages);
if (vm_page_free_count < vm_page_free_count_minimum)
vm_page_free_count_minimum = vm_page_free_count;
wakeup = ((vm_page_free_count < vm_page_free_min) ||
((vm_page_free_count < vm_page_free_target) &&
(vm_page_inactive_count < vm_page_inactive_target)));
mutex_unlock(&vm_page_queue_free_lock);
if (pages == VM_PAGE_NULL) {
vm_page_unlock_queues();
return KERN_NO_SPACE;
}
if (wire == TRUE)
for (m = pages; m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
assert(!m->active);
assert(!m->inactive);
assert(!m->private);
assert(!m->fictitious);
assert(m->wire_count == 0);
assert(m->gobbled);
m->gobbled = FALSE;
m->wire_count++;
--vm_page_gobble_count;
}
vm_page_unlock_queues();
if (wakeup)
thread_wakeup((event_t) &vm_page_free_wanted);
assert(vm_page_verify_contiguous(pages, npages));
*list = pages;
return KERN_SUCCESS;
}
#include <mach_vm_debug.h>
#if MACH_VM_DEBUG
#include <mach_debug/hash_info.h>
#include <vm/vm_debug.h>
unsigned int
vm_page_info(
hash_info_bucket_t *info,
unsigned int count)
{
int i;
if (vm_page_bucket_count < count)
count = vm_page_bucket_count;
for (i = 0; i < count; i++) {
vm_page_bucket_t *bucket = &vm_page_buckets[i];
unsigned int bucket_count = 0;
vm_page_t m;
simple_lock(&vm_page_bucket_lock);
for (m = bucket->pages; m != VM_PAGE_NULL; m = m->next)
bucket_count++;
simple_unlock(&vm_page_bucket_lock);
info[i].hib_count = bucket_count;
}
return vm_page_bucket_count;
}
#endif
#include <mach_kdb.h>
#if MACH_KDB
#include <ddb/db_output.h>
#include <vm/vm_print.h>
#define printf kdbprintf
void
vm_page_print(
vm_page_t p)
{
extern db_indent;
iprintf("page 0x%x\n", p);
db_indent += 2;
iprintf("object=0x%x", p->object);
printf(", offset=0x%x", p->offset);
printf(", wire_count=%d", p->wire_count);
iprintf("%sinactive, %sactive, %sgobbled, %slaundry, %sfree, %sref, %sdiscard\n",
(p->inactive ? "" : "!"),
(p->active ? "" : "!"),
(p->gobbled ? "" : "!"),
(p->laundry ? "" : "!"),
(p->free ? "" : "!"),
(p->reference ? "" : "!"),
(p->discard_request ? "" : "!"));
iprintf("%sbusy, %swanted, %stabled, %sfictitious, %sprivate, %sprecious\n",
(p->busy ? "" : "!"),
(p->wanted ? "" : "!"),
(p->tabled ? "" : "!"),
(p->fictitious ? "" : "!"),
(p->private ? "" : "!"),
(p->precious ? "" : "!"));
iprintf("%sabsent, %serror, %sdirty, %scleaning, %spageout, %sclustered\n",
(p->absent ? "" : "!"),
(p->error ? "" : "!"),
(p->dirty ? "" : "!"),
(p->cleaning ? "" : "!"),
(p->pageout ? "" : "!"),
(p->clustered ? "" : "!"));
iprintf("%slock_supplied, %soverwriting, %srestart, %sunusual\n",
(p->lock_supplied ? "" : "!"),
(p->overwriting ? "" : "!"),
(p->restart ? "" : "!"),
(p->unusual ? "" : "!"));
iprintf("phys_page=0x%x", p->phys_page);
printf(", page_error=0x%x", p->page_error);
printf(", page_lock=0x%x", p->page_lock);
printf(", unlock_request=%d\n", p->unlock_request);
db_indent -= 2;
}
#endif