#ifndef _VM_VM_PAGE_H_
#define _VM_VM_PAGE_H_
#include <debug.h>
#include <mach/boolean.h>
#include <mach/vm_prot.h>
#include <mach/vm_param.h>
#include <vm/vm_object.h>
#include <kern/queue.h>
#include <kern/lock.h>
#include <kern/macro_help.h>
#include <libkern/OSAtomic.h>
#define VM_PAGE_MAX_SPECULATIVE_AGE_Q 10
#define VM_PAGE_MIN_SPECULATIVE_AGE_Q 1
#define VM_PAGE_SPECULATIVE_AGED_Q 0
#define VM_PAGE_SPECULATIVE_Q_AGE_MS 500
struct vm_speculative_age_q {
queue_head_t age_q;
mach_timespec_t age_ts;
};
extern
struct vm_speculative_age_q vm_page_queue_speculative[];
extern int speculative_steal_index;
extern int speculative_age_index;
extern unsigned int vm_page_speculative_q_age_ms;
struct vm_page {
queue_chain_t pageq;
queue_chain_t listq;
struct vm_page *next;
vm_object_t object;
vm_object_offset_t offset;
#define local_id wire_count
unsigned int wire_count:16,
inactive:1,
zero_fill:1,
active:1,
pageout_queue:1,
speculative:1,
laundry:1,
free:1,
reference:1,
gobbled:1,
private:1,
throttled:1,
local:1,
no_cache:1,
__unused_pageq_bits:3;
ppnum_t phys_page;
unsigned int
busy:1,
wanted:1,
tabled:1,
fictitious:1,
pmapped:1,
wpmapped:1,
pageout:1,
absent:1,
error:1,
dirty:1,
cleaning:1,
precious:1,
clustered:1,
overwriting:1,
restart:1,
unusual:1,
encrypted:1,
encrypted_cleaning:1,
list_req_pending:1,
dump_cleaning:1,
cs_validated:1,
cs_tainted:1,
reusable:1,
lopage:1,
slid:1,
__unused_object_bits:7;
#if __LP64__
unsigned int __unused_padding;
#endif
};
#define DEBUG_ENCRYPTED_SWAP 1
#if DEBUG_ENCRYPTED_SWAP
#define ASSERT_PAGE_DECRYPTED(page) \
MACRO_BEGIN \
if ((page)->encrypted) { \
panic("VM page %p should not be encrypted here\n", \
(page)); \
} \
MACRO_END
#else
#define ASSERT_PAGE_DECRYPTED(page) assert(!(page)->encrypted)
#endif
typedef struct vm_page *vm_page_t;
typedef struct vm_locks_array {
char pad __attribute__ ((aligned (64)));
lck_mtx_t vm_page_queue_lock2 __attribute__ ((aligned (64)));
lck_mtx_t vm_page_queue_free_lock2 __attribute__ ((aligned (64)));
char pad2 __attribute__ ((aligned (64)));
} vm_locks_array_t;
#define VM_PAGE_WIRED(m) ((!(m)->local && (m)->wire_count))
#define VM_PAGE_NULL ((vm_page_t) 0)
#define NEXT_PAGE(m) ((vm_page_t) (m)->pageq.next)
#define NEXT_PAGE_PTR(m) ((vm_page_t *) &(m)->pageq.next)
#define VM_PAGE_CHECK(mem) \
MACRO_BEGIN \
VM_PAGE_QUEUES_ASSERT(mem, 1); \
MACRO_END
#define MAX_COLORS 128
#define DEFAULT_COLORS 32
extern
unsigned int vm_colors;
extern
unsigned int vm_color_mask;
extern
unsigned int vm_cache_geometry_colors;
#define VM_NOT_USER_WIREABLE (64*1024*1024)
extern
vm_map_size_t vm_user_wire_limit;
extern
vm_map_size_t vm_global_user_wire_limit;
extern
vm_map_size_t vm_global_no_user_wire_amount;
#define VPL_LOCK_SPIN 1
struct vpl {
unsigned int vpl_count;
queue_head_t vpl_queue;
#ifdef VPL_LOCK_SPIN
lck_spin_t vpl_lock;
#else
lck_mtx_t vpl_lock;
lck_mtx_ext_t vpl_lock_ext;
#endif
};
struct vplq {
union {
char cache_line_pad[128];
struct vpl vpl;
} vpl_un;
};
extern
unsigned int vm_page_local_q_count;
extern
struct vplq *vm_page_local_q;
extern
unsigned int vm_page_local_q_soft_limit;
extern
unsigned int vm_page_local_q_hard_limit;
extern
vm_locks_array_t vm_page_locks;
extern
queue_head_t vm_page_queue_free[MAX_COLORS];
extern
queue_head_t vm_lopage_queue_free;
extern
queue_head_t vm_page_queue_active;
extern
queue_head_t vm_page_queue_inactive;
extern
queue_head_t vm_page_queue_zf;
extern
queue_head_t vm_page_queue_throttled;
extern
vm_offset_t first_phys_addr;
extern
vm_offset_t last_phys_addr;
extern
unsigned int vm_page_free_count;
extern
unsigned int vm_page_fictitious_count;
extern
unsigned int vm_page_active_count;
extern
unsigned int vm_page_inactive_count;
extern
unsigned int vm_page_throttled_count;
extern
unsigned int vm_page_speculative_count;
extern
unsigned int vm_page_wire_count;
extern
unsigned int vm_page_free_target;
extern
unsigned int vm_page_free_min;
extern
unsigned int vm_page_throttle_limit;
extern
uint32_t vm_page_creation_throttle;
extern
unsigned int vm_page_inactive_target;
extern
unsigned int vm_page_inactive_min;
extern
unsigned int vm_page_free_reserved;
extern
unsigned int vm_page_throttle_count;
extern
unsigned int vm_page_gobble_count;
#if DEVELOPMENT || DEBUG
extern
unsigned int vm_page_speculative_used;
#endif
extern
unsigned int vm_page_purgeable_count;
extern
unsigned int vm_page_purgeable_wired_count;
extern
uint64_t vm_page_purged_count;
extern unsigned int vm_page_free_wanted;
extern unsigned int vm_page_free_wanted_privileged;
extern ppnum_t vm_page_fictitious_addr;
extern ppnum_t vm_page_guard_addr;
extern boolean_t vm_page_deactivate_hint;
extern int vm_himemory_mode;
extern boolean_t vm_lopage_needed;
extern uint32_t vm_lopage_free_count;
extern uint32_t vm_lopage_free_limit;
extern uint32_t vm_lopage_lowater;
extern boolean_t vm_lopage_refill;
extern uint64_t max_valid_dma_address;
extern ppnum_t max_valid_low_ppnum;
extern void vm_page_bootstrap(
vm_offset_t *startp,
vm_offset_t *endp) __attribute__((section("__TEXT, initcode")));
extern void vm_page_module_init(void) __attribute__((section("__TEXT, initcode")));
extern void vm_page_init_local_q(void);
extern void vm_page_create(
ppnum_t start,
ppnum_t end);
extern vm_page_t vm_page_lookup(
vm_object_t object,
vm_object_offset_t offset);
extern vm_page_t vm_page_grab_fictitious(void);
extern vm_page_t vm_page_grab_guard(void);
extern void vm_page_release_fictitious(
vm_page_t page);
extern void vm_page_more_fictitious(void);
extern int vm_pool_low(void);
extern vm_page_t vm_page_grab(void);
extern vm_page_t vm_page_grablo(void);
extern void vm_page_release(
vm_page_t page);
extern boolean_t vm_page_wait(
int interruptible );
extern vm_page_t vm_page_alloc(
vm_object_t object,
vm_object_offset_t offset);
extern vm_page_t vm_page_alloclo(
vm_object_t object,
vm_object_offset_t offset);
extern vm_page_t vm_page_alloc_guard(
vm_object_t object,
vm_object_offset_t offset);
extern void vm_page_init(
vm_page_t page,
ppnum_t phys_page,
boolean_t lopage);
extern void vm_page_free(
vm_page_t page);
extern void vm_page_free_unlocked(
vm_page_t page,
boolean_t remove_from_hash);
extern void vm_page_activate(
vm_page_t page);
extern void vm_page_deactivate(
vm_page_t page);
extern void vm_page_deactivate_internal(
vm_page_t page,
boolean_t clear_hw_reference);
extern void vm_page_lru(
vm_page_t page);
extern void vm_page_speculate(
vm_page_t page,
boolean_t new);
extern void vm_page_speculate_ageit(
struct vm_speculative_age_q *aq);
extern void vm_page_reactivate_all_throttled(void);
extern void vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t nolocks);
extern void vm_page_rename(
vm_page_t page,
vm_object_t new_object,
vm_object_offset_t new_offset,
boolean_t encrypted_ok);
extern void vm_page_insert(
vm_page_t page,
vm_object_t object,
vm_object_offset_t offset);
extern void vm_page_insert_internal(
vm_page_t page,
vm_object_t object,
vm_object_offset_t offset,
boolean_t queues_lock_held,
boolean_t insert_in_hash);
extern void vm_page_replace(
vm_page_t mem,
vm_object_t object,
vm_object_offset_t offset);
extern void vm_page_remove(
vm_page_t page,
boolean_t remove_from_hash);
extern void vm_page_zero_fill(
vm_page_t page);
extern void vm_page_part_zero_fill(
vm_page_t m,
vm_offset_t m_pa,
vm_size_t len);
extern void vm_page_copy(
vm_page_t src_page,
vm_page_t dest_page);
extern void vm_page_part_copy(
vm_page_t src_m,
vm_offset_t src_pa,
vm_page_t dst_m,
vm_offset_t dst_pa,
vm_size_t len);
extern void vm_page_wire(
vm_page_t page);
extern void vm_page_unwire(
vm_page_t page,
boolean_t queueit);
extern void vm_set_page_size(void);
extern void vm_page_gobble(
vm_page_t page);
extern void vm_page_validate_cs(vm_page_t page);
extern void vm_page_validate_cs_mapped(
vm_page_t page,
const void *kaddr);
extern void vm_page_free_prepare_queues(
vm_page_t page);
extern void vm_page_free_prepare_object(
vm_page_t page,
boolean_t remove_from_hash);
extern void vm_check_memorystatus(void);
#define PAGE_ASSERT_WAIT(m, interruptible) \
(((m)->wanted = TRUE), \
assert_wait((event_t) (m), (interruptible)))
#define PAGE_SLEEP(o, m, interruptible) \
(((m)->wanted = TRUE), \
thread_sleep_vm_object((o), (m), (interruptible)))
#define PAGE_WAKEUP_DONE(m) \
MACRO_BEGIN \
(m)->busy = FALSE; \
if ((m)->wanted) { \
(m)->wanted = FALSE; \
thread_wakeup((event_t) (m)); \
} \
MACRO_END
#define PAGE_WAKEUP(m) \
MACRO_BEGIN \
if ((m)->wanted) { \
(m)->wanted = FALSE; \
thread_wakeup((event_t) (m)); \
} \
MACRO_END
#define VM_PAGE_FREE(p) \
MACRO_BEGIN \
vm_page_free_unlocked(p, TRUE); \
MACRO_END
#define VM_PAGE_GRAB_FICTITIOUS(M) \
MACRO_BEGIN \
while ((M = vm_page_grab_fictitious()) == VM_PAGE_NULL) \
vm_page_more_fictitious(); \
MACRO_END
#define VM_PAGE_WAIT() ((void)vm_page_wait(THREAD_UNINT))
#define vm_page_queue_lock (vm_page_locks.vm_page_queue_lock2)
#define vm_page_queue_free_lock (vm_page_locks.vm_page_queue_free_lock2)
#define vm_page_lock_queues() lck_mtx_lock(&vm_page_queue_lock)
#define vm_page_unlock_queues() lck_mtx_unlock(&vm_page_queue_lock)
#define vm_page_lockspin_queues() lck_mtx_lock_spin(&vm_page_queue_lock)
#define vm_page_trylockspin_queues() lck_mtx_try_lock_spin(&vm_page_queue_lock)
#define vm_page_lockconvert_queues() lck_mtx_convert_spin(&vm_page_queue_lock)
#ifdef VPL_LOCK_SPIN
#define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_spin_init(&vlq->vpl_lock, vpl_grp, vpl_attr)
#define VPL_LOCK(vpl) lck_spin_lock(vpl)
#define VPL_UNLOCK(vpl) lck_spin_unlock(vpl)
#else
#define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_mtx_init_ext(&vlq->vpl_lock, &vlq->vpl_lock_ext, vpl_grp, vpl_attr)
#define VPL_LOCK(vpl) lck_mtx_lock_spin(vpl)
#define VPL_UNLOCK(vpl) lck_mtx_unlock(vpl)
#endif
#if MACH_ASSERT
extern void vm_page_queues_assert(vm_page_t mem, int val);
#define VM_PAGE_QUEUES_ASSERT(mem, val) vm_page_queues_assert((mem), (val))
#else
#define VM_PAGE_QUEUES_ASSERT(mem, val)
#endif
#define VM_PAGE_QUEUES_REMOVE(mem) \
MACRO_BEGIN \
VM_PAGE_QUEUES_ASSERT(mem, 1); \
assert(!mem->laundry); \
assert(!mem->pageout_queue); \
if (mem->local) { \
struct vpl *lq; \
assert(mem->object != kernel_object); \
assert(!mem->inactive && !mem->speculative); \
assert(!mem->active && !mem->throttled); \
assert(!mem->fictitious); \
lq = &vm_page_local_q[mem->local_id].vpl_un.vpl; \
VPL_LOCK(&lq->vpl_lock); \
queue_remove(&lq->vpl_queue, \
mem, vm_page_t, pageq); \
mem->local = FALSE; \
mem->local_id = 0; \
lq->vpl_count--; \
VPL_UNLOCK(&lq->vpl_lock); \
} \
\
else if (mem->active) { \
assert(mem->object != kernel_object); \
assert(!mem->inactive && !mem->speculative); \
assert(!mem->throttled); \
assert(!mem->fictitious); \
queue_remove(&vm_page_queue_active, \
mem, vm_page_t, pageq); \
mem->active = FALSE; \
vm_page_active_count--; \
} \
\
else if (mem->inactive) { \
assert(mem->object != kernel_object); \
assert(!mem->active && !mem->speculative); \
assert(!mem->throttled); \
assert(!mem->fictitious); \
if (mem->zero_fill) { \
queue_remove(&vm_page_queue_zf, \
mem, vm_page_t, pageq); \
vm_zf_queue_count--; \
} else { \
queue_remove(&vm_page_queue_inactive, \
mem, vm_page_t, pageq); \
} \
mem->inactive = FALSE; \
vm_page_inactive_count--; \
vm_purgeable_q_advance_all(); \
} \
\
else if (mem->throttled) { \
assert(!mem->active && !mem->inactive); \
assert(!mem->speculative); \
assert(!mem->fictitious); \
queue_remove(&vm_page_queue_throttled, \
mem, vm_page_t, pageq); \
mem->throttled = FALSE; \
vm_page_throttled_count--; \
} \
\
else if (mem->speculative) { \
assert(!mem->active && !mem->inactive); \
assert(!mem->throttled); \
assert(!mem->fictitious); \
remque(&mem->pageq); \
mem->speculative = FALSE; \
vm_page_speculative_count--; \
} \
\
else if (mem->pageq.next || mem->pageq.prev) \
panic("VM_PAGE_QUEUES_REMOVE: unmarked page on Q"); \
mem->pageq.next = NULL; \
mem->pageq.prev = NULL; \
VM_PAGE_QUEUES_ASSERT(mem, 0); \
MACRO_END
#define VM_PAGE_ENQUEUE_INACTIVE(mem, first) \
MACRO_BEGIN \
VM_PAGE_QUEUES_ASSERT(mem, 0); \
assert(!mem->fictitious); \
assert(!mem->laundry); \
assert(!mem->pageout_queue); \
if (mem->zero_fill) { \
if (first == TRUE) \
queue_enter_first(&vm_page_queue_zf, mem, vm_page_t, pageq); \
else \
queue_enter(&vm_page_queue_zf, mem, vm_page_t, pageq); \
vm_zf_queue_count++; \
} else { \
if (first == TRUE) \
queue_enter_first(&vm_page_queue_inactive, mem, vm_page_t, pageq); \
else \
queue_enter(&vm_page_queue_inactive, mem, vm_page_t, pageq); \
} \
mem->inactive = TRUE; \
vm_page_inactive_count++; \
token_new_pagecount++; \
MACRO_END
#if DEVELOPMENT || DEBUG
#define VM_PAGE_SPECULATIVE_USED_ADD() \
MACRO_BEGIN \
OSAddAtomic(1, &vm_page_speculative_used); \
MACRO_END
#else
#define VM_PAGE_SPECULATIVE_USED_ADD()
#endif
#define VM_PAGE_CONSUME_CLUSTERED(mem) \
MACRO_BEGIN \
if (mem->clustered) { \
assert(mem->object); \
mem->object->pages_used++; \
mem->clustered = FALSE; \
VM_PAGE_SPECULATIVE_USED_ADD(); \
} \
MACRO_END
#define DW_vm_page_unwire 0x01
#define DW_vm_page_wire 0x02
#define DW_vm_page_free 0x04
#define DW_vm_page_activate 0x08
#define DW_vm_page_deactivate_internal 0x10
#define DW_vm_page_speculate 0x20
#define DW_vm_page_lru 0x40
#define DW_vm_pageout_throttle_up 0x80
#define DW_PAGE_WAKEUP 0x100
#define DW_clear_busy 0x200
#define DW_clear_reference 0x400
#define DW_set_reference 0x800
#define DW_move_page 0x1000
#define DW_VM_PAGE_QUEUES_REMOVE 0x2000
#define DW_set_list_req_pending 0x4000
struct vm_page_delayed_work {
vm_page_t dw_m;
int dw_mask;
};
void vm_page_do_delayed_work(vm_object_t object, struct vm_page_delayed_work *dwp, int dw_count);
extern unsigned int vm_max_delayed_work_limit;
#define DEFAULT_DELAYED_WORK_LIMIT 32
#define DELAYED_WORK_LIMIT(max) ((vm_max_delayed_work_limit >= max ? max : vm_max_delayed_work_limit))
#define VM_PAGE_ADD_DELAYED_WORK(dwp, mem, dw_cnt) \
MACRO_BEGIN \
if (mem->busy == FALSE) { \
mem->busy = TRUE; \
if ( !(dwp->dw_mask & DW_vm_page_free)) \
dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); \
} \
if (mem->list_req_pending) { \
mem->list_req_pending = FALSE; \
dwp->dw_mask |= DW_set_list_req_pending; \
} \
dwp->dw_m = mem; \
dwp++; \
dw_count++; \
MACRO_END
extern vm_page_t vm_object_page_grab(vm_object_t);
#endif