#ifndef _VM_VM_PAGE_H_
#define _VM_VM_PAGE_H_
#include <debug.h>
#include <vm/vm_options.h>
#include <mach/boolean.h>
#include <mach/vm_prot.h>
#include <mach/vm_param.h>
#include <vm/vm_object.h>
#include <kern/queue.h>
#include <kern/locks.h>
#include <kern/macro_help.h>
#include <libkern/OSAtomic.h>
#define VM_PAGE_MAX_SPECULATIVE_AGE_Q 10
#define VM_PAGE_MIN_SPECULATIVE_AGE_Q 1
#define VM_PAGE_SPECULATIVE_AGED_Q 0
#define VM_PAGE_SPECULATIVE_Q_AGE_MS 500
struct vm_speculative_age_q {
queue_head_t age_q;
mach_timespec_t age_ts;
};
extern
struct vm_speculative_age_q vm_page_queue_speculative[];
extern int speculative_steal_index;
extern int speculative_age_index;
extern unsigned int vm_page_speculative_q_age_ms;
#define VM_PAGE_COMPRESSOR_COUNT (compressor_object->resident_page_count)
#if defined(__LP64__)
typedef uint32_t vm_page_packed_t;
#define VM_PAGE_PACK_PTR(m) (!(m) ? (vm_page_packed_t)0 : ((vm_page_packed_t)((uintptr_t)(((uintptr_t)(m) - (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS)) >> 6)))
#define VM_PAGE_UNPACK_PTR(p) (!(p) ? VM_PAGE_NULL : ((vm_page_t)((((uintptr_t)(p)) << 6) + (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS)))
#else
typedef struct vm_page *vm_page_packed_t;
#define VM_PAGE_PACK_PTR(m) ((vm_page_packed_t)(m))
#define VM_PAGE_UNPACK_PTR(p) ((vm_page_t)(p))
#endif
struct vm_page {
queue_chain_t pageq;
queue_chain_t listq;
vm_object_offset_t offset;
vm_object_t object;
vm_page_packed_t next_m;
#define local_id wire_count
unsigned int wire_count:16,
active:1,
inactive:1,
clean_queue:1,
local:1,
speculative:1,
throttled:1,
free:1,
pageout_queue:1,
laundry:1,
reference:1,
gobbled:1,
private:1,
no_cache:1,
__unused_pageq_bits:3;
ppnum_t phys_page;
unsigned int
busy:1,
wanted:1,
tabled:1,
hashed:1,
fictitious:1,
clustered:1,
pmapped:1,
xpmapped:1,
wpmapped:1,
pageout:1,
absent:1,
error:1,
dirty:1,
cleaning:1,
precious:1,
overwriting:1,
restart:1,
unusual:1,
encrypted:1,
encrypted_cleaning:1,
cs_validated:1,
cs_tainted:1,
cs_nx:1,
reusable:1,
lopage:1,
slid:1,
compressor:1,
written_by_kernel:1,
__unused_object_bits:4;
};
#define DEBUG_ENCRYPTED_SWAP 1
#if DEBUG_ENCRYPTED_SWAP
#define ASSERT_PAGE_DECRYPTED(page) \
MACRO_BEGIN \
if ((page)->encrypted) { \
panic("VM page %p should not be encrypted here\n", \
(page)); \
} \
MACRO_END
#else
#define ASSERT_PAGE_DECRYPTED(page) assert(!(page)->encrypted)
#endif
typedef struct vm_page *vm_page_t;
typedef struct vm_locks_array {
char pad __attribute__ ((aligned (64)));
lck_mtx_t vm_page_queue_lock2 __attribute__ ((aligned (64)));
lck_mtx_t vm_page_queue_free_lock2 __attribute__ ((aligned (64)));
char pad2 __attribute__ ((aligned (64)));
} vm_locks_array_t;
#define VM_PAGE_WIRED(m) ((!(m)->local && (m)->wire_count))
#define VM_PAGE_NULL ((vm_page_t) 0)
#define NEXT_PAGE(m) ((vm_page_t) (m)->pageq.next)
#define NEXT_PAGE_PTR(m) ((vm_page_t *) &(m)->pageq.next)
#define VM_PAGE_CHECK(mem) \
MACRO_BEGIN \
VM_PAGE_QUEUES_ASSERT(mem, 1); \
MACRO_END
#define MAX_COLORS 128
#define DEFAULT_COLORS 32
extern
unsigned int vm_colors;
extern
unsigned int vm_color_mask;
extern
unsigned int vm_cache_geometry_colors;
#define VM_NOT_USER_WIREABLE (64*1024*1024)
extern
vm_map_size_t vm_user_wire_limit;
extern
vm_map_size_t vm_global_user_wire_limit;
extern
vm_map_size_t vm_global_no_user_wire_amount;
#define VPL_LOCK_SPIN 1
struct vpl {
unsigned int vpl_count;
unsigned int vpl_internal_count;
unsigned int vpl_external_count;
queue_head_t vpl_queue;
#ifdef VPL_LOCK_SPIN
lck_spin_t vpl_lock;
#else
lck_mtx_t vpl_lock;
lck_mtx_ext_t vpl_lock_ext;
#endif
};
struct vplq {
union {
char cache_line_pad[128];
struct vpl vpl;
} vpl_un;
};
extern
unsigned int vm_page_local_q_count;
extern
struct vplq *vm_page_local_q;
extern
unsigned int vm_page_local_q_soft_limit;
extern
unsigned int vm_page_local_q_hard_limit;
extern
vm_locks_array_t vm_page_locks;
extern
queue_head_t vm_page_queue_free[MAX_COLORS];
extern
queue_head_t vm_lopage_queue_free;
extern
queue_head_t vm_page_queue_active;
extern
queue_head_t vm_page_queue_inactive;
extern
queue_head_t vm_page_queue_cleaned;
extern
queue_head_t vm_page_queue_anonymous;
extern
queue_head_t vm_page_queue_throttled;
extern
queue_head_t vm_objects_wired;
extern
lck_spin_t vm_objects_wired_lock;
extern
vm_offset_t first_phys_addr;
extern
vm_offset_t last_phys_addr;
extern
unsigned int vm_page_free_count;
extern
unsigned int vm_page_fictitious_count;
extern
unsigned int vm_page_active_count;
extern
unsigned int vm_page_inactive_count;
extern
unsigned int vm_page_cleaned_count;
extern
unsigned int vm_page_throttled_count;
extern
unsigned int vm_page_speculative_count;
extern unsigned int vm_page_pageable_internal_count;
extern unsigned int vm_page_pageable_external_count;
extern
unsigned int vm_page_xpmapped_external_count;
extern
unsigned int vm_page_external_count;
extern
unsigned int vm_page_internal_count;
extern
unsigned int vm_page_wire_count;
extern
unsigned int vm_page_wire_count_initial;
extern
unsigned int vm_page_free_target;
extern
unsigned int vm_page_free_min;
extern
unsigned int vm_page_throttle_limit;
extern
uint32_t vm_page_creation_throttle;
extern
unsigned int vm_page_inactive_target;
extern
unsigned int vm_page_anonymous_min;
extern
unsigned int vm_page_inactive_min;
extern
unsigned int vm_page_free_reserved;
extern
unsigned int vm_page_throttle_count;
extern
unsigned int vm_page_gobble_count;
extern
unsigned int vm_page_stolen_count;
#if DEVELOPMENT || DEBUG
extern
unsigned int vm_page_speculative_used;
#endif
extern
unsigned int vm_page_purgeable_count;
extern
unsigned int vm_page_purgeable_wired_count;
extern
uint64_t vm_page_purged_count;
extern unsigned int vm_page_free_wanted;
extern unsigned int vm_page_free_wanted_privileged;
extern ppnum_t vm_page_fictitious_addr;
extern ppnum_t vm_page_guard_addr;
extern boolean_t vm_page_deactivate_hint;
extern int vm_compressor_mode;
extern int vm_himemory_mode;
extern boolean_t vm_lopage_needed;
extern uint32_t vm_lopage_free_count;
extern uint32_t vm_lopage_free_limit;
extern uint32_t vm_lopage_lowater;
extern boolean_t vm_lopage_refill;
extern uint64_t max_valid_dma_address;
extern ppnum_t max_valid_low_ppnum;
extern void vm_page_bootstrap(
vm_offset_t *startp,
vm_offset_t *endp);
extern void vm_page_module_init(void);
extern void vm_page_init_local_q(void);
extern void vm_page_create(
ppnum_t start,
ppnum_t end);
extern vm_page_t kdp_vm_page_lookup(
vm_object_t object,
vm_object_offset_t offset);
extern vm_page_t vm_page_lookup(
vm_object_t object,
vm_object_offset_t offset);
extern vm_page_t vm_page_grab_fictitious(void);
extern vm_page_t vm_page_grab_guard(void);
extern void vm_page_release_fictitious(
vm_page_t page);
extern void vm_page_more_fictitious(void);
extern int vm_pool_low(void);
extern vm_page_t vm_page_grab(void);
extern vm_page_t vm_page_grablo(void);
extern void vm_page_release(
vm_page_t page);
extern boolean_t vm_page_wait(
int interruptible );
extern vm_page_t vm_page_alloc(
vm_object_t object,
vm_object_offset_t offset);
extern vm_page_t vm_page_alloc_guard(
vm_object_t object,
vm_object_offset_t offset);
extern void vm_page_init(
vm_page_t page,
ppnum_t phys_page,
boolean_t lopage);
extern void vm_page_free(
vm_page_t page);
extern void vm_page_free_unlocked(
vm_page_t page,
boolean_t remove_from_hash);
extern void vm_page_activate(
vm_page_t page);
extern void vm_page_deactivate(
vm_page_t page);
extern void vm_page_deactivate_internal(
vm_page_t page,
boolean_t clear_hw_reference);
extern void vm_page_enqueue_cleaned(vm_page_t page);
extern void vm_page_lru(
vm_page_t page);
extern void vm_page_speculate(
vm_page_t page,
boolean_t new);
extern void vm_page_speculate_ageit(
struct vm_speculative_age_q *aq);
extern void vm_page_reactivate_all_throttled(void);
extern void vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t nolocks);
extern void vm_page_rename(
vm_page_t page,
vm_object_t new_object,
vm_object_offset_t new_offset,
boolean_t encrypted_ok);
extern void vm_page_insert(
vm_page_t page,
vm_object_t object,
vm_object_offset_t offset);
extern void vm_page_insert_wired(
vm_page_t page,
vm_object_t object,
vm_object_offset_t offset,
vm_tag_t tag);
extern void vm_page_insert_internal(
vm_page_t page,
vm_object_t object,
vm_object_offset_t offset,
vm_tag_t tag,
boolean_t queues_lock_held,
boolean_t insert_in_hash,
boolean_t batch_pmap_op,
boolean_t delayed_accounting,
uint64_t *delayed_ledger_update);
extern void vm_page_replace(
vm_page_t mem,
vm_object_t object,
vm_object_offset_t offset);
extern void vm_page_remove(
vm_page_t page,
boolean_t remove_from_hash);
extern void vm_page_zero_fill(
vm_page_t page);
extern void vm_page_part_zero_fill(
vm_page_t m,
vm_offset_t m_pa,
vm_size_t len);
extern void vm_page_copy(
vm_page_t src_page,
vm_page_t dest_page);
extern void vm_page_part_copy(
vm_page_t src_m,
vm_offset_t src_pa,
vm_page_t dst_m,
vm_offset_t dst_pa,
vm_size_t len);
extern void vm_page_wire(
vm_page_t page,
vm_tag_t tag,
boolean_t check_memorystatus);
extern void vm_page_unwire(
vm_page_t page,
boolean_t queueit);
extern void vm_set_page_size(void);
extern void vm_page_gobble(
vm_page_t page);
extern void vm_page_validate_cs(vm_page_t page);
extern void vm_page_validate_cs_mapped(
vm_page_t page,
const void *kaddr);
extern void vm_page_validate_cs_mapped_chunk(
vm_page_t page,
const void *kaddr,
vm_offset_t chunk_offset,
boolean_t *validated,
unsigned *tainted);
extern void vm_page_free_prepare_queues(
vm_page_t page);
extern void vm_page_free_prepare_object(
vm_page_t page,
boolean_t remove_from_hash);
#if CONFIG_IOSCHED
extern wait_result_t vm_page_sleep(
vm_object_t object,
vm_page_t m,
int interruptible);
#endif
extern void vm_pressure_response(void);
#if CONFIG_JETSAM
extern void memorystatus_pages_update(unsigned int pages_avail);
#define VM_CHECK_MEMORYSTATUS do { \
memorystatus_pages_update( \
vm_page_pageable_external_count + \
vm_page_free_count + \
(VM_DYNAMIC_PAGING_ENABLED(memory_manager_default) ? 0 : vm_page_purgeable_count) \
); \
} while(0)
#else
#define VM_CHECK_MEMORYSTATUS vm_pressure_response()
#endif
#define SET_PAGE_DIRTY(m, set_pmap_modified) \
MACRO_BEGIN \
vm_page_t __page__ = (m); \
__page__->dirty = TRUE; \
MACRO_END
#define PAGE_ASSERT_WAIT(m, interruptible) \
(((m)->wanted = TRUE), \
assert_wait((event_t) (m), (interruptible)))
#if CONFIG_IOSCHED
#define PAGE_SLEEP(o, m, interruptible) \
vm_page_sleep(o, m, interruptible)
#else
#define PAGE_SLEEP(o, m, interruptible) \
(((m)->wanted = TRUE), \
thread_sleep_vm_object((o), (m), (interruptible)))
#endif
#define PAGE_WAKEUP_DONE(m) \
MACRO_BEGIN \
(m)->busy = FALSE; \
if ((m)->wanted) { \
(m)->wanted = FALSE; \
thread_wakeup((event_t) (m)); \
} \
MACRO_END
#define PAGE_WAKEUP(m) \
MACRO_BEGIN \
if ((m)->wanted) { \
(m)->wanted = FALSE; \
thread_wakeup((event_t) (m)); \
} \
MACRO_END
#define VM_PAGE_FREE(p) \
MACRO_BEGIN \
vm_page_free_unlocked(p, TRUE); \
MACRO_END
#define VM_PAGE_GRAB_FICTITIOUS(M) \
MACRO_BEGIN \
while ((M = vm_page_grab_fictitious()) == VM_PAGE_NULL) \
vm_page_more_fictitious(); \
MACRO_END
#define VM_PAGE_WAIT() ((void)vm_page_wait(THREAD_UNINT))
#define vm_page_queue_lock (vm_page_locks.vm_page_queue_lock2)
#define vm_page_queue_free_lock (vm_page_locks.vm_page_queue_free_lock2)
#define vm_page_lock_queues() lck_mtx_lock(&vm_page_queue_lock)
#define vm_page_unlock_queues() lck_mtx_unlock(&vm_page_queue_lock)
#define vm_page_lockspin_queues() lck_mtx_lock_spin(&vm_page_queue_lock)
#define vm_page_trylockspin_queues() lck_mtx_try_lock_spin(&vm_page_queue_lock)
#define vm_page_lockconvert_queues() lck_mtx_convert_spin(&vm_page_queue_lock)
#ifdef VPL_LOCK_SPIN
#define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_spin_init(&vlq->vpl_lock, vpl_grp, vpl_attr)
#define VPL_LOCK(vpl) lck_spin_lock(vpl)
#define VPL_UNLOCK(vpl) lck_spin_unlock(vpl)
#else
#define VPL_LOCK_INIT(vlq, vpl_grp, vpl_attr) lck_mtx_init_ext(&vlq->vpl_lock, &vlq->vpl_lock_ext, vpl_grp, vpl_attr)
#define VPL_LOCK(vpl) lck_mtx_lock_spin(vpl)
#define VPL_UNLOCK(vpl) lck_mtx_unlock(vpl)
#endif
#if MACH_ASSERT
extern void vm_page_queues_assert(vm_page_t mem, int val);
#define VM_PAGE_QUEUES_ASSERT(mem, val) vm_page_queues_assert((mem), (val))
#else
#define VM_PAGE_QUEUES_ASSERT(mem, val)
#endif
#if DEVELOPMENT || DEBUG
#define VM_PAGE_SPECULATIVE_USED_ADD() \
MACRO_BEGIN \
OSAddAtomic(1, &vm_page_speculative_used); \
MACRO_END
#else
#define VM_PAGE_SPECULATIVE_USED_ADD()
#endif
#define VM_PAGE_CONSUME_CLUSTERED(mem) \
MACRO_BEGIN \
pmap_lock_phys_page(mem->phys_page); \
if (mem->clustered) { \
assert(mem->object); \
mem->object->pages_used++; \
mem->clustered = FALSE; \
VM_PAGE_SPECULATIVE_USED_ADD(); \
} \
pmap_unlock_phys_page(mem->phys_page); \
MACRO_END
#define VM_PAGE_COUNT_AS_PAGEIN(mem) \
MACRO_BEGIN \
DTRACE_VM2(pgin, int, 1, (uint64_t *), NULL); \
current_task()->pageins++; \
if (mem->object->internal) { \
DTRACE_VM2(anonpgin, int, 1, (uint64_t *), NULL); \
} else { \
DTRACE_VM2(fspgin, int, 1, (uint64_t *), NULL); \
} \
MACRO_END
#define VM_PAGE_MOVE_STOLEN(page_count) \
MACRO_BEGIN \
vm_page_stolen_count -= (page_count); \
vm_page_wire_count_initial -= (page_count); \
MACRO_END
#define DW_vm_page_unwire 0x01
#define DW_vm_page_wire 0x02
#define DW_vm_page_free 0x04
#define DW_vm_page_activate 0x08
#define DW_vm_page_deactivate_internal 0x10
#define DW_vm_page_speculate 0x20
#define DW_vm_page_lru 0x40
#define DW_vm_pageout_throttle_up 0x80
#define DW_PAGE_WAKEUP 0x100
#define DW_clear_busy 0x200
#define DW_clear_reference 0x400
#define DW_set_reference 0x800
#define DW_move_page 0x1000
#define DW_VM_PAGE_QUEUES_REMOVE 0x2000
#define DW_enqueue_cleaned 0x4000
#define DW_vm_phantom_cache_update 0x8000
struct vm_page_delayed_work {
vm_page_t dw_m;
int dw_mask;
};
void vm_page_do_delayed_work(vm_object_t object, vm_tag_t tag, struct vm_page_delayed_work *dwp, int dw_count);
extern unsigned int vm_max_delayed_work_limit;
#define DEFAULT_DELAYED_WORK_LIMIT 32
#define DELAYED_WORK_LIMIT(max) ((vm_max_delayed_work_limit >= max ? max : vm_max_delayed_work_limit))
#define VM_PAGE_ADD_DELAYED_WORK(dwp, mem, dw_cnt) \
MACRO_BEGIN \
if (mem->busy == FALSE) { \
mem->busy = TRUE; \
if ( !(dwp->dw_mask & DW_vm_page_free)) \
dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP); \
} \
dwp->dw_m = mem; \
dwp++; \
dw_cnt++; \
MACRO_END
extern vm_page_t vm_object_page_grab(vm_object_t);
#if VM_PAGE_BUCKETS_CHECK
extern void vm_page_buckets_check(void);
#endif
extern void vm_page_queues_remove(vm_page_t mem);
extern void vm_page_remove_internal(vm_page_t page);
extern void vm_page_enqueue_inactive(vm_page_t mem, boolean_t first);
extern void vm_page_check_pageable_safe(vm_page_t page);
#endif