#include <vm/vm_compressor.h>
#include <vm/vm_map.h>
#include <vm/vm_pageout.h>
#include <vm/memory_object.h>
#include <mach/mach_host.h>
#include <kern/ledger.h>
#include <default_pager/default_pager_alerts.h>
#include <default_pager/default_pager_object_server.h>
#include <IOKit/IOHibernatePrivate.h>
extern ipc_port_t min_pages_trigger_port;
extern lck_mtx_t paging_segments_lock;
#define PSL_LOCK() lck_mtx_lock(&paging_segments_lock)
#define PSL_UNLOCK() lck_mtx_unlock(&paging_segments_lock)
int vm_compressor_mode = VM_PAGER_COMPRESSOR_WITH_SWAP;
int vm_scale = 16;
int vm_compression_limit = 0;
extern boolean_t vm_swap_up;
extern void vm_pageout_io_throttle(void);
#if CHECKSUM_THE_DATA || CHECKSUM_THE_SWAP || CHECKSUM_THE_COMPRESSED_DATA
extern unsigned int hash_string(char *cp, int len);
#endif
struct c_slot {
uint64_t c_offset:C_SEG_OFFSET_BITS,
c_size:12,
c_packed_ptr:36;
#if CHECKSUM_THE_DATA
unsigned int c_hash_data;
#endif
#if CHECKSUM_THE_COMPRESSED_DATA
unsigned int c_hash_compressed_data;
#endif
};
#define UNPACK_C_SIZE(cs) ((cs->c_size == (PAGE_SIZE-1)) ? 4096 : cs->c_size)
#define PACK_C_SIZE(cs, size) (cs->c_size = ((size == PAGE_SIZE) ? PAGE_SIZE - 1 : size))
struct c_slot_mapping {
uint32_t s_cseg:22,
s_cindx:10;
};
typedef struct c_slot_mapping *c_slot_mapping_t;
union c_segu {
c_segment_t c_seg;
uint32_t c_segno;
};
#define C_SLOT_PACK_PTR(ptr) (((uintptr_t)ptr - (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS) >> 2)
#define C_SLOT_UNPACK_PTR(cslot) ((uintptr_t)(cslot->c_packed_ptr << 2) + (uintptr_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS)
uint32_t c_segment_count = 0;
uint64_t c_generation_id = 0;
uint64_t c_generation_id_flush_barrier;
#define HIBERNATE_FLUSHING_SECS_TO_COMPLETE 120
boolean_t hibernate_no_swapspace = FALSE;
clock_sec_t hibernate_flushing_deadline = 0;
#if TRACK_BAD_C_SEGMENTS
queue_head_t c_bad_list_head;
uint32_t c_bad_count = 0;
#endif
queue_head_t c_age_list_head;
queue_head_t c_swapout_list_head;
queue_head_t c_swappedin_list_head;
queue_head_t c_swappedout_list_head;
queue_head_t c_swappedout_sparse_list_head;
uint32_t c_age_count = 0;
uint32_t c_swapout_count = 0;
uint32_t c_swappedin_count = 0;
uint32_t c_swappedout_count = 0;
uint32_t c_swappedout_sparse_count = 0;
queue_head_t c_minor_list_head;
uint32_t c_minor_count = 0;
union c_segu *c_segments;
caddr_t c_segments_next_page;
boolean_t c_segments_busy;
uint32_t c_segments_available;
uint32_t c_segments_limit;
uint32_t c_segment_pages_compressed;
uint32_t c_segment_pages_compressed_limit;
uint32_t c_free_segno_head = (uint32_t)-1;
uint32_t vm_compressor_minorcompact_threshold_divisor = 10;
uint32_t vm_compressor_majorcompact_threshold_divisor = 10;
uint32_t vm_compressor_unthrottle_threshold_divisor = 10;
uint32_t vm_compressor_catchup_threshold_divisor = 10;
#define C_SEGMENTS_PER_PAGE (PAGE_SIZE / sizeof(union c_segu))
lck_grp_attr_t vm_compressor_lck_grp_attr;
lck_attr_t vm_compressor_lck_attr;
lck_grp_t vm_compressor_lck_grp;
#if __i386__ || __x86_64__
lck_mtx_t *c_list_lock;
#else
lck_spin_t *c_list_lock;
#endif
lck_rw_t c_master_lock;
lck_rw_t c_decompressor_lock;
zone_t compressor_segment_zone;
int c_compressor_swap_trigger = 0;
uint32_t compressor_cpus;
char *compressor_scratch_bufs;
clock_sec_t start_of_sample_period_sec = 0;
clock_nsec_t start_of_sample_period_nsec = 0;
clock_sec_t start_of_eval_period_sec = 0;
clock_nsec_t start_of_eval_period_nsec = 0;
uint32_t sample_period_decompression_count = 0;
uint32_t sample_period_compression_count = 0;
uint32_t last_eval_decompression_count = 0;
uint32_t last_eval_compression_count = 0;
#define DECOMPRESSION_SAMPLE_MAX_AGE (60 * 30)
uint32_t swapout_target_age = 0;
uint32_t age_of_decompressions_during_sample_period[DECOMPRESSION_SAMPLE_MAX_AGE];
uint32_t overage_decompressions_during_sample_period = 0;
void do_fastwake_warmup(void);
boolean_t fastwake_warmup = FALSE;
boolean_t fastwake_recording_in_progress = FALSE;
clock_sec_t dont_trim_until_ts = 0;
uint64_t c_segment_warmup_count;
uint64_t first_c_segment_to_warm_generation_id = 0;
uint64_t last_c_segment_to_warm_generation_id = 0;
boolean_t hibernate_flushing = FALSE;
int64_t c_segment_input_bytes = 0;
int64_t c_segment_compressed_bytes = 0;
int64_t compressor_bytes_used = 0;
static boolean_t compressor_needs_to_swap(void);
static void vm_compressor_swap_trigger_thread(void);
static void vm_compressor_do_delayed_compactions(boolean_t);
static void vm_compressor_compact_and_swap(boolean_t);
static void vm_compressor_age_swapped_in_segments(boolean_t);
static uint64_t compute_elapsed_msecs(clock_sec_t, clock_nsec_t, clock_sec_t, clock_nsec_t);
boolean_t vm_compressor_low_on_space(void);
void compute_swapout_target_age(void);
boolean_t c_seg_major_compact(c_segment_t, c_segment_t);
boolean_t c_seg_major_compact_ok(c_segment_t, c_segment_t);
int c_seg_minor_compaction_and_unlock(c_segment_t, boolean_t);
int c_seg_do_minor_compaction_and_unlock(c_segment_t, boolean_t, boolean_t, boolean_t);
void c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg);
void c_seg_need_delayed_compaction(c_segment_t);
void c_seg_move_to_sparse_list(c_segment_t);
void c_seg_insert_into_q(queue_head_t *, c_segment_t);
boolean_t c_seg_try_free(c_segment_t);
void c_seg_free(c_segment_t);
void c_seg_free_locked(c_segment_t);
uint64_t vm_available_memory(void);
extern unsigned int dp_pages_free, dp_pages_reserve;
uint64_t
vm_available_memory(void)
{
return (((uint64_t)AVAILABLE_NON_COMPRESSED_MEMORY) * PAGE_SIZE_64);
}
boolean_t
vm_compression_available(void)
{
if ( !(COMPRESSED_PAGER_IS_ACTIVE || DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE))
return (FALSE);
if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit)
return (FALSE);
return (TRUE);
}
boolean_t
vm_compressor_low_on_space(void)
{
if ((c_segment_pages_compressed > (c_segment_pages_compressed_limit - 20000)) ||
(c_segment_count > (c_segments_limit - 250)))
return (TRUE);
return (FALSE);
}
int
vm_low_on_space(void)
{
if (vm_compressor_mode == COMPRESSED_PAGER_IS_ACTIVE || vm_compressor_mode == DEFAULT_FREEZER_COMPRESSED_PAGER_IS_ACTIVE) {
if (vm_compressor_low_on_space() || HARD_THROTTLE_LIMIT_REACHED())
return (1);
} else {
if (((dp_pages_free + dp_pages_reserve < 2000) && VM_DYNAMIC_PAGING_ENABLED(memory_manager_default)))
return (1);
}
return (0);
}
void
vm_compressor_init_locks(void)
{
lck_grp_attr_setdefault(&vm_compressor_lck_grp_attr);
lck_grp_init(&vm_compressor_lck_grp, "vm_compressor", &vm_compressor_lck_grp_attr);
lck_attr_setdefault(&vm_compressor_lck_attr);
lck_rw_init(&c_master_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
lck_rw_init(&c_decompressor_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
}
void
vm_decompressor_lock(void)
{
lck_rw_lock_exclusive(&c_decompressor_lock);
}
void
vm_decompressor_unlock(void)
{
lck_rw_done(&c_decompressor_lock);
}
void
vm_compressor_init(void)
{
thread_t thread;
assert((C_SEGMENTS_PER_PAGE * sizeof(union c_segu)) == PAGE_SIZE);
PE_parse_boot_argn("vm_compression_limit", &vm_compression_limit, sizeof (vm_compression_limit));
if (max_mem <= (3ULL * 1024ULL * 1024ULL * 1024ULL)) {
vm_compressor_minorcompact_threshold_divisor = 11;
vm_compressor_majorcompact_threshold_divisor = 13;
vm_compressor_unthrottle_threshold_divisor = 20;
vm_compressor_catchup_threshold_divisor = 35;
} else {
vm_compressor_minorcompact_threshold_divisor = 20;
vm_compressor_majorcompact_threshold_divisor = 25;
vm_compressor_unthrottle_threshold_divisor = 35;
vm_compressor_catchup_threshold_divisor = 50;
}
#if __i386__ || __x86_64__
c_list_lock = lck_mtx_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
#else
c_list_lock = lck_spin_alloc_init(&vm_compressor_lck_grp, &vm_compressor_lck_attr);
#endif
#if TRACK_BAD_C_SEGMENTS
queue_init(&c_bad_list_head);
#endif
queue_init(&c_age_list_head);
queue_init(&c_minor_list_head);
queue_init(&c_swapout_list_head);
queue_init(&c_swappedin_list_head);
queue_init(&c_swappedout_list_head);
queue_init(&c_swappedout_sparse_list_head);
compressor_segment_zone = zinit(sizeof (struct c_segment),
128000 * sizeof (struct c_segment),
8192, "compressor_segment");
zone_change(compressor_segment_zone, Z_CALLERACCT, FALSE);
zone_change(compressor_segment_zone, Z_NOENCRYPT, TRUE);
c_free_segno_head = -1;
c_segments_available = 0;
if (vm_compression_limit == 0) {
c_segment_pages_compressed_limit = (uint32_t)((max_mem / PAGE_SIZE)) * vm_scale;
#define OLD_SWAP_LIMIT (1024 * 1024 * 16)
#define MAX_SWAP_LIMIT (1024 * 1024 * 128)
if (c_segment_pages_compressed_limit > (OLD_SWAP_LIMIT))
c_segment_pages_compressed_limit = OLD_SWAP_LIMIT;
if (c_segment_pages_compressed_limit < (uint32_t)(max_mem / PAGE_SIZE_64))
c_segment_pages_compressed_limit = (uint32_t)(max_mem / PAGE_SIZE_64);
} else {
if (vm_compression_limit < MAX_SWAP_LIMIT)
c_segment_pages_compressed_limit = vm_compression_limit;
else
c_segment_pages_compressed_limit = MAX_SWAP_LIMIT;
}
if ((c_segments_limit = c_segment_pages_compressed_limit / (C_SEG_BUFSIZE / PAGE_SIZE)) > C_SEG_MAX_LIMIT)
c_segments_limit = C_SEG_MAX_LIMIT;
c_segments_busy = FALSE;
if (kernel_memory_allocate(kernel_map, (vm_offset_t *)(&c_segments), (sizeof(union c_segu) * c_segments_limit), 0, KMA_KOBJECT | KMA_VAONLY) != KERN_SUCCESS)
panic("vm_compressor_init: kernel_memory_allocate failed\n");
c_segments_next_page = (caddr_t)c_segments;
{
host_basic_info_data_t hinfo;
mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
#define BSD_HOST 1
host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
compressor_cpus = hinfo.max_cpus;
compressor_scratch_bufs = kalloc(compressor_cpus * WKdm_SCRATCH_BUF_SIZE);
}
if (kernel_thread_start_priority((thread_continue_t)vm_compressor_swap_trigger_thread, NULL,
BASEPRI_PREEMPT - 1, &thread) != KERN_SUCCESS) {
panic("vm_compressor_swap_trigger_thread: create failed");
}
thread->options |= TH_OPT_VMPRIV;
thread_deallocate(thread);
assert(default_pager_init_flag == 0);
if (vm_pageout_internal_start() != KERN_SUCCESS) {
panic("vm_compressor_init: Failed to start the internal pageout thread.\n");
}
#if CONFIG_FREEZE
memorystatus_freeze_enabled = TRUE;
#endif
default_pager_init_flag = 1;
vm_page_reactivate_all_throttled();
}
#if VALIDATE_C_SEGMENTS
static void
c_seg_validate(c_segment_t c_seg, boolean_t must_be_compact)
{
int c_indx;
int32_t bytes_used;
int32_t bytes_unused;
uint32_t c_rounded_size;
uint32_t c_size;
c_slot_t cs;
if (c_seg->c_firstemptyslot < c_seg->c_nextslot) {
c_indx = c_seg->c_firstemptyslot;
cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
if (cs == NULL)
panic("c_seg_validate: no slot backing c_firstemptyslot");
if (cs->c_size)
panic("c_seg_validate: c_firstemptyslot has non-zero size (%d)\n", cs->c_size);
}
bytes_used = 0;
bytes_unused = 0;
for (c_indx = 0; c_indx < c_seg->c_nextslot; c_indx++) {
cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
c_size = UNPACK_C_SIZE(cs);
c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
bytes_used += c_rounded_size;
#if CHECKSUM_THE_COMPRESSED_DATA
if (c_size && cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
panic("compressed data doesn't match original");
#endif
}
if (bytes_used != c_seg->c_bytes_used)
panic("c_seg_validate: bytes_used mismatch - found %d, segment has %d\n", bytes_used, c_seg->c_bytes_used);
if (c_seg->c_bytes_used > C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
panic("c_seg_validate: c_bytes_used > c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
(int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
if (must_be_compact) {
if (c_seg->c_bytes_used != C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset))
panic("c_seg_validate: c_bytes_used doesn't match c_nextoffset - c_nextoffset = %d, c_bytes_used = %d\n",
(int32_t)C_SEG_OFFSET_TO_BYTES((int32_t)c_seg->c_nextoffset), c_seg->c_bytes_used);
}
}
#endif
void
c_seg_need_delayed_compaction(c_segment_t c_seg)
{
boolean_t clear_busy = FALSE;
if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
c_seg->c_busy = 1;
lck_mtx_unlock_always(&c_seg->c_lock);
lck_mtx_lock_spin_always(c_list_lock);
lck_mtx_lock_spin_always(&c_seg->c_lock);
clear_busy = TRUE;
}
if (!c_seg->c_on_minorcompact_q && !c_seg->c_ondisk && !c_seg->c_on_swapout_q) {
queue_enter(&c_minor_list_head, c_seg, c_segment_t, c_list);
c_seg->c_on_minorcompact_q = 1;
c_minor_count++;
}
lck_mtx_unlock_always(c_list_lock);
if (clear_busy == TRUE)
C_SEG_WAKEUP_DONE(c_seg);
}
unsigned int c_seg_moved_to_sparse_list = 0;
void
c_seg_move_to_sparse_list(c_segment_t c_seg)
{
boolean_t clear_busy = FALSE;
if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
c_seg->c_busy = 1;
lck_mtx_unlock_always(&c_seg->c_lock);
lck_mtx_lock_spin_always(c_list_lock);
lck_mtx_lock_spin_always(&c_seg->c_lock);
clear_busy = TRUE;
}
assert(c_seg->c_ondisk);
assert(c_seg->c_on_swappedout_q);
assert(!c_seg->c_on_swappedout_sparse_q);
queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
c_seg->c_on_swappedout_q = 0;
c_swappedout_count--;
c_seg_insert_into_q(&c_swappedout_sparse_list_head, c_seg);
c_seg->c_on_swappedout_sparse_q = 1;
c_swappedout_sparse_count++;
c_seg_moved_to_sparse_list++;
lck_mtx_unlock_always(c_list_lock);
if (clear_busy == TRUE)
C_SEG_WAKEUP_DONE(c_seg);
}
void
c_seg_insert_into_q(queue_head_t *qhead, c_segment_t c_seg)
{
c_segment_t c_seg_next;
if (queue_empty(qhead)) {
queue_enter(qhead, c_seg, c_segment_t, c_age_list);
} else {
c_seg_next = (c_segment_t)queue_first(qhead);
while (TRUE) {
if (c_seg->c_generation_id < c_seg_next->c_generation_id) {
queue_insert_before(qhead, c_seg, c_seg_next, c_segment_t, c_age_list);
break;
}
c_seg_next = (c_segment_t) queue_next(&c_seg_next->c_age_list);
if (queue_end(qhead, (queue_entry_t) c_seg_next)) {
queue_enter(qhead, c_seg, c_segment_t, c_age_list);
break;
}
}
}
}
int try_minor_compaction_failed = 0;
int try_minor_compaction_succeeded = 0;
void
c_seg_try_minor_compaction_and_unlock(c_segment_t c_seg)
{
assert(c_seg->c_on_minorcompact_q);
if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
try_minor_compaction_failed++;
lck_mtx_unlock_always(&c_seg->c_lock);
} else {
try_minor_compaction_succeeded++;
c_seg->c_busy = 1;
c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, FALSE);
}
}
int
c_seg_do_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy, boolean_t need_list_lock, boolean_t disallow_page_replacement)
{
int c_seg_freed;
assert(c_seg->c_busy);
if (!c_seg->c_on_minorcompact_q) {
if (clear_busy == TRUE)
C_SEG_WAKEUP_DONE(c_seg);
lck_mtx_unlock_always(&c_seg->c_lock);
return (0);
}
queue_remove(&c_minor_list_head, c_seg, c_segment_t, c_list);
c_seg->c_on_minorcompact_q = 0;
c_minor_count--;
lck_mtx_unlock_always(c_list_lock);
if (disallow_page_replacement == TRUE) {
lck_mtx_unlock_always(&c_seg->c_lock);
PAGE_REPLACEMENT_DISALLOWED(TRUE);
lck_mtx_lock_spin_always(&c_seg->c_lock);
}
c_seg_freed = c_seg_minor_compaction_and_unlock(c_seg, clear_busy);
if (disallow_page_replacement == TRUE)
PAGE_REPLACEMENT_DISALLOWED(FALSE);
if (need_list_lock == TRUE)
lck_mtx_lock_spin_always(c_list_lock);
return (c_seg_freed);
}
void
c_seg_wait_on_busy(c_segment_t c_seg)
{
c_seg->c_wanted = 1;
assert_wait((event_t) (c_seg), THREAD_UNINT);
lck_mtx_unlock_always(&c_seg->c_lock);
thread_block(THREAD_CONTINUE_NULL);
}
int try_free_succeeded = 0;
int try_free_failed = 0;
boolean_t
c_seg_try_free(c_segment_t c_seg)
{
if ( !lck_mtx_try_lock_spin_always(c_list_lock)) {
try_free_failed++;
return (FALSE);
}
if (c_seg->c_on_minorcompact_q) {
queue_remove(&c_minor_list_head, c_seg, c_segment_t, c_list);
c_seg->c_on_minorcompact_q = 0;
c_minor_count--;
} else {
assert(c_seg->c_on_swappedout_sparse_q);
}
if (!c_seg->c_busy_swapping)
c_seg->c_busy = 1;
c_seg_free_locked(c_seg);
try_free_succeeded++;
return (TRUE);
}
void
c_seg_free(c_segment_t c_seg)
{
if (!c_seg->c_busy_swapping)
c_seg->c_busy = 1;
lck_mtx_unlock_always(&c_seg->c_lock);
lck_mtx_lock_spin_always(c_list_lock);
lck_mtx_lock_spin_always(&c_seg->c_lock);
c_seg_free_locked(c_seg);
}
void
c_seg_free_locked(c_segment_t c_seg)
{
int segno, i;
int pages_populated;
int32_t *c_buffer = NULL;
uint64_t c_swap_handle;
assert(!c_seg->c_on_minorcompact_q);
if (c_seg->c_on_age_q) {
queue_remove(&c_age_list_head, c_seg, c_segment_t, c_age_list);
c_seg->c_on_age_q = 0;
c_age_count--;
} else if (c_seg->c_on_swappedin_q) {
queue_remove(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
c_seg->c_on_swappedin_q = 0;
c_swappedin_count--;
} else if (c_seg->c_on_swapout_q) {
queue_remove(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
c_seg->c_on_swapout_q = 0;
c_swapout_count--;
thread_wakeup((event_t)&compaction_swapper_running);
} else if (c_seg->c_on_swappedout_q) {
queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
c_seg->c_on_swappedout_q = 0;
c_swappedout_count--;
} else if (c_seg->c_on_swappedout_sparse_q) {
queue_remove(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
c_seg->c_on_swappedout_sparse_q = 0;
c_swappedout_sparse_count--;
}
#if TRACK_BAD_C_SEGMENTS
else if (c_seg->c_on_bad_q) {
queue_remove(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
c_seg->c_on_bad_q = 0;
c_bad_count--;
}
#endif
segno = c_seg->c_mysegno;
c_segments[segno].c_segno = c_free_segno_head;
c_free_segno_head = segno;
c_segment_count--;
lck_mtx_unlock_always(c_list_lock);
if (c_seg->c_wanted) {
thread_wakeup((event_t) (c_seg));
c_seg->c_wanted = 0;
}
if (c_seg->c_busy_swapping) {
c_seg->c_must_free = 1;
lck_mtx_unlock_always(&c_seg->c_lock);
return;
}
if (c_seg->c_ondisk == 0) {
pages_populated = (round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE;
c_buffer = c_seg->c_store.c_buffer;
c_seg->c_store.c_buffer = NULL;
} else {
c_swap_handle = c_seg->c_store.c_swap_handle;
c_seg->c_store.c_swap_handle = (uint64_t)-1;
}
lck_mtx_unlock_always(&c_seg->c_lock);
if (c_buffer) {
kernel_memory_depopulate(kernel_map, (vm_offset_t) c_buffer, pages_populated * PAGE_SIZE, KMA_COMPRESSOR);
kmem_free(kernel_map, (vm_offset_t) c_buffer, C_SEG_ALLOCSIZE);
} else if (c_swap_handle)
vm_swap_free(c_swap_handle);
#if __i386__ || __x86_64__
lck_mtx_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
#else
lck_spin_destroy(&c_seg->c_lock, &vm_compressor_lck_grp);
#endif
for (i = 0; i < C_SEG_SLOT_ARRAYS; i++) {
if (c_seg->c_slots[i] == 0)
break;
kfree((char *)c_seg->c_slots[i], sizeof(struct c_slot) * C_SEG_SLOT_ARRAY_SIZE);
}
zfree(compressor_segment_zone, c_seg);
}
int c_seg_trim_page_count = 0;
void
c_seg_trim_tail(c_segment_t c_seg)
{
c_slot_t cs;
uint32_t c_size;
uint32_t c_offset;
uint32_t c_rounded_size;
uint16_t current_nextslot;
uint32_t current_populated_offset;
if (c_seg->c_bytes_used == 0)
return;
current_nextslot = c_seg->c_nextslot;
current_populated_offset = c_seg->c_populated_offset;
while (c_seg->c_nextslot) {
cs = C_SEG_SLOT_FROM_INDEX(c_seg, (c_seg->c_nextslot - 1));
c_size = UNPACK_C_SIZE(cs);
if (c_size) {
if (current_nextslot != c_seg->c_nextslot) {
c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
c_offset = cs->c_offset + C_SEG_BYTES_TO_OFFSET(c_rounded_size);
c_seg->c_nextoffset = c_offset;
c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
if (c_seg->c_firstemptyslot > c_seg->c_nextslot)
c_seg->c_firstemptyslot = c_seg->c_nextslot;
c_seg_trim_page_count += ((round_page_32(C_SEG_OFFSET_TO_BYTES(current_populated_offset)) -
round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset))) / PAGE_SIZE);
}
break;
}
c_seg->c_nextslot--;
}
assert(c_seg->c_nextslot);
}
int
c_seg_minor_compaction_and_unlock(c_segment_t c_seg, boolean_t clear_busy)
{
c_slot_mapping_t slot_ptr;
uint32_t c_offset = 0;
uint32_t old_populated_offset;
uint32_t c_rounded_size;
uint32_t c_size;
int c_indx = 0;
int i;
c_slot_t c_dst;
c_slot_t c_src;
boolean_t need_unlock = TRUE;
assert(c_seg->c_busy);
#if VALIDATE_C_SEGMENTS
c_seg_validate(c_seg, FALSE);
#endif
if (c_seg->c_bytes_used == 0) {
c_seg_free(c_seg);
return (1);
}
if (c_seg->c_firstemptyslot >= c_seg->c_nextslot || C_SEG_UNUSED_BYTES(c_seg) < PAGE_SIZE)
goto done;
#if VALIDATE_C_SEGMENTS
c_seg->c_was_minor_compacted++;
#endif
c_indx = c_seg->c_firstemptyslot;
c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
old_populated_offset = c_seg->c_populated_offset;
c_offset = c_dst->c_offset;
for (i = c_indx + 1; i < c_seg->c_nextslot && c_offset < c_seg->c_nextoffset; i++) {
c_src = C_SEG_SLOT_FROM_INDEX(c_seg, i);
c_size = UNPACK_C_SIZE(c_src);
if (c_size == 0)
continue;
memcpy(&c_seg->c_store.c_buffer[c_offset], &c_seg->c_store.c_buffer[c_src->c_offset], c_size);
#if CHECKSUM_THE_DATA
c_dst->c_hash_data = c_src->c_hash_data;
#endif
#if CHECKSUM_THE_COMPRESSED_DATA
c_dst->c_hash_compressed_data = c_src->c_hash_compressed_data;
#endif
c_dst->c_size = c_src->c_size;
c_dst->c_packed_ptr = c_src->c_packed_ptr;
c_dst->c_offset = c_offset;
slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
slot_ptr->s_cindx = c_indx;
c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
c_offset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
PACK_C_SIZE(c_src, 0);
c_indx++;
c_dst = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
}
c_seg->c_firstemptyslot = c_indx;
c_seg->c_nextslot = c_indx;
c_seg->c_nextoffset = c_offset;
c_seg->c_populated_offset = (c_offset + (C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1)) & ~(C_SEG_BYTES_TO_OFFSET(PAGE_SIZE) - 1);
c_seg->c_bytes_unused = 0;
#if VALIDATE_C_SEGMENTS
c_seg_validate(c_seg, TRUE);
#endif
if (old_populated_offset > c_seg->c_populated_offset) {
uint32_t gc_size;
int32_t *gc_ptr;
gc_size = C_SEG_OFFSET_TO_BYTES(old_populated_offset - c_seg->c_populated_offset);
gc_ptr = &c_seg->c_store.c_buffer[c_seg->c_populated_offset];
lck_mtx_unlock_always(&c_seg->c_lock);
kernel_memory_depopulate(kernel_map, (vm_offset_t)gc_ptr, gc_size, KMA_COMPRESSOR);
if (clear_busy == TRUE)
lck_mtx_lock_spin_always(&c_seg->c_lock);
else
need_unlock = FALSE;
}
done:
if (need_unlock == TRUE) {
if (clear_busy == TRUE)
C_SEG_WAKEUP_DONE(c_seg);
lck_mtx_unlock_always(&c_seg->c_lock);
}
return (0);
}
struct {
uint64_t asked_permission;
uint64_t compactions;
uint64_t moved_slots;
uint64_t moved_bytes;
uint64_t wasted_space_in_swapouts;
uint64_t count_of_swapouts;
} c_seg_major_compact_stats;
#define C_MAJOR_COMPACTION_AGE_APPROPRIATE 30
#define C_MAJOR_COMPACTION_OLD_ENOUGH 300
#define C_MAJOR_COMPACTION_SIZE_APPROPRIATE ((C_SEG_BUFSIZE * 80) / 100)
boolean_t
c_seg_major_compact_ok(
c_segment_t c_seg_dst,
c_segment_t c_seg_src)
{
c_seg_major_compact_stats.asked_permission++;
if (c_seg_src->c_filling) {
return (FALSE);
}
if (c_seg_src->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE &&
c_seg_dst->c_bytes_used >= C_MAJOR_COMPACTION_SIZE_APPROPRIATE)
return (FALSE);
if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX) {
return (FALSE);
}
return (TRUE);
}
boolean_t
c_seg_major_compact(
c_segment_t c_seg_dst,
c_segment_t c_seg_src)
{
c_slot_mapping_t slot_ptr;
uint32_t c_rounded_size;
uint32_t c_size;
uint16_t dst_slot;
int i;
c_slot_t c_dst;
c_slot_t c_src;
int slotarray;
boolean_t keep_compacting = TRUE;
#if VALIDATE_C_SEGMENTS
c_seg_dst->c_was_major_compacted++;
c_seg_src->c_was_major_donor++;
#endif
c_seg_major_compact_stats.compactions++;
dst_slot = c_seg_dst->c_nextslot;
for (i = 0; i < c_seg_src->c_nextslot; i++) {
c_src = C_SEG_SLOT_FROM_INDEX(c_seg_src, i);
c_size = UNPACK_C_SIZE(c_src);
if (c_size == 0) {
continue;
}
if (C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset - c_seg_dst->c_nextoffset) < (unsigned) c_size) {
if ((C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset) == C_SEG_BUFSIZE)) {
keep_compacting = FALSE;
break;
}
kernel_memory_populate(kernel_map,
(vm_offset_t) &c_seg_dst->c_store.c_buffer[c_seg_dst->c_populated_offset],
PAGE_SIZE,
KMA_COMPRESSOR);
c_seg_dst->c_populated_offset += C_SEG_BYTES_TO_OFFSET(PAGE_SIZE);
assert(C_SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset) <= C_SEG_BUFSIZE);
}
slotarray = C_SEG_SLOTARRAY_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
if (c_seg_dst->c_slots[slotarray] == 0) {
KERNEL_DEBUG(0xe0400008 | DBG_FUNC_START, 0, 0, 0, 0, 0);
c_seg_dst->c_slots[slotarray] = (struct c_slot *)
kalloc(sizeof(struct c_slot) *
C_SEG_SLOT_ARRAY_SIZE);
KERNEL_DEBUG(0xe0400008 | DBG_FUNC_END, 0, 0, 0, 0, 0);
}
c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, c_seg_dst->c_nextslot);
memcpy(&c_seg_dst->c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);
c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
c_seg_major_compact_stats.moved_slots++;
c_seg_major_compact_stats.moved_bytes += c_size;
#if CHECKSUM_THE_DATA
c_dst->c_hash_data = c_src->c_hash_data;
#endif
#if CHECKSUM_THE_COMPRESSED_DATA
c_dst->c_hash_compressed_data = c_src->c_hash_compressed_data;
#endif
c_dst->c_size = c_src->c_size;
c_dst->c_packed_ptr = c_src->c_packed_ptr;
c_dst->c_offset = c_seg_dst->c_nextoffset;
if (c_seg_dst->c_firstemptyslot == c_seg_dst->c_nextslot)
c_seg_dst->c_firstemptyslot++;
c_seg_dst->c_nextslot++;
c_seg_dst->c_bytes_used += c_rounded_size;
c_seg_dst->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
PACK_C_SIZE(c_src, 0);
c_seg_src->c_bytes_used -= c_rounded_size;
c_seg_src->c_bytes_unused += c_rounded_size;
c_seg_src->c_firstemptyslot = 0;
if (c_seg_dst->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg_dst->c_nextslot >= C_SLOT_MAX) {
keep_compacting = FALSE;
break;
}
}
if (dst_slot < c_seg_dst->c_nextslot) {
PAGE_REPLACEMENT_ALLOWED(TRUE);
while (dst_slot < c_seg_dst->c_nextslot) {
c_dst = C_SEG_SLOT_FROM_INDEX(c_seg_dst, dst_slot);
slot_ptr = (c_slot_mapping_t)C_SLOT_UNPACK_PTR(c_dst);
slot_ptr->s_cseg = c_seg_dst->c_mysegno + 1;
slot_ptr->s_cindx = dst_slot++;
}
PAGE_REPLACEMENT_ALLOWED(FALSE);
}
return (keep_compacting);
}
static uint64_t
compute_elapsed_msecs(clock_sec_t end_sec, clock_nsec_t end_nsec, clock_sec_t start_sec, clock_nsec_t start_nsec)
{
uint64_t end_msecs;
uint64_t start_msecs;
end_msecs = (end_sec * 1000) + end_nsec / 1000000;
start_msecs = (start_sec * 1000) + start_nsec / 1000000;
return (end_msecs - start_msecs);
}
uint32_t compressor_eval_period_in_msecs = 250;
uint32_t compressor_sample_min_in_msecs = 500;
uint32_t compressor_sample_max_in_msecs = 10000;
uint32_t compressor_thrashing_threshold_per_10msecs = 50;
uint32_t compressor_thrashing_min_per_10msecs = 20;
extern uint32_t vm_page_filecache_min;
void
compute_swapout_target_age(void)
{
clock_sec_t cur_ts_sec;
clock_nsec_t cur_ts_nsec;
uint32_t min_operations_needed_in_this_sample;
uint64_t elapsed_msecs_in_eval;
uint64_t elapsed_msecs_in_sample;
boolean_t need_sample_reset = FALSE;
boolean_t need_eval_reset = FALSE;
clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
elapsed_msecs_in_sample = compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_sample_period_sec, start_of_sample_period_nsec);
if (elapsed_msecs_in_sample >= compressor_sample_max_in_msecs) {
need_sample_reset = TRUE;
need_eval_reset = TRUE;
goto done;
}
elapsed_msecs_in_eval = compute_elapsed_msecs(cur_ts_sec, cur_ts_nsec, start_of_eval_period_sec, start_of_eval_period_nsec);
if (elapsed_msecs_in_eval < compressor_eval_period_in_msecs)
goto done;
need_eval_reset = TRUE;
KERNEL_DEBUG(0xe0400020 | DBG_FUNC_START, elapsed_msecs_in_eval, sample_period_compression_count, sample_period_decompression_count, 0, 0);
min_operations_needed_in_this_sample = (compressor_thrashing_min_per_10msecs * (uint32_t)elapsed_msecs_in_eval) / 10;
if ((sample_period_compression_count - last_eval_compression_count) < min_operations_needed_in_this_sample ||
(sample_period_decompression_count - last_eval_decompression_count) < min_operations_needed_in_this_sample) {
KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_compression_count - last_eval_compression_count,
sample_period_decompression_count - last_eval_decompression_count, 0, 1, 0);
swapout_target_age = 0;
need_sample_reset = TRUE;
need_eval_reset = TRUE;
goto done;
}
last_eval_compression_count = sample_period_compression_count;
last_eval_decompression_count = sample_period_decompression_count;
if (elapsed_msecs_in_sample < compressor_sample_min_in_msecs) {
KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, 0, 0, 5, 0);
goto done;
}
if (sample_period_decompression_count > ((compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10)) {
uint64_t running_total;
uint64_t working_target;
uint64_t aging_target;
uint32_t oldest_age_of_csegs_sampled = 0;
uint64_t working_set_approximation = 0;
swapout_target_age = 0;
working_target = (sample_period_decompression_count / 100) * 95;
aging_target = (sample_period_decompression_count / 100) * 1;
running_total = 0;
for (oldest_age_of_csegs_sampled = 0; oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE; oldest_age_of_csegs_sampled++) {
running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
working_set_approximation += oldest_age_of_csegs_sampled * age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
if (running_total >= working_target)
break;
}
if (oldest_age_of_csegs_sampled < DECOMPRESSION_SAMPLE_MAX_AGE) {
working_set_approximation = (working_set_approximation * 1000) / elapsed_msecs_in_sample;
if (working_set_approximation < VM_PAGE_COMPRESSOR_COUNT) {
running_total = overage_decompressions_during_sample_period;
for (oldest_age_of_csegs_sampled = DECOMPRESSION_SAMPLE_MAX_AGE - 1; oldest_age_of_csegs_sampled; oldest_age_of_csegs_sampled--) {
running_total += age_of_decompressions_during_sample_period[oldest_age_of_csegs_sampled];
if (running_total >= aging_target)
break;
}
swapout_target_age = (uint32_t)cur_ts_sec - oldest_age_of_csegs_sampled;
KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, swapout_target_age, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 2, 0);
} else {
KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_set_approximation, VM_PAGE_COMPRESSOR_COUNT, 0, 3, 0);
}
} else
KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, working_target, running_total, 0, 4, 0);
need_sample_reset = TRUE;
need_eval_reset = TRUE;
} else
KERNEL_DEBUG(0xe0400020 | DBG_FUNC_END, sample_period_decompression_count, (compressor_thrashing_threshold_per_10msecs * elapsed_msecs_in_sample) / 10, 0, 6, 0);
done:
if (need_sample_reset == TRUE) {
bzero(age_of_decompressions_during_sample_period, sizeof(age_of_decompressions_during_sample_period));
overage_decompressions_during_sample_period = 0;
start_of_sample_period_sec = cur_ts_sec;
start_of_sample_period_nsec = cur_ts_nsec;
sample_period_decompression_count = 0;
sample_period_compression_count = 0;
last_eval_decompression_count = 0;
last_eval_compression_count = 0;
}
if (need_eval_reset == TRUE) {
start_of_eval_period_sec = cur_ts_sec;
start_of_eval_period_nsec = cur_ts_nsec;
}
}
int calls_since_last_considered = 0;
int compaction_swapper_running = 0;
int compaction_swapper_abort = 0;
#if CONFIG_JETSAM
boolean_t memorystatus_kill_on_VM_thrashing(boolean_t);
int compressor_thrashing_induced_jetsam = 0;
boolean_t vm_compressor_thrashing_detected = FALSE;
#endif
static boolean_t
compressor_needs_to_swap(void)
{
boolean_t should_swap = FALSE;
if (vm_swap_up == TRUE) {
if (COMPRESSOR_NEEDS_TO_SWAP()) {
return (TRUE);
}
if (VM_PAGE_Q_THROTTLED(&vm_pageout_queue_external) && vm_page_anonymous_count < (vm_page_inactive_count / 20)) {
return (TRUE);
}
if (vm_page_free_count < (vm_page_free_reserved - COMPRESSOR_FREE_RESERVED_LIMIT))
return (TRUE);
}
compute_swapout_target_age();
if (swapout_target_age) {
c_segment_t c_seg;
lck_mtx_lock_spin_always(c_list_lock);
if (!queue_empty(&c_age_list_head)) {
c_seg = (c_segment_t) queue_first(&c_age_list_head);
if (c_seg->c_creation_ts <= swapout_target_age)
should_swap = TRUE;
else
swapout_target_age = 0;
}
lck_mtx_unlock_always(c_list_lock);
}
if (vm_swap_up == FALSE) {
#if CONFIG_JETSAM
if (should_swap) {
if (vm_compressor_thrashing_detected == FALSE) {
vm_compressor_thrashing_detected = TRUE;
memorystatus_kill_on_VM_thrashing(TRUE );
compressor_thrashing_induced_jetsam++;
should_swap = FALSE;
}
} else
#endif
if (COMPRESSOR_NEEDS_TO_MAJOR_COMPACT())
should_swap = TRUE;
}
return (should_swap);
}
uint64_t
vm_compressor_total_compressions(void)
{
processor_t processor = processor_list;
vm_statistics64_t stat = &PROCESSOR_DATA(processor, vm_stat);
uint64_t compressions = stat->compressions;
if (processor_count > 1) {
simple_lock(&processor_list_lock);
while ((processor = processor->processor_list) != NULL) {
stat = &PROCESSOR_DATA(processor, vm_stat);
compressions += stat->compressions;
}
simple_unlock(&processor_list_lock);
}
return compressions;
}
uint32_t vm_wake_compactor_swapper_calls = 0;
void
vm_wake_compactor_swapper(void)
{
if (compaction_swapper_running)
return;
if (c_minor_count == 0)
return;
lck_mtx_lock_spin_always(c_list_lock);
fastwake_warmup = FALSE;
if (compaction_swapper_running == 0) {
vm_wake_compactor_swapper_calls++;
thread_wakeup((event_t)&c_compressor_swap_trigger);
compaction_swapper_running = 1;
}
lck_mtx_unlock_always(c_list_lock);
}
void
vm_consider_waking_compactor_swapper(void)
{
boolean_t need_wakeup = FALSE;
if (calls_since_last_considered++ < 1000 || compaction_swapper_running)
return;
calls_since_last_considered = 0;
if (c_minor_count && (COMPRESSOR_NEEDS_TO_MINOR_COMPACT())) {
need_wakeup = TRUE;
} else if (compressor_needs_to_swap()) {
need_wakeup = TRUE;
} else if (c_minor_count) {
uint64_t total_bytes;
total_bytes = compressor_object->resident_page_count * PAGE_SIZE_64;
if ((total_bytes - compressor_bytes_used) > total_bytes / 10)
need_wakeup = TRUE;
}
if (need_wakeup == TRUE) {
lck_mtx_lock_spin_always(c_list_lock);
fastwake_warmup = FALSE;
if (compaction_swapper_running == 0) {
memoryshot(VM_WAKEUP_COMPACTOR_SWAPPER, DBG_FUNC_NONE);
thread_wakeup((event_t)&c_compressor_swap_trigger);
compaction_swapper_running = 1;
}
lck_mtx_unlock_always(c_list_lock);
}
}
#define C_SWAPOUT_LIMIT 4
#define DELAYED_COMPACTIONS_PER_PASS 30
void
vm_compressor_do_delayed_compactions(boolean_t flush_all)
{
c_segment_t c_seg;
int number_compacted = 0;
boolean_t needs_to_swap = FALSE;
lck_mtx_assert(c_list_lock, LCK_MTX_ASSERT_OWNED);
while (!queue_empty(&c_minor_list_head) && needs_to_swap == FALSE) {
c_seg = (c_segment_t)queue_first(&c_minor_list_head);
lck_mtx_lock_spin_always(&c_seg->c_lock);
c_seg->c_busy = 1;
c_seg_do_minor_compaction_and_unlock(c_seg, TRUE, FALSE, TRUE);
if (vm_swap_up == TRUE && (number_compacted++ > DELAYED_COMPACTIONS_PER_PASS)) {
if ((flush_all == TRUE || compressor_needs_to_swap() == TRUE) && c_swapout_count < C_SWAPOUT_LIMIT)
needs_to_swap = TRUE;
number_compacted = 0;
}
lck_mtx_lock_spin_always(c_list_lock);
}
}
#define C_SEGMENT_SWAPPEDIN_AGE_LIMIT 10
static void
vm_compressor_age_swapped_in_segments(boolean_t flush_all)
{
c_segment_t c_seg;
clock_sec_t now;
clock_nsec_t nsec;
clock_get_system_nanotime(&now, &nsec);
while (!queue_empty(&c_swappedin_list_head)) {
c_seg = (c_segment_t)queue_first(&c_swappedin_list_head);
if (flush_all == FALSE && (now - c_seg->c_swappedin_ts) < C_SEGMENT_SWAPPEDIN_AGE_LIMIT)
break;
lck_mtx_lock_spin_always(&c_seg->c_lock);
queue_remove(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
c_seg->c_on_swappedin_q = 0;
c_swappedin_count--;
c_seg_insert_into_q(&c_age_list_head, c_seg);
c_seg->c_on_age_q = 1;
c_age_count++;
lck_mtx_unlock_always(&c_seg->c_lock);
}
}
void
vm_compressor_flush(void)
{
uint64_t vm_swap_put_failures_at_start;
wait_result_t wait_result = 0;
AbsoluteTime startTime, endTime;
clock_sec_t now_sec;
clock_nsec_t now_nsec;
uint64_t nsec;
HIBLOG("vm_compressor_flush - starting\n");
clock_get_uptime(&startTime);
lck_mtx_lock_spin_always(c_list_lock);
fastwake_warmup = FALSE;
compaction_swapper_abort = 1;
while (compaction_swapper_running) {
assert_wait((event_t)&compaction_swapper_running, THREAD_UNINT);
lck_mtx_unlock_always(c_list_lock);
thread_block(THREAD_CONTINUE_NULL);
lck_mtx_lock_spin_always(c_list_lock);
}
compaction_swapper_abort = 0;
compaction_swapper_running = 1;
hibernate_flushing = TRUE;
hibernate_no_swapspace = FALSE;
c_generation_id_flush_barrier = c_generation_id + 1000;
clock_get_system_nanotime(&now_sec, &now_nsec);
hibernate_flushing_deadline = now_sec + HIBERNATE_FLUSHING_SECS_TO_COMPLETE;
vm_swap_put_failures_at_start = vm_swap_put_failures;
vm_compressor_compact_and_swap(TRUE);
while (!queue_empty(&c_swapout_list_head)) {
assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 5000, 1000*NSEC_PER_USEC);
lck_mtx_unlock_always(c_list_lock);
wait_result = thread_block(THREAD_CONTINUE_NULL);
lck_mtx_lock_spin_always(c_list_lock);
if (wait_result == THREAD_TIMED_OUT)
break;
}
hibernate_flushing = FALSE;
compaction_swapper_running = 0;
if (vm_swap_put_failures > vm_swap_put_failures_at_start)
HIBLOG("vm_compressor_flush failed to clean %llu segments - vm_page_compressor_count(%d)\n",
vm_swap_put_failures - vm_swap_put_failures_at_start, VM_PAGE_COMPRESSOR_COUNT);
lck_mtx_unlock_always(c_list_lock);
clock_get_uptime(&endTime);
SUB_ABSOLUTETIME(&endTime, &startTime);
absolutetime_to_nanoseconds(endTime, &nsec);
HIBLOG("vm_compressor_flush completed - took %qd msecs\n", nsec / 1000000ULL);
}
int compaction_swap_trigger_thread_awakened = 0;
static void
vm_compressor_swap_trigger_thread(void)
{
lck_mtx_lock_spin_always(c_list_lock);
compaction_swap_trigger_thread_awakened++;
vm_compressor_compact_and_swap(FALSE);
assert_wait((event_t)&c_compressor_swap_trigger, THREAD_UNINT);
compaction_swapper_running = 0;
thread_wakeup((event_t)&compaction_swapper_running);
lck_mtx_unlock_always(c_list_lock);
thread_block((thread_continue_t)vm_compressor_swap_trigger_thread);
}
void
vm_compressor_record_warmup_start(void)
{
c_segment_t c_seg;
lck_mtx_lock_spin_always(c_list_lock);
if (!queue_empty(&c_age_list_head)) {
c_seg = (c_segment_t)queue_last(&c_age_list_head);
first_c_segment_to_warm_generation_id = c_seg->c_generation_id;
} else
first_c_segment_to_warm_generation_id = 0;
fastwake_recording_in_progress = TRUE;
lck_mtx_unlock_always(c_list_lock);
}
void
vm_compressor_record_warmup_end(void)
{
c_segment_t c_seg;
lck_mtx_lock_spin_always(c_list_lock);
if (!queue_empty(&c_age_list_head)) {
c_seg = (c_segment_t)queue_last(&c_age_list_head);
last_c_segment_to_warm_generation_id = c_seg->c_generation_id;
} else
last_c_segment_to_warm_generation_id = first_c_segment_to_warm_generation_id;
fastwake_recording_in_progress = FALSE;
lck_mtx_unlock_always(c_list_lock);
}
#define DELAY_TRIM_ON_WAKE_SECS 4
void
vm_compressor_do_warmup(void)
{
clock_sec_t sec;
clock_nsec_t nsec;
clock_get_system_nanotime(&sec, &nsec);
dont_trim_until_ts = sec + DELAY_TRIM_ON_WAKE_SECS;
if (first_c_segment_to_warm_generation_id == last_c_segment_to_warm_generation_id)
return;
lck_mtx_lock_spin_always(c_list_lock);
if (compaction_swapper_running == 0) {
fastwake_warmup = TRUE;
compaction_swapper_running = 1;
thread_wakeup((event_t)&c_compressor_swap_trigger);
}
lck_mtx_unlock_always(c_list_lock);
}
void
do_fastwake_warmup(void)
{
uint64_t my_thread_id;
c_segment_t c_seg = NULL;
lck_mtx_unlock_always(c_list_lock);
my_thread_id = current_thread()->thread_id;
proc_set_task_policy_thread(kernel_task, my_thread_id,
TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
PAGE_REPLACEMENT_DISALLOWED(TRUE);
lck_mtx_lock_spin_always(c_list_lock);
while (!queue_empty(&c_swappedout_list_head) && fastwake_warmup == TRUE) {
c_seg = (c_segment_t) queue_first(&c_swappedout_list_head);
if (c_seg->c_generation_id < first_c_segment_to_warm_generation_id ||
c_seg->c_generation_id > last_c_segment_to_warm_generation_id)
break;
lck_mtx_lock_spin_always(&c_seg->c_lock);
lck_mtx_unlock_always(c_list_lock);
if (c_seg->c_busy)
c_seg_wait_on_busy(c_seg);
else {
c_seg_swapin(c_seg, TRUE);
lck_mtx_unlock_always(&c_seg->c_lock);
c_segment_warmup_count++;
vm_pageout_io_throttle();
}
lck_mtx_lock_spin_always(c_list_lock);
}
lck_mtx_unlock_always(c_list_lock);
PAGE_REPLACEMENT_DISALLOWED(FALSE);
proc_set_task_policy_thread(kernel_task, my_thread_id,
TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER0);
lck_mtx_lock_spin_always(c_list_lock);
}
void
vm_compressor_compact_and_swap(boolean_t flush_all)
{
c_segment_t c_seg, c_seg_next;
boolean_t keep_compacting;
if (fastwake_warmup == TRUE) {
uint64_t starting_warmup_count;
starting_warmup_count = c_segment_warmup_count;
KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_START, c_segment_warmup_count,
first_c_segment_to_warm_generation_id, last_c_segment_to_warm_generation_id, 0, 0);
do_fastwake_warmup();
KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 11) | DBG_FUNC_END, c_segment_warmup_count, c_segment_warmup_count - starting_warmup_count, 0, 0, 0);
fastwake_warmup = FALSE;
}
while (!queue_empty(&c_age_list_head) && compaction_swapper_abort == 0) {
if (hibernate_flushing == TRUE) {
clock_sec_t sec;
clock_nsec_t nsec;
if (hibernate_should_abort()) {
HIBLOG("vm_compressor_flush - hibernate_should_abort returned TRUE\n");
break;
}
if (hibernate_no_swapspace == TRUE) {
HIBLOG("vm_compressor_flush - out of swap space\n");
break;
}
clock_get_system_nanotime(&sec, &nsec);
if (sec > hibernate_flushing_deadline) {
HIBLOG("vm_compressor_flush - failed to finish before deadline\n");
break;
}
}
if (c_swapout_count >= C_SWAPOUT_LIMIT) {
assert_wait_timeout((event_t) &compaction_swapper_running, THREAD_INTERRUPTIBLE, 100, 1000*NSEC_PER_USEC);
lck_mtx_unlock_always(c_list_lock);
thread_block(THREAD_CONTINUE_NULL);
lck_mtx_lock_spin_always(c_list_lock);
}
vm_compressor_do_delayed_compactions(flush_all);
vm_compressor_age_swapped_in_segments(flush_all);
if (c_swapout_count >= C_SWAPOUT_LIMIT) {
continue;
}
if (flush_all == FALSE) {
boolean_t needs_to_swap;
lck_mtx_unlock_always(c_list_lock);
needs_to_swap = compressor_needs_to_swap();
lck_mtx_lock_spin_always(c_list_lock);
if (needs_to_swap == FALSE)
break;
}
if (queue_empty(&c_age_list_head))
break;
c_seg = (c_segment_t) queue_first(&c_age_list_head);
if (flush_all == TRUE && c_seg->c_generation_id > c_generation_id_flush_barrier)
break;
if (c_seg->c_filling) {
break;
}
lck_mtx_lock_spin_always(&c_seg->c_lock);
if (c_seg->c_busy) {
lck_mtx_unlock_always(c_list_lock);
c_seg_wait_on_busy(c_seg);
lck_mtx_lock_spin_always(c_list_lock);
continue;
}
c_seg->c_busy = 1;
if (c_seg_do_minor_compaction_and_unlock(c_seg, FALSE, TRUE, TRUE)) {
continue;
}
keep_compacting = TRUE;
while (keep_compacting == TRUE) {
assert(c_seg->c_busy);
c_seg_next = (c_segment_t) queue_next(&c_seg->c_age_list);
if (queue_end(&c_age_list_head, (queue_entry_t)c_seg_next))
break;
if (c_seg_major_compact_ok(c_seg, c_seg_next) == FALSE)
break;
lck_mtx_lock_spin_always(&c_seg_next->c_lock);
if (c_seg_next->c_busy) {
lck_mtx_unlock_always(c_list_lock);
c_seg_wait_on_busy(c_seg_next);
lck_mtx_lock_spin_always(c_list_lock);
continue;
}
c_seg_next->c_busy = 1;
if (c_seg_do_minor_compaction_and_unlock(c_seg_next, FALSE, TRUE, TRUE)) {
continue;
}
lck_mtx_unlock_always(c_list_lock);
keep_compacting = c_seg_major_compact(c_seg, c_seg_next);
PAGE_REPLACEMENT_DISALLOWED(TRUE);
lck_mtx_lock_spin_always(&c_seg_next->c_lock);
c_seg_minor_compaction_and_unlock(c_seg_next, TRUE);
PAGE_REPLACEMENT_DISALLOWED(FALSE);
lck_mtx_lock_spin_always(c_list_lock);
}
c_seg_major_compact_stats.wasted_space_in_swapouts += C_SEG_BUFSIZE - c_seg->c_bytes_used;
c_seg_major_compact_stats.count_of_swapouts++;
lck_mtx_lock_spin_always(&c_seg->c_lock);
assert(c_seg->c_busy);
assert(c_seg->c_on_age_q);
assert(!c_seg->c_on_minorcompact_q);
queue_remove(&c_age_list_head, c_seg, c_segment_t, c_age_list);
c_seg->c_on_age_q = 0;
c_age_count--;
if (vm_swap_up == TRUE) {
queue_enter(&c_swapout_list_head, c_seg, c_segment_t, c_age_list);
c_seg->c_on_swapout_q = 1;
c_swapout_count++;
} else {
queue_enter(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
c_seg->c_on_swappedout_q = 1;
c_swappedout_count++;
}
C_SEG_WAKEUP_DONE(c_seg);
lck_mtx_unlock_always(&c_seg->c_lock);
if (c_swapout_count) {
lck_mtx_unlock_always(c_list_lock);
thread_wakeup((event_t)&c_swapout_list_head);
lck_mtx_lock_spin_always(c_list_lock);
}
}
}
static c_segment_t
c_seg_allocate(c_segment_t *current_chead)
{
clock_sec_t sec;
clock_nsec_t nsec;
c_segment_t c_seg;
int slotarray;
if ( (c_seg = *current_chead) == NULL ) {
uint32_t c_segno;
KERNEL_DEBUG(0xe0400004 | DBG_FUNC_START, 0, 0, 0, 0, 0);
lck_mtx_lock_spin_always(c_list_lock);
while (c_segments_busy == TRUE) {
assert_wait((event_t) (&c_segments_busy), THREAD_UNINT);
lck_mtx_unlock_always(c_list_lock);
thread_block(THREAD_CONTINUE_NULL);
lck_mtx_lock_spin_always(c_list_lock);
}
if (c_free_segno_head == (uint32_t)-1) {
if (c_segments_available >= c_segments_limit || c_segment_pages_compressed >= c_segment_pages_compressed_limit) {
lck_mtx_unlock_always(c_list_lock);
KERNEL_DEBUG(0xe0400004 | DBG_FUNC_END, 0, 0, 0, 1, 0);
return (NULL);
}
c_segments_busy = TRUE;
lck_mtx_unlock_always(c_list_lock);
kernel_memory_populate(kernel_map, (vm_offset_t)c_segments_next_page, PAGE_SIZE, KMA_KOBJECT);
c_segments_next_page += PAGE_SIZE;
for (c_segno = c_segments_available + 1; c_segno < (c_segments_available + C_SEGMENTS_PER_PAGE); c_segno++)
c_segments[c_segno - 1].c_segno = c_segno;
lck_mtx_lock_spin_always(c_list_lock);
c_segments[c_segno - 1].c_segno = c_free_segno_head;
c_free_segno_head = c_segments_available;
c_segments_available += C_SEGMENTS_PER_PAGE;
c_segments_busy = FALSE;
thread_wakeup((event_t) (&c_segments_busy));
}
c_segno = c_free_segno_head;
c_free_segno_head = c_segments[c_segno].c_segno;
lck_mtx_unlock_always(c_list_lock);
c_seg = (c_segment_t)zalloc(compressor_segment_zone);
bzero((char *)c_seg, sizeof(struct c_segment));
if (kernel_memory_allocate(kernel_map, (vm_offset_t *)(&c_seg->c_store.c_buffer), C_SEG_ALLOCSIZE, 0, KMA_COMPRESSOR | KMA_VAONLY) != KERN_SUCCESS) {
zfree(compressor_segment_zone, c_seg);
lck_mtx_lock_spin_always(c_list_lock);
c_segments[c_segno].c_segno = c_free_segno_head;
c_free_segno_head = c_segno;
lck_mtx_unlock_always(c_list_lock);
KERNEL_DEBUG(0xe0400004 | DBG_FUNC_END, 0, 0, 0, 2, 0);
return (NULL);
}
#if __i386__ || __x86_64__
lck_mtx_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
#else
lck_spin_init(&c_seg->c_lock, &vm_compressor_lck_grp, &vm_compressor_lck_attr);
#endif
kernel_memory_populate(kernel_map, (vm_offset_t)(c_seg->c_store.c_buffer), 3 * PAGE_SIZE, KMA_COMPRESSOR);
c_seg->c_populated_offset = C_SEG_BYTES_TO_OFFSET(3 * PAGE_SIZE);
c_seg->c_firstemptyslot = C_SLOT_MAX;
c_seg->c_mysegno = c_segno;
c_seg->c_filling = 1;
lck_mtx_lock_spin_always(c_list_lock);
c_segment_count++;
c_segments[c_segno].c_seg = c_seg;
c_seg->c_generation_id = c_generation_id++;
queue_enter(&c_age_list_head, c_seg, c_segment_t, c_age_list);
c_seg->c_on_age_q = 1;
c_age_count++;
lck_mtx_unlock_always(c_list_lock);
clock_get_system_nanotime(&sec, &nsec);
c_seg->c_creation_ts = (uint32_t)sec;
*current_chead = c_seg;
KERNEL_DEBUG(0xe0400004 | DBG_FUNC_END, c_seg, 0, 0, 3, 0);
}
slotarray = C_SEG_SLOTARRAY_FROM_INDEX(c_seg, c_seg->c_nextslot);
if (c_seg->c_slots[slotarray] == 0) {
KERNEL_DEBUG(0xe0400008 | DBG_FUNC_START, 0, 0, 0, 0, 0);
c_seg->c_slots[slotarray] = (struct c_slot *)kalloc(sizeof(struct c_slot) * C_SEG_SLOT_ARRAY_SIZE);
KERNEL_DEBUG(0xe0400008 | DBG_FUNC_END, 0, 0, 0, 0, 0);
}
PAGE_REPLACEMENT_DISALLOWED(TRUE);
lck_mtx_lock_spin_always(&c_seg->c_lock);
return (c_seg);
}
static void
c_current_seg_filled(c_segment_t c_seg, c_segment_t *current_chead)
{
uint32_t unused_bytes;
uint32_t offset_to_depopulate;
unused_bytes = trunc_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset - c_seg->c_nextoffset));
if (unused_bytes) {
offset_to_depopulate = C_SEG_BYTES_TO_OFFSET(round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset)));
lck_mtx_unlock_always(&c_seg->c_lock);
kernel_memory_depopulate(
kernel_map,
(vm_offset_t) &c_seg->c_store.c_buffer[offset_to_depopulate],
unused_bytes,
KMA_COMPRESSOR);
lck_mtx_lock_spin_always(&c_seg->c_lock);
c_seg->c_populated_offset = offset_to_depopulate;
}
c_seg->c_filling = 0;
if (C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE)
c_seg_need_delayed_compaction(c_seg);
lck_mtx_unlock_always(&c_seg->c_lock);
*current_chead = NULL;
}
void
c_seg_swapin_requeue(c_segment_t c_seg)
{
clock_sec_t sec;
clock_nsec_t nsec;
clock_get_system_nanotime(&sec, &nsec);
lck_mtx_lock_spin_always(c_list_lock);
lck_mtx_lock_spin_always(&c_seg->c_lock);
if (c_seg->c_on_swappedout_q) {
queue_remove(&c_swappedout_list_head, c_seg, c_segment_t, c_age_list);
c_seg->c_on_swappedout_q = 0;
c_swappedout_count--;
} else {
assert(c_seg->c_on_swappedout_sparse_q);
queue_remove(&c_swappedout_sparse_list_head, c_seg, c_segment_t, c_age_list);
c_seg->c_on_swappedout_sparse_q = 0;
c_swappedout_sparse_count--;
}
if (c_seg->c_store.c_buffer) {
queue_enter(&c_swappedin_list_head, c_seg, c_segment_t, c_age_list);
c_seg->c_on_swappedin_q = 1;
c_swappedin_count++;
}
#if TRACK_BAD_C_SEGMENTS
else {
queue_enter(&c_bad_list_head, c_seg, c_segment_t, c_age_list);
c_seg->c_on_bad_q = 1;
c_bad_count++;
}
#endif
c_seg->c_swappedin_ts = (uint32_t)sec;
c_seg->c_ondisk = 0;
c_seg->c_was_swapped_in = 1;
lck_mtx_unlock_always(c_list_lock);
}
void
c_seg_swapin(c_segment_t c_seg, boolean_t force_minor_compaction)
{
vm_offset_t addr = 0;
uint32_t io_size = 0;
uint64_t f_offset;
#if !CHECKSUM_THE_SWAP
if (c_seg->c_ondisk)
c_seg_trim_tail(c_seg);
#endif
io_size = round_page_32(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset));
f_offset = c_seg->c_store.c_swap_handle;
c_seg->c_busy = 1;
lck_mtx_unlock_always(&c_seg->c_lock);
if (c_seg->c_ondisk) {
PAGE_REPLACEMENT_DISALLOWED(FALSE);
if (kernel_memory_allocate(kernel_map, &addr, C_SEG_ALLOCSIZE, 0, KMA_COMPRESSOR | KMA_VAONLY) != KERN_SUCCESS)
panic("c_seg_swapin: kernel_memory_allocate failed\n");
kernel_memory_populate(kernel_map, addr, io_size, KMA_COMPRESSOR);
if (vm_swap_get(addr, f_offset, io_size) != KERN_SUCCESS) {
PAGE_REPLACEMENT_DISALLOWED(TRUE);
kernel_memory_depopulate(kernel_map, addr, io_size, KMA_COMPRESSOR);
kmem_free(kernel_map, addr, C_SEG_ALLOCSIZE);
c_seg->c_store.c_buffer = (int32_t*) NULL;
} else {
c_seg->c_store.c_buffer = (int32_t*) addr;
#if CRYPTO
vm_swap_decrypt(c_seg);
#endif
#if CHECKSUM_THE_SWAP
if (c_seg->cseg_swap_size != io_size)
panic("swapin size doesn't match swapout size");
if (c_seg->cseg_hash != hash_string((char*) c_seg->c_store.c_buffer, (int)io_size)) {
panic("c_seg_swapin - Swap hash mismatch\n");
}
#endif
PAGE_REPLACEMENT_DISALLOWED(TRUE);
if (force_minor_compaction == TRUE) {
lck_mtx_lock_spin_always(&c_seg->c_lock);
c_seg_minor_compaction_and_unlock(c_seg, FALSE);
}
OSAddAtomic64(c_seg->c_bytes_used, &compressor_bytes_used);
}
}
c_seg_swapin_requeue(c_seg);
C_SEG_WAKEUP_DONE(c_seg);
}
static int
c_compress_page(char *src, c_slot_mapping_t slot_ptr, c_segment_t *current_chead, char *scratch_buf)
{
int c_size;
int c_rounded_size;
int max_csize;
c_slot_t cs;
c_segment_t c_seg;
KERNEL_DEBUG(0xe0400000 | DBG_FUNC_START, *current_chead, 0, 0, 0, 0);
retry:
if ((c_seg = c_seg_allocate(current_chead)) == NULL)
return (1);
cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_seg->c_nextslot);
cs->c_packed_ptr = C_SLOT_PACK_PTR(slot_ptr);
cs->c_offset = c_seg->c_nextoffset;
max_csize = C_SEG_BUFSIZE - C_SEG_OFFSET_TO_BYTES((int32_t)cs->c_offset);
if (max_csize > PAGE_SIZE)
max_csize = PAGE_SIZE;
if (C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset -
c_seg->c_nextoffset)
< (unsigned) max_csize + PAGE_SIZE &&
(C_SEG_OFFSET_TO_BYTES(c_seg->c_populated_offset)
< C_SEG_ALLOCSIZE)) {
lck_mtx_unlock_always(&c_seg->c_lock);
kernel_memory_populate(kernel_map,
(vm_offset_t) &c_seg->c_store.c_buffer[c_seg->c_populated_offset],
PAGE_SIZE,
KMA_COMPRESSOR);
lck_mtx_lock_spin_always(&c_seg->c_lock);
c_seg->c_populated_offset += C_SEG_BYTES_TO_OFFSET(PAGE_SIZE);
}
#if CHECKSUM_THE_DATA
cs->c_hash_data = hash_string(src, PAGE_SIZE);
#endif
c_size = WKdm_compress_new((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
(WK_word *)(uintptr_t)scratch_buf, max_csize - 4);
assert(c_size <= (max_csize - 4) && c_size >= -1);
if (c_size == -1) {
if (max_csize < PAGE_SIZE) {
c_current_seg_filled(c_seg, current_chead);
PAGE_REPLACEMENT_DISALLOWED(FALSE);
goto retry;
}
c_size = PAGE_SIZE;
memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);
}
#if CHECKSUM_THE_COMPRESSED_DATA
cs->c_hash_compressed_data = hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);
#endif
c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
PACK_C_SIZE(cs, c_size);
c_seg->c_bytes_used += c_rounded_size;
c_seg->c_nextoffset += C_SEG_BYTES_TO_OFFSET(c_rounded_size);
slot_ptr->s_cindx = c_seg->c_nextslot++;
slot_ptr->s_cseg = c_seg->c_mysegno + 1;
if (c_seg->c_nextoffset >= C_SEG_OFF_LIMIT || c_seg->c_nextslot >= C_SLOT_MAX)
c_current_seg_filled(c_seg, current_chead);
else
lck_mtx_unlock_always(&c_seg->c_lock);
PAGE_REPLACEMENT_DISALLOWED(FALSE);
OSAddAtomic64(c_rounded_size, &compressor_bytes_used);
OSAddAtomic64(PAGE_SIZE, &c_segment_input_bytes);
OSAddAtomic64(c_size, &c_segment_compressed_bytes);
OSAddAtomic(1, &c_segment_pages_compressed);
OSAddAtomic(1, &sample_period_compression_count);
KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_compressed_bytes, 0);
if (vm_compressor_low_on_space()) {
ipc_port_t trigger = IP_NULL;
PSL_LOCK();
if (IP_VALID(min_pages_trigger_port)) {
trigger = min_pages_trigger_port;
min_pages_trigger_port = IP_NULL;
}
PSL_UNLOCK();
if (IP_VALID(trigger)) {
no_paging_space_action();
default_pager_space_alert(trigger, HI_WAT_ALERT);
ipc_port_release_send(trigger);
}
}
return (0);
}
static int
c_decompress_page(char *dst, volatile c_slot_mapping_t slot_ptr, int flags, int *zeroslot)
{
c_slot_t cs;
c_segment_t c_seg;
int c_indx;
int c_rounded_size;
uint32_t c_size;
int retval = 0;
boolean_t c_seg_has_data = TRUE;
boolean_t c_seg_swappedin = FALSE;
boolean_t need_unlock = TRUE;
boolean_t consider_defragmenting = FALSE;
ReTry:
#if HIBERNATION
if (dst) {
if (lck_rw_try_lock_shared(&c_decompressor_lock) == 0) {
if (flags & C_DONT_BLOCK) {
*zeroslot = 0;
return (-2);
}
lck_rw_lock_shared(&c_decompressor_lock);
}
}
#endif
PAGE_REPLACEMENT_DISALLOWED(TRUE);
c_seg = c_segments[slot_ptr->s_cseg - 1].c_seg;
lck_mtx_lock_spin_always(&c_seg->c_lock);
if (flags & C_DONT_BLOCK) {
if (c_seg->c_busy || c_seg->c_ondisk) {
retval = -2;
*zeroslot = 0;
goto done;
}
}
if (c_seg->c_busy) {
PAGE_REPLACEMENT_DISALLOWED(FALSE);
#if HIBERNATION
if (dst)
lck_rw_done(&c_decompressor_lock);
#endif
c_seg_wait_on_busy(c_seg);
goto ReTry;
}
c_indx = slot_ptr->s_cindx;
cs = C_SEG_SLOT_FROM_INDEX(c_seg, c_indx);
c_size = UNPACK_C_SIZE(cs);
c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;
if (dst) {
uint32_t age_of_cseg;
clock_sec_t cur_ts_sec;
clock_nsec_t cur_ts_nsec;
if (c_seg->c_on_swappedout_q || c_seg->c_on_swappedout_sparse_q) {
if (c_seg->c_ondisk)
c_seg_swappedin = TRUE;
c_seg_swapin(c_seg, FALSE);
}
if (c_seg->c_store.c_buffer == NULL) {
c_seg_has_data = FALSE;
goto c_seg_invalid_data;
}
#if CHECKSUM_THE_COMPRESSED_DATA
if (cs->c_hash_compressed_data != hash_string((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))
panic("compressed data doesn't match original");
#endif
if (c_rounded_size == PAGE_SIZE) {
memcpy(dst, &c_seg->c_store.c_buffer[cs->c_offset], PAGE_SIZE);
} else {
uint32_t my_cpu_no;
char *scratch_buf;
my_cpu_no = cpu_number();
assert(my_cpu_no < compressor_cpus);
scratch_buf = &compressor_scratch_bufs[my_cpu_no * WKdm_SCRATCH_BUF_SIZE];
WKdm_decompress_new((WK_word *)(uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset],
(WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);
}
#if CHECKSUM_THE_DATA
if (cs->c_hash_data != hash_string(dst, PAGE_SIZE))
panic("decompressed data doesn't match original");
#endif
if (!c_seg->c_was_swapped_in) {
clock_get_system_nanotime(&cur_ts_sec, &cur_ts_nsec);
age_of_cseg = (uint32_t)cur_ts_sec - c_seg->c_creation_ts;
if (age_of_cseg < DECOMPRESSION_SAMPLE_MAX_AGE)
OSAddAtomic(1, &age_of_decompressions_during_sample_period[age_of_cseg]);
else
OSAddAtomic(1, &overage_decompressions_during_sample_period);
OSAddAtomic(1, &sample_period_decompression_count);
}
} else {
if (c_seg->c_store.c_buffer == NULL)
c_seg_has_data = FALSE;
}
c_seg_invalid_data:
if (c_seg_has_data == TRUE) {
if (c_seg_swappedin == TRUE)
retval = 1;
else
retval = 0;
} else
retval = -1;
if (flags & C_KEEP) {
*zeroslot = 0;
goto done;
}
c_seg->c_bytes_unused += c_rounded_size;
c_seg->c_bytes_used -= c_rounded_size;
PACK_C_SIZE(cs, 0);
if (c_indx < c_seg->c_firstemptyslot)
c_seg->c_firstemptyslot = c_indx;
OSAddAtomic(-1, &c_segment_pages_compressed);
if (c_seg_has_data == TRUE && !c_seg->c_ondisk) {
OSAddAtomic64(-c_rounded_size, &compressor_bytes_used);
}
if (!c_seg->c_filling) {
if (c_seg->c_bytes_used == 0) {
if (c_seg->c_on_minorcompact_q || c_seg->c_on_swappedout_sparse_q) {
if (c_seg_try_free(c_seg) == TRUE)
need_unlock = FALSE;
} else {
c_seg_free(c_seg);
need_unlock = FALSE;
}
} else if (c_seg->c_on_minorcompact_q) {
if (C_SEG_INCORE_IS_SPARSE(c_seg)) {
c_seg_try_minor_compaction_and_unlock(c_seg);
need_unlock = FALSE;
}
} else if (!c_seg->c_ondisk) {
if (c_seg_has_data == TRUE && !c_seg->c_on_swapout_q && C_SEG_UNUSED_BYTES(c_seg) >= PAGE_SIZE) {
c_seg_need_delayed_compaction(c_seg);
}
} else if (!c_seg->c_on_swappedout_sparse_q && C_SEG_ONDISK_IS_SPARSE(c_seg)) {
c_seg_move_to_sparse_list(c_seg);
consider_defragmenting = TRUE;
}
}
done:
if (need_unlock == TRUE)
lck_mtx_unlock_always(&c_seg->c_lock);
PAGE_REPLACEMENT_DISALLOWED(FALSE);
if (consider_defragmenting == TRUE)
vm_swap_consider_defragmenting();
#if HIBERNATION
if (dst)
lck_rw_done(&c_decompressor_lock);
#endif
return (retval);
}
int
vm_compressor_get(ppnum_t pn, int *slot, int flags)
{
char *dst;
int zeroslot = 1;
int retval;
#if __x86_64__
dst = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
#else
#error "unsupported architecture"
#endif
retval = c_decompress_page(dst, (c_slot_mapping_t)slot, flags, &zeroslot);
if (zeroslot) {
pmap_ledger_debit(current_task()->map->pmap, task_ledgers.phys_footprint, PAGE_SIZE);
*slot = 0;
}
return (retval);
}
void
vm_compressor_free(int *slot)
{
int zeroslot = 1;
(void)c_decompress_page(NULL, (c_slot_mapping_t)slot, 0, &zeroslot);
*slot = 0;
}
int
vm_compressor_put(ppnum_t pn, int *slot, void **current_chead, char *scratch_buf)
{
char *src;
int retval;
if ((vm_offset_t)slot < VM_MIN_KERNEL_AND_KEXT_ADDRESS || (vm_offset_t)slot >= VM_MAX_KERNEL_ADDRESS)
panic("vm_compressor_put: slot 0x%llx address out of range [0x%llx:0x%llx]",
(uint64_t)(vm_offset_t) slot,
(uint64_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS,
(uint64_t) VM_MAX_KERNEL_ADDRESS);
#if __x86_64__
src = PHYSMAP_PTOV((uint64_t)pn << (uint64_t)PAGE_SHIFT);
#else
#error "unsupported architecture"
#endif
retval = c_compress_page(src, (c_slot_mapping_t)slot, (c_segment_t *)current_chead, scratch_buf);
return (retval);
}