#include "auto_zone.h"
#include "auto_impl_utilities.h"
#include "auto_weak.h"
#include "auto_trace.h"
#include "auto_dtrace.h"
#include "Zone.h"
#include "Locks.h"
#include "InUseEnumerator.h"
#include "ThreadLocalCollector.h"
#include "auto_tester/auto_tester.h"
#include "BlockIterator.h"
#include <stdlib.h>
#include <libc.h>
#include <sys/types.h>
#include <sys/event.h>
#include <sys/time.h>
#ifdef __BLOCKS__
#include <Block.h>
#include <notify.h>
#include <dispatch/source_private.h>
#endif
#define USE_INTERPOSING 0
#if USE_INTERPOSING
#include <mach-o/dyld-interposing.h>
#endif
using namespace Auto;
static char *b2s(uint64_t bytes, char *buf, int bufsize);
#ifdef AUTO_TESTER
AutoProbeFunctions *auto_probe_functions = NULL;
#endif
bool auto_set_probe_functions(AutoProbeFunctions *functions) {
#ifdef AUTO_TESTER
auto_probe_functions = functions;
return true;
#else
return false;
#endif
}
void (*__auto_reference_logger)(uint32_t eventtype, void *ptr, uintptr_t data) = NULL;
#define VM_COPY_THRESHOLD (40 * 1024)
boolean_t auto_zone_is_finalized(auto_zone_t *zone, const void *ptr) {
Zone *azone = (Zone *)zone;
return (ptr && azone->block_is_garbage((void *)ptr));
}
static void auto_collect_internal(Zone *zone, boolean_t generational) {
if (zone->_collector_disable_count) return;
CollectionTimer timer;
Statistics &zone_stats = zone->statistics();
timer.total_time().start();
zone_stats.idle_timer().stop();
if (zone->control.log & AUTO_LOG_TIMINGS) timer.enable_scan_timer();
zone_stats.reset_for_heap_collection();
AUTO_PROBE(auto_probe_begin_heap_scan(generational));
vm_address_t stack_bottom = auto_get_sp();
if (zone->control.disable_generational) generational = false;
GARBAGE_COLLECTION_COLLECTION_BEGIN((auto_zone_t*)zone, generational ? AUTO_TRACE_GENERATIONAL : AUTO_TRACE_FULL);
zone->set_state(scanning);
Thread &collector_thread = zone->register_thread();
collector_thread.set_in_collector(true);
zone->collect_begin();
zone->collect((bool)generational, (void *)stack_bottom, timer);
PointerList &list = zone->garbage_list();
size_t garbage_count = list.count();
void **garbage = list.buffer();
size_t large_garbage_count = zone->large_garbage_count();
void **large_garbage = (large_garbage_count ? garbage + garbage_count - large_garbage_count : NULL);
AUTO_PROBE(auto_probe_end_heap_scan(garbage_count, garbage));
size_t bytes_freed = 0;
GARBAGE_COLLECTION_COLLECTION_PHASE_BEGIN((auto_zone_t*)zone, AUTO_TRACE_FINALIZING_PHASE);
zone->set_state(finalizing);
size_t block_count = garbage_count, byte_count = 0;
zone->invalidate_garbage(garbage_count, garbage);
GARBAGE_COLLECTION_COLLECTION_PHASE_END((auto_zone_t*)zone, AUTO_TRACE_FINALIZING_PHASE, (uint64_t)block_count, (uint64_t)byte_count);
zone->set_state(reclaiming);
GARBAGE_COLLECTION_COLLECTION_PHASE_BEGIN((auto_zone_t*)zone, AUTO_TRACE_SCAVENGING_PHASE);
bytes_freed = zone->free_garbage(garbage_count - large_garbage_count, garbage, large_garbage_count, large_garbage, block_count, byte_count);
zone->clear_zombies();
GARBAGE_COLLECTION_COLLECTION_PHASE_END((auto_zone_t*)zone, AUTO_TRACE_SCAVENGING_PHASE, (uint64_t)block_count, (uint64_t)bytes_freed);
timer.total_time().stop();
zone->collect_end(timer, bytes_freed);
collector_thread.set_in_collector(false);
GARBAGE_COLLECTION_COLLECTION_END((auto_zone_t*)zone, (uint64_t)garbage_count, (uint64_t)bytes_freed, (uint64_t)zone_stats.count(), (uint64_t)zone_stats.size());
zone->set_state(idle);
AUTO_PROBE(auto_probe_heap_collection_complete());
WallClockTimer &idle_timer = zone->statistics().idle_timer();
if (zone->control.log & AUTO_LOG_TIMINGS) {
const char *idle = idle_timer.time_string();
char bytes[16];
b2s(zone->statistics().bytes_scanned(), bytes, sizeof(bytes));
malloc_printf("%s: %s GC completed in %s after %s idle. scanned %5llu blocks (%s) in %s\n",
auto_prelude(), (generational ? "gen." : "full"), timer.total_time().time_string(), idle,
zone->statistics().blocks_scanned(), bytes, timer.scan_timer().time_string());
}
if (zone->control.log & AUTO_LOG_COLLECTIONS) {
malloc_statistics_t stats;
zone->malloc_statistics(&stats);
char freed[16], in_use[16];
b2s(zone->statistics().size(), in_use, sizeof(in_use));
b2s(bytes_freed, freed, sizeof(freed));
malloc_printf("%s: %s GC collected %5llu blocks (%s). blocks in use: %7llu (%s)\n",
auto_prelude(), (generational ? "gen." : "full"),
(unsigned long long)garbage_count, freed,
zone->statistics().count(), in_use);
}
#ifdef MEASURE_TLC_STATS
zone->statistics().print_tlc_stats();
#endif
idle_timer.reset();
idle_timer.start();
}
void auto_collect(auto_zone_t *zone, auto_collection_mode_t mode, void *collection_context) {
auto_collection_mode_t heap_mode = mode & 0x3;
if ((mode & AUTO_COLLECT_IF_NEEDED) || (mode == 0)) {
auto_zone_collect(zone, AUTO_ZONE_COLLECT_NO_OPTIONS);
} else {
static uintptr_t options_translation[] = {AUTO_ZONE_COLLECT_RATIO_COLLECTION, AUTO_ZONE_COLLECT_GENERATIONAL_COLLECTION, AUTO_ZONE_COLLECT_FULL_COLLECTION, AUTO_ZONE_COLLECT_EXHAUSTIVE_COLLECTION};
auto_zone_options_t request_mode = options_translation[heap_mode];
auto_zone_collect(zone, request_mode);
if (mode & AUTO_COLLECT_SYNCHRONOUS) {
Zone *azone = (Zone *)zone;
dispatch_group_t group = dispatch_group_create();
dispatch_group_async(group, azone->_collection_queue, ^{});
dispatch_group_wait(group, dispatch_time(0, 10*NSEC_PER_SEC));
dispatch_release(group);
}
}
}
static inline bool _increment_pending_count(Zone *azone, auto_zone_options_t global_mode, bool coalesce_requested) {
bool did_coalesce = true;
Mutex lock(&azone->_collection_mutex);
if (global_mode < AUTO_ZONE_COLLECT_GLOBAL_MODE_COUNT) {
if (!coalesce_requested || azone->_pending_collections[global_mode] == 0) {
if (azone->_pending_collections[global_mode] == UINT8_MAX) {
auto_error(azone, "pending collection count overflowed", NULL);
} else {
azone->_pending_collections[global_mode]++;
did_coalesce = false;
}
}
}
return did_coalesce;
}
static inline void _decrement_pending_count(Zone *azone, auto_zone_options_t global_mode) {
Mutex lock(&azone->_collection_mutex);
assert(global_mode < AUTO_ZONE_COLLECT_GLOBAL_MODE_COUNT);
assert(azone->_pending_collections[global_mode] > 0);
azone->_pending_collections[global_mode]--;
AUTO_PROBE(auto_probe_collection_complete());
}
static void auto_zone_generational_collection(Zone *zone)
{
auto_collect_internal(zone, true);
_decrement_pending_count(zone, AUTO_ZONE_COLLECT_GENERATIONAL_COLLECTION);
}
static void auto_zone_full_collection(Zone *zone)
{
auto_collect_internal(zone, false);
_decrement_pending_count(zone, AUTO_ZONE_COLLECT_FULL_COLLECTION);
if (zone->collection_checking_enabled())
zone->increment_check_counts();
}
static void auto_zone_exhaustive_collection(Zone *zone)
{
Statistics &stats = zone->statistics();
uint64_t count, collections = 0;
do {
count = stats.count();
auto_collect_internal(zone, false);
} while (stats.count() < count && ((Environment::exhaustive_collection_limit == 0) || (++collections < Environment::exhaustive_collection_limit)));
_decrement_pending_count(zone, AUTO_ZONE_COLLECT_EXHAUSTIVE_COLLECTION);
if (zone->collection_checking_enabled())
zone->increment_check_counts();
}
static void auto_zone_ratio_collection(Zone *zone)
{
if (zone->_collection_count++ == zone->control.full_vs_gen_frequency) {
zone->_collection_count = 0;
auto_collect_internal(zone, false);
} else {
auto_collect_internal(zone, true);
}
_decrement_pending_count(zone, AUTO_ZONE_COLLECT_RATIO_COLLECTION);
}
void auto_zone_collect(auto_zone_t *zone, auto_zone_options_t options)
{
AUTO_PROBE(auto_probe_auto_zone_collect(options));
Zone *azone = (Zone *)zone;
Thread &thread = azone->registered_thread();
if (options == AUTO_ZONE_COLLECT_NO_OPTIONS) {
if (azone->should_collect())
options = AUTO_ZONE_COLLECT_COALESCE|AUTO_ZONE_COLLECT_RATIO_COLLECTION;
if (ThreadLocalCollector::should_collect(azone, thread, true))
options |= AUTO_ZONE_COLLECT_LOCAL_COLLECTION;
}
if (options & AUTO_ZONE_COLLECT_LOCAL_COLLECTION) {
ThreadLocalCollector tlc(azone, (void *)auto_get_sp(), thread);
tlc.collect(true);
}
if (!pthread_main_np()) azone->volunteer_for_work(true);
auto_zone_options_t global_mode = options & AUTO_ZONE_COLLECT_GLOBAL_COLLECTION_MODE_MASK;
if (global_mode != 0) {
if (!_increment_pending_count(azone, global_mode, options & AUTO_ZONE_COLLECT_COALESCE)) {
dispatch_block_t collect_func;
switch (global_mode) {
case AUTO_ZONE_COLLECT_NO_OPTIONS:
collect_func = NULL;
break;
case AUTO_ZONE_COLLECT_RATIO_COLLECTION:
collect_func = ^{
auto_zone_ratio_collection((Zone *)dispatch_get_context(dispatch_get_current_queue()));
};
break;
case AUTO_ZONE_COLLECT_GENERATIONAL_COLLECTION:
collect_func = ^{
auto_zone_generational_collection((Zone *)dispatch_get_context(dispatch_get_current_queue()));
};
break;
case AUTO_ZONE_COLLECT_FULL_COLLECTION:
collect_func = ^{
auto_zone_full_collection((Zone *)dispatch_get_context(dispatch_get_current_queue()));
};
break;
case AUTO_ZONE_COLLECT_EXHAUSTIVE_COLLECTION:
collect_func = ^{
auto_zone_exhaustive_collection((Zone *)dispatch_get_context(dispatch_get_current_queue()));
};
break;
default:
collect_func = NULL;
malloc_printf("%s: Unknown mode %d passed to auto_zone_collect() ignored.\n", auto_prelude(), global_mode);
break;
}
if (collect_func) {
dispatch_async(azone->_collection_queue, collect_func);
}
}
}
}
extern void auto_zone_reap_all_local_blocks(auto_zone_t *zone)
{
Zone *azone = (Zone *)zone;
Thread *thread = azone->current_thread();
if (thread)
thread->reap_all_local_blocks();
}
void auto_zone_collect_and_notify(auto_zone_t *zone, auto_zone_options_t options, dispatch_queue_t callback_queue, dispatch_block_t completion_callback) {
Zone *azone = (Zone *)zone;
auto_zone_collect(zone, options);
if (callback_queue && completion_callback) {
dispatch_retain(callback_queue);
completion_callback = Block_copy(completion_callback);
dispatch_async(azone->_collection_queue, ^{
dispatch_async(callback_queue, completion_callback);
Block_release(completion_callback);
dispatch_release(callback_queue);
});
}
}
void auto_zone_compact(auto_zone_t *zone, auto_zone_compact_options_t options, dispatch_queue_t callback_queue, dispatch_block_t completion_callback) {
Zone *azone = (Zone *)zone;
if (!azone->compaction_disabled()) {
switch (options) {
case AUTO_ZONE_COMPACT_ANALYZE: {
if (callback_queue && completion_callback) {
dispatch_retain(callback_queue);
completion_callback = Block_copy(completion_callback);
}
dispatch_async(azone->_collection_queue, ^{
Zone *zone = (Zone *)dispatch_get_context(dispatch_get_current_queue());
static const char *analyze_name = Environment::get("AUTO_ANALYZE_NOTIFICATION");
zone->analyze_heap(analyze_name);
if (callback_queue && completion_callback) {
dispatch_async(callback_queue, completion_callback);
Block_release(completion_callback);
dispatch_release(callback_queue);
}
});
break;
}
case AUTO_ZONE_COMPACT_IF_IDLE: {
if (azone->_compaction_timer && !azone->_compaction_pending) {
dispatch_time_t when = dispatch_time(0, 10 * NSEC_PER_SEC);
if (when < azone->_compaction_next_time)
when = azone->_compaction_next_time;
if (when != DISPATCH_TIME_FOREVER) {
dispatch_source_set_timer(azone->_compaction_timer, when, 0, 0);
azone->_compaction_pending = true;
}
}
break;
}
case AUTO_ZONE_COMPACT_NO_OPTIONS: {
if (callback_queue && completion_callback) {
dispatch_retain(callback_queue);
completion_callback = Block_copy(completion_callback);
}
dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 10 * NSEC_PER_SEC), azone->_collection_queue, ^{
Zone *zone = (Zone *)dispatch_get_context(dispatch_get_current_queue());
zone->compact_heap();
if (callback_queue && completion_callback) {
dispatch_async(callback_queue, completion_callback);
Block_release(completion_callback);
dispatch_release(callback_queue);
}
});
break;
}
}
}
}
void auto_zone_disable_compaction(auto_zone_t *zone) {
Zone *azone = (Zone *)zone;
azone->disable_compaction();
}
void auto_zone_register_resource_tracker(auto_zone_t *zone, const char *description, boolean_t (^should_collect)(void))
{
Zone *azone = (Zone *)zone;
azone->register_resource_tracker(description, should_collect);
}
void auto_zone_unregister_resource_tracker(auto_zone_t *zone, const char *description)
{
Zone *azone = (Zone *)zone;
azone->unregister_resource_tracker(description);
}
boolean_t auto_zone_is_valid_pointer(auto_zone_t *zone, const void *ptr) {
auto_block_info_sieve<AUTO_BLOCK_INFO_IS_BLOCK> sieve((Zone *)zone, (void *)ptr);
return sieve.is_block();
}
size_t auto_zone_size(auto_zone_t *zone, const void *ptr) {
auto_block_info_sieve<AUTO_BLOCK_INFO_SIZE> sieve((Zone *)zone, (void *)ptr);
return sieve.size();
}
const void *auto_zone_base_pointer(auto_zone_t *zone, const void *ptr) {
auto_block_info_sieve<AUTO_BLOCK_INFO_BASE_POINTER> sieve((Zone *)zone, (void *)ptr);
return sieve.base();
}
#if DEBUG
void *WatchPoint = (void *)-1L;
void blainer() {
sleep(0);
}
#endif
static inline void *auto_malloc(auto_zone_t *zone, size_t size) {
Zone *azone = (Zone *)zone;
Thread &thread = azone->registered_thread();
void *result = azone->block_allocate(thread, size, AUTO_MEMORY_UNSCANNED, false, true);
return result;
}
class auto_free_sieve : public sieve_base {
Zone *_zone;
public:
auto_free_sieve(Zone *zone, const void *ptr) __attribute__((always_inline)) : _zone(zone) {
sieve_base_pointer(zone, ptr, *this);
}
template <class BlockRef> inline void processBlock(BlockRef ref) TEMPLATE_INLINE {
unsigned refcount = ref.refcount();
if (refcount != 1) {
malloc_printf("*** free() called on collectable block with %p with refcount %d (ignored)\n", ref.address(), refcount);
} else {
_zone->block_deallocate(ref);
}
}
inline void nonBlock(const void *ptr) {
if (ptr != NULL)
error("Deallocating a non-block", ptr);
}
};
static void auto_free(auto_zone_t *azone, void *ptr) {
auto_free_sieve sieve((Zone *)azone, (void *)ptr);
}
static void *auto_calloc(auto_zone_t *zone, size_t size1, size_t size2) {
Zone *azone = (Zone *)zone;
size1 *= size2;
void *ptr;
Thread &thread = azone->registered_thread();
ptr = azone->block_allocate(thread, size1, AUTO_MEMORY_UNSCANNED, true, true);
return ptr;
}
static void *auto_valloc(auto_zone_t *zone, size_t size) {
Zone *azone = (Zone *)zone;
Thread &thread = azone->registered_thread();
void *result = azone->block_allocate(thread, auto_round_page(size), AUTO_MEMORY_UNSCANNED, true, true);
return result;
}
static void *auto_realloc(auto_zone_t *zone, void *ptr, size_t size) {
Zone *azone = (Zone*)zone;
if (!ptr) return auto_malloc(zone, size);
auto_block_info_sieve<AUTO_BLOCK_INFO_SIZE|AUTO_BLOCK_INFO_LAYOUT|AUTO_BLOCK_INFO_REFCOUNT> block_info(azone, (void *)ptr);
size_t block_size = block_info.size();
auto_memory_type_t layout = block_info.layout();
if (!block_info.is_block()) {
auto_error(azone, "auto_realloc: can't get type or retain count, ptr from ordinary malloc zone?", ptr);
return malloc_zone_realloc(malloc_zone_from_ptr(ptr), ptr, size);
}
if (size == 0) size = allocate_quantum_small;
if (block_size >= size) {
size_t delta = block_size - size;
if ((block_size <= allocate_quantum_medium && delta < allocate_quantum_small) ||
(block_size <= allocate_quantum_large && size >= allocate_quantum_medium && delta < allocate_quantum_medium) ||
(size > allocate_quantum_large && auto_round_page(block_size) == auto_round_page(size))) {
if (layout == AUTO_MEMORY_SCANNED)
bzero(displace(ptr,size), delta);
else if (layout == AUTO_MEMORY_ALL_WEAK_POINTERS)
weak_unregister_range(azone, (void **)displace(ptr, size), delta / sizeof(void*));
return ptr;
}
}
Thread &thread = azone->registered_thread();
void *new_ptr = azone->block_allocate(thread, size, layout, is_allocated_cleared(layout), (block_info.refcount() != 0));
if (new_ptr) {
size_t min_size = MIN(size, block_size);
if (is_scanned(layout)) {
auto_zone_write_barrier_memmove((auto_zone_t *)azone, new_ptr, ptr, min_size);
} else if (layout == AUTO_MEMORY_ALL_WEAK_POINTERS) {
memmove(new_ptr, ptr, min_size);
Auto::SpinLock lock(&azone->weak_refs_table_lock);
weak_transfer_weak_contents_unscanned(azone, (void **)ptr, (void **)new_ptr, min_size, false);
if (block_size > size) weak_unregister_range_no_lock(azone, (void **)displace(ptr, size), (block_size - size) / sizeof(void*));
} else {
memmove(new_ptr, ptr, min_size);
}
if (block_info.refcount() != 0) auto_zone_release(zone, ptr); }
return new_ptr;
}
static unsigned auto_batch_malloc(auto_zone_t *zone, size_t size, void **results, unsigned num_requested) {
return auto_zone_batch_allocate(zone, size, AUTO_MEMORY_UNSCANNED, true, false, results, num_requested);
}
static void auto_zone_destroy(auto_zone_t *zone) {
Zone *azone = (Zone*)zone;
auto_error(azone, "auto_zone_destroy", zone);
}
static kern_return_t auto_default_reader(task_t task, vm_address_t address, vm_size_t size, void **ptr) {
*ptr = (void *)address;
return KERN_SUCCESS;
}
static kern_return_t auto_in_use_enumerator(task_t task, void *context, unsigned type_mask, vm_address_t zone_address, memory_reader_t reader, vm_range_recorder_t recorder) {
kern_return_t err;
if (!reader) reader = auto_default_reader;
union {
unsigned *version;
void *voidStarVersion;
} u;
err = reader(task, zone_address + offsetof(malloc_zone_t, version), sizeof(unsigned), &u.voidStarVersion);
if (err != KERN_SUCCESS || *u.version != AUTO_ZONE_VERSION) return KERN_FAILURE;
InUseEnumerator enumerator(task, context, type_mask, zone_address, reader, recorder);
err = enumerator.scan();
return err;
}
static size_t auto_good_size(malloc_zone_t *azone, size_t size) {
return ((Zone *)azone)->good_block_size(size);
}
unsigned auto_check_counter = 0;
unsigned auto_check_start = 0;
unsigned auto_check_modulo = 1;
static boolean_t auto_check(malloc_zone_t *zone) {
if (! (++auto_check_counter % 10000)) {
malloc_printf("%s: At auto_check counter=%d\n", auto_prelude(), auto_check_counter);
}
if (auto_check_counter < auto_check_start) return 1;
if (auto_check_counter % auto_check_modulo) return 1;
return 1;
}
static char *b2s(uint64_t bytes, char *buf, int bufsize) {
if (bytes < 1024) {
snprintf(buf, bufsize, "%4llu bytes", bytes);
} else if (bytes < 1024*1024) {
snprintf(buf, bufsize, "%4.3g Kb", (float)bytes / 1024);
} else if (bytes < 1024*1024*1024) {
snprintf(buf, bufsize, "%4.3g Mb", (float)bytes / (1024*1024));
} else {
snprintf(buf, bufsize, "%4.3g Gb", (float)bytes / (1024*1024*1024));
}
return buf;
}
static void auto_zone_print(malloc_zone_t *zone, boolean_t verbose) {
malloc_statistics_t stats;
Zone *azone = (Zone *)zone;
azone->malloc_statistics(&stats);
char buf1[256];
char buf2[256];
printf("auto zone %p: in_use=%u used=%s allocated=%s\n", azone, stats.blocks_in_use, b2s(stats.size_in_use, buf1, sizeof(buf1)), b2s(stats.size_allocated, buf2, sizeof(buf2)));
if (verbose) azone->print_all_blocks();
}
static void auto_zone_log(malloc_zone_t *zone, void *log_address) {
}
static void auto_zone_force_lock(malloc_zone_t *zone) {
}
static void auto_zone_force_unlock(malloc_zone_t *zone) {
}
static void auto_malloc_statistics(malloc_zone_t *zone, malloc_statistics_t *stats) {
Zone *azone = (Zone *)zone;
azone->malloc_statistics(stats);
}
static boolean_t auto_malloc_zone_locked(malloc_zone_t *zone) {
return ((Zone *)zone)->is_locked();
}
static struct malloc_introspection_t auto_zone_introspect = {
auto_in_use_enumerator,
auto_good_size,
auto_check,
auto_zone_print,
auto_zone_log,
auto_zone_force_lock,
auto_zone_force_unlock,
auto_malloc_statistics,
auto_malloc_zone_locked,
auto_zone_enable_collection_checking,
auto_zone_disable_collection_checking,
auto_zone_track_pointer,
(void (*)(malloc_zone_t *, void (^)(void *,void *)))auto_zone_enumerate_uncollected
};
struct malloc_introspection_t auto_zone_introspection() {
return auto_zone_introspect;
}
static auto_zone_t *gc_zone = NULL;
auto_zone_t *auto_zone(void) {
return gc_zone;
}
auto_zone_t *auto_zone_from_pointer(void *pointer) {
malloc_zone_t *zone = malloc_zone_from_ptr(pointer);
return (zone && zone->introspect == &auto_zone_introspect) ? zone : NULL;
}
static void * volatile queues[__PTK_FRAMEWORK_GC_KEY9-__PTK_FRAMEWORK_GC_KEY0+1];
static void * volatile pressure_sources[__PTK_FRAMEWORK_GC_KEY9-__PTK_FRAMEWORK_GC_KEY0+1];
static void * volatile compaction_timers[__PTK_FRAMEWORK_GC_KEY9-__PTK_FRAMEWORK_GC_KEY0+1];
auto_zone_t *auto_zone_create(const char *name) {
aux_init();
pthread_key_t key = Zone::allocate_thread_key();
if (key == 0) return NULL;
Zone *azone = new Zone(key);
azone->basic_zone.size = auto_zone_size;
azone->basic_zone.malloc = auto_malloc;
azone->basic_zone.free = auto_free;
azone->basic_zone.calloc = auto_calloc;
azone->basic_zone.valloc = auto_valloc;
azone->basic_zone.realloc = auto_realloc;
azone->basic_zone.destroy = auto_zone_destroy;
azone->basic_zone.batch_malloc = auto_batch_malloc;
azone->basic_zone.zone_name = name; azone->basic_zone.introspect = &auto_zone_introspect;
azone->basic_zone.version = AUTO_ZONE_VERSION;
azone->basic_zone.memalign = NULL;
azone->control.version = sizeof(auto_collection_control_t);
azone->control.disable_generational = Environment::read_bool("AUTO_DISABLE_GENERATIONAL", false);
azone->control.malloc_stack_logging = (Environment::get("MallocStackLogging") != NULL || Environment::get("MallocStackLoggingNoCompact") != NULL);
azone->control.log = AUTO_LOG_NONE;
if (Environment::read_bool("AUTO_LOG_TIMINGS")) azone->control.log |= AUTO_LOG_TIMINGS;
if (Environment::read_bool("AUTO_LOG_ALL")) azone->control.log |= AUTO_LOG_ALL;
if (Environment::read_bool("AUTO_LOG_COLLECTIONS")) azone->control.log |= AUTO_LOG_COLLECTIONS;
if (Environment::read_bool("AUTO_LOG_REGIONS")) azone->control.log |= AUTO_LOG_REGIONS;
if (Environment::read_bool("AUTO_LOG_UNUSUAL")) azone->control.log |= AUTO_LOG_UNUSUAL;
if (Environment::read_bool("AUTO_LOG_WEAK")) azone->control.log |= AUTO_LOG_WEAK;
azone->control.collection_threshold = (size_t)Environment::read_long("AUTO_COLLECTION_THRESHOLD", 1024L * 1024L);
azone->control.full_vs_gen_frequency = Environment::read_long("AUTO_COLLECTION_RATIO", 10);
malloc_zone_register((auto_zone_t*)azone);
pthread_mutex_init(&azone->_collection_mutex, NULL);
azone->register_thread();
azone->_collection_queue = dispatch_queue_create("Garbage Collection Work Queue", NULL);
dispatch_queue_t target_queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, DISPATCH_QUEUE_OVERCOMMIT);
dispatch_set_target_queue(azone->_collection_queue, target_queue);
dispatch_set_context(azone->_collection_queue, azone);
const char *notify_name;
#if COMPACTION_ENABLED
notify_name = Environment::get("AUTO_COMPACT_NOTIFICATION");
if (notify_name != NULL) {
int compact_token_unused = 0;
notify_register_dispatch(notify_name, &compact_token_unused, azone->_collection_queue, ^(int token) {
Zone *zone = (Zone *)dispatch_get_context(dispatch_get_current_queue());
auto_date_t start = auto_date_now();
zone->compact_heap();
auto_date_t end = auto_date_now();
if (Environment::log_compactions) malloc_printf("compaction took %lld microseconds.\n", (end - start));
});
} else {
azone->_compaction_timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, dispatch_get_main_queue());
dispatch_source_set_timer(azone->_compaction_timer, DISPATCH_TIME_FOREVER, 0, 0);
dispatch_source_set_event_handler(azone->_compaction_timer, ^{
if (!azone->compaction_disabled()) {
azone->_compaction_next_time = DISPATCH_TIME_FOREVER;
dispatch_source_set_timer(azone->_compaction_timer, DISPATCH_TIME_FOREVER, 0, 0);
dispatch_async(azone->_collection_queue, ^{
auto_date_t start = auto_date_now();
azone->compact_heap();
auto_date_t end = auto_date_now();
malloc_printf("compaction took %lld microseconds.\n", (end - start));
azone->_compaction_next_time = dispatch_time(0, 30 * NSEC_PER_SEC);
azone->_compaction_pending = false;
});
}
});
dispatch_resume(azone->_compaction_timer);
}
notify_name = Environment::get("AUTO_ANALYZE_NOTIFICATION");
if (notify_name != NULL) {
int analyze_token_unused = 0;
notify_register_dispatch(notify_name, &analyze_token_unused, azone->_collection_queue, ^(int token) {
Zone *zone = (Zone *)dispatch_get_context(dispatch_get_current_queue());
static const char *analyze_name = Environment::get("AUTO_ANALYZE_NOTIFICATION");
zone->analyze_heap(analyze_name);
static const char *reply_name = Environment::get("AUTO_ANALYZE_REPLY");
if (reply_name) notify_post(reply_name);
});
}
#endif
notify_name = Environment::get("AUTO_MEMORY_PRESSURE_NOTIFICATION");
if (notify_name != NULL) {
int pressure_token_unused = 0;
notify_register_dispatch(notify_name, &pressure_token_unused, azone->_collection_queue, ^(int token) {
Zone *zone = (Zone *)dispatch_get_context(dispatch_get_current_queue());
usword_t size = zone->purge_free_space();
printf("purged %ld bytes.\n", size);
});
} else {
#if TARGET_OS_IPHONE
# warning no memory pressure dispatch source on iOS
#else
azone->_pressure_source = dispatch_source_create(DISPATCH_SOURCE_TYPE_VM, 0, DISPATCH_VM_PRESSURE, azone->_collection_queue);
if (azone->_pressure_source != NULL) {
dispatch_source_set_event_handler(azone->_pressure_source, ^{
Zone *zone = (Zone *)dispatch_get_context(dispatch_get_current_queue());
zone->purge_free_space();
});
dispatch_resume(azone->_pressure_source);
}
#endif
}
notify_name = Environment::get("AUTO_COLLECT_NOTIFICATION");
if (notify_name != NULL) {
int collect_token_unused = 0;
notify_register_dispatch(notify_name, &collect_token_unused, dispatch_get_main_queue(), ^(int token) {
malloc_printf("collecting on demand.\n");
auto_zone_collect((auto_zone_t *)azone, AUTO_ZONE_COLLECT_LOCAL_COLLECTION | AUTO_ZONE_COLLECT_EXHAUSTIVE_COLLECTION);
const char *reply_name = Environment::get("AUTO_COLLECT_REPLY");
if (reply_name) notify_post(reply_name);
});
}
if (!gc_zone) gc_zone = (auto_zone_t *)azone;
queues[key-__PTK_FRAMEWORK_GC_KEY0] = azone->_collection_queue;
pressure_sources[key-__PTK_FRAMEWORK_GC_KEY0] = azone->_pressure_source;
compaction_timers[key-__PTK_FRAMEWORK_GC_KEY0] = azone->_compaction_timer;
return (auto_zone_t*)azone;
}
void auto_zone_retain(auto_zone_t *zone, void *ptr) {
Zone *azone = (Zone *)zone;
auto_refcount_sieve<AUTO_REFCOUNT_INCREMENT> refcount_sieve(azone, ptr);
#if DEBUG
if (ptr == WatchPoint) {
malloc_printf("auto_zone_retain watchpoint: %p\n", WatchPoint);
blainer();
}
#endif
if (__auto_reference_logger) __auto_reference_logger(AUTO_RETAIN_EVENT, ptr, uintptr_t(refcount_sieve.refcount));
if (Environment::log_reference_counting && malloc_logger) {
malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, uintptr_t(zone), auto_zone_size(zone, ptr), 0, uintptr_t(ptr), 0);
}
}
unsigned int auto_zone_release(auto_zone_t *zone, void *ptr) {
Zone *azone = (Zone *)zone;
auto_refcount_sieve<AUTO_REFCOUNT_DECREMENT> refcount_sieve(azone, ptr);
#if DEBUG
if (ptr == WatchPoint) {
malloc_printf("auto_zone_release watchpoint: %p\n", WatchPoint);
blainer();
}
#endif
if (__auto_reference_logger) __auto_reference_logger(AUTO_RELEASE_EVENT, ptr, uintptr_t(refcount_sieve.refcount));
if (Environment::log_reference_counting && malloc_logger) {
malloc_logger(MALLOC_LOG_TYPE_DEALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, uintptr_t(zone), uintptr_t(ptr), 0, 0, 0);
}
return refcount_sieve.refcount;
}
unsigned int auto_zone_retain_count(auto_zone_t *zone, const void *ptr) {
auto_block_info_sieve<AUTO_BLOCK_INFO_REFCOUNT> refcount_sieve((Zone *)zone, ptr);
return refcount_sieve.refcount();
}
static void handle_resurrection(Zone *azone, const void *recipient, bool recipient_is_block, const void *new_value, size_t offset)
{
if (!recipient_is_block || ((auto_memory_type_t)azone->block_layout((void*)recipient) & AUTO_UNSCANNED) != AUTO_UNSCANNED) {
auto_memory_type_t new_value_type = (auto_memory_type_t) azone->block_layout((void*)new_value);
char msg[256];
snprintf(msg, sizeof(msg), "resurrection error for block %p while assigning %p[%d] = %p", new_value, recipient, (int)offset, new_value);
if ((new_value_type & AUTO_OBJECT_UNSCANNED) == AUTO_OBJECT) {
bool thread_local = false;
if (azone->in_subzone_memory((void*)new_value)) {
Subzone *sz = Subzone::subzone((void*)new_value);
usword_t q = sz->quantum_index((void*)new_value);
if (sz->is_thread_local(q)) {
thread_local = true;
Thread &thread = azone->registered_thread();
ThreadLocalCollector *tlc = thread.thread_local_collector();
if (tlc) {
thread.thread_local_collector()->add_zombie((void*)new_value);
} else {
auto_error(azone, "resurrection of thread local garbage belonging to another thread", new_value);
}
}
}
auto_zone_retain((auto_zone_t*)azone, (void*)new_value); if (!thread_local) {
azone->add_zombie((void*)new_value);
}
if (azone->control.name_for_address) {
char *recipient_name = azone->control.name_for_address((auto_zone_t *)azone, (vm_address_t)recipient, offset);
char *new_value_name = azone->control.name_for_address((auto_zone_t *)azone, (vm_address_t)new_value, 0);
snprintf(msg, sizeof(msg), "resurrection error for object %p while assigning %s(%p)[%d] = %s(%p)",
new_value, recipient_name, recipient, (int)offset, new_value_name, new_value);
free(recipient_name);
free(new_value_name);
}
}
malloc_printf("%s\ngarbage pointer stored into reachable memory, break on auto_zone_resurrection_error to debug\n", msg);
auto_zone_resurrection_error();
}
}
template <class BlockRef> static void handle_resurrection(Zone *azone, void *recipient, BlockRef new_value, size_t offset)
{
char msg[256];
snprintf(msg, sizeof(msg), "resurrection error for block %p while assigning %p[%d] = %p", new_value.address(), recipient, (int)offset, new_value.address());
if (new_value.is_object()) {
bool thread_local = false;
if (new_value.is_thread_local()) {
thread_local = true;
Thread &thread = azone->registered_thread();
ThreadLocalCollector *tlc = thread.thread_local_collector();
if (tlc) {
thread.thread_local_collector()->add_zombie((void*)new_value.address());
} else {
auto_error(azone, "resurrection of thread local garbage belonging to another thread", new_value.address());
}
}
auto_zone_retain((auto_zone_t*)azone, (void*)new_value.address()); if (!thread_local) {
azone->add_zombie((void*)new_value.address());
}
if (azone->control.name_for_address) {
char *recipient_name = azone->control.name_for_address((auto_zone_t *)azone, (vm_address_t)recipient, offset);
char *new_value_name = azone->control.name_for_address((auto_zone_t *)azone, (vm_address_t)new_value.address(), 0);
snprintf(msg, sizeof(msg), "resurrection error for object %p while assigning %s(%p)[%d] = %s(%p)",
new_value.address(), recipient_name, recipient, (int)offset, new_value_name, new_value.address());
free(recipient_name);
free(new_value_name);
}
}
malloc_printf("%s\ngarbage pointer stored into reachable memory, break on auto_zone_resurrection_error to debug\n", msg);
auto_zone_resurrection_error();
}
template <class DestBlockRef, class ValueBlockRef> static void handle_resurrection(Zone *azone, DestBlockRef recipient, ValueBlockRef new_value, size_t offset)
{
if (recipient.is_scanned()) {
handle_resurrection(azone, recipient.address(), new_value, offset);
}
}
inline static bool check_resurrection(Thread &thread, Zone *azone, void *recipient, bool recipient_is_block, const void *new_value, size_t offset) {
if (new_value &&
azone->is_block((void *)new_value) &&
azone->block_is_garbage((void *)new_value) &&
(!recipient_is_block || !azone->block_is_garbage(recipient))) {
handle_resurrection(azone, recipient, recipient_is_block, new_value, offset);
return true;
}
return false;
}
template <class DestBlockRef, class ValueBlockRef> inline static bool check_resurrection(Thread &thread, Zone *azone, DestBlockRef recipient, ValueBlockRef new_value, size_t offset) {
if (new_value.is_garbage() && (!recipient.is_garbage())) {
handle_resurrection(azone, recipient, new_value, offset);
return true;
}
return false;
}
template <class BlockRef> inline static bool check_resurrection(Thread &thread, Zone *azone, void *global_recipient, BlockRef new_value, size_t offset) {
if (new_value.is_garbage()) {
handle_resurrection(azone, global_recipient, new_value, offset);
return true;
}
return false;
}
template <class ValueBlockRef> class set_write_barrier_dest_sieve : public sieve_base {
public:
Zone *_zone;
const void *_dest;
ValueBlockRef _new_value;
const void *_new_value_addr;
bool _result;
set_write_barrier_dest_sieve(Zone *zone, const void *dest, ValueBlockRef new_value, const void *new_value_addr) __attribute__((always_inline)) : _zone(zone), _dest(dest), _new_value(new_value), _new_value_addr(new_value_addr), _result(true) {
sieve_interior_pointer(_zone, _dest, *this);
}
template <class DestBlockRef> inline void processBlock(DestBlockRef ref) TEMPLATE_INLINE {
Thread &thread = _zone->registered_thread();
size_t offset_in_bytes = (char *)_dest - (char *)ref.address();
check_resurrection(thread, _zone, ref, _new_value, offset_in_bytes);
if (Environment::unscanned_store_warning && _zone->compaction_enabled() && !ref.is_scanned() && !_new_value.has_refcount()) {
auto_error(_zone, "auto_zone_set_write_barrier: Storing a GC-managed pointer in unscanned memory location. Break on auto_zone_unscanned_store_error() to debug.", _new_value_addr);
auto_zone_unscanned_store_error(_dest, _new_value_addr);
}
_zone->set_write_barrier(thread, ref, (const void **)_dest, _new_value, _new_value_addr);
}
inline void nonBlock(const void *ptr) __attribute__((always_inline)) {
Thread &thread = _zone->registered_thread();
if (thread.is_stack_address((void *)_dest)) {
*(const void **)_dest = _new_value_addr;
} else {
if (Environment::unscanned_store_warning && _zone->compaction_enabled() && !_new_value.has_refcount() && !_zone->is_global_address((void*)_dest)) {
auto_error(_zone, "auto_zone_set_write_barrier: Storing a GC-managed pointer in unscanned memory location. Break on auto_zone_unscanned_store_error() to debug.", _new_value_addr);
auto_zone_unscanned_store_error(_dest, _new_value_addr);
}
_result = false;
}
}
};
class set_write_barrier_value_sieve : public sieve_base {
public:
Zone *_zone;
const void *_dest;
const void *_new_value;
bool _result;
set_write_barrier_value_sieve(Zone *zone, const void *dest, const void *new_value) __attribute__((always_inline)) : _zone(zone), _dest(dest), _new_value(new_value), _result(true) {
sieve_base_pointer(_zone, _new_value, *this);
}
template <class BlockRef> inline void processBlock(BlockRef ref) TEMPLATE_INLINE {
set_write_barrier_dest_sieve<BlockRef> dest(_zone, _dest, ref, _new_value);
_result = dest._result;
}
inline void nonBlock(const void *ptr) __attribute__((always_inline)) {
*(void **)_dest = (void *)ptr;
}
};
boolean_t auto_zone_set_write_barrier(auto_zone_t *zone, const void *dest, const void *new_value) {
set_write_barrier_value_sieve value((Zone *)zone, dest, new_value);
return value._result;
}
void *auto_zone_write_barrier_memmove(auto_zone_t *zone, void *dst, const void *src, size_t size) {
if (size == 0 || dst == src)
return dst;
Zone *azone = (Zone *)zone;
void *base = azone->block_start(dst);
if (base && is_scanned(azone->block_layout(base))) {
size_t block_size = auto_zone_size(zone, base);
ptrdiff_t block_overrun = (ptrdiff_t(dst) + size) - (ptrdiff_t(base) + block_size);
if (block_overrun > 0) {
auto_fatal("auto_zone_write_barrier_memmove: will overwrite block %p, size %ld by %ld bytes.", base, block_size, block_overrun);
}
const void *ptrSrc;
size_t ptrSize;
if (is_pointer_aligned((void *)src) && ((size & (sizeof(void *)-1)) == 0)) {
ptrSrc = src;
ptrSize = size;
} else {
ptrSrc = align_up((void *)src, pointer_alignment);
const void *ptrEnd = align_down(displace((void *)src, size), pointer_alignment);
if ((vm_address_t)ptrEnd > (vm_address_t)ptrSrc) {
ptrSize = (vm_address_t)ptrEnd - (vm_address_t)ptrSrc;
} else {
ptrSize = 0; }
}
if (ptrSize >= sizeof(void *)) {
Thread &thread = azone->registered_thread();
thread.track_local_memcopy(ptrSrc, dst, ptrSize);
if (azone->set_write_barrier_range(dst, size)) {
UnconditionalBarrier barrier(thread.needs_enlivening());
if (barrier) {
void **start = (void **)ptrSrc;
void **end = (void **)displace(start, ptrSize);
while (start < end) {
void *candidate = *start++;
if (azone->is_block(candidate)) thread.enliven_block(candidate);
}
}
return memmove(dst, src, size);
}
}
} else if (base == NULL) {
void *srcbase = azone->block_start((void*)src);
if (srcbase && is_scanned(azone->block_layout(srcbase)) && azone->is_global_address(dst)) {
auto_error(zone, "auto_zone_write_barrier_memmove: Copying a scanned block into global data. Break on auto_zone_global_data_memmove_error() to debug.", dst);
auto_zone_global_data_memmove_error();
}
}
return memmove(dst, src, size);
}
#define CHECK_STACK_READS 0
void *auto_zone_strong_read_barrier(auto_zone_t *zone, void **source) {
void *volatile *location = (void *volatile *)source;
void *value = *location;
Zone *azone = (Zone*)zone;
if (azone->in_subzone_memory(value)) {
Thread &thread = azone->registered_thread();
if (CHECK_STACK_READS) {
if (thread.is_stack_address(source)) return value;
}
SpinLock lock(&thread.in_compaction().lock);
value = *location;
usword_t q;
Subzone *subzone = Subzone::subzone(value);
if (subzone->block_start(value, q)) subzone->mark_pinned(q);
}
return value;
}
void* auto_zone_allocate_object(auto_zone_t *zone, size_t size, auto_memory_type_t type, boolean_t initial_refcount_to_one, boolean_t clear) {
void *ptr;
Zone *azone = (Zone *)zone;
Thread &thread = azone->registered_thread();
ptr = azone->block_allocate(thread, size, type, clear || is_allocated_cleared(type), initial_refcount_to_one);
if (ptr && malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE | (clear ? MALLOC_LOG_TYPE_CLEARED : 0), uintptr_t(zone), size, initial_refcount_to_one ? 1 : 0, uintptr_t(ptr), 0);
return ptr;
}
extern unsigned auto_zone_batch_allocate(auto_zone_t *zone, size_t size, auto_memory_type_t type, boolean_t initial_refcount_to_one, boolean_t clear, void **results, unsigned num_requested) {
Zone *azone = (Zone *)zone;
Thread &thread = azone->registered_thread();
unsigned count = azone->batch_allocate(thread, size, type, clear || is_allocated_cleared(type), initial_refcount_to_one, results, num_requested);
if (count && malloc_logger) {
for (unsigned i=0; i<count; i++)
malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE | MALLOC_LOG_TYPE_CLEARED, uintptr_t(zone), size, initial_refcount_to_one ? 1 : 0, uintptr_t(results[i]), 0);
}
return count;
}
extern "C" void *auto_zone_create_copy(auto_zone_t *zone, void *ptr) {
Zone *azone = (Zone *)zone;
auto_block_info_sieve<AUTO_BLOCK_INFO_SIZE|AUTO_BLOCK_INFO_LAYOUT|AUTO_BLOCK_INFO_REFCOUNT> block_info(azone, ptr);
auto_memory_type_t type;
int rc = 0;
size_t size;
if (block_info.is_block()) {
type = block_info.layout();
rc = block_info.refcount();
size = block_info.size();
} else {
type = AUTO_MEMORY_UNSCANNED;
rc = 0;
size = malloc_size(ptr);
}
if (type & AUTO_OBJECT) {
auto_error(azone, "auto_zone_copy_memory called on object", ptr);
return (void *)0;
}
void *result = auto_zone_allocate_object(zone, size, type, (rc == 1), false);
if (result) memmove(result, ptr, size);
return result;
}
void auto_zone_set_nofinalize(auto_zone_t *zone, void *ptr) {
Zone *azone = (Zone *)zone;
auto_memory_type_t type = azone->block_layout(ptr);
if (type == AUTO_TYPE_UNKNOWN) return;
if ((type & AUTO_OBJECT) && azone->weak_layout_map_for_block(ptr))
return; azone->block_set_layout(ptr, type & ~AUTO_OBJECT);
}
void auto_zone_set_unscanned(auto_zone_t *zone, void *ptr) {
Zone *azone = (Zone *)zone;
auto_memory_type_t type = azone->block_layout(ptr);
if (type == AUTO_TYPE_UNKNOWN) return;
azone->block_set_layout(ptr, type|AUTO_UNSCANNED);
}
void auto_zone_set_scan_exactly(auto_zone_t *zone, void *ptr) {
Zone *azone = (Zone *)zone;
auto_memory_type_t type = azone->block_layout(ptr);
if (type & AUTO_UNSCANNED) return; azone->block_set_layout(ptr, type|AUTO_POINTERS_ONLY);
}
extern void auto_zone_clear_stack(auto_zone_t *zone, unsigned long options)
{
Zone *azone = (Zone *)zone;
Thread *thread = azone->current_thread();
if (thread) {
thread->clear_stack();
}
}
auto_memory_type_t auto_zone_get_layout_type(auto_zone_t *zone, void *ptr) {
auto_block_info_sieve<AUTO_BLOCK_INFO_LAYOUT> block_info((Zone *)zone, ptr);
return block_info.layout();
}
void auto_zone_register_thread(auto_zone_t *zone) {
((Zone *)zone)->register_thread();
}
void auto_zone_unregister_thread(auto_zone_t *zone) {
((Zone *)zone)->unregister_thread();
}
void auto_zone_assert_thread_registered(auto_zone_t *zone) {
Zone *azone = (Zone *)zone;
azone->registered_thread();
}
void auto_zone_register_datasegment(auto_zone_t *zone, void *address, size_t size) {
((Zone *)zone)->add_datasegment(address, size);
}
void auto_zone_unregister_datasegment(auto_zone_t *zone, void *address, size_t size) {
((Zone *)zone)->remove_datasegment(address, size);
}
auto_collection_control_t *auto_collection_parameters(auto_zone_t *zone) {
Zone *azone = (Zone *)zone;
return &azone->control;
}
void auto_zone_statistics(auto_zone_t *zone, auto_statistics_t *stats) {
if (!stats) return;
bzero(stats, sizeof(auto_statistics_t));
}
typedef struct {
FILE *f;
char *buff;
size_t buff_size;
size_t buff_pos;
} AutoZonePrintInfo;
__private_extern__ malloc_zone_t *aux_zone;
void auto_collector_reenable(auto_zone_t *zone) {
Zone *azone = (Zone *)zone;
Mutex lock(&azone->_collection_mutex);
if (azone->_collector_disable_count == 0) return;
azone->_collector_disable_count--;
}
void auto_collector_disable(auto_zone_t *zone) {
Zone *azone = (Zone *)zone;
Mutex lock(&azone->_collection_mutex);
azone->_collector_disable_count++;
}
boolean_t auto_zone_is_enabled(auto_zone_t *zone) {
Zone *azone = (Zone *)zone;
Mutex lock(&azone->_collection_mutex);
return azone->_collector_disable_count == 0;
}
boolean_t auto_zone_is_collecting(auto_zone_t *zone) {
Zone *azone = (Zone *)zone;
return !azone->is_state(idle);
}
void auto_collect_multithreaded(auto_zone_t *zone) {
}
void auto_enumerate_references(auto_zone_t *zone, void *referent,
auto_reference_recorder_t callback, void *stack_bottom, void *ctx)
{
}
void auto_assign_weak_reference(auto_zone_t *zone, const void *value, const void **location, auto_weak_callback_block_t *block) {
Zone *azone = (Zone *)zone;
Thread &thread = azone->registered_thread();
void *base = azone->block_start((void *)location);
if (value) {
if ((base && azone->block_is_garbage(base)) || check_resurrection(thread, azone, (void *)base, base != NULL, value, (size_t)location - (size_t)base)) {
value = NULL;
}
}
if (!thread.is_stack_address((void *)location)) {
weak_register(azone, value, (void **)location, block);
if (value != NULL) {
if (base) thread.block_escaped(base);
thread.block_escaped((void *)value);
}
} else {
*location = value;
}
}
void *auto_read_weak_reference(auto_zone_t *zone, void **referrer) {
void *result = *referrer;
if (result != NULL) {
Zone *azone = (Zone*)zone;
Thread &thread = azone->registered_thread();
ConditionBarrier barrier(thread.needs_enlivening());
if (barrier) {
result = *referrer;
if (result) thread.enliven_block(result);
} else {
result = *referrer;
}
}
return result;
}
extern char CompactionObserverKey;
void auto_zone_set_compaction_observer(auto_zone_t *zone, void *block, void (^observer) (void)) {
if (observer) {
observer = Block_copy(observer);
Block_release(observer);
}
auto_zone_set_associative_ref(zone, block, &CompactionObserverKey, observer);
}
void auto_zone_set_associative_ref(auto_zone_t *zone, void *object, void *key, void *value) {
Zone *azone = (Zone*)zone;
Thread &thread = azone->registered_thread();
bool object_is_block = azone->is_block(object);
if (!object_is_block && !azone->is_global_address(object)) {
auto_error(zone, "auto_zone_set_associative_ref: object should point to a GC block or a global address, otherwise associations will leak. Break on auto_zone_association_error() to debug.", object);
auto_zone_association_error(object);
return;
}
check_resurrection(thread, azone, object, object_is_block, value, 0);
azone->set_associative_ref(object, key, value);
}
void *auto_zone_get_associative_ref(auto_zone_t *zone, void *object, void *key) {
Zone *azone = (Zone*)zone;
return azone->get_associative_ref(object, key);
}
void auto_zone_erase_associative_refs(auto_zone_t *zone, void *object) {
Zone *azone = (Zone*)zone;
return azone->erase_associations(object);
}
void auto_zone_enumerate_associative_refs(auto_zone_t *zone, void *key, boolean_t (^block) (void *object, void *value)) {
Zone *azone = (Zone*)zone;
azone->visit_associations_for_key(key, block);
}
size_t auto_zone_get_associative_hash(auto_zone_t *zone, void *object) {
Zone *azone = (Zone*)zone;
return azone->get_associative_hash(object);
}
class auto_zone_add_root_sieve : public sieve_base {
public:
Zone * const _zone;
void * const _root;
auto_zone_add_root_sieve(Zone *zone, void *root, void *ptr) __attribute__((always_inline)) : _zone(zone), _root(root) {
sieve_base_pointer(zone, ptr, *this);
}
template <class BlockRef> inline void processBlock(BlockRef ref) TEMPLATE_INLINE {
Thread &thread = _zone->registered_thread();
check_resurrection(thread, _zone, _root, ref, 0);
_zone->add_root(_root, ref);
}
inline void nonBlock(const void *ptr) __attribute__((always_inline)) {
*(void **)_root = (void *)ptr;
}
};
void auto_zone_add_root(auto_zone_t *zone, void *root, void *value)
{
auto_zone_add_root_sieve((Zone *)zone, root, value);
}
void auto_zone_remove_root(auto_zone_t *zone, void *root) {
((Zone *)zone)->remove_root(root);
}
void auto_zone_root_write_barrier(auto_zone_t *zone, void *address_of_possible_root_ptr, void *value) {
if (!value) {
*(void **)address_of_possible_root_ptr = NULL;
return;
}
Zone *azone = (Zone *)zone;
Thread &thread = azone->registered_thread();
if (thread.is_stack_address(address_of_possible_root_ptr)) {
*(void **)address_of_possible_root_ptr = value;
} else if (azone->is_root(address_of_possible_root_ptr)) {
thread.block_escaped(value);
UnconditionalBarrier barrier(thread.needs_enlivening());
if (barrier) thread.enliven_block(value);
check_resurrection(thread, azone, address_of_possible_root_ptr, false, value, 0);
*(void **)address_of_possible_root_ptr = value;
} else if (azone->is_global_address(address_of_possible_root_ptr)) {
auto_zone_add_root(zone, address_of_possible_root_ptr, value);
} else {
thread.block_escaped(value);
check_resurrection(thread, azone, address_of_possible_root_ptr, false, value, 0);
*(void **)address_of_possible_root_ptr = value;
if (Environment::unscanned_store_warning && azone->compaction_enabled()) {
auto_block_info_sieve<AUTO_BLOCK_INFO_REFCOUNT> info(azone, value);
if (info.is_block() && info.refcount() == 0) {
auto_error(zone, "auto_zone_root_write_barrier: Storing a GC-managed pointer in unscanned memory location. Break on auto_zone_unscanned_store_error() to debug.", value);
auto_zone_unscanned_store_error(address_of_possible_root_ptr, value);
}
}
}
}
void auto_zone_print_roots(auto_zone_t *zone) {
Zone *azone = (Zone *)zone;
PointerList roots;
azone->copy_roots(roots);
usword_t count = roots.count();
printf("### %lu roots. ###\n", count);
void ***buffer = (void ***)roots.buffer();
for (usword_t i = 0; i < count; ++i) {
void **root = buffer[i];
printf("%p -> %p\n", root, *root);
}
}
boolean_t auto_zone_atomicCompareAndSwap(auto_zone_t *zone, void *existingValue, void *newValue, void *volatile *location, boolean_t isGlobal, boolean_t issueBarrier) {
Zone *azone = (Zone *)zone;
Thread &thread = azone->registered_thread();
if (isGlobal) {
azone->add_root_no_barrier((void *)location);
}
check_resurrection(thread, azone, (void *)location, !isGlobal, newValue, 0);
thread.block_escaped(newValue);
UnconditionalBarrier barrier(thread.needs_enlivening());
boolean_t result;
if (issueBarrier)
result = OSAtomicCompareAndSwapPtrBarrier(existingValue, newValue, location);
else
result = OSAtomicCompareAndSwapPtr(existingValue, newValue, location);
if (!isGlobal) {
azone->set_write_barrier((char*)location);
}
if (result && barrier) thread.enliven_block(newValue);
return result;
}
boolean_t auto_zone_enable_collection_checking(auto_zone_t *zone) {
Zone *azone = (Zone *)zone;
azone->enable_collection_checking();
return true;
}
void auto_zone_disable_collection_checking(auto_zone_t *zone) {
Zone *azone = (Zone *)zone;
azone->disable_collection_checking();
}
void auto_zone_track_pointer(auto_zone_t *zone, void *pointer) {
Zone *azone = (Zone *)zone;
if (azone->collection_checking_enabled())
azone->track_pointer(pointer);
}
void auto_zone_enumerate_uncollected(auto_zone_t *zone, auto_zone_collection_checking_callback_t callback) {
Zone *azone = (Zone *)zone;
azone->enumerate_uncollected(callback);
}
#ifdef __BLOCKS__
void auto_zone_dump(auto_zone_t *zone,
auto_zone_stack_dump stack_dump,
auto_zone_register_dump register_dump,
auto_zone_node_dump thread_local_node_dump,
auto_zone_root_dump root_dump,
auto_zone_node_dump global_node_dump,
auto_zone_weak_dump weak_dump
) {
Auto::Zone *azone = (Auto::Zone *)zone;
azone->dump_zone(stack_dump, register_dump, thread_local_node_dump, root_dump, global_node_dump, weak_dump);
}
void auto_zone_visit(auto_zone_t *zone, auto_zone_visitor_t *visitor) {
Auto::Zone *azone = (Auto::Zone *)zone;
azone->visit_zone(visitor);
}
auto_probe_results_t auto_zone_probe_unlocked(auto_zone_t *zone, void *address) {
Zone *azone = (Zone *)zone;
auto_probe_results_t result = azone->block_is_start(address) ? auto_is_auto : auto_is_not_auto;
if ((result & auto_is_auto) && azone->is_local(address))
result |= auto_is_local;
return result;
}
void auto_zone_scan_exact(auto_zone_t *zone, void *address, void (^callback)(void *base, unsigned long byte_offset, void *candidate)) {
Zone *azone = (Zone *)zone;
auto_memory_type_t layout = azone->block_layout(address);
if ((layout & AUTO_UNSCANNED) == 0) {
const unsigned char *map = NULL;
if (layout & AUTO_OBJECT) {
map = azone->layout_map_for_block((void *)address);
}
unsigned int byte_offset = 0;
unsigned int size = auto_zone_size(zone, address);
if (map) {
while (*map) {
int skip = (*map >> 4) & 0xf;
int refs = (*map) & 0xf;
byte_offset = byte_offset + skip*sizeof(void *);
while (refs--) {
callback(address, byte_offset, *(void **)(((char *)address)+byte_offset));
byte_offset += sizeof(void *);
}
++map;
}
}
while (byte_offset < size) {
callback(address, byte_offset, *(void **)(((char *)address)+byte_offset));
byte_offset += sizeof(void *);
}
}
}
#endif
boolean_t auto_zone_atomicCompareAndSwapPtr(auto_zone_t *zone, void *existingValue, void *newValue, void *volatile *location, boolean_t issueBarrier) {
Zone *azone = (Zone *)zone;
return auto_zone_atomicCompareAndSwap(zone, existingValue, newValue, location, azone->is_global_address((void*)location), issueBarrier);
}
#if DEBUG
static void range_check(void *pointer, size_t size) {
Zone *azone = (Zone *)gc_zone;
if (azone) {
void *base_pointer = azone->block_start(pointer);
if (base_pointer) {
size_t block_size = auto_zone_size((auto_zone_t *)azone,base_pointer);
if ((uintptr_t(pointer) + size) > (uintptr_t(base_pointer) + block_size)) {
malloc_printf("SmashMonitor: range check violation for pointer = %p, size = %lu", pointer, size);
__builtin_trap();
}
}
}
}
void *SmashMonitor_memcpy(void *dst, const void* src, size_t size) {
range_check(dst, size);
return memcpy(dst, src, size);
}
void *SmashMonitor_memmove(void *dst, const void* src, size_t size) {
range_check(dst, size);
return memmove(dst, src, size);
}
void *SmashMonitor_memset(void *pointer, int value, size_t size) {
range_check(pointer, size);
return memset(pointer, value, size);
}
void SmashMonitor_bzero(void *pointer, size_t size) {
range_check(pointer, size);
bzero(pointer, size);
}
#if USE_INTERPOSING
DYLD_INTERPOSE(SmashMonitor_memcpy, memcpy)
DYLD_INTERPOSE(SmashMonitor_memmove, memmove)
DYLD_INTERPOSE(SmashMonitor_memset, memset)
DYLD_INTERPOSE(SmashMonitor_bzero, bzero)
#endif
#endif