AutoThreadLocalCollector.cpp [plain text]
#include "AutoThreadLocalCollector.h"
#include "auto_trace.h"
#include "auto_dtrace.h"
namespace Auto {
void ThreadLocalCollector::scan_local_block(void *startingBlock) {
SimplePointerBuffer scannedBlocks;
set_marked_blocks_buffer(&scannedBlocks);
void *block = startingBlock;
do {
int32_t index = _localBlocks.slotIndex(block);
if (!_localBlocks.wasScanned(index)) {
Subzone *subzone = Subzone::subzone(block);
Range range(block, subzone->size(block));
const unsigned char *map = (subzone->layout(block) & AUTO_OBJECT) ? _zone->layout_map_for_block(block) : NULL;
if (map)
scan_with_layout(range, map, NULL); else
scan_range(range, NULL);
if (index != -1) {
_localBlocks.setScanned(index);
_localBlocks.setMarked(index);
}
}
bool shouldScan;
do {
block = scannedBlocks.pop();
if (block) {
Subzone *rsz = Subzone::subzone(block);
shouldScan = rsz->should_scan_local_block(block);
}
} while (block && !shouldScan);
} while (block != NULL);
}
void ThreadLocalCollector::scan_marked_blocks() {
for (uint32_t i = _localBlocks.firstOccupiedSlot(), last = _localBlocks.lastOccupiedSlot(); i <= last; i++) {
void *block = _localBlocks.markedUnscannedPointerAtIndex(i);
if (block) {
Subzone *subzone = Subzone::subzone(block);
if (subzone->should_scan_local_block(block)) {
scan_local_block(block);
}
}
}
}
struct garbage_list {
Zone *zone;
size_t count;
vm_address_t garbage[0];
};
#if DEBUG
static size_t blocks_scavenged_locally, bytes_scavenged_locally;
#endif
void ThreadLocalCollector::scavenge_local(size_t count, vm_address_t garbage[]) {
size_t blocks_freed = 0;
size_t bytes_freed = 0;
GARBAGE_COLLECTION_COLLECTION_PHASE_BEGIN((auto_zone_t*)_zone, AUTO_TRACE_SCAVENGING_PHASE);
for (size_t index = 0; index < count; index++) {
void *ptr = reinterpret_cast<void*>(garbage[index]);
Subzone *subzone = Subzone::subzone(ptr);
usword_t q = subzone->quantum_index_unchecked(ptr, allocate_quantum_small_log2);
if (!subzone->has_refcount(q)) {
blocks_freed++;
bytes_freed += subzone->size(q);
if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_DEALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, uintptr_t(_zone), uintptr_t(ptr), 0, 0, 0);
_thread.thread_cache_add(ptr);
} else {
_zone->handle_overretained_garbage(ptr, _zone->block_refcount(ptr));
}
}
GARBAGE_COLLECTION_COLLECTION_PHASE_END((auto_zone_t*)_zone, AUTO_TRACE_SCAVENGING_PHASE, (uint64_t)blocks_freed, (uint64_t)bytes_freed);
__sync_add_and_fetch(&_zone->stats.thread_collections_total, 1);
__sync_add_and_fetch(&_zone->stats.thread_blocks_recovered_total, blocks_freed);
__sync_add_and_fetch(&_zone->stats.thread_bytes_recovered_total, bytes_freed);
#if DEBUG
__sync_add_and_fetch(&blocks_scavenged_locally, blocks_freed);
__sync_add_and_fetch(&bytes_scavenged_locally, bytes_freed);
#endif
}
#if DEBUG
static size_t blocks_scavenged_globally, bytes_scavenged_globally;
#endif
static void finalize_work(void *args) {
garbage_list *list = (garbage_list *)args;
size_t blocks_freed = 0, bytes_freed = 0;
Zone *zone = list->zone;
zone->invalidate_garbage(list->count, list->garbage);
zone->free_garbage(false, list->count, list->garbage, blocks_freed, bytes_freed); zone->clear_zombies();
aux_free(list);
#if DEBUG
__sync_add_and_fetch(&blocks_scavenged_globally, blocks_freed);
__sync_add_and_fetch(&bytes_scavenged_globally, bytes_freed);
#endif
}
void ThreadLocalCollector::evict_local_garbage(size_t count, vm_address_t garbage[]) {
set_marked_blocks_buffer(NULL);
_markedBlocksCounter = 0;
for (size_t i = 0; i < count; ++i) {
void *block = (void*)garbage[i];
Subzone *sz = Subzone::subzone(block);
usword_t layout = sz->layout(block);
if (!(layout & AUTO_UNSCANNED)) {
Range range(block, sz->size(block));
const unsigned char *map = (layout & AUTO_OBJECT) ? _zone->layout_map_for_block(block) : NULL;
if (map)
scan_with_layout(range, map, NULL); else
scan_range(range, NULL);
}
}
if (_markedBlocksCounter == 0) return;
scan_marked_blocks();
for (uint32_t i = _localBlocks.firstOccupiedSlot(); i <= _localBlocks.lastOccupiedSlot(); i++) {
if (!_localBlocks.validPointerAtIndex(i))
continue;
if (_localBlocks.wasMarked(i)) {
void *block = _localBlocks[i];
Subzone *subzone = Subzone::subzone(block);
subzone->make_global(block);
_localBlocks.remove(i);
}
}
}
void ThreadLocalCollector::process_local_garbage(bool finalizeNow) {
usword_t garbage_count = _localBlocks.count() - _markedBlocksCounter;
if (garbage_count == 0) {
_localBlocks.clearFlags(); GARBAGE_COLLECTION_COLLECTION_END((auto_zone_t*)_zone, 0ull, 0ull, _localBlocks.count(), (uint64_t)(-1));
return;
}
garbage_list *list = (garbage_list *)aux_malloc(sizeof(garbage_list) + garbage_count * sizeof(vm_address_t));
list->count = 0;
list->zone = _zone;
size_t list_count = 0;
size_t remaining_size = 0, scavenged_size = 0;
for (uint32_t i = _localBlocks.firstOccupiedSlot(), last = _localBlocks.lastOccupiedSlot(); i <= last; i++) {
void *block = _localBlocks.unmarkedPointerAtIndex(i);
if (block) {
Subzone *subzone = Subzone::subzone(block);
if (subzone->is_thread_local(block)) {
if (GARBAGE_COLLECTION_COLLECTION_END_ENABLED()) {
scavenged_size += subzone->size(block);
}
list->garbage[list_count++] = reinterpret_cast<vm_address_t>(block);
_localBlocks.remove(i);
subzone->mark_local_garbage(block);
} else {
auto_error(_zone, "not thread local garbage", (const void *)block);
}
} else if (GARBAGE_COLLECTION_COLLECTION_END_ENABLED()) {
block = _localBlocks.markedPointerAtIndex(i);
if (block) {
Subzone *subzone = Subzone::subzone(block);
if (subzone->is_thread_local(block)) {
remaining_size += subzone->size(block);
}
}
}
}
list->count = list_count;
_localBlocks.clearFlagsCompact();
if (!finalizeNow) evict_local_garbage(list_count, list->garbage);
AUTO_PROBE(auto_probe_end_local_scan(list_count, list->garbage));
if (finalizeNow) {
_zone->invalidate_garbage(list_count, list->garbage);
scavenge_local(list->count, list->garbage);
aux_free(list);
} else {
#if USE_DISPATCH_QUEUE
dispatch_async(_zone->collection_queue, ^{ finalize_work(list); });
#else
__builtin_trap();
#endif
}
GARBAGE_COLLECTION_COLLECTION_END((auto_zone_t*)_zone, list_count, (uint64_t)scavenged_size, _localBlocks.count(), (uint64_t)remaining_size);
}
bool ThreadLocalCollector::should_collect(Zone *zone, Thread &thread, bool canFinalizeNow) {
if (canFinalizeNow) {
return (thread.locals().count() >= thread.localAllocationThreshold());
} else {
#if USE_DISPATCH_QUEUE
if (zone->collection_queue) {
return (thread.locals().count() >= (10 * thread.localAllocationThreshold()));
}
#endif
return false;
}
}
void ThreadLocalCollector::collect(bool finalizeNow) {
AUTO_PROBE(auto_probe_begin_local_scan());
GARBAGE_COLLECTION_COLLECTION_BEGIN((auto_zone_t*)_zone, AUTO_TRACE_LOCAL);
GARBAGE_COLLECTION_COLLECTION_PHASE_BEGIN((auto_zone_t*)_zone, AUTO_TRACE_SCANNING_PHASE);
_thread.scan_current_thread(*this);
scan_marked_blocks();
GARBAGE_COLLECTION_COLLECTION_PHASE_END((auto_zone_t*)_zone, AUTO_TRACE_SCANNING_PHASE, (uint64_t)this->blocks_scanned(), (uint64_t)this->bytes_scanned());
process_local_garbage(finalizeNow);
uint32_t newThreshold = 3 * _localBlocks.count() / 2;
if (newThreshold < 100) newThreshold = 100;
_thread.setLocalAllocationThreshold(newThreshold);
AUTO_PROBE(auto_probe_local_collection_complete());
}
void ThreadLocalCollector::eject_local_block(void *startingBlock) {
SimplePointerBuffer scannedBlocks;
set_marked_blocks_buffer(&scannedBlocks);
usword_t count = 0, totsize = 0;
void *block = startingBlock;
do {
int32_t index = _localBlocks.slotIndex(block);
if (index != -1) {
Subzone *subzone = Subzone::subzone(block);
usword_t size = subzone->size(block);
totsize += size;
++count;
Range range(block, size);
const unsigned char *map = (subzone->layout(block) & AUTO_OBJECT) ? _zone->layout_map_for_block(block) : NULL;
if (map)
scan_with_layout(range, map, NULL);
else
scan_range(range, NULL);
subzone->make_global(block);
_localBlocks.remove(index);
}
bool shouldScan;
do {
block = scannedBlocks.pop();
if (block) {
Subzone *rsz = Subzone::subzone(block);
if (rsz->is_thread_local_block(block)) { shouldScan = rsz->should_scan_local_block(block);
int32_t index = _localBlocks.slotIndex(block);
if (!shouldScan) {
rsz->make_global(block);
totsize += rsz->size(block);
++count;
_localBlocks.remove(index);
index = -1;
}
} else {
shouldScan = false;
}
}
} while (block && !shouldScan);
} while (block != NULL);
Statistics &zone_stats = _zone->statistics();
zone_stats.add_count(count);
zone_stats.add_size(totsize);
_zone->adjust_threshold_counter(totsize);
}
#if DEBUG
extern "C" void print_TLC_stats() {
printf("TLC scavenged %ld bytes (%ld blocks) locally.\n", bytes_scavenged_locally, blocks_scavenged_locally);
printf("TLC scavenged %ld bytes (%ld blocks) globally.\n", bytes_scavenged_globally, blocks_scavenged_globally);
}
#endif
}