#include "AutoAdmin.h"
#include "AutoBitmap.h"
#include "AutoConfiguration.h"
#include "AutoDefs.h"
#include "AutoZone.h"
#include "AutoLock.h"
#include <stdio.h>
#define COALESCE_BLOCKS
extern "C" char *__crashreporter_info__;
namespace Auto {
void Admin::initialize(Zone *zone, Region *region, const usword_t quantum_log2) {
_zone = zone;
_region = region;
_quantum_log2 = quantum_log2;
_admin_lock = 0;
}
usword_t Admin::unused_count() {
return _active_subzone->allocation_limit() - _active_subzone->allocation_count();
}
usword_t Admin::free_space() {
SpinLock lock(&_admin_lock);
usword_t empty_space = 0;
for (usword_t m = 0; m < cache_size; m++) {
for (FreeListNode *node = _cache[m].head(); node != NULL; node = node->next()) {
empty_space += node->size();
}
}
return empty_space;
}
usword_t Admin::empty_space() {
SpinLock lock(&_admin_lock);
usword_t empty_space = 0;
for (FreeListNode *node = _cache[0].head(); node != NULL; node = node->next()) {
empty_space += node->size();
}
return empty_space;
}
bool Admin::test_node_integrity(FreeListNode *node) {
bool node_is_valid = false;
const Range &coverage = _zone->coverage();
do {
if (!coverage.in_range(node)) break;
Subzone *subzone = Subzone::subzone((void *)node);
usword_t q = subzone->quantum_index(node->address());
if (q >= subzone->allocation_limit()) break;
if (subzone->quantum_address(q) != node->address()) break;
if (subzone->is_used(q)) break;
FreeListNode *next = node->next();
if (next && !coverage.in_range(next)) break;
FreeListNode *prev = node->prev();
if (prev && !coverage.in_range(prev)) break;
if (node->size() != node->size_again()) break;
node_is_valid = true;
} while (0);
if (!node_is_valid) {
static char buffer[256];
if (coverage.in_range(node)) {
snprintf(buffer, sizeof(buffer), "test_node_integrity: FreeListNode %p { _prev = %p, _next = %p, _size = %lu } failed integrity check.\n",
node, node->prev(), node->next(), node->size());
} else {
snprintf(buffer, sizeof(buffer), "test_node_integrity: FreeListNode %p failed integrity check.\n", node);
}
__crashreporter_info__ = buffer;
malloc_printf("%s", buffer);
__builtin_trap();
}
return node_is_valid;
}
bool Admin::test_freelist_integrity() {
SpinLock lock(&_admin_lock);
for (usword_t m = 0; m < cache_size; m++) {
for (FreeListNode *node = _cache[m].head(), *prev_node = NULL; node; node = node->next()) {
Subzone *subzone = Subzone::subzone((void *)node);
usword_t q = subzone->quantum_index(node->address());
if (q >= subzone->allocation_limit()) return false;
if (subzone->quantum_address(q) != node->address()) return false;
if (subzone->is_used(q)) return false;
if (node->prev() != prev_node) return false;
if (node->size() != node->size_again()) return false;
prev_node = node;
}
}
return true;
}
inline FreeListNode *Admin::pop_node(usword_t index) {
FreeListNode *head = _cache[index].head();
return (head && test_node_integrity(head) ? _cache[index].pop() : NULL);
}
inline void Admin::mark_allocated(void *address, const usword_t n, const unsigned layout, const bool refcount_is_one) {
Subzone *subzone = Subzone::subzone(address);
*(void **)address = NULL;
subzone->allocate(subzone->quantum_index(address), n, layout, refcount_is_one);
}
void *Admin::find_allocation(const usword_t size, const unsigned layout, const bool refcount_is_one, bool &did_grow) {
SpinLock lock(&_admin_lock);
usword_t n = quantum_count(size);
ASSERTION(n < cache_size);
FreeListNode *node = pop_node(n);
void *address = node->address();
if (address) {
ConditionBarrier barrier(_zone->needs_enlivening(), _zone->enlivening_lock());
mark_allocated(address, n, layout, refcount_is_one);
if (barrier) _zone->enlivening_queue().add(address);
return address;
}
for (usword_t i = n + 1; node == NULL && i < cache_size; i++) {
node = pop_node(i);
}
if (!node) node = pop_node(0);
if (node) {
address = node->address();
Subzone *subzone = Subzone::subzone(address);
usword_t allocation_size = subzone->quantum_size(n);
ASSERTION(node->size() >= allocation_size);
usword_t remainder_size = node->size() - allocation_size;
if (remainder_size) {
void *remainder_address = displace(address, allocation_size);
usword_t m = cache_slot(remainder_size);
_cache[m].push(remainder_address, remainder_size);
}
}
else if (_active_subzone) {
did_grow = true;
usword_t top = _active_subzone->allocation_count();
usword_t unused = _active_subzone->allocation_limit() - top;
ASSERTION(unused >= n);
address = _active_subzone->quantum_address(top);
*(void **)address = NULL;
_active_subzone->raise_allocation_count(n);
_zone->statistics().add_dirty(_active_subzone->quantum_size(n));
unused -= n;
if (unused == 0) {
set_active_subzone(NULL);
}
else if (unused < cache_size) {
_cache[unused].push(_active_subzone->quantum_address(top+n), _active_subzone->quantum_size(unused));
_active_subzone->raise_allocation_count(unused);
set_active_subzone(NULL);
}
}
else {
return NULL;
}
ConditionBarrier barrier(_zone->needs_enlivening(), _zone->enlivening_lock());
mark_allocated(address, n, layout, refcount_is_one);
if (barrier) _zone->enlivening_queue().add(address);
return address;
}
void Admin::deallocate(void *address) {
SpinLock lock(&_admin_lock);
Subzone *subzone = Subzone::subzone(address);
usword_t q = subzone->quantum_index(address);
usword_t n = subzone->length(q);
ASSERTION(!subzone->is_free(q));
if (subzone->is_free(q)) {
malloc_printf("Admin::deallocate: attempting to free already freed block %p\n", address);
return;
}
void *free_address = address;
usword_t free_size = subzone->quantum_size(n);
usword_t next_q = q + n;
usword_t highwater = subzone->allocation_count();
if (next_q < highwater && subzone->is_free(next_q)) {
FreeListNode *next_node = (FreeListNode *)displace(free_address, free_size);
if (test_node_integrity(next_node)) {
usword_t next_size = next_node->size();
usword_t m = cache_slot(next_size);
_cache[m].remove(next_node);
free_size += next_size;
}
}
if (q && subzone->is_free(q - 1)) {
FreeListNode *this_node = (FreeListNode *)address;
FreeListNode *prev_node = this_node->prior_node();
if (test_node_integrity(prev_node)) {
free_address = prev_node->address();
usword_t prev_size = prev_node->size();
free_size += prev_size;
usword_t m = cache_slot(prev_size);
_cache[m].remove(prev_node);
}
}
if (Environment::_agc_env._dirty_all_deleted) {
memset(free_address, 0x55, free_size);
}
if (next_q == highwater && highwater < subzone->allocation_limit()) {
subzone->lower_allocation_count(quantum_count(free_size));
_zone->statistics().add_dirty(-free_size); }
else {
usword_t m = cache_slot(free_size);
_cache[m].push(free_address, free_size);
}
subzone->deallocate(q, n);
}
bool Admin::is_pending(usword_t q) { return _region->is_pending(q); }
void Admin::clear_pending(usword_t q) { _region->clear_pending(q); }
void Admin::set_pending(usword_t q) { _region->set_pending(q); }
void Admin::set_mark(usword_t q) { _region->set_mark(q); }
bool Admin::is_marked(usword_t q) { return _region->is_marked(q); }
void Admin::clear_mark(usword_t q) { _region->clear_mark(q); }
bool Admin::test_set_mark(usword_t q) { return _region->test_set_mark(q); }
};