LocalAllocator.cpp [plain text]
#include "config.h"
#include "LocalAllocator.h"
#include "AllocatingScope.h"
#include "LocalAllocatorInlines.h"
#include "Options.h"
namespace JSC {
LocalAllocator::LocalAllocator(BlockDirectory* directory)
: m_directory(directory)
, m_cellSize(directory->m_cellSize)
, m_freeList(m_cellSize)
{
auto locker = holdLock(directory->m_localAllocatorsLock);
directory->m_localAllocators.append(this);
}
void LocalAllocator::reset()
{
m_freeList.clear();
m_currentBlock = nullptr;
m_lastActiveBlock = nullptr;
m_allocationCursor = 0;
}
LocalAllocator::~LocalAllocator()
{
if (isOnList()) {
auto locker = holdLock(m_directory->m_localAllocatorsLock);
remove();
}
bool ok = true;
if (!m_freeList.allocationWillFail()) {
dataLog("FATAL: ", RawPointer(this), "->~LocalAllocator has non-empty free-list.\n");
ok = false;
}
if (m_currentBlock) {
dataLog("FATAL: ", RawPointer(this), "->~LocalAllocator has non-null current block.\n");
ok = false;
}
if (m_lastActiveBlock) {
dataLog("FATAL: ", RawPointer(this), "->~LocalAllocator has non-null last active block.\n");
ok = false;
}
RELEASE_ASSERT(ok);
}
void LocalAllocator::stopAllocating()
{
ASSERT(!m_lastActiveBlock);
if (!m_currentBlock) {
ASSERT(m_freeList.allocationWillFail());
return;
}
m_currentBlock->stopAllocating(m_freeList);
m_lastActiveBlock = m_currentBlock;
m_currentBlock = nullptr;
m_freeList.clear();
}
void LocalAllocator::resumeAllocating()
{
if (!m_lastActiveBlock)
return;
m_lastActiveBlock->resumeAllocating(m_freeList);
m_currentBlock = m_lastActiveBlock;
m_lastActiveBlock = nullptr;
}
void LocalAllocator::prepareForAllocation()
{
reset();
}
void LocalAllocator::stopAllocatingForGood()
{
stopAllocating();
reset();
}
void* LocalAllocator::allocateSlowCase(GCDeferralContext* deferralContext, AllocationFailureMode failureMode)
{
SuperSamplerScope superSamplerScope(false);
Heap& heap = *m_directory->m_heap;
ASSERT(heap.vm()->currentThreadIsHoldingAPILock());
doTestCollectionsIfNeeded(deferralContext);
ASSERT(!m_directory->markedSpace().isIterating());
heap.didAllocate(m_freeList.originalSize());
didConsumeFreeList();
AllocatingScope helpingHeap(heap);
heap.collectIfNecessaryOrDefer(deferralContext);
if (UNLIKELY(m_currentBlock))
return allocate(deferralContext, failureMode);
void* result = tryAllocateWithoutCollecting();
if (LIKELY(result != 0))
return result;
MarkedBlock::Handle* block = m_directory->tryAllocateBlock();
if (!block) {
if (failureMode == AllocationFailureMode::Assert)
RELEASE_ASSERT_NOT_REACHED();
else
return nullptr;
}
m_directory->addBlock(block);
result = allocateIn(block);
ASSERT(result);
return result;
}
void LocalAllocator::didConsumeFreeList()
{
if (m_currentBlock)
m_currentBlock->didConsumeFreeList();
m_freeList.clear();
m_currentBlock = nullptr;
}
void* LocalAllocator::tryAllocateWithoutCollecting()
{
SuperSamplerScope superSamplerScope(false);
ASSERT(!m_currentBlock);
ASSERT(m_freeList.allocationWillFail());
for (;;) {
MarkedBlock::Handle* block = m_directory->findBlockForAllocation(*this);
if (!block)
break;
if (void* result = tryAllocateIn(block))
return result;
}
if (Options::stealEmptyBlocksFromOtherAllocators()
&& (Options::tradeDestructorBlocks() || !m_directory->needsDestruction())) {
if (MarkedBlock::Handle* block = m_directory->m_subspace->findEmptyBlockToSteal()) {
RELEASE_ASSERT(block->alignedMemoryAllocator() == m_directory->m_subspace->alignedMemoryAllocator());
block->sweep(nullptr);
block->removeFromDirectory();
m_directory->addBlock(block);
return allocateIn(block);
}
}
return nullptr;
}
void* LocalAllocator::allocateIn(MarkedBlock::Handle* block)
{
void* result = tryAllocateIn(block);
RELEASE_ASSERT(result);
return result;
}
void* LocalAllocator::tryAllocateIn(MarkedBlock::Handle* block)
{
ASSERT(block);
ASSERT(!block->isFreeListed());
block->sweep(&m_freeList);
if (m_freeList.allocationWillFail()) {
ASSERT(block->isFreeListed());
block->unsweepWithNoNewlyAllocated();
ASSERT(!block->isFreeListed());
ASSERT(!m_directory->isEmpty(NoLockingNecessary, block));
ASSERT(!m_directory->isCanAllocateButNotEmpty(NoLockingNecessary, block));
return nullptr;
}
m_currentBlock = block;
void* result = m_freeList.allocate(
[] () -> HeapCell* {
RELEASE_ASSERT_NOT_REACHED();
return nullptr;
});
m_directory->setIsEden(NoLockingNecessary, m_currentBlock, true);
m_directory->markedSpace().didAllocateInBlock(m_currentBlock);
return result;
}
void LocalAllocator::doTestCollectionsIfNeeded(GCDeferralContext* deferralContext)
{
if (!Options::slowPathAllocsBetweenGCs())
return;
static unsigned allocationCount = 0;
if (!allocationCount) {
if (!m_directory->m_heap->isDeferred()) {
if (deferralContext)
deferralContext->m_shouldGC = true;
else
m_directory->m_heap->collectNow(Sync, CollectionScope::Full);
}
}
if (++allocationCount >= Options::slowPathAllocsBetweenGCs())
allocationCount = 0;
}
bool LocalAllocator::isFreeListedCell(const void* target) const
{
return m_freeList.contains(bitwise_cast<HeapCell*>(target));
}
}