#include "config.h"
#include "MarkedBlock.h"
#include "FreeListInlines.h"
#include "JSCell.h"
#include "JSDestructibleObject.h"
#include "JSCInlines.h"
#include "MarkedAllocatorInlines.h"
#include "MarkedBlockInlines.h"
#include "SuperSampler.h"
#include "SweepingScope.h"
#include <wtf/CommaPrinter.h>
namespace JSC {
const size_t MarkedBlock::blockSize;
static const bool computeBalance = false;
static size_t balance;
MarkedBlock::Handle* MarkedBlock::tryCreate(Heap& heap)
{
if (computeBalance) {
balance++;
if (!(balance % 10))
dataLog("MarkedBlock Balance: ", balance, "\n");
}
void* blockSpace = tryFastAlignedMalloc(blockSize, blockSize);
if (!blockSpace)
return nullptr;
if (scribbleFreeCells())
scribble(blockSpace, blockSize);
return new Handle(heap, blockSpace);
}
MarkedBlock::Handle::Handle(Heap& heap, void* blockSpace)
: m_weakSet(heap.vm(), CellContainer())
, m_newlyAllocatedVersion(MarkedSpace::nullVersion)
{
m_block = new (NotNull, blockSpace) MarkedBlock(*heap.vm(), *this);
m_weakSet.setContainer(*m_block);
heap.didAllocateBlock(blockSize);
}
MarkedBlock::Handle::~Handle()
{
Heap& heap = *this->heap();
if (computeBalance) {
balance--;
if (!(balance % 10))
dataLog("MarkedBlock Balance: ", balance, "\n");
}
removeFromAllocator();
m_block->~MarkedBlock();
fastAlignedFree(m_block);
heap.didFreeBlock(blockSize);
}
MarkedBlock::MarkedBlock(VM& vm, Handle& handle)
: m_markingVersion(MarkedSpace::nullVersion)
, m_handle(handle)
, m_vm(&vm)
{
if (false)
dataLog(RawPointer(this), ": Allocated.\n");
}
void MarkedBlock::Handle::unsweepWithNoNewlyAllocated()
{
RELEASE_ASSERT(m_isFreeListed);
m_isFreeListed = false;
}
void MarkedBlock::Handle::setIsFreeListed()
{
m_allocator->setIsEmpty(NoLockingNecessary, this, false);
m_isFreeListed = true;
}
void MarkedBlock::Handle::stopAllocating(const FreeList& freeList)
{
auto locker = holdLock(block().m_lock);
if (false)
dataLog(RawPointer(this), ": MarkedBlock::Handle::stopAllocating!\n");
ASSERT(!allocator()->isAllocated(NoLockingNecessary, this));
if (!isFreeListed()) {
if (false)
dataLog("There ain't no newly allocated.\n");
ASSERT(freeList.allocationWillFail());
return;
}
if (false)
dataLog("Free list: ", freeList, "\n");
m_newlyAllocated.clearAll();
m_newlyAllocatedVersion = heap()->objectSpace().newlyAllocatedVersion();
forEachCell(
[&] (HeapCell* cell, HeapCell::Kind) -> IterationStatus {
setNewlyAllocated(cell);
return IterationStatus::Continue;
});
freeList.forEach(
[&] (HeapCell* cell) {
if (false)
dataLog("Free cell: ", RawPointer(cell), "\n");
if (m_attributes.destruction == NeedsDestruction)
cell->zap();
clearNewlyAllocated(cell);
});
m_isFreeListed = false;
}
void MarkedBlock::Handle::lastChanceToFinalize()
{
allocator()->setIsAllocated(NoLockingNecessary, this, false);
m_block->m_marks.clearAll();
m_block->clearHasAnyMarked();
m_block->m_markingVersion = heap()->objectSpace().markingVersion();
m_weakSet.lastChanceToFinalize();
m_newlyAllocated.clearAll();
m_newlyAllocatedVersion = heap()->objectSpace().newlyAllocatedVersion();
sweep(nullptr);
}
void MarkedBlock::Handle::resumeAllocating(FreeList& freeList)
{
{
auto locker = holdLock(block().m_lock);
if (false)
dataLog(RawPointer(this), ": MarkedBlock::Handle::resumeAllocating!\n");
ASSERT(!allocator()->isAllocated(NoLockingNecessary, this));
ASSERT(!isFreeListed());
if (!hasAnyNewlyAllocated()) {
if (false)
dataLog("There ain't no newly allocated.\n");
freeList.clear();
return;
}
}
sweep(&freeList);
}
void MarkedBlock::Handle::zap(const FreeList& freeList)
{
freeList.forEach(
[&] (HeapCell* cell) {
if (m_attributes.destruction == NeedsDestruction)
cell->zap();
});
}
void MarkedBlock::aboutToMarkSlow(HeapVersion markingVersion)
{
ASSERT(vm()->heap.objectSpace().isMarking());
LockHolder locker(m_lock);
if (!areMarksStale(markingVersion))
return;
MarkedAllocator* allocator = handle().allocator();
if (handle().allocator()->isAllocated(holdLock(allocator->bitvectorLock()), &handle())
|| !marksConveyLivenessDuringMarking(markingVersion)) {
if (false)
dataLog(RawPointer(this), ": Clearing marks without doing anything else.\n");
m_marks.clearAll();
} else {
if (false)
dataLog(RawPointer(this), ": Doing things.\n");
HeapVersion newlyAllocatedVersion = space()->newlyAllocatedVersion();
if (handle().m_newlyAllocatedVersion == newlyAllocatedVersion) {
handle().m_newlyAllocated.mergeAndClear(m_marks);
} else {
handle().m_newlyAllocated.setAndClear(m_marks);
}
handle().m_newlyAllocatedVersion = newlyAllocatedVersion;
}
clearHasAnyMarked();
WTF::storeStoreFence();
m_markingVersion = markingVersion;
allocator->setIsMarkingNotEmpty(holdLock(allocator->bitvectorLock()), &handle(), true);
}
void MarkedBlock::Handle::resetAllocated()
{
m_newlyAllocated.clearAll();
m_newlyAllocatedVersion = MarkedSpace::nullVersion;
}
void MarkedBlock::resetMarks()
{
if (areMarksStale())
m_marks.clearAll();
m_markingVersion = MarkedSpace::nullVersion;
}
#if !ASSERT_DISABLED
void MarkedBlock::assertMarksNotStale()
{
ASSERT(m_markingVersion == vm()->heap.objectSpace().markingVersion());
}
#endif // !ASSERT_DISABLED
bool MarkedBlock::areMarksStale()
{
return areMarksStale(vm()->heap.objectSpace().markingVersion());
}
bool MarkedBlock::Handle::areMarksStale()
{
return m_block->areMarksStale();
}
bool MarkedBlock::isMarked(const void* p)
{
return isMarked(vm()->heap.objectSpace().markingVersion(), p);
}
void MarkedBlock::Handle::didConsumeFreeList()
{
auto locker = holdLock(block().m_lock);
if (false)
dataLog(RawPointer(this), ": MarkedBlock::Handle::didConsumeFreeList!\n");
ASSERT(isFreeListed());
m_isFreeListed = false;
allocator()->setIsAllocated(NoLockingNecessary, this, true);
}
size_t MarkedBlock::markCount()
{
return areMarksStale() ? 0 : m_marks.count();
}
bool MarkedBlock::Handle::isEmpty()
{
return m_allocator->isEmpty(NoLockingNecessary, this);
}
void MarkedBlock::clearHasAnyMarked()
{
m_biasedMarkCount = m_markCountBias;
}
void MarkedBlock::noteMarkedSlow()
{
MarkedAllocator* allocator = handle().allocator();
allocator->setIsMarkingRetired(holdLock(allocator->bitvectorLock()), &handle(), true);
}
void MarkedBlock::Handle::removeFromAllocator()
{
if (!m_allocator)
return;
m_allocator->removeBlock(this);
}
void MarkedBlock::updateNeedsDestruction()
{
m_needsDestruction = handle().needsDestruction();
}
void MarkedBlock::Handle::didAddToAllocator(MarkedAllocator* allocator, size_t index)
{
ASSERT(m_index == std::numeric_limits<size_t>::max());
ASSERT(!m_allocator);
m_index = index;
m_allocator = allocator;
size_t cellSize = allocator->cellSize();
m_atomsPerCell = (cellSize + atomSize - 1) / atomSize;
m_endAtom = atomsPerBlock - m_atomsPerCell + 1;
m_attributes = allocator->attributes();
if (m_attributes.cellKind != HeapCell::JSCell)
RELEASE_ASSERT(m_attributes.destruction == DoesNotNeedDestruction);
block().updateNeedsDestruction();
double markCountBias = -(Options::minMarkedBlockUtilization() * cellsPerBlock());
RELEASE_ASSERT(markCountBias > static_cast<double>(std::numeric_limits<int16_t>::min()));
RELEASE_ASSERT(markCountBias < 0);
block().m_biasedMarkCount = block().m_markCountBias = static_cast<int16_t>(markCountBias);
}
void MarkedBlock::Handle::didRemoveFromAllocator()
{
ASSERT(m_index != std::numeric_limits<size_t>::max());
ASSERT(m_allocator);
m_index = std::numeric_limits<size_t>::max();
m_allocator = nullptr;
}
bool MarkedBlock::Handle::isLive(const HeapCell* cell)
{
return isLive(space()->markingVersion(), space()->isMarking(), cell);
}
bool MarkedBlock::Handle::isLiveCell(const void* p)
{
return isLiveCell(space()->markingVersion(), space()->isMarking(), p);
}
#if !ASSERT_DISABLED
void MarkedBlock::assertValidCell(VM& vm, HeapCell* cell) const
{
RELEASE_ASSERT(&vm == this->vm());
RELEASE_ASSERT(const_cast<MarkedBlock*>(this)->handle().cellAlign(cell) == cell);
}
#endif
void MarkedBlock::Handle::dumpState(PrintStream& out)
{
CommaPrinter comma;
allocator()->forEachBitVectorWithName(
holdLock(allocator()->bitvectorLock()),
[&] (FastBitVector& bitvector, const char* name) {
out.print(comma, name, ":", bitvector[index()] ? "YES" : "no");
});
}
Subspace* MarkedBlock::Handle::subspace() const
{
return allocator()->subspace();
}
void MarkedBlock::Handle::sweep(FreeList* freeList)
{
SweepingScope sweepingScope(*heap());
SweepMode sweepMode = freeList ? SweepToFreeList : SweepOnly;
m_allocator->setIsUnswept(NoLockingNecessary, this, false);
m_weakSet.sweep();
if (sweepMode == SweepOnly && m_attributes.destruction == DoesNotNeedDestruction)
return;
if (UNLIKELY(m_isFreeListed)) {
RELEASE_ASSERT(sweepMode == SweepToFreeList);
return;
}
ASSERT(!m_allocator->isAllocated(NoLockingNecessary, this));
if (space()->isMarking())
block().m_lock.lock();
if (m_attributes.destruction == NeedsDestruction) {
subspace()->finishSweep(*this, freeList);
return;
}
EmptyMode emptyMode = this->emptyMode();
ScribbleMode scribbleMode = this->scribbleMode();
NewlyAllocatedMode newlyAllocatedMode = this->newlyAllocatedMode();
MarksMode marksMode = this->marksMode();
auto trySpecialized = [&] () -> bool {
if (sweepMode != SweepToFreeList)
return false;
if (scribbleMode != DontScribble)
return false;
if (newlyAllocatedMode != DoesNotHaveNewlyAllocated)
return false;
switch (emptyMode) {
case IsEmpty:
switch (marksMode) {
case MarksNotStale:
specializedSweep<true, IsEmpty, SweepToFreeList, BlockHasNoDestructors, DontScribble, DoesNotHaveNewlyAllocated, MarksNotStale>(freeList, IsEmpty, SweepToFreeList, BlockHasNoDestructors, DontScribble, DoesNotHaveNewlyAllocated, MarksNotStale, [] (VM&, JSCell*) { });
return true;
case MarksStale:
specializedSweep<true, IsEmpty, SweepToFreeList, BlockHasNoDestructors, DontScribble, DoesNotHaveNewlyAllocated, MarksStale>(freeList, IsEmpty, SweepToFreeList, BlockHasNoDestructors, DontScribble, DoesNotHaveNewlyAllocated, MarksStale, [] (VM&, JSCell*) { });
return true;
}
break;
case NotEmpty:
switch (marksMode) {
case MarksNotStale:
specializedSweep<true, NotEmpty, SweepToFreeList, BlockHasNoDestructors, DontScribble, DoesNotHaveNewlyAllocated, MarksNotStale>(freeList, IsEmpty, SweepToFreeList, BlockHasNoDestructors, DontScribble, DoesNotHaveNewlyAllocated, MarksNotStale, [] (VM&, JSCell*) { });
return true;
case MarksStale:
specializedSweep<true, NotEmpty, SweepToFreeList, BlockHasNoDestructors, DontScribble, DoesNotHaveNewlyAllocated, MarksStale>(freeList, IsEmpty, SweepToFreeList, BlockHasNoDestructors, DontScribble, DoesNotHaveNewlyAllocated, MarksStale, [] (VM&, JSCell*) { });
return true;
}
break;
}
return false;
};
if (trySpecialized())
return;
specializedSweep<false, IsEmpty, SweepOnly, BlockHasNoDestructors, DontScribble, HasNewlyAllocated, MarksStale>(freeList, emptyMode, sweepMode, BlockHasNoDestructors, scribbleMode, newlyAllocatedMode, marksMode, [] (VM&, JSCell*) { });
}
bool MarkedBlock::Handle::isFreeListedCell(const void* target) const
{
ASSERT(isFreeListed());
return m_allocator->isFreeListedCell(target);
}
}
namespace WTF {
void printInternal(PrintStream& out, JSC::MarkedBlock::Handle::SweepMode mode)
{
switch (mode) {
case JSC::MarkedBlock::Handle::SweepToFreeList:
out.print("SweepToFreeList");
return;
case JSC::MarkedBlock::Handle::SweepOnly:
out.print("SweepOnly");
return;
}
RELEASE_ASSERT_NOT_REACHED();
}
}