MarkedBlockInlines.h [plain text]
#pragma once
#include "JSCell.h"
#include "MarkedAllocator.h"
#include "MarkedBlock.h"
#include "MarkedSpace.h"
#include "Operations.h"
#include "SuperSampler.h"
#include "VM.h"
namespace JSC {
inline unsigned MarkedBlock::Handle::cellsPerBlock()
{
return MarkedSpace::blockPayload / cellSize();
}
inline bool MarkedBlock::Handle::isNewlyAllocatedStale() const
{
return m_newlyAllocatedVersion != space()->newlyAllocatedVersion();
}
inline bool MarkedBlock::Handle::hasAnyNewlyAllocated()
{
return !isNewlyAllocatedStale();
}
inline Heap* MarkedBlock::heap() const
{
return &vm()->heap;
}
inline MarkedSpace* MarkedBlock::space() const
{
return &heap()->objectSpace();
}
inline MarkedSpace* MarkedBlock::Handle::space() const
{
return &heap()->objectSpace();
}
inline bool MarkedBlock::marksConveyLivenessDuringMarking(HeapVersion markingVersion)
{
ASSERT(space()->isMarking());
if (heap()->collectionScope() != CollectionScope::Full)
return false;
return m_markingVersion == MarkedSpace::nullVersion
|| MarkedSpace::nextVersion(m_markingVersion) == markingVersion;
}
inline bool MarkedBlock::Handle::isLive(HeapVersion markingVersion, bool isMarking, const HeapCell* cell)
{
ASSERT(!isFreeListed());
if (UNLIKELY(hasAnyNewlyAllocated())) {
if (isNewlyAllocated(cell))
return true;
}
if (allocator()->isAllocated(NoLockingNecessary, this))
return true;
MarkedBlock& block = this->block();
if (block.areMarksStale()) {
if (!isMarking)
return false;
if (!block.marksConveyLivenessDuringMarking(markingVersion))
return false;
}
return block.m_marks.get(block.atomNumber(cell));
}
inline bool MarkedBlock::Handle::isLiveCell(HeapVersion markingVersion, bool isMarking, const void* p)
{
if (!m_block->isAtom(p))
return false;
return isLive(markingVersion, isMarking, static_cast<const HeapCell*>(p));
}
template<bool specialize, MarkedBlock::Handle::EmptyMode specializedEmptyMode, MarkedBlock::Handle::SweepMode specializedSweepMode, MarkedBlock::Handle::SweepDestructionMode specializedDestructionMode, MarkedBlock::Handle::ScribbleMode specializedScribbleMode, MarkedBlock::Handle::NewlyAllocatedMode specializedNewlyAllocatedMode, MarkedBlock::Handle::MarksMode specializedMarksMode, typename DestroyFunc>
void MarkedBlock::Handle::specializedSweep(FreeList* freeList, MarkedBlock::Handle::EmptyMode emptyMode, MarkedBlock::Handle::SweepMode sweepMode, MarkedBlock::Handle::SweepDestructionMode destructionMode, MarkedBlock::Handle::ScribbleMode scribbleMode, MarkedBlock::Handle::NewlyAllocatedMode newlyAllocatedMode, MarkedBlock::Handle::MarksMode marksMode, const DestroyFunc& destroyFunc)
{
if (specialize) {
emptyMode = specializedEmptyMode;
sweepMode = specializedSweepMode;
destructionMode = specializedDestructionMode;
scribbleMode = specializedScribbleMode;
newlyAllocatedMode = specializedNewlyAllocatedMode;
marksMode = specializedMarksMode;
}
RELEASE_ASSERT(!(destructionMode == BlockHasNoDestructors && sweepMode == SweepOnly));
SuperSamplerScope superSamplerScope(false);
MarkedBlock& block = this->block();
if (false)
dataLog(RawPointer(this), "/", RawPointer(&block), ": MarkedBlock::Handle::specializedSweep!\n");
unsigned cellSize = this->cellSize();
if (Options::useBumpAllocator()
&& emptyMode == IsEmpty
&& newlyAllocatedMode == DoesNotHaveNewlyAllocated) {
if (marksMode == MarksNotStale && !block.m_marks.isEmpty()) {
WTF::dataFile().atomically(
[&] (PrintStream& out) {
out.print("Block ", RawPointer(&block), ": marks not empty!\n");
out.print("Block lock is held: ", block.m_lock.isHeld(), "\n");
out.print("Marking version of block: ", block.m_markingVersion, "\n");
out.print("Marking version of heap: ", space()->markingVersion(), "\n");
UNREACHABLE_FOR_PLATFORM();
});
}
char* startOfLastCell = static_cast<char*>(cellAlign(block.atoms() + m_endAtom - 1));
char* payloadEnd = startOfLastCell + cellSize;
RELEASE_ASSERT(payloadEnd - MarkedBlock::blockSize <= bitwise_cast<char*>(&block));
char* payloadBegin = bitwise_cast<char*>(block.atoms() + firstAtom());
if (scribbleMode == Scribble)
scribble(payloadBegin, payloadEnd - payloadBegin);
if (sweepMode == SweepToFreeList)
setIsFreeListed();
else
m_allocator->setIsEmpty(NoLockingNecessary, this, true);
if (space()->isMarking())
block.m_lock.unlock();
if (sweepMode == SweepToFreeList)
freeList->initializeBump(payloadEnd, payloadEnd - payloadBegin);
if (false)
dataLog("Quickly swept block ", RawPointer(this), " with cell size ", cellSize, " and attributes ", m_attributes, ": ", pointerDump(freeList), "\n");
return;
}
FreeCell* head = 0;
size_t count = 0;
uintptr_t secret;
cryptographicallyRandomValues(&secret, sizeof(uintptr_t));
bool isEmpty = true;
Vector<size_t> deadCells;
VM& vm = *this->vm();
auto handleDeadCell = [&] (size_t i) {
HeapCell* cell = reinterpret_cast_ptr<HeapCell*>(&block.atoms()[i]);
if (destructionMode != BlockHasNoDestructors && emptyMode == NotEmpty) {
JSCell* jsCell = static_cast<JSCell*>(cell);
if (!jsCell->isZapped()) {
destroyFunc(vm, jsCell);
jsCell->zap();
}
}
if (sweepMode == SweepToFreeList) {
FreeCell* freeCell = reinterpret_cast_ptr<FreeCell*>(cell);
if (scribbleMode == Scribble)
scribble(freeCell, cellSize);
freeCell->setNext(head, secret);
head = freeCell;
++count;
}
};
for (size_t i = firstAtom(); i < m_endAtom; i += m_atomsPerCell) {
if (emptyMode == NotEmpty
&& ((marksMode == MarksNotStale && block.m_marks.get(i))
|| (newlyAllocatedMode == HasNewlyAllocated && m_newlyAllocated.get(i)))) {
isEmpty = false;
continue;
}
if (destructionMode == BlockHasDestructorsAndCollectorIsRunning)
deadCells.append(i);
else
handleDeadCell(i);
}
if (sweepMode == SweepToFreeList && newlyAllocatedMode == HasNewlyAllocated)
m_newlyAllocatedVersion = MarkedSpace::nullVersion;
if (space()->isMarking())
block.m_lock.unlock();
if (destructionMode == BlockHasDestructorsAndCollectorIsRunning) {
for (size_t i : deadCells)
handleDeadCell(i);
}
if (sweepMode == SweepToFreeList) {
freeList->initializeList(head, secret, count * cellSize);
setIsFreeListed();
} else if (isEmpty)
m_allocator->setIsEmpty(NoLockingNecessary, this, true);
if (false)
dataLog("Slowly swept block ", RawPointer(&block), " with cell size ", cellSize, " and attributes ", m_attributes, ": ", pointerDump(freeList), "\n");
}
template<typename DestroyFunc>
void MarkedBlock::Handle::finishSweepKnowingSubspace(FreeList* freeList, const DestroyFunc& destroyFunc)
{
SweepMode sweepMode = freeList ? SweepToFreeList : SweepOnly;
SweepDestructionMode destructionMode = this->sweepDestructionMode();
EmptyMode emptyMode = this->emptyMode();
ScribbleMode scribbleMode = this->scribbleMode();
NewlyAllocatedMode newlyAllocatedMode = this->newlyAllocatedMode();
MarksMode marksMode = this->marksMode();
auto trySpecialized = [&] () -> bool {
if (sweepMode != SweepToFreeList)
return false;
if (scribbleMode != DontScribble)
return false;
if (newlyAllocatedMode != DoesNotHaveNewlyAllocated)
return false;
if (destructionMode != BlockHasDestructors)
return false;
if (emptyMode == IsEmpty)
return false;
switch (marksMode) {
case MarksNotStale:
specializedSweep<true, NotEmpty, SweepToFreeList, BlockHasDestructors, DontScribble, DoesNotHaveNewlyAllocated, MarksNotStale>(freeList, IsEmpty, SweepToFreeList, BlockHasDestructors, DontScribble, DoesNotHaveNewlyAllocated, MarksNotStale, destroyFunc);
return true;
case MarksStale:
specializedSweep<true, NotEmpty, SweepToFreeList, BlockHasDestructors, DontScribble, DoesNotHaveNewlyAllocated, MarksStale>(freeList, IsEmpty, SweepToFreeList, BlockHasDestructors, DontScribble, DoesNotHaveNewlyAllocated, MarksStale, destroyFunc);
return true;
}
return false;
};
if (trySpecialized())
return;
specializedSweep<false, IsEmpty, SweepOnly, BlockHasNoDestructors, DontScribble, HasNewlyAllocated, MarksStale>(freeList, emptyMode, sweepMode, destructionMode, scribbleMode, newlyAllocatedMode, marksMode, destroyFunc);
}
inline MarkedBlock::Handle::SweepDestructionMode MarkedBlock::Handle::sweepDestructionMode()
{
if (m_attributes.destruction == NeedsDestruction) {
if (space()->isMarking())
return BlockHasDestructorsAndCollectorIsRunning;
return BlockHasDestructors;
}
return BlockHasNoDestructors;
}
inline MarkedBlock::Handle::EmptyMode MarkedBlock::Handle::emptyMode()
{
return m_allocator->isEmpty(NoLockingNecessary, this) ? IsEmpty : NotEmpty;
}
inline MarkedBlock::Handle::ScribbleMode MarkedBlock::Handle::scribbleMode()
{
return scribbleFreeCells() ? Scribble : DontScribble;
}
inline MarkedBlock::Handle::NewlyAllocatedMode MarkedBlock::Handle::newlyAllocatedMode()
{
return hasAnyNewlyAllocated() ? HasNewlyAllocated : DoesNotHaveNewlyAllocated;
}
inline MarkedBlock::Handle::MarksMode MarkedBlock::Handle::marksMode()
{
HeapVersion markingVersion = space()->markingVersion();
bool marksAreUseful = !block().areMarksStale(markingVersion);
if (space()->isMarking())
marksAreUseful |= block().marksConveyLivenessDuringMarking(markingVersion);
return marksAreUseful ? MarksNotStale : MarksStale;
}
template <typename Functor>
inline IterationStatus MarkedBlock::Handle::forEachLiveCell(const Functor& functor)
{
HeapCell::Kind kind = m_attributes.cellKind;
for (size_t i = firstAtom(); i < m_endAtom; i += m_atomsPerCell) {
HeapCell* cell = reinterpret_cast_ptr<HeapCell*>(&m_block->atoms()[i]);
if (!isLive(cell))
continue;
if (functor(cell, kind) == IterationStatus::Done)
return IterationStatus::Done;
}
return IterationStatus::Continue;
}
template <typename Functor>
inline IterationStatus MarkedBlock::Handle::forEachDeadCell(const Functor& functor)
{
HeapCell::Kind kind = m_attributes.cellKind;
for (size_t i = firstAtom(); i < m_endAtom; i += m_atomsPerCell) {
HeapCell* cell = reinterpret_cast_ptr<HeapCell*>(&m_block->atoms()[i]);
if (isLive(cell))
continue;
if (functor(cell, kind) == IterationStatus::Done)
return IterationStatus::Done;
}
return IterationStatus::Continue;
}
template <typename Functor>
inline IterationStatus MarkedBlock::Handle::forEachMarkedCell(const Functor& functor)
{
HeapCell::Kind kind = m_attributes.cellKind;
MarkedBlock& block = this->block();
bool areMarksStale = block.areMarksStale();
WTF::loadLoadFence();
if (areMarksStale)
return IterationStatus::Continue;
for (size_t i = firstAtom(); i < m_endAtom; i += m_atomsPerCell) {
HeapCell* cell = reinterpret_cast_ptr<HeapCell*>(&m_block->atoms()[i]);
if (!block.isMarkedRaw(cell))
continue;
if (functor(cell, kind) == IterationStatus::Done)
return IterationStatus::Done;
}
return IterationStatus::Continue;
}
}