PolymorphicAccess.cpp [plain text]
#include "config.h"
#include "PolymorphicAccess.h"
#if ENABLE(JIT)
#include "BinarySwitch.h"
#include "CCallHelpers.h"
#include "CodeBlock.h"
#include "DirectArguments.h"
#include "GetterSetter.h"
#include "Heap.h"
#include "JITOperations.h"
#include "JSCInlines.h"
#include "LinkBuffer.h"
#include "ScopedArguments.h"
#include "ScratchRegisterAllocator.h"
#include "StructureStubClearingWatchpoint.h"
#include "StructureStubInfo.h"
#include <wtf/CommaPrinter.h>
#include <wtf/ListDump.h>
namespace JSC {
static const bool verbose = false;
void AccessGenerationResult::dump(PrintStream& out) const
{
out.print(m_kind);
if (m_code)
out.print(":", m_code);
}
Watchpoint* AccessGenerationState::addWatchpoint(const ObjectPropertyCondition& condition)
{
return WatchpointsOnStructureStubInfo::ensureReferenceAndAddWatchpoint(
watchpoints, jit->codeBlock(), stubInfo, condition);
}
void AccessGenerationState::restoreScratch()
{
allocator->restoreReusedRegistersByPopping(*jit, preservedReusedRegisterState);
}
void AccessGenerationState::succeed()
{
restoreScratch();
success.append(jit->jump());
}
void AccessGenerationState::calculateLiveRegistersForCallAndExceptionHandling(const RegisterSet& extra)
{
if (!m_calculatedRegistersForCallAndExceptionHandling) {
m_calculatedRegistersForCallAndExceptionHandling = true;
m_liveRegistersToPreserveAtExceptionHandlingCallSite = jit->codeBlock()->jitCode()->liveRegistersToPreserveAtExceptionHandlingCallSite(jit->codeBlock(), stubInfo->callSiteIndex);
m_needsToRestoreRegistersIfException = m_liveRegistersToPreserveAtExceptionHandlingCallSite.numberOfSetRegisters() > 0;
if (m_needsToRestoreRegistersIfException)
RELEASE_ASSERT(JITCode::isOptimizingJIT(jit->codeBlock()->jitType()));
m_liveRegistersForCall = RegisterSet(m_liveRegistersToPreserveAtExceptionHandlingCallSite, allocator->usedRegisters());
m_liveRegistersForCall.merge(extra);
m_liveRegistersForCall.exclude(RegisterSet::registersToNotSaveForJSCall());
m_liveRegistersForCall.merge(extra);
}
}
void AccessGenerationState::preserveLiveRegistersToStackForCall(const RegisterSet& extra)
{
calculateLiveRegistersForCallAndExceptionHandling(extra);
unsigned extraStackPadding = 0;
unsigned numberOfStackBytesUsedForRegisterPreservation = ScratchRegisterAllocator::preserveRegistersToStackForCall(*jit, liveRegistersForCall(), extraStackPadding);
if (m_numberOfStackBytesUsedForRegisterPreservation != std::numeric_limits<unsigned>::max())
RELEASE_ASSERT(numberOfStackBytesUsedForRegisterPreservation == m_numberOfStackBytesUsedForRegisterPreservation);
m_numberOfStackBytesUsedForRegisterPreservation = numberOfStackBytesUsedForRegisterPreservation;
}
void AccessGenerationState::restoreLiveRegistersFromStackForCall(bool isGetter)
{
RegisterSet dontRestore;
if (isGetter) {
dontRestore.set(valueRegs);
}
restoreLiveRegistersFromStackForCall(dontRestore);
}
void AccessGenerationState::restoreLiveRegistersFromStackForCallWithThrownException()
{
RegisterSet dontRestore = liveRegistersForCall();
dontRestore.exclude(liveRegistersToPreserveAtExceptionHandlingCallSite());
restoreLiveRegistersFromStackForCall(dontRestore);
}
void AccessGenerationState::restoreLiveRegistersFromStackForCall(const RegisterSet& dontRestore)
{
unsigned extraStackPadding = 0;
ScratchRegisterAllocator::restoreRegistersFromStackForCall(*jit, liveRegistersForCall(), dontRestore, m_numberOfStackBytesUsedForRegisterPreservation, extraStackPadding);
}
CallSiteIndex AccessGenerationState::callSiteIndexForExceptionHandlingOrOriginal()
{
RELEASE_ASSERT(m_calculatedRegistersForCallAndExceptionHandling);
if (!m_calculatedCallSiteIndex) {
m_calculatedCallSiteIndex = true;
if (m_needsToRestoreRegistersIfException)
m_callSiteIndex = jit->codeBlock()->newExceptionHandlingCallSiteIndex(stubInfo->callSiteIndex);
else
m_callSiteIndex = originalCallSiteIndex();
}
return m_callSiteIndex;
}
const HandlerInfo& AccessGenerationState::originalExceptionHandler() const
{
RELEASE_ASSERT(m_needsToRestoreRegistersIfException);
HandlerInfo* exceptionHandler = jit->codeBlock()->handlerForIndex(stubInfo->callSiteIndex.bits());
RELEASE_ASSERT(exceptionHandler);
return *exceptionHandler;
}
CallSiteIndex AccessGenerationState::originalCallSiteIndex() const { return stubInfo->callSiteIndex; }
void AccessGenerationState::emitExplicitExceptionHandler()
{
restoreScratch();
jit->copyCalleeSavesToVMEntryFrameCalleeSavesBuffer();
if (needsToRestoreRegistersIfException()) {
jit->storePtr(GPRInfo::callFrameRegister, jit->vm()->addressOfCallFrameForCatch());
CCallHelpers::Jump jumpToOSRExitExceptionHandler = jit->jump();
HandlerInfo originalHandler = originalExceptionHandler();
jit->addLinkTask(
[=] (LinkBuffer& linkBuffer) {
linkBuffer.link(jumpToOSRExitExceptionHandler, originalHandler.nativeCode);
});
} else {
jit->setupArguments(CCallHelpers::TrustedImmPtr(jit->vm()), GPRInfo::callFrameRegister);
CCallHelpers::Call lookupExceptionHandlerCall = jit->call();
jit->addLinkTask(
[=] (LinkBuffer& linkBuffer) {
linkBuffer.link(lookupExceptionHandlerCall, lookupExceptionHandler);
});
jit->jumpToExceptionHandler();
}
}
AccessCase::AccessCase()
{
}
std::unique_ptr<AccessCase> AccessCase::tryGet(
VM& vm, JSCell* owner, AccessType type, PropertyOffset offset, Structure* structure,
const ObjectPropertyConditionSet& conditionSet, bool viaProxy, WatchpointSet* additionalSet)
{
std::unique_ptr<AccessCase> result(new AccessCase());
result->m_type = type;
result->m_offset = offset;
result->m_structure.set(vm, owner, structure);
result->m_conditionSet = conditionSet;
if (viaProxy || additionalSet) {
result->m_rareData = std::make_unique<RareData>();
result->m_rareData->viaProxy = viaProxy;
result->m_rareData->additionalSet = additionalSet;
}
return result;
}
std::unique_ptr<AccessCase> AccessCase::get(
VM& vm, JSCell* owner, AccessType type, PropertyOffset offset, Structure* structure,
const ObjectPropertyConditionSet& conditionSet, bool viaProxy, WatchpointSet* additionalSet,
PropertySlot::GetValueFunc customGetter, JSObject* customSlotBase)
{
std::unique_ptr<AccessCase> result(new AccessCase());
result->m_type = type;
result->m_offset = offset;
result->m_structure.set(vm, owner, structure);
result->m_conditionSet = conditionSet;
if (viaProxy || additionalSet || result->doesCalls() || customGetter || customSlotBase) {
result->m_rareData = std::make_unique<RareData>();
result->m_rareData->viaProxy = viaProxy;
result->m_rareData->additionalSet = additionalSet;
result->m_rareData->customAccessor.getter = customGetter;
result->m_rareData->customSlotBase.setMayBeNull(vm, owner, customSlotBase);
}
return result;
}
std::unique_ptr<AccessCase> AccessCase::megamorphicLoad(VM& vm, JSCell* owner)
{
UNUSED_PARAM(vm);
UNUSED_PARAM(owner);
if (GPRInfo::numberOfRegisters < 9)
return nullptr;
std::unique_ptr<AccessCase> result(new AccessCase());
result->m_type = MegamorphicLoad;
return result;
}
std::unique_ptr<AccessCase> AccessCase::replace(
VM& vm, JSCell* owner, Structure* structure, PropertyOffset offset)
{
std::unique_ptr<AccessCase> result(new AccessCase());
result->m_type = Replace;
result->m_offset = offset;
result->m_structure.set(vm, owner, structure);
return result;
}
std::unique_ptr<AccessCase> AccessCase::transition(
VM& vm, JSCell* owner, Structure* oldStructure, Structure* newStructure, PropertyOffset offset,
const ObjectPropertyConditionSet& conditionSet)
{
RELEASE_ASSERT(oldStructure == newStructure->previousID());
if (GPRInfo::numberOfRegisters < 6
&& oldStructure->outOfLineCapacity() != newStructure->outOfLineCapacity()
&& oldStructure->outOfLineCapacity()) {
return nullptr;
}
std::unique_ptr<AccessCase> result(new AccessCase());
result->m_type = Transition;
result->m_offset = offset;
result->m_structure.set(vm, owner, newStructure);
result->m_conditionSet = conditionSet;
return result;
}
std::unique_ptr<AccessCase> AccessCase::setter(
VM& vm, JSCell* owner, AccessType type, Structure* structure, PropertyOffset offset,
const ObjectPropertyConditionSet& conditionSet, PutPropertySlot::PutValueFunc customSetter,
JSObject* customSlotBase)
{
std::unique_ptr<AccessCase> result(new AccessCase());
result->m_type = type;
result->m_offset = offset;
result->m_structure.set(vm, owner, structure);
result->m_conditionSet = conditionSet;
result->m_rareData = std::make_unique<RareData>();
result->m_rareData->customAccessor.setter = customSetter;
result->m_rareData->customSlotBase.setMayBeNull(vm, owner, customSlotBase);
return result;
}
std::unique_ptr<AccessCase> AccessCase::in(
VM& vm, JSCell* owner, AccessType type, Structure* structure,
const ObjectPropertyConditionSet& conditionSet)
{
std::unique_ptr<AccessCase> result(new AccessCase());
result->m_type = type;
result->m_structure.set(vm, owner, structure);
result->m_conditionSet = conditionSet;
return result;
}
std::unique_ptr<AccessCase> AccessCase::getLength(VM&, JSCell*, AccessType type)
{
std::unique_ptr<AccessCase> result(new AccessCase());
result->m_type = type;
return result;
}
std::unique_ptr<AccessCase> AccessCase::getIntrinsic(
VM& vm, JSCell* owner, JSFunction* getter, PropertyOffset offset,
Structure* structure, const ObjectPropertyConditionSet& conditionSet)
{
std::unique_ptr<AccessCase> result(new AccessCase());
result->m_type = IntrinsicGetter;
result->m_structure.set(vm, owner, structure);
result->m_conditionSet = conditionSet;
result->m_offset = offset;
result->m_rareData = std::make_unique<RareData>();
result->m_rareData->intrinsicFunction.set(vm, owner, getter);
return result;
}
AccessCase::~AccessCase()
{
}
std::unique_ptr<AccessCase> AccessCase::fromStructureStubInfo(
VM& vm, JSCell* owner, StructureStubInfo& stubInfo)
{
switch (stubInfo.cacheType) {
case CacheType::GetByIdSelf:
return get(
vm, owner, Load, stubInfo.u.byIdSelf.offset,
stubInfo.u.byIdSelf.baseObjectStructure.get());
case CacheType::PutByIdReplace:
return replace(
vm, owner, stubInfo.u.byIdSelf.baseObjectStructure.get(), stubInfo.u.byIdSelf.offset);
default:
return nullptr;
}
}
std::unique_ptr<AccessCase> AccessCase::clone() const
{
std::unique_ptr<AccessCase> result(new AccessCase());
result->m_type = m_type;
result->m_offset = m_offset;
result->m_structure = m_structure;
result->m_conditionSet = m_conditionSet;
if (RareData* rareData = m_rareData.get()) {
result->m_rareData = std::make_unique<RareData>();
result->m_rareData->viaProxy = rareData->viaProxy;
result->m_rareData->additionalSet = rareData->additionalSet;
result->m_rareData->customAccessor.opaque = rareData->customAccessor.opaque;
result->m_rareData->customSlotBase = rareData->customSlotBase;
result->m_rareData->intrinsicFunction = rareData->intrinsicFunction;
}
return result;
}
Vector<WatchpointSet*, 2> AccessCase::commit(VM& vm, const Identifier& ident)
{
RELEASE_ASSERT(m_state == Primordial || m_state == Committed);
Vector<WatchpointSet*, 2> result;
if ((structure() && structure()->needImpurePropertyWatchpoint())
|| m_conditionSet.needImpurePropertyWatchpoint())
result.append(vm.ensureWatchpointSetForImpureProperty(ident));
if (additionalSet())
result.append(additionalSet());
m_state = Committed;
return result;
}
bool AccessCase::guardedByStructureCheck() const
{
if (viaProxy())
return false;
switch (m_type) {
case MegamorphicLoad:
case ArrayLength:
case StringLength:
case DirectArgumentsLength:
case ScopedArgumentsLength:
return false;
default:
return true;
}
}
JSObject* AccessCase::alternateBase() const
{
if (customSlotBase())
return customSlotBase();
return conditionSet().slotBaseCondition().object();
}
bool AccessCase::doesCalls(Vector<JSCell*>* cellsToMark) const
{
switch (type()) {
case Getter:
case Setter:
case CustomValueGetter:
case CustomAccessorGetter:
case CustomValueSetter:
case CustomAccessorSetter:
return true;
case Transition:
if (newStructure()->outOfLineCapacity() != structure()->outOfLineCapacity()
&& structure()->couldHaveIndexingHeader()) {
if (cellsToMark)
cellsToMark->append(newStructure());
return true;
}
return false;
default:
return false;
}
}
bool AccessCase::couldStillSucceed() const
{
return m_conditionSet.structuresEnsureValidityAssumingImpurePropertyWatchpoint();
}
bool AccessCase::canBeReplacedByMegamorphicLoad() const
{
if (type() == MegamorphicLoad)
return true;
return type() == Load
&& !viaProxy()
&& conditionSet().isEmpty()
&& !additionalSet()
&& !customSlotBase();
}
bool AccessCase::canReplace(const AccessCase& other) const
{
switch (type()) {
case MegamorphicLoad:
return other.canBeReplacedByMegamorphicLoad();
case ArrayLength:
case StringLength:
case DirectArgumentsLength:
case ScopedArgumentsLength:
return other.type() == type();
default:
if (!guardedByStructureCheck() || !other.guardedByStructureCheck())
return false;
return structure() == other.structure();
}
}
void AccessCase::dump(PrintStream& out) const
{
out.print(m_type, ":(");
CommaPrinter comma;
out.print(comma, m_state);
if (m_type == Transition)
out.print(comma, "structure = ", pointerDump(structure()), " -> ", pointerDump(newStructure()));
else if (m_structure)
out.print(comma, "structure = ", pointerDump(m_structure.get()));
if (isValidOffset(m_offset))
out.print(comma, "offset = ", m_offset);
if (!m_conditionSet.isEmpty())
out.print(comma, "conditions = ", m_conditionSet);
if (RareData* rareData = m_rareData.get()) {
if (rareData->viaProxy)
out.print(comma, "viaProxy = ", rareData->viaProxy);
if (rareData->additionalSet)
out.print(comma, "additionalSet = ", RawPointer(rareData->additionalSet.get()));
if (rareData->callLinkInfo)
out.print(comma, "callLinkInfo = ", RawPointer(rareData->callLinkInfo.get()));
if (rareData->customAccessor.opaque)
out.print(comma, "customAccessor = ", RawPointer(rareData->customAccessor.opaque));
if (rareData->customSlotBase)
out.print(comma, "customSlotBase = ", RawPointer(rareData->customSlotBase.get()));
}
out.print(")");
}
bool AccessCase::visitWeak(VM& vm) const
{
if (m_structure && !Heap::isMarked(m_structure.get()))
return false;
if (!m_conditionSet.areStillLive())
return false;
if (m_rareData) {
if (m_rareData->callLinkInfo)
m_rareData->callLinkInfo->visitWeak(vm);
if (m_rareData->customSlotBase && !Heap::isMarked(m_rareData->customSlotBase.get()))
return false;
if (m_rareData->intrinsicFunction && !Heap::isMarked(m_rareData->intrinsicFunction.get()))
return false;
}
return true;
}
bool AccessCase::propagateTransitions(SlotVisitor& visitor) const
{
bool result = true;
if (m_structure)
result &= m_structure->markIfCheap(visitor);
switch (m_type) {
case Transition:
if (Heap::isMarked(m_structure->previousID()))
visitor.appendUnbarrieredReadOnlyPointer(m_structure.get());
else
result = false;
break;
default:
break;
}
return result;
}
void AccessCase::generateWithGuard(
AccessGenerationState& state, CCallHelpers::JumpList& fallThrough)
{
SuperSamplerScope superSamplerScope(false);
RELEASE_ASSERT(m_state == Committed);
m_state = Generated;
CCallHelpers& jit = *state.jit;
VM& vm = *jit.vm();
const Identifier& ident = *state.ident;
StructureStubInfo& stubInfo = *state.stubInfo;
JSValueRegs valueRegs = state.valueRegs;
GPRReg baseGPR = state.baseGPR;
GPRReg scratchGPR = state.scratchGPR;
UNUSED_PARAM(vm);
switch (m_type) {
case ArrayLength: {
ASSERT(!viaProxy());
jit.load8(CCallHelpers::Address(baseGPR, JSCell::indexingTypeOffset()), scratchGPR);
fallThrough.append(
jit.branchTest32(
CCallHelpers::Zero, scratchGPR, CCallHelpers::TrustedImm32(IsArray)));
fallThrough.append(
jit.branchTest32(
CCallHelpers::Zero, scratchGPR, CCallHelpers::TrustedImm32(IndexingShapeMask)));
break;
}
case StringLength: {
ASSERT(!viaProxy());
fallThrough.append(
jit.branch8(
CCallHelpers::NotEqual,
CCallHelpers::Address(baseGPR, JSCell::typeInfoTypeOffset()),
CCallHelpers::TrustedImm32(StringType)));
break;
}
case DirectArgumentsLength: {
ASSERT(!viaProxy());
fallThrough.append(
jit.branch8(
CCallHelpers::NotEqual,
CCallHelpers::Address(baseGPR, JSCell::typeInfoTypeOffset()),
CCallHelpers::TrustedImm32(DirectArgumentsType)));
fallThrough.append(
jit.branchTestPtr(
CCallHelpers::NonZero,
CCallHelpers::Address(baseGPR, DirectArguments::offsetOfOverrides())));
jit.load32(
CCallHelpers::Address(baseGPR, DirectArguments::offsetOfLength()),
valueRegs.payloadGPR());
jit.boxInt32(valueRegs.payloadGPR(), valueRegs);
state.succeed();
return;
}
case ScopedArgumentsLength: {
ASSERT(!viaProxy());
fallThrough.append(
jit.branch8(
CCallHelpers::NotEqual,
CCallHelpers::Address(baseGPR, JSCell::typeInfoTypeOffset()),
CCallHelpers::TrustedImm32(ScopedArgumentsType)));
fallThrough.append(
jit.branchTest8(
CCallHelpers::NonZero,
CCallHelpers::Address(baseGPR, ScopedArguments::offsetOfOverrodeThings())));
jit.load32(
CCallHelpers::Address(baseGPR, ScopedArguments::offsetOfTotalLength()),
valueRegs.payloadGPR());
jit.boxInt32(valueRegs.payloadGPR(), valueRegs);
state.succeed();
return;
}
case MegamorphicLoad: {
UniquedStringImpl* key = ident.impl();
unsigned hash = IdentifierRepHash::hash(key);
ScratchRegisterAllocator allocator(stubInfo.patch.usedRegisters);
allocator.lock(baseGPR);
#if USE(JSVALUE32_64)
allocator.lock(static_cast<GPRReg>(stubInfo.patch.baseTagGPR));
#endif
allocator.lock(valueRegs);
allocator.lock(scratchGPR);
GPRReg intermediateGPR = scratchGPR;
GPRReg maskGPR = allocator.allocateScratchGPR();
GPRReg maskedHashGPR = allocator.allocateScratchGPR();
GPRReg indexGPR = allocator.allocateScratchGPR();
GPRReg offsetGPR = allocator.allocateScratchGPR();
if (verbose) {
dataLog("baseGPR = ", baseGPR, "\n");
dataLog("valueRegs = ", valueRegs, "\n");
dataLog("scratchGPR = ", scratchGPR, "\n");
dataLog("intermediateGPR = ", intermediateGPR, "\n");
dataLog("maskGPR = ", maskGPR, "\n");
dataLog("maskedHashGPR = ", maskedHashGPR, "\n");
dataLog("indexGPR = ", indexGPR, "\n");
dataLog("offsetGPR = ", offsetGPR, "\n");
}
ScratchRegisterAllocator::PreservedState preservedState =
allocator.preserveReusedRegistersByPushing(jit, ScratchRegisterAllocator::ExtraStackSpace::SpaceForCCall);
CCallHelpers::JumpList myFailAndIgnore;
CCallHelpers::JumpList myFallThrough;
jit.emitLoadStructure(baseGPR, intermediateGPR, maskGPR);
jit.loadPtr(
CCallHelpers::Address(intermediateGPR, Structure::propertyTableUnsafeOffset()),
intermediateGPR);
myFailAndIgnore.append(jit.branchTestPtr(CCallHelpers::Zero, intermediateGPR));
jit.load32(CCallHelpers::Address(intermediateGPR, PropertyTable::offsetOfIndexMask()), maskGPR);
jit.loadPtr(CCallHelpers::Address(intermediateGPR, PropertyTable::offsetOfIndex()), indexGPR);
jit.load32(
CCallHelpers::Address(intermediateGPR, PropertyTable::offsetOfIndexSize()),
intermediateGPR);
jit.move(maskGPR, maskedHashGPR);
jit.and32(CCallHelpers::TrustedImm32(hash), maskedHashGPR);
jit.lshift32(CCallHelpers::TrustedImm32(2), intermediateGPR);
jit.addPtr(indexGPR, intermediateGPR);
CCallHelpers::Label loop = jit.label();
jit.load32(CCallHelpers::BaseIndex(indexGPR, maskedHashGPR, CCallHelpers::TimesFour), offsetGPR);
myFallThrough.append(
jit.branch32(
CCallHelpers::Equal,
offsetGPR,
CCallHelpers::TrustedImm32(PropertyTable::EmptyEntryIndex)));
jit.sub32(CCallHelpers::TrustedImm32(1), offsetGPR);
jit.mul32(CCallHelpers::TrustedImm32(sizeof(PropertyMapEntry)), offsetGPR, offsetGPR);
jit.addPtr(intermediateGPR, offsetGPR);
CCallHelpers::Jump collision = jit.branchPtr(
CCallHelpers::NotEqual,
CCallHelpers::Address(offsetGPR, OBJECT_OFFSETOF(PropertyMapEntry, key)),
CCallHelpers::TrustedImmPtr(key));
myFallThrough.append(
jit.branchTest32(
CCallHelpers::NonZero,
CCallHelpers::Address(offsetGPR, OBJECT_OFFSETOF(PropertyMapEntry, attributes)),
CCallHelpers::TrustedImm32(Accessor | CustomAccessor)));
jit.load32(CCallHelpers::Address(offsetGPR, OBJECT_OFFSETOF(PropertyMapEntry, offset)), offsetGPR);
jit.loadProperty(baseGPR, offsetGPR, valueRegs);
allocator.restoreReusedRegistersByPopping(jit, preservedState);
state.succeed();
collision.link(&jit);
jit.add32(CCallHelpers::TrustedImm32(1), maskedHashGPR);
jit.and32(maskGPR, maskedHashGPR);
jit.jump().linkTo(loop, &jit);
if (allocator.didReuseRegisters()) {
myFailAndIgnore.link(&jit);
allocator.restoreReusedRegistersByPopping(jit, preservedState);
state.failAndIgnore.append(jit.jump());
myFallThrough.link(&jit);
allocator.restoreReusedRegistersByPopping(jit, preservedState);
fallThrough.append(jit.jump());
} else {
state.failAndIgnore.append(myFailAndIgnore);
fallThrough.append(myFallThrough);
}
return;
}
default: {
if (viaProxy()) {
fallThrough.append(
jit.branch8(
CCallHelpers::NotEqual,
CCallHelpers::Address(baseGPR, JSCell::typeInfoTypeOffset()),
CCallHelpers::TrustedImm32(PureForwardingProxyType)));
jit.loadPtr(CCallHelpers::Address(baseGPR, JSProxy::targetOffset()), scratchGPR);
fallThrough.append(
jit.branchStructure(
CCallHelpers::NotEqual,
CCallHelpers::Address(scratchGPR, JSCell::structureIDOffset()),
structure()));
} else {
fallThrough.append(
jit.branchStructure(
CCallHelpers::NotEqual,
CCallHelpers::Address(baseGPR, JSCell::structureIDOffset()),
structure()));
}
break;
} };
generateImpl(state);
}
void AccessCase::generate(AccessGenerationState& state)
{
RELEASE_ASSERT(m_state == Committed);
m_state = Generated;
generateImpl(state);
}
void AccessCase::generateImpl(AccessGenerationState& state)
{
SuperSamplerScope superSamplerScope(false);
if (verbose)
dataLog("Generating code for: ", *this, "\n");
ASSERT(m_state == Generated);
CCallHelpers& jit = *state.jit;
VM& vm = *jit.vm();
CodeBlock* codeBlock = jit.codeBlock();
StructureStubInfo& stubInfo = *state.stubInfo;
const Identifier& ident = *state.ident;
JSValueRegs valueRegs = state.valueRegs;
GPRReg baseGPR = state.baseGPR;
GPRReg scratchGPR = state.scratchGPR;
ASSERT(m_conditionSet.structuresEnsureValidityAssumingImpurePropertyWatchpoint());
for (const ObjectPropertyCondition& condition : m_conditionSet) {
Structure* structure = condition.object()->structure();
if (condition.isWatchableAssumingImpurePropertyWatchpoint()) {
structure->addTransitionWatchpoint(state.addWatchpoint(condition));
continue;
}
if (!condition.structureEnsuresValidityAssumingImpurePropertyWatchpoint(structure)) {
dataLog("This condition is no longer met: ", condition, "\n");
RELEASE_ASSERT_NOT_REACHED();
}
state.weakReferences.append(WriteBarrier<JSCell>(vm, codeBlock, structure));
jit.move(CCallHelpers::TrustedImmPtr(condition.object()), scratchGPR);
state.failAndRepatch.append(
jit.branchStructure(
CCallHelpers::NotEqual,
CCallHelpers::Address(scratchGPR, JSCell::structureIDOffset()),
structure));
}
switch (m_type) {
case InHit:
case InMiss:
jit.boxBooleanPayload(m_type == InHit, valueRegs.payloadGPR());
state.succeed();
return;
case Miss:
jit.moveTrustedValue(jsUndefined(), valueRegs);
state.succeed();
return;
case Load:
case GetGetter:
case Getter:
case Setter:
case CustomValueGetter:
case CustomAccessorGetter:
case CustomValueSetter:
case CustomAccessorSetter: {
if (isValidOffset(m_offset)) {
Structure* currStructure;
if (m_conditionSet.isEmpty())
currStructure = structure();
else
currStructure = m_conditionSet.slotBaseCondition().object()->structure();
currStructure->startWatchingPropertyForReplacements(vm, offset());
}
GPRReg baseForGetGPR;
if (viaProxy()) {
baseForGetGPR = valueRegs.payloadGPR();
jit.loadPtr(
CCallHelpers::Address(baseGPR, JSProxy::targetOffset()),
baseForGetGPR);
} else
baseForGetGPR = baseGPR;
GPRReg baseForAccessGPR;
if (!m_conditionSet.isEmpty()) {
jit.move(
CCallHelpers::TrustedImmPtr(alternateBase()),
scratchGPR);
baseForAccessGPR = scratchGPR;
} else
baseForAccessGPR = baseForGetGPR;
GPRReg loadedValueGPR = InvalidGPRReg;
if (m_type != CustomValueGetter && m_type != CustomAccessorGetter && m_type != CustomValueSetter && m_type != CustomAccessorSetter) {
if (m_type == Load || m_type == GetGetter)
loadedValueGPR = valueRegs.payloadGPR();
else
loadedValueGPR = scratchGPR;
GPRReg storageGPR;
if (isInlineOffset(m_offset))
storageGPR = baseForAccessGPR;
else {
jit.loadPtr(
CCallHelpers::Address(baseForAccessGPR, JSObject::butterflyOffset()),
loadedValueGPR);
storageGPR = loadedValueGPR;
}
#if USE(JSVALUE64)
jit.load64(
CCallHelpers::Address(storageGPR, offsetRelativeToBase(m_offset)), loadedValueGPR);
#else
if (m_type == Load || m_type == GetGetter) {
jit.load32(
CCallHelpers::Address(storageGPR, offsetRelativeToBase(m_offset) + TagOffset),
valueRegs.tagGPR());
}
jit.load32(
CCallHelpers::Address(storageGPR, offsetRelativeToBase(m_offset) + PayloadOffset),
loadedValueGPR);
#endif
}
if (m_type == Load || m_type == GetGetter) {
state.succeed();
return;
}
CCallHelpers::Call operationCall;
CCallHelpers::DataLabelPtr addressOfLinkFunctionCheck;
CCallHelpers::Call fastPathCall;
CCallHelpers::Call slowPathCall;
CCallHelpers::Jump success;
CCallHelpers::Jump fail;
state.preserveLiveRegistersToStackForCall();
jit.store32(
CCallHelpers::TrustedImm32(state.callSiteIndexForExceptionHandlingOrOriginal().bits()),
CCallHelpers::tagFor(static_cast<VirtualRegister>(CallFrameSlot::argumentCount)));
if (m_type == Getter || m_type == Setter) {
RELEASE_ASSERT(!m_rareData->callLinkInfo);
m_rareData->callLinkInfo = std::make_unique<CallLinkInfo>();
m_rareData->callLinkInfo->disallowStubs();
m_rareData->callLinkInfo->setUpCall(
CallLinkInfo::Call, stubInfo.codeOrigin, loadedValueGPR);
CCallHelpers::JumpList done;
unsigned numberOfParameters = 1;
if (m_type == Setter)
numberOfParameters++;
if (m_type == Setter) {
jit.loadPtr(
CCallHelpers::Address(loadedValueGPR, GetterSetter::offsetOfSetter()),
loadedValueGPR);
} else {
jit.loadPtr(
CCallHelpers::Address(loadedValueGPR, GetterSetter::offsetOfGetter()),
loadedValueGPR);
}
CCallHelpers::Jump returnUndefined = jit.branchTestPtr(
CCallHelpers::Zero, loadedValueGPR);
unsigned numberOfRegsForCall = CallFrame::headerSizeInRegisters + numberOfParameters;
unsigned numberOfBytesForCall =
numberOfRegsForCall * sizeof(Register) - sizeof(CallerFrameAndPC);
unsigned alignedNumberOfBytesForCall =
WTF::roundUpToMultipleOf(stackAlignmentBytes(), numberOfBytesForCall);
jit.subPtr(
CCallHelpers::TrustedImm32(alignedNumberOfBytesForCall),
CCallHelpers::stackPointerRegister);
CCallHelpers::Address calleeFrame = CCallHelpers::Address(
CCallHelpers::stackPointerRegister,
-static_cast<ptrdiff_t>(sizeof(CallerFrameAndPC)));
jit.store32(
CCallHelpers::TrustedImm32(numberOfParameters),
calleeFrame.withOffset(CallFrameSlot::argumentCount * sizeof(Register) + PayloadOffset));
jit.storeCell(
loadedValueGPR, calleeFrame.withOffset(CallFrameSlot::callee * sizeof(Register)));
jit.storeCell(
baseForGetGPR,
calleeFrame.withOffset(virtualRegisterForArgument(0).offset() * sizeof(Register)));
if (m_type == Setter) {
jit.storeValue(
valueRegs,
calleeFrame.withOffset(
virtualRegisterForArgument(1).offset() * sizeof(Register)));
}
CCallHelpers::Jump slowCase = jit.branchPtrWithPatch(
CCallHelpers::NotEqual, loadedValueGPR, addressOfLinkFunctionCheck,
CCallHelpers::TrustedImmPtr(0));
fastPathCall = jit.nearCall();
if (m_type == Getter)
jit.setupResults(valueRegs);
done.append(jit.jump());
slowCase.link(&jit);
jit.move(loadedValueGPR, GPRInfo::regT0);
#if USE(JSVALUE32_64)
jit.move(CCallHelpers::TrustedImm32(JSValue::CellTag), GPRInfo::regT1);
#endif
jit.move(CCallHelpers::TrustedImmPtr(m_rareData->callLinkInfo.get()), GPRInfo::regT2);
slowPathCall = jit.nearCall();
if (m_type == Getter)
jit.setupResults(valueRegs);
done.append(jit.jump());
returnUndefined.link(&jit);
if (m_type == Getter)
jit.moveTrustedValue(jsUndefined(), valueRegs);
done.link(&jit);
jit.addPtr(CCallHelpers::TrustedImm32((codeBlock->stackPointerOffset() * sizeof(Register)) - state.preservedReusedRegisterState.numberOfBytesPreserved - state.numberOfStackBytesUsedForRegisterPreservation()),
GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister);
state.restoreLiveRegistersFromStackForCall(isGetter());
jit.addLinkTask(
[=, &vm] (LinkBuffer& linkBuffer) {
m_rareData->callLinkInfo->setCallLocations(
linkBuffer.locationOfNearCall(slowPathCall),
linkBuffer.locationOf(addressOfLinkFunctionCheck),
linkBuffer.locationOfNearCall(fastPathCall));
linkBuffer.link(
slowPathCall,
CodeLocationLabel(vm.getCTIStub(linkCallThunkGenerator).code()));
});
} else {
jit.makeSpaceOnStackForCCall();
GPRReg baseForCustomValue = m_type == CustomValueGetter || m_type == CustomValueSetter ? baseForAccessGPR : baseForGetGPR;
#if USE(JSVALUE64)
if (m_type == CustomValueGetter || m_type == CustomAccessorGetter) {
jit.setupArgumentsWithExecState(
baseForCustomValue,
CCallHelpers::TrustedImmPtr(ident.impl()));
} else
jit.setupArgumentsWithExecState(baseForCustomValue, valueRegs.gpr());
#else
if (m_type == CustomValueGetter || m_type == CustomAccessorGetter) {
jit.setupArgumentsWithExecState(
EABI_32BIT_DUMMY_ARG baseForCustomValue,
CCallHelpers::TrustedImm32(JSValue::CellTag),
CCallHelpers::TrustedImmPtr(ident.impl()));
} else {
jit.setupArgumentsWithExecState(
EABI_32BIT_DUMMY_ARG baseForCustomValue,
CCallHelpers::TrustedImm32(JSValue::CellTag),
valueRegs.payloadGPR(), valueRegs.tagGPR());
}
#endif
jit.storePtr(GPRInfo::callFrameRegister, &vm.topCallFrame);
operationCall = jit.call();
jit.addLinkTask(
[=] (LinkBuffer& linkBuffer) {
linkBuffer.link(operationCall, FunctionPtr(m_rareData->customAccessor.opaque));
});
if (m_type == CustomValueGetter || m_type == CustomAccessorGetter)
jit.setupResults(valueRegs);
jit.reclaimSpaceOnStackForCCall();
CCallHelpers::Jump noException =
jit.emitExceptionCheck(CCallHelpers::InvertedExceptionCheck);
state.restoreLiveRegistersFromStackForCallWithThrownException();
state.emitExplicitExceptionHandler();
noException.link(&jit);
state.restoreLiveRegistersFromStackForCall(isGetter());
}
state.succeed();
return;
}
case Replace: {
if (InferredType* type = structure()->inferredTypeFor(ident.impl())) {
if (verbose)
dataLog("Have type: ", type->descriptor(), "\n");
state.failAndRepatch.append(
jit.branchIfNotType(valueRegs, scratchGPR, type->descriptor()));
} else if (verbose)
dataLog("Don't have type.\n");
if (isInlineOffset(m_offset)) {
jit.storeValue(
valueRegs,
CCallHelpers::Address(
baseGPR,
JSObject::offsetOfInlineStorage() +
offsetInInlineStorage(m_offset) * sizeof(JSValue)));
} else {
jit.loadPtr(CCallHelpers::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR);
jit.storeValue(
valueRegs,
CCallHelpers::Address(
scratchGPR, offsetInButterfly(m_offset) * sizeof(JSValue)));
}
state.succeed();
return;
}
case Transition: {
RELEASE_ASSERT(GPRInfo::numberOfRegisters >= 6 || !structure()->outOfLineCapacity() || structure()->outOfLineCapacity() == newStructure()->outOfLineCapacity());
if (InferredType* type = newStructure()->inferredTypeFor(ident.impl())) {
if (verbose)
dataLog("Have type: ", type->descriptor(), "\n");
state.failAndRepatch.append(
jit.branchIfNotType(valueRegs, scratchGPR, type->descriptor()));
} else if (verbose)
dataLog("Don't have type.\n");
bool allocating = newStructure()->outOfLineCapacity() != structure()->outOfLineCapacity();
bool reallocating = allocating && structure()->outOfLineCapacity();
bool allocatingInline = allocating && !structure()->couldHaveIndexingHeader();
ScratchRegisterAllocator allocator(stubInfo.patch.usedRegisters);
allocator.lock(baseGPR);
#if USE(JSVALUE32_64)
allocator.lock(static_cast<GPRReg>(stubInfo.patch.baseTagGPR));
#endif
allocator.lock(valueRegs);
allocator.lock(scratchGPR);
GPRReg scratchGPR2 = InvalidGPRReg;
GPRReg scratchGPR3 = InvalidGPRReg;
if (allocatingInline) {
scratchGPR2 = allocator.allocateScratchGPR();
scratchGPR3 = allocator.allocateScratchGPR();
}
ScratchRegisterAllocator::PreservedState preservedState =
allocator.preserveReusedRegistersByPushing(jit, ScratchRegisterAllocator::ExtraStackSpace::SpaceForCCall);
CCallHelpers::JumpList slowPath;
ASSERT(structure()->transitionWatchpointSetHasBeenInvalidated());
if (allocating) {
size_t newSize = newStructure()->outOfLineCapacity() * sizeof(JSValue);
if (allocatingInline) {
CopiedAllocator* copiedAllocator = &vm.heap.storageAllocator();
if (!reallocating) {
jit.loadPtr(&copiedAllocator->m_currentRemaining, scratchGPR);
slowPath.append(
jit.branchSubPtr(
CCallHelpers::Signed, CCallHelpers::TrustedImm32(newSize), scratchGPR));
jit.storePtr(scratchGPR, &copiedAllocator->m_currentRemaining);
jit.negPtr(scratchGPR);
jit.addPtr(
CCallHelpers::AbsoluteAddress(&copiedAllocator->m_currentPayloadEnd), scratchGPR);
jit.addPtr(CCallHelpers::TrustedImm32(sizeof(JSValue)), scratchGPR);
} else {
size_t oldSize = structure()->outOfLineCapacity() * sizeof(JSValue);
ASSERT(newSize > oldSize);
jit.loadPtr(CCallHelpers::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR3);
jit.loadPtr(&copiedAllocator->m_currentRemaining, scratchGPR);
slowPath.append(
jit.branchSubPtr(
CCallHelpers::Signed, CCallHelpers::TrustedImm32(newSize), scratchGPR));
jit.storePtr(scratchGPR, &copiedAllocator->m_currentRemaining);
jit.negPtr(scratchGPR);
jit.addPtr(
CCallHelpers::AbsoluteAddress(&copiedAllocator->m_currentPayloadEnd), scratchGPR);
jit.addPtr(CCallHelpers::TrustedImm32(sizeof(JSValue)), scratchGPR);
for (size_t offset = 0; offset < oldSize; offset += sizeof(void*)) {
jit.loadPtr(
CCallHelpers::Address(
scratchGPR3,
-static_cast<ptrdiff_t>(
offset + sizeof(JSValue) + sizeof(void*))),
scratchGPR2);
jit.storePtr(
scratchGPR2,
CCallHelpers::Address(
scratchGPR,
-static_cast<ptrdiff_t>(offset + sizeof(JSValue) + sizeof(void*))));
}
}
} else {
RegisterSet extraRegistersToPreserve;
extraRegistersToPreserve.set(baseGPR);
extraRegistersToPreserve.set(valueRegs);
state.preserveLiveRegistersToStackForCall(extraRegistersToPreserve);
jit.store32(
CCallHelpers::TrustedImm32(
state.callSiteIndexForExceptionHandlingOrOriginal().bits()),
CCallHelpers::tagFor(static_cast<VirtualRegister>(CallFrameSlot::argumentCount)));
jit.makeSpaceOnStackForCCall();
if (!reallocating) {
jit.setupArgumentsWithExecState(baseGPR);
CCallHelpers::Call operationCall = jit.call();
jit.addLinkTask(
[=] (LinkBuffer& linkBuffer) {
linkBuffer.link(
operationCall,
FunctionPtr(operationReallocateButterflyToHavePropertyStorageWithInitialCapacity));
});
} else {
jit.setupArgumentsWithExecState(
baseGPR, CCallHelpers::TrustedImm32(newSize / sizeof(JSValue)));
CCallHelpers::Call operationCall = jit.call();
jit.addLinkTask(
[=] (LinkBuffer& linkBuffer) {
linkBuffer.link(
operationCall,
FunctionPtr(operationReallocateButterflyToGrowPropertyStorage));
});
}
jit.reclaimSpaceOnStackForCCall();
jit.move(GPRInfo::returnValueGPR, scratchGPR);
CCallHelpers::Jump noException =
jit.emitExceptionCheck(CCallHelpers::InvertedExceptionCheck);
state.restoreLiveRegistersFromStackForCallWithThrownException();
state.emitExplicitExceptionHandler();
noException.link(&jit);
state.restoreLiveRegistersFromStackForCall();
}
}
if (isInlineOffset(m_offset)) {
jit.storeValue(
valueRegs,
CCallHelpers::Address(
baseGPR,
JSObject::offsetOfInlineStorage() +
offsetInInlineStorage(m_offset) * sizeof(JSValue)));
} else {
if (!allocating)
jit.loadPtr(CCallHelpers::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR);
jit.storeValue(
valueRegs,
CCallHelpers::Address(scratchGPR, offsetInButterfly(m_offset) * sizeof(JSValue)));
}
if (allocatingInline) {
CCallHelpers::Jump ownerIsRememberedOrInEden = jit.jumpIfIsRememberedOrInEden(baseGPR);
WriteBarrierBuffer& writeBarrierBuffer = jit.vm()->heap.writeBarrierBuffer();
jit.load32(writeBarrierBuffer.currentIndexAddress(), scratchGPR2);
slowPath.append(
jit.branch32(
CCallHelpers::AboveOrEqual, scratchGPR2,
CCallHelpers::TrustedImm32(writeBarrierBuffer.capacity())));
jit.add32(CCallHelpers::TrustedImm32(1), scratchGPR2);
jit.store32(scratchGPR2, writeBarrierBuffer.currentIndexAddress());
jit.move(CCallHelpers::TrustedImmPtr(writeBarrierBuffer.buffer()), scratchGPR3);
jit.storePtr(
baseGPR,
CCallHelpers::BaseIndex(
scratchGPR3, scratchGPR2, CCallHelpers::ScalePtr,
static_cast<int32_t>(-sizeof(void*))));
ownerIsRememberedOrInEden.link(&jit);
jit.storePtr(scratchGPR, CCallHelpers::Address(baseGPR, JSObject::butterflyOffset()));
}
uint32_t structureBits = bitwise_cast<uint32_t>(newStructure()->id());
jit.store32(
CCallHelpers::TrustedImm32(structureBits),
CCallHelpers::Address(baseGPR, JSCell::structureIDOffset()));
allocator.restoreReusedRegistersByPopping(jit, preservedState);
state.succeed();
if (allocatingInline) {
if (allocator.didReuseRegisters()) {
slowPath.link(&jit);
allocator.restoreReusedRegistersByPopping(jit, preservedState);
state.failAndIgnore.append(jit.jump());
} else
state.failAndIgnore.append(slowPath);
} else
RELEASE_ASSERT(slowPath.empty());
return;
}
case ArrayLength: {
jit.loadPtr(CCallHelpers::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR);
jit.load32(CCallHelpers::Address(scratchGPR, ArrayStorage::lengthOffset()), scratchGPR);
state.failAndIgnore.append(
jit.branch32(CCallHelpers::LessThan, scratchGPR, CCallHelpers::TrustedImm32(0)));
jit.boxInt32(scratchGPR, valueRegs);
state.succeed();
return;
}
case StringLength: {
jit.load32(CCallHelpers::Address(baseGPR, JSString::offsetOfLength()), valueRegs.payloadGPR());
jit.boxInt32(valueRegs.payloadGPR(), valueRegs);
state.succeed();
return;
}
case IntrinsicGetter: {
RELEASE_ASSERT(isValidOffset(offset()));
Structure* currStructure;
if (m_conditionSet.isEmpty())
currStructure = structure();
else
currStructure = m_conditionSet.slotBaseCondition().object()->structure();
currStructure->startWatchingPropertyForReplacements(vm, offset());
emitIntrinsicGetter(state);
return;
}
case DirectArgumentsLength:
case ScopedArgumentsLength:
case MegamorphicLoad:
RELEASE_ASSERT_NOT_REACHED();
}
RELEASE_ASSERT_NOT_REACHED();
}
PolymorphicAccess::PolymorphicAccess() { }
PolymorphicAccess::~PolymorphicAccess() { }
AccessGenerationResult PolymorphicAccess::addCases(
VM& vm, CodeBlock* codeBlock, StructureStubInfo& stubInfo, const Identifier& ident,
Vector<std::unique_ptr<AccessCase>, 2> originalCasesToAdd)
{
SuperSamplerScope superSamplerScope(false);
Vector<std::unique_ptr<AccessCase>> casesToAdd;
for (unsigned i = 0; i < originalCasesToAdd.size(); ++i) {
std::unique_ptr<AccessCase> myCase = WTFMove(originalCasesToAdd[i]);
bool found = false;
for (unsigned j = i + 1; j < originalCasesToAdd.size(); ++j) {
if (originalCasesToAdd[j]->canReplace(*myCase)) {
found = true;
break;
}
}
if (found)
continue;
casesToAdd.append(WTFMove(myCase));
}
if (verbose)
dataLog("casesToAdd: ", listDump(casesToAdd), "\n");
if (casesToAdd.isEmpty())
return AccessGenerationResult::MadeNoChanges;
for (auto& caseToAdd : casesToAdd) {
commit(vm, m_watchpoints, codeBlock, stubInfo, ident, *caseToAdd);
m_list.append(WTFMove(caseToAdd));
}
if (verbose)
dataLog("After addCases: m_list: ", listDump(m_list), "\n");
return AccessGenerationResult::Buffered;
}
AccessGenerationResult PolymorphicAccess::addCase(
VM& vm, CodeBlock* codeBlock, StructureStubInfo& stubInfo, const Identifier& ident,
std::unique_ptr<AccessCase> newAccess)
{
Vector<std::unique_ptr<AccessCase>, 2> newAccesses;
newAccesses.append(WTFMove(newAccess));
return addCases(vm, codeBlock, stubInfo, ident, WTFMove(newAccesses));
}
bool PolymorphicAccess::visitWeak(VM& vm) const
{
for (unsigned i = 0; i < size(); ++i) {
if (!at(i).visitWeak(vm))
return false;
}
if (Vector<WriteBarrier<JSCell>>* weakReferences = m_weakReferences.get()) {
for (WriteBarrier<JSCell>& weakReference : *weakReferences) {
if (!Heap::isMarked(weakReference.get()))
return false;
}
}
return true;
}
bool PolymorphicAccess::propagateTransitions(SlotVisitor& visitor) const
{
bool result = true;
for (unsigned i = 0; i < size(); ++i)
result &= at(i).propagateTransitions(visitor);
return result;
}
void PolymorphicAccess::dump(PrintStream& out) const
{
out.print(RawPointer(this), ":[");
CommaPrinter comma;
for (auto& entry : m_list)
out.print(comma, *entry);
out.print("]");
}
void PolymorphicAccess::commit(
VM& vm, std::unique_ptr<WatchpointsOnStructureStubInfo>& watchpoints, CodeBlock* codeBlock,
StructureStubInfo& stubInfo, const Identifier& ident, AccessCase& accessCase)
{
for (WatchpointSet* set : accessCase.commit(vm, ident)) {
Watchpoint* watchpoint =
WatchpointsOnStructureStubInfo::ensureReferenceAndAddWatchpoint(
watchpoints, codeBlock, &stubInfo, ObjectPropertyCondition());
set->add(watchpoint);
}
}
AccessGenerationResult PolymorphicAccess::regenerate(
VM& vm, CodeBlock* codeBlock, StructureStubInfo& stubInfo, const Identifier& ident)
{
SuperSamplerScope superSamplerScope(false);
if (verbose)
dataLog("Regenerate with m_list: ", listDump(m_list), "\n");
AccessGenerationState state;
state.access = this;
state.stubInfo = &stubInfo;
state.ident = &ident;
state.baseGPR = static_cast<GPRReg>(stubInfo.patch.baseGPR);
state.valueRegs = stubInfo.valueRegs();
ScratchRegisterAllocator allocator(stubInfo.patch.usedRegisters);
state.allocator = &allocator;
allocator.lock(state.baseGPR);
allocator.lock(state.valueRegs);
#if USE(JSVALUE32_64)
allocator.lock(static_cast<GPRReg>(stubInfo.patch.baseTagGPR));
#endif
state.scratchGPR = allocator.allocateScratchGPR();
CCallHelpers jit(&vm, codeBlock);
state.jit = &jit;
state.preservedReusedRegisterState =
allocator.preserveReusedRegistersByPushing(jit, ScratchRegisterAllocator::ExtraStackSpace::NoExtraSpace);
ListType cases;
unsigned srcIndex = 0;
unsigned dstIndex = 0;
while (srcIndex < m_list.size()) {
std::unique_ptr<AccessCase> someCase = WTFMove(m_list[srcIndex++]);
bool isGenerated = someCase->state() == AccessCase::Generated;
[&] () {
if (!someCase->couldStillSucceed())
return;
for (unsigned j = srcIndex; j < m_list.size(); ++j) {
if (m_list[j]->canReplace(*someCase))
return;
}
if (isGenerated)
cases.append(someCase->clone());
else
cases.append(WTFMove(someCase));
}();
if (isGenerated)
m_list[dstIndex++] = WTFMove(someCase);
}
m_list.resize(dstIndex);
if (verbose)
dataLog("In regenerate: cases: ", listDump(cases), "\n");
if (cases.size() >= Options::maxAccessVariantListSize()) {
unsigned numSelfLoads = 0;
for (auto& newCase : cases) {
if (newCase->canBeReplacedByMegamorphicLoad())
numSelfLoads++;
}
if (numSelfLoads >= Options::megamorphicLoadCost()) {
if (auto mega = AccessCase::megamorphicLoad(vm, codeBlock)) {
cases.removeAllMatching(
[&] (std::unique_ptr<AccessCase>& newCase) -> bool {
return newCase->canBeReplacedByMegamorphicLoad();
});
cases.append(WTFMove(mega));
}
}
}
if (verbose)
dataLog("Optimized cases: ", listDump(cases), "\n");
bool allGuardedByStructureCheck = true;
bool hasJSGetterSetterCall = false;
for (auto& newCase : cases) {
commit(vm, state.watchpoints, codeBlock, stubInfo, ident, *newCase);
allGuardedByStructureCheck &= newCase->guardedByStructureCheck();
if (newCase->type() == AccessCase::Getter || newCase->type() == AccessCase::Setter)
hasJSGetterSetterCall = true;
}
if (cases.isEmpty()) {
state.failAndRepatch.append(jit.jump());
} else if (!allGuardedByStructureCheck || cases.size() == 1) {
CCallHelpers::JumpList fallThrough;
for (unsigned i = cases.size(); i--;) {
fallThrough.link(&jit);
cases[i]->generateWithGuard(state, fallThrough);
}
state.failAndRepatch.append(fallThrough);
} else {
jit.load32(
CCallHelpers::Address(state.baseGPR, JSCell::structureIDOffset()),
state.scratchGPR);
Vector<int64_t> caseValues(cases.size());
for (unsigned i = 0; i < cases.size(); ++i)
caseValues[i] = bitwise_cast<int32_t>(cases[i]->structure()->id());
BinarySwitch binarySwitch(state.scratchGPR, caseValues, BinarySwitch::Int32);
while (binarySwitch.advance(jit))
cases[binarySwitch.caseIndex()]->generate(state);
state.failAndRepatch.append(binarySwitch.fallThrough());
}
if (!state.failAndIgnore.empty()) {
state.failAndIgnore.link(&jit);
#if CPU(X86) || CPU(X86_64)
jit.move(CCallHelpers::TrustedImmPtr(&stubInfo.countdown), state.scratchGPR);
jit.add8(CCallHelpers::TrustedImm32(1), CCallHelpers::Address(state.scratchGPR));
#else
jit.load8(&stubInfo.countdown, state.scratchGPR);
jit.add32(CCallHelpers::TrustedImm32(1), state.scratchGPR);
jit.store8(state.scratchGPR, &stubInfo.countdown);
#endif
}
CCallHelpers::JumpList failure;
if (allocator.didReuseRegisters()) {
state.failAndRepatch.link(&jit);
state.restoreScratch();
} else
failure = state.failAndRepatch;
failure.append(jit.jump());
CodeBlock* codeBlockThatOwnsExceptionHandlers = nullptr;
CallSiteIndex callSiteIndexForExceptionHandling;
if (state.needsToRestoreRegistersIfException() && hasJSGetterSetterCall) {
MacroAssembler::Label makeshiftCatchHandler = jit.label();
int stackPointerOffset = codeBlock->stackPointerOffset() * sizeof(EncodedJSValue);
stackPointerOffset -= state.preservedReusedRegisterState.numberOfBytesPreserved;
stackPointerOffset -= state.numberOfStackBytesUsedForRegisterPreservation();
jit.loadPtr(vm.addressOfCallFrameForCatch(), GPRInfo::callFrameRegister);
jit.addPtr(CCallHelpers::TrustedImm32(stackPointerOffset), GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister);
state.restoreLiveRegistersFromStackForCallWithThrownException();
state.restoreScratch();
CCallHelpers::Jump jumpToOSRExitExceptionHandler = jit.jump();
HandlerInfo oldHandler = state.originalExceptionHandler();
CallSiteIndex newExceptionHandlingCallSite = state.callSiteIndexForExceptionHandling();
jit.addLinkTask(
[=] (LinkBuffer& linkBuffer) {
linkBuffer.link(jumpToOSRExitExceptionHandler, oldHandler.nativeCode);
HandlerInfo handlerToRegister = oldHandler;
handlerToRegister.nativeCode = linkBuffer.locationOf(makeshiftCatchHandler);
handlerToRegister.start = newExceptionHandlingCallSite.bits();
handlerToRegister.end = newExceptionHandlingCallSite.bits() + 1;
codeBlock->appendExceptionHandler(handlerToRegister);
});
codeBlockThatOwnsExceptionHandlers = codeBlock;
ASSERT(JITCode::isOptimizingJIT(codeBlockThatOwnsExceptionHandlers->jitType()));
callSiteIndexForExceptionHandling = state.callSiteIndexForExceptionHandling();
}
LinkBuffer linkBuffer(vm, jit, codeBlock, JITCompilationCanFail);
if (linkBuffer.didFailToAllocate()) {
if (verbose)
dataLog("Did fail to allocate.\n");
return AccessGenerationResult::GaveUp;
}
CodeLocationLabel successLabel = stubInfo.doneLocation();
linkBuffer.link(state.success, successLabel);
linkBuffer.link(failure, stubInfo.slowPathStartLocation());
if (verbose)
dataLog(*codeBlock, " ", stubInfo.codeOrigin, ": Generating polymorphic access stub for ", listDump(cases), "\n");
MacroAssemblerCodeRef code = FINALIZE_CODE_FOR(
codeBlock, linkBuffer,
("%s", toCString("Access stub for ", *codeBlock, " ", stubInfo.codeOrigin, " with return point ", successLabel, ": ", listDump(cases)).data()));
bool doesCalls = false;
Vector<JSCell*> cellsToMark;
for (auto& entry : cases)
doesCalls |= entry->doesCalls(&cellsToMark);
m_stubRoutine = createJITStubRoutine(code, vm, codeBlock, doesCalls, cellsToMark, codeBlockThatOwnsExceptionHandlers, callSiteIndexForExceptionHandling);
m_watchpoints = WTFMove(state.watchpoints);
if (!state.weakReferences.isEmpty())
m_weakReferences = std::make_unique<Vector<WriteBarrier<JSCell>>>(WTFMove(state.weakReferences));
if (verbose)
dataLog("Returning: ", code.code(), "\n");
m_list = WTFMove(cases);
AccessGenerationResult::Kind resultKind;
if (m_list.size() >= Options::maxAccessVariantListSize())
resultKind = AccessGenerationResult::GeneratedFinalCode;
else
resultKind = AccessGenerationResult::GeneratedNewCode;
return AccessGenerationResult(resultKind, code.code());
}
void PolymorphicAccess::aboutToDie()
{
if (m_stubRoutine)
m_stubRoutine->aboutToDie();
}
}
namespace WTF {
using namespace JSC;
void printInternal(PrintStream& out, AccessGenerationResult::Kind kind)
{
switch (kind) {
case AccessGenerationResult::MadeNoChanges:
out.print("MadeNoChanges");
return;
case AccessGenerationResult::GaveUp:
out.print("GaveUp");
return;
case AccessGenerationResult::Buffered:
out.print("Buffered");
return;
case AccessGenerationResult::GeneratedNewCode:
out.print("GeneratedNewCode");
return;
case AccessGenerationResult::GeneratedFinalCode:
out.print("GeneratedFinalCode");
return;
}
RELEASE_ASSERT_NOT_REACHED();
}
void printInternal(PrintStream& out, AccessCase::AccessType type)
{
switch (type) {
case AccessCase::Load:
out.print("Load");
return;
case AccessCase::MegamorphicLoad:
out.print("MegamorphicLoad");
return;
case AccessCase::Transition:
out.print("Transition");
return;
case AccessCase::Replace:
out.print("Replace");
return;
case AccessCase::Miss:
out.print("Miss");
return;
case AccessCase::GetGetter:
out.print("GetGetter");
return;
case AccessCase::Getter:
out.print("Getter");
return;
case AccessCase::Setter:
out.print("Setter");
return;
case AccessCase::CustomValueGetter:
out.print("CustomValueGetter");
return;
case AccessCase::CustomAccessorGetter:
out.print("CustomAccessorGetter");
return;
case AccessCase::CustomValueSetter:
out.print("CustomValueSetter");
return;
case AccessCase::CustomAccessorSetter:
out.print("CustomAccessorSetter");
return;
case AccessCase::IntrinsicGetter:
out.print("IntrinsicGetter");
return;
case AccessCase::InHit:
out.print("InHit");
return;
case AccessCase::InMiss:
out.print("InMiss");
return;
case AccessCase::ArrayLength:
out.print("ArrayLength");
return;
case AccessCase::StringLength:
out.print("StringLength");
return;
case AccessCase::DirectArgumentsLength:
out.print("DirectArgumentsLength");
return;
case AccessCase::ScopedArgumentsLength:
out.print("ScopedArgumentsLength");
return;
}
RELEASE_ASSERT_NOT_REACHED();
}
void printInternal(PrintStream& out, AccessCase::State state)
{
switch (state) {
case AccessCase::Primordial:
out.print("Primordial");
return;
case AccessCase::Committed:
out.print("Committed");
return;
case AccessCase::Generated:
out.print("Generated");
return;
}
RELEASE_ASSERT_NOT_REACHED();
}
}
#endif // ENABLE(JIT)