#include "config.h"
#include "DFGOSREntry.h"
#if ENABLE(DFG_JIT)
#include "BytecodeStructs.h"
#include "CallFrame.h"
#include "CodeBlock.h"
#include "DFGJITCode.h"
#include "DFGNode.h"
#include "JSCJSValueInlines.h"
#include "RegisterAtOffsetList.h"
#include "VMInlines.h"
#include <wtf/CommaPrinter.h>
namespace JSC { namespace DFG {
void OSREntryData::dumpInContext(PrintStream& out, DumpContext* context) const
{
out.print(m_bytecodeIndex, ", machine code = ", RawPointer(m_machineCode.executableAddress()));
out.print(", stack rules = [");
auto printOperand = [&] (VirtualRegister reg) {
out.print(inContext(m_expectedValues.operand(reg), context), " (");
VirtualRegister toReg;
bool overwritten = false;
for (OSREntryReshuffling reshuffling : m_reshufflings) {
if (reg == VirtualRegister(reshuffling.fromOffset)) {
toReg = VirtualRegister(reshuffling.toOffset);
break;
}
if (reg == VirtualRegister(reshuffling.toOffset))
overwritten = true;
}
if (!overwritten && !toReg.isValid())
toReg = reg;
if (toReg.isValid()) {
if (toReg.isLocal() && !m_machineStackUsed.get(toReg.toLocal()))
out.print("ignored");
else
out.print("maps to ", toReg);
} else
out.print("overwritten");
if (reg.isLocal() && m_localsForcedDouble.get(reg.toLocal()))
out.print(", forced double");
if (reg.isLocal() && m_localsForcedAnyInt.get(reg.toLocal()))
out.print(", forced machine int");
out.print(")");
};
CommaPrinter comma;
for (size_t argumentIndex = m_expectedValues.numberOfArguments(); argumentIndex--;) {
out.print(comma, "arg", argumentIndex, ":");
printOperand(virtualRegisterForArgumentIncludingThis(argumentIndex));
}
for (size_t localIndex = 0; localIndex < m_expectedValues.numberOfLocals(); ++localIndex) {
out.print(comma, "loc", localIndex, ":");
printOperand(virtualRegisterForLocal(localIndex));
}
out.print("], machine stack used = ", m_machineStackUsed);
}
void OSREntryData::dump(PrintStream& out) const
{
dumpInContext(out, nullptr);
}
SUPPRESS_ASAN
void* prepareOSREntry(VM& vm, CallFrame* callFrame, CodeBlock* codeBlock, BytecodeIndex bytecodeIndex)
{
ASSERT(JITCode::isOptimizingJIT(codeBlock->jitType()));
ASSERT(codeBlock->alternative());
ASSERT(codeBlock->alternative()->jitType() == JITType::BaselineJIT);
ASSERT(!codeBlock->jitCodeMap());
ASSERT(codeBlock->jitCode()->dfgCommon()->isStillValid);
if (!Options::useOSREntryToDFG())
return nullptr;
dataLogLnIf(Options::verboseOSR(),
"DFG OSR in ", *codeBlock->alternative(), " -> ", *codeBlock,
" from ", bytecodeIndex);
sanitizeStackForVM(vm);
if (bytecodeIndex)
codeBlock->ownerExecutable()->setDidTryToEnterInLoop(true);
if (codeBlock->jitType() != JITType::DFGJIT) {
RELEASE_ASSERT(codeBlock->jitType() == JITType::FTLJIT);
dataLogLnIf(Options::verboseOSR(), " OSR failed because the target code block is not DFG.");
return nullptr;
}
JITCode* jitCode = codeBlock->jitCode()->dfg();
OSREntryData* entry = jitCode->osrEntryDataForBytecodeIndex(bytecodeIndex);
if (!entry) {
dataLogLnIf(Options::verboseOSR(), " OSR failed because the entrypoint was optimized out.");
return nullptr;
}
ASSERT(entry->m_bytecodeIndex == bytecodeIndex);
for (size_t argument = 0; argument < entry->m_expectedValues.numberOfArguments(); ++argument) {
JSValue value;
if (!argument)
value = callFrame->thisValue();
else
value = callFrame->argument(argument - 1);
if (!entry->m_expectedValues.argument(argument).validateOSREntryValue(value, FlushedJSValue)) {
dataLogLnIf(Options::verboseOSR(),
" OSR failed because argument ", argument, " is ", value,
", expected ", entry->m_expectedValues.argument(argument));
return nullptr;
}
}
for (size_t local = 0; local < entry->m_expectedValues.numberOfLocals(); ++local) {
int localOffset = virtualRegisterForLocal(local).offset();
JSValue value = callFrame->registers()[localOffset].asanUnsafeJSValue();
FlushFormat format = FlushedJSValue;
if (entry->m_localsForcedAnyInt.get(local)) {
if (!value.isAnyInt()) {
dataLogLnIf(Options::verboseOSR(),
" OSR failed because variable ", localOffset, " is ",
value, ", expected ",
"machine int.");
return nullptr;
}
value = jsDoubleNumber(value.asAnyInt());
format = FlushedInt52;
}
if (entry->m_localsForcedDouble.get(local)) {
if (!value.isNumber()) {
dataLogLnIf(Options::verboseOSR(),
" OSR failed because variable ", localOffset, " is ",
value, ", expected number.");
return nullptr;
}
value = jsDoubleNumber(value.asNumber());
format = FlushedDouble;
}
if (!entry->m_expectedValues.local(local).validateOSREntryValue(value, format)) {
dataLogLnIf(Options::verboseOSR(),
" OSR failed because variable ", VirtualRegister(localOffset), " is ",
value, ", expected ",
entry->m_expectedValues.local(local), ".");
return nullptr;
}
}
unsigned frameSizeForCheck = jitCode->common.requiredRegisterCountForExecutionAndExit();
if (UNLIKELY(!vm.ensureStackCapacityFor(&callFrame->registers()[virtualRegisterForLocal(frameSizeForCheck - 1).offset()]))) {
dataLogLnIf(Options::verboseOSR(), " OSR failed because stack growth failed.");
return nullptr;
}
dataLogLnIf(Options::verboseOSR(), " OSR should succeed.");
unsigned frameSize = jitCode->common.frameRegisterCount;
unsigned baselineFrameSize = entry->m_expectedValues.numberOfLocals();
unsigned maxFrameSize = std::max(frameSize, baselineFrameSize);
Register* scratch = bitwise_cast<Register*>(vm.scratchBufferForSize(sizeof(Register) * (2 + CallFrame::headerSizeInRegisters + maxFrameSize))->dataBuffer());
*bitwise_cast<size_t*>(scratch + 0) = frameSize;
void* targetPC = entry->m_machineCode.executableAddress();
RELEASE_ASSERT(codeBlock->jitCode()->contains(entry->m_machineCode.untaggedExecutableAddress()));
dataLogLnIf(Options::verboseOSR(), " OSR using target PC ", RawPointer(targetPC));
RELEASE_ASSERT(targetPC);
*bitwise_cast<void**>(scratch + 1) = tagCodePtrWithStackPointerForJITCall(untagCodePtr<OSREntryPtrTag>(targetPC), callFrame);
Register* pivot = scratch + 2 + CallFrame::headerSizeInRegisters;
for (int index = -CallFrame::headerSizeInRegisters; index < static_cast<int>(baselineFrameSize); ++index) {
VirtualRegister reg(-1 - index);
if (reg.isLocal()) {
if (entry->m_localsForcedDouble.get(reg.toLocal())) {
*bitwise_cast<double*>(pivot + index) = callFrame->registers()[reg.offset()].asanUnsafeJSValue().asNumber();
continue;
}
if (entry->m_localsForcedAnyInt.get(reg.toLocal())) {
*bitwise_cast<int64_t*>(pivot + index) = callFrame->registers()[reg.offset()].asanUnsafeJSValue().asAnyInt() << JSValue::int52ShiftAmount;
continue;
}
}
pivot[index] = callFrame->registers()[reg.offset()].asanUnsafeJSValue();
}
Vector<JSValue> temporaryLocals(entry->m_reshufflings.size());
for (unsigned i = entry->m_reshufflings.size(); i--;)
temporaryLocals[i] = pivot[VirtualRegister(entry->m_reshufflings[i].fromOffset).toLocal()].asanUnsafeJSValue();
for (unsigned i = entry->m_reshufflings.size(); i--;)
pivot[VirtualRegister(entry->m_reshufflings[i].toOffset).toLocal()] = temporaryLocals[i];
for (unsigned i = frameSize; i--;) {
if (entry->m_machineStackUsed.get(i))
continue;
pivot[i] = JSValue();
}
#if NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
const RegisterAtOffsetList* registerSaveLocations = codeBlock->calleeSaveRegisters();
RegisterAtOffsetList* allCalleeSaves = RegisterSet::vmCalleeSaveRegisterOffsets();
RegisterSet dontSaveRegisters = RegisterSet(RegisterSet::stackRegisters(), RegisterSet::allFPRs());
unsigned registerCount = registerSaveLocations->size();
VMEntryRecord* record = vmEntryRecord(vm.topEntryFrame);
for (unsigned i = 0; i < registerCount; i++) {
RegisterAtOffset currentEntry = registerSaveLocations->at(i);
if (dontSaveRegisters.get(currentEntry.reg()))
continue;
RegisterAtOffset* calleeSavesEntry = allCalleeSaves->find(currentEntry.reg());
if constexpr (CallerFrameAndPC::sizeInRegisters == 2)
*(bitwise_cast<intptr_t*>(pivot - 1) - currentEntry.offsetAsIndex()) = record->calleeSaveRegistersBuffer[calleeSavesEntry->offsetAsIndex()];
else {
ASSERT(sizeof(intptr_t) == 4);
ASSERT(CallerFrameAndPC::sizeInRegisters == 1);
ASSERT(currentEntry.offsetAsIndex() < 0);
int offsetAsIndex = currentEntry.offsetAsIndex();
int properIndex = offsetAsIndex % 2 ? offsetAsIndex - 1 : offsetAsIndex + 1;
*(bitwise_cast<intptr_t*>(pivot - 1) + 1 - properIndex) = record->calleeSaveRegistersBuffer[calleeSavesEntry->offsetAsIndex()];
}
}
#endif
*bitwise_cast<CodeBlock**>(pivot - (CallFrameSlot::codeBlock + 1)) = codeBlock;
dataLogLnIf(Options::verboseOSR(), " OSR returning data buffer ", RawPointer(scratch));
return scratch;
}
MacroAssemblerCodePtr<ExceptionHandlerPtrTag> prepareCatchOSREntry(VM& vm, CallFrame* callFrame, CodeBlock* baselineCodeBlock, CodeBlock* optimizedCodeBlock, BytecodeIndex bytecodeIndex)
{
ASSERT(optimizedCodeBlock->jitType() == JITType::DFGJIT || optimizedCodeBlock->jitType() == JITType::FTLJIT);
ASSERT(optimizedCodeBlock->jitCode()->dfgCommon()->isStillValid);
if (!Options::useOSREntryToDFG() && optimizedCodeBlock->jitCode()->jitType() == JITType::DFGJIT)
return nullptr;
if (!Options::useOSREntryToFTL() && optimizedCodeBlock->jitCode()->jitType() == JITType::FTLJIT)
return nullptr;
CommonData* dfgCommon = optimizedCodeBlock->jitCode()->dfgCommon();
RELEASE_ASSERT(dfgCommon);
DFG::CatchEntrypointData* catchEntrypoint = dfgCommon->catchOSREntryDataForBytecodeIndex(bytecodeIndex);
if (!catchEntrypoint) {
return nullptr;
}
for (unsigned argument = 0; argument < catchEntrypoint->argumentFormats.size(); ++argument) {
JSValue value = callFrame->uncheckedR(virtualRegisterForArgumentIncludingThis(argument)).jsValue();
switch (catchEntrypoint->argumentFormats[argument]) {
case DFG::FlushedInt32:
if (!value.isInt32())
return nullptr;
break;
case DFG::FlushedCell:
if (!value.isCell())
return nullptr;
break;
case DFG::FlushedBoolean:
if (!value.isBoolean())
return nullptr;
break;
case DFG::DeadFlush:
break;
case DFG::FlushedJSValue:
break;
default:
RELEASE_ASSERT_NOT_REACHED();
}
}
unsigned frameSizeForCheck = dfgCommon->requiredRegisterCountForExecutionAndExit();
if (UNLIKELY(!vm.ensureStackCapacityFor(&callFrame->registers()[virtualRegisterForLocal(frameSizeForCheck).offset()])))
return nullptr;
auto instruction = baselineCodeBlock->instructions().at(callFrame->bytecodeIndex());
ASSERT(instruction->is<OpCatch>());
ValueProfileAndVirtualRegisterBuffer* buffer = instruction->as<OpCatch>().metadata(baselineCodeBlock).m_buffer;
JSValue* dataBuffer = reinterpret_cast<JSValue*>(dfgCommon->catchOSREntryBuffer->dataBuffer());
unsigned index = 0;
buffer->forEach([&] (ValueProfileAndVirtualRegister& profile) {
if (!VirtualRegister(profile.m_operand).isLocal())
return;
dataBuffer[index] = callFrame->uncheckedR(profile.m_operand).jsValue();
++index;
});
dfgCommon->catchOSREntryBuffer->setActiveLength(sizeof(JSValue) * index);
return catchEntrypoint->machineCode;
}
} }
#endif // ENABLE(DFG_JIT)