FTLOSRExitCompiler.cpp [plain text]
#include "config.h"
#include "FTLOSRExitCompiler.h"
#if ENABLE(FTL_JIT)
#include "DFGOSRExitCompilerCommon.h"
#include "DFGOSRExitPreparation.h"
#include "FTLExitArgumentForOperand.h"
#include "FTLJITCode.h"
#include "FTLOSRExit.h"
#include "FTLOperations.h"
#include "FTLState.h"
#include "FTLSaveRestore.h"
#include "LinkBuffer.h"
#include "MaxFrameExtentForSlowPathCall.h"
#include "OperandsInlines.h"
#include "JSCInlines.h"
#include "RegisterPreservationWrapperGenerator.h"
#include "RepatchBuffer.h"
namespace JSC { namespace FTL {
using namespace DFG;
static void compileRecovery(
CCallHelpers& jit, const ExitValue& value, StackMaps::Record* record, StackMaps& stackmaps,
char* registerScratch,
const HashMap<ExitTimeObjectMaterialization*, EncodedJSValue*>& materializationToPointer)
{
switch (value.kind()) {
case ExitValueDead:
jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())), GPRInfo::regT0);
break;
case ExitValueConstant:
jit.move(MacroAssembler::TrustedImm64(JSValue::encode(value.constant())), GPRInfo::regT0);
break;
case ExitValueArgument:
record->locations[value.exitArgument().argument()].restoreInto(
jit, stackmaps, registerScratch, GPRInfo::regT0);
break;
case ExitValueInJSStack:
case ExitValueInJSStackAsInt32:
case ExitValueInJSStackAsInt52:
case ExitValueInJSStackAsDouble:
jit.load64(AssemblyHelpers::addressFor(value.virtualRegister()), GPRInfo::regT0);
break;
case ExitValueRecovery:
record->locations[value.rightRecoveryArgument()].restoreInto(
jit, stackmaps, registerScratch, GPRInfo::regT1);
record->locations[value.leftRecoveryArgument()].restoreInto(
jit, stackmaps, registerScratch, GPRInfo::regT0);
switch (value.recoveryOpcode()) {
case AddRecovery:
switch (value.recoveryFormat()) {
case ValueFormatInt32:
jit.add32(GPRInfo::regT1, GPRInfo::regT0);
break;
case ValueFormatInt52:
jit.add64(GPRInfo::regT1, GPRInfo::regT0);
break;
default:
RELEASE_ASSERT_NOT_REACHED();
break;
}
break;
case SubRecovery:
switch (value.recoveryFormat()) {
case ValueFormatInt32:
jit.sub32(GPRInfo::regT1, GPRInfo::regT0);
break;
case ValueFormatInt52:
jit.sub64(GPRInfo::regT1, GPRInfo::regT0);
break;
default:
RELEASE_ASSERT_NOT_REACHED();
break;
}
break;
default:
RELEASE_ASSERT_NOT_REACHED();
break;
}
break;
case ExitValueMaterializeNewObject:
jit.loadPtr(materializationToPointer.get(value.objectMaterialization()), GPRInfo::regT0);
break;
default:
RELEASE_ASSERT_NOT_REACHED();
break;
}
reboxAccordingToFormat(
value.valueFormat(), jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
}
static void compileStub(
unsigned exitID, JITCode* jitCode, OSRExit& exit, VM* vm, CodeBlock* codeBlock)
{
StackMaps::Record* record = nullptr;
for (unsigned i = jitCode->stackmaps.records.size(); i--;) {
record = &jitCode->stackmaps.records[i];
if (record->patchpointID == exit.m_stackmapID)
break;
}
RELEASE_ASSERT(record->patchpointID == exit.m_stackmapID);
static_assert(MacroAssembler::framePointerRegister == GPRInfo::callFrameRegister, "MacroAssembler::framePointerRegister and GPRInfo::callFrameRegister must be the same");
CCallHelpers jit(vm, codeBlock);
unsigned numMaterializations = 0;
size_t maxMaterializationNumArguments = 0;
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) {
numMaterializations++;
maxMaterializationNumArguments = std::max(
maxMaterializationNumArguments,
materialization->properties().size());
}
ScratchBuffer* scratchBuffer = vm->scratchBufferForSize(
sizeof(EncodedJSValue) * (
exit.m_values.size() + numMaterializations + maxMaterializationNumArguments) +
requiredScratchMemorySizeInBytes() +
jitCode->unwindInfo.m_registers.size() * sizeof(uint64_t));
EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
EncodedJSValue* materializationPointers = scratch + exit.m_values.size();
EncodedJSValue* materializationArguments = materializationPointers + numMaterializations;
char* registerScratch = bitwise_cast<char*>(materializationArguments + maxMaterializationNumArguments);
uint64_t* unwindScratch = bitwise_cast<uint64_t*>(registerScratch + requiredScratchMemorySizeInBytes());
HashMap<ExitTimeObjectMaterialization*, EncodedJSValue*> materializationToPointer;
unsigned materializationCount = 0;
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) {
materializationToPointer.add(
materialization, materializationPointers + materializationCount++);
}
saveAllRegisters(jit, registerScratch);
jit.popToRestore(GPRInfo::regT0);
jit.checkStackPointerAlignment();
if (vm->m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation) {
Profiler::Database& database = *vm->m_perBytecodeProfiler;
Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get();
Profiler::OSRExit* profilerExit = compilation->addOSRExit(
exitID, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
exit.m_kind, exit.m_kind == UncountableInvalidation);
jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
}
jit.move(MacroAssembler::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister);
jit.move(MacroAssembler::TrustedImm64(TagMask), GPRInfo::tagMaskRegister);
if (exit.m_profileValueFormat != InvalidValueFormat) {
record->locations[0].restoreInto(jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
reboxAccordingToFormat(
exit.m_profileValueFormat, jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) {
CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
if (ArrayProfile* arrayProfile = jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) {
jit.load32(MacroAssembler::Address(GPRInfo::regT0, JSCell::structureIDOffset()), GPRInfo::regT1);
jit.store32(GPRInfo::regT1, arrayProfile->addressOfLastSeenStructureID());
jit.load8(MacroAssembler::Address(GPRInfo::regT0, JSCell::indexingTypeOffset()), GPRInfo::regT1);
jit.move(MacroAssembler::TrustedImm32(1), GPRInfo::regT2);
jit.lshift32(GPRInfo::regT1, GPRInfo::regT2);
jit.or32(GPRInfo::regT2, MacroAssembler::AbsoluteAddress(arrayProfile->addressOfArrayModes()));
}
}
if (!!exit.m_valueProfile)
jit.store64(GPRInfo::regT0, exit.m_valueProfile.getSpecFailBucket(0));
}
HashSet<ExitTimeObjectMaterialization*> toMaterialize;
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations)
toMaterialize.add(materialization);
while (!toMaterialize.isEmpty()) {
unsigned previousToMaterializeSize = toMaterialize.size();
Vector<ExitTimeObjectMaterialization*> worklist;
worklist.appendRange(toMaterialize.begin(), toMaterialize.end());
for (ExitTimeObjectMaterialization* materialization : worklist) {
bool allGood = true;
for (ExitPropertyValue value : materialization->properties()) {
if (!value.value().isObjectMaterialization())
continue;
if (toMaterialize.contains(value.value().objectMaterialization())) {
allGood = false;
break;
}
}
if (!allGood)
continue;
for (unsigned propertyIndex = materialization->properties().size(); propertyIndex--;) {
const ExitValue& value = materialization->properties()[propertyIndex].value();
compileRecovery(
jit, value, record, jitCode->stackmaps, registerScratch,
materializationToPointer);
jit.storePtr(GPRInfo::regT0, materializationArguments + propertyIndex);
}
jit.setupArgumentsWithExecState(
CCallHelpers::TrustedImmPtr(materialization),
CCallHelpers::TrustedImmPtr(materializationArguments));
jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(operationMaterializeObjectInOSR)), GPRInfo::nonArgGPR0);
jit.call(GPRInfo::nonArgGPR0);
jit.storePtr(GPRInfo::returnValueGPR, materializationToPointer.get(materialization));
toMaterialize.remove(materialization);
}
RELEASE_ASSERT(toMaterialize.size() < previousToMaterializeSize);
}
for (unsigned index = exit.m_values.size(); index--;) {
compileRecovery(
jit, exit.m_values[index], record, jitCode->stackmaps, registerScratch,
materializationToPointer);
jit.store64(GPRInfo::regT0, scratch + index);
}
ptrdiff_t offset = registerPreservationOffset();
RegisterSet toSave = registersToPreserve();
for (unsigned i = jitCode->unwindInfo.m_registers.size(); i--;) {
RegisterAtOffset entry = jitCode->unwindInfo.m_registers[i];
jit.load64(
MacroAssembler::Address(MacroAssembler::framePointerRegister, entry.offset()),
GPRInfo::regT0);
jit.store64(GPRInfo::regT0, unwindScratch + i);
}
jit.load32(CCallHelpers::payloadFor(JSStack::ArgumentCount), GPRInfo::regT2);
RELEASE_ASSERT(codeBlock->codeType() == FunctionCode);
jit.add32(
MacroAssembler::TrustedImm32(-codeBlock->numParameters()), GPRInfo::regT2,
GPRInfo::regT3);
MacroAssembler::Jump arityIntact = jit.branch32(
MacroAssembler::GreaterThanOrEqual, GPRInfo::regT3, MacroAssembler::TrustedImm32(0));
jit.neg32(GPRInfo::regT3);
jit.add32(MacroAssembler::TrustedImm32(1 + stackAlignmentRegisters() - 1), GPRInfo::regT3);
jit.and32(MacroAssembler::TrustedImm32(-stackAlignmentRegisters()), GPRInfo::regT3);
jit.add32(GPRInfo::regT3, GPRInfo::regT2);
arityIntact.link(&jit);
unsigned conservativeStackDelta =
registerPreservationOffset() +
exit.m_values.numberOfLocals() * sizeof(Register) +
maxFrameExtentForSlowPathCall;
conservativeStackDelta = WTF::roundUpToMultipleOf(
stackAlignmentBytes(), conservativeStackDelta);
jit.addPtr(
MacroAssembler::TrustedImm32(-conservativeStackDelta),
MacroAssembler::framePointerRegister, MacroAssembler::stackPointerRegister);
jit.checkStackPointerAlignment();
jit.subPtr(
MacroAssembler::TrustedImm32(registerPreservationOffset()),
MacroAssembler::framePointerRegister);
jit.add32(MacroAssembler::TrustedImm32(JSStack::CallFrameHeaderSize), GPRInfo::regT2);
jit.move(MacroAssembler::framePointerRegister, GPRInfo::regT1);
MacroAssembler::Label loop = jit.label();
jit.sub32(MacroAssembler::TrustedImm32(1), GPRInfo::regT2);
jit.load64(MacroAssembler::Address(GPRInfo::regT1, offset), GPRInfo::regT0);
jit.store64(GPRInfo::regT0, GPRInfo::regT1);
jit.addPtr(MacroAssembler::TrustedImm32(sizeof(Register)), GPRInfo::regT1);
jit.branchTest32(MacroAssembler::NonZero, GPRInfo::regT2).linkTo(loop, &jit);
ptrdiff_t currentOffset = 0;
for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
if (!toSave.get(reg))
continue;
currentOffset += sizeof(Register);
unsigned unwindIndex = jitCode->unwindInfo.indexOf(reg);
if (unwindIndex == UINT_MAX) {
jit.load64(registerScratch + offsetOfReg(reg), GPRInfo::regT0);
} else {
jit.load64(unwindScratch + unwindIndex, GPRInfo::regT0);
}
jit.store64(GPRInfo::regT0, AssemblyHelpers::Address(GPRInfo::regT1, currentOffset));
}
MacroAssembler::Jump arityIntactForReturnPC = jit.branch32(
MacroAssembler::GreaterThanOrEqual,
CCallHelpers::payloadFor(JSStack::ArgumentCount),
MacroAssembler::TrustedImm32(codeBlock->numParameters()));
jit.loadPtr(MacroAssembler::Address(GPRInfo::regT1, -static_cast<ptrdiff_t>(sizeof(Register))), GPRInfo::regT0);
jit.storePtr(GPRInfo::regT0, GPRInfo::regT1);
jit.storePtr(
MacroAssembler::TrustedImmPtr(vm->getCTIStub(registerRestorationThunkGenerator).code().executableAddress()),
MacroAssembler::Address(GPRInfo::regT1, -static_cast<ptrdiff_t>(sizeof(Register))));
MacroAssembler::Jump arityReturnPCReady = jit.jump();
arityIntactForReturnPC.link(&jit);
jit.loadPtr(MacroAssembler::Address(MacroAssembler::framePointerRegister, CallFrame::returnPCOffset()), GPRInfo::regT0);
jit.storePtr(GPRInfo::regT0, GPRInfo::regT1);
jit.storePtr(
MacroAssembler::TrustedImmPtr(vm->getCTIStub(registerRestorationThunkGenerator).code().executableAddress()),
MacroAssembler::Address(MacroAssembler::framePointerRegister, CallFrame::returnPCOffset()));
arityReturnPCReady.link(&jit);
for (unsigned index = exit.m_values.size(); index--;) {
int operand = exit.m_values.operandForIndex(index);
jit.load64(scratch + index, GPRInfo::regT0);
jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(static_cast<VirtualRegister>(operand)));
}
handleExitCounts(jit, exit);
reifyInlinedCallFrames(jit, exit);
adjustAndJumpToTarget(jit, exit);
LinkBuffer patchBuffer(*vm, jit, codeBlock);
exit.m_code = FINALIZE_CODE_IF(
shouldShowDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit(),
patchBuffer,
("FTL OSR exit #%u (%s, %s) from %s, with operands = %s, and record = %s",
exitID, toCString(exit.m_codeOrigin).data(),
exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
toCString(ignoringContext<DumpContext>(exit.m_values)).data(),
toCString(*record).data()));
}
extern "C" void* compileFTLOSRExit(ExecState* exec, unsigned exitID)
{
SamplingRegion samplingRegion("FTL OSR Exit Compilation");
if (shouldShowDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit())
dataLog("Compiling OSR exit with exitID = ", exitID, "\n");
CodeBlock* codeBlock = exec->codeBlock();
ASSERT(codeBlock);
ASSERT(codeBlock->jitType() == JITCode::FTLJIT);
VM* vm = &exec->vm();
DeferGCForAWhile deferGC(vm->heap);
JITCode* jitCode = codeBlock->jitCode()->ftl();
OSRExit& exit = jitCode->osrExit[exitID];
if (shouldShowDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit()) {
dataLog(" Owning block: ", pointerDump(codeBlock), "\n");
dataLog(" Origin: ", exit.m_codeOrigin, "\n");
if (exit.m_codeOriginForExitProfile != exit.m_codeOrigin)
dataLog(" Origin for exit profile: ", exit.m_codeOriginForExitProfile, "\n");
dataLog(" Exit values: ", exit.m_values, "\n");
if (!exit.m_materializations.isEmpty()) {
dataLog(" Materializations:\n");
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations)
dataLog(" ", pointerDump(materialization), "\n");
}
}
prepareCodeOriginForOSRExit(exec, exit.m_codeOrigin);
compileStub(exitID, jitCode, exit, vm, codeBlock);
RepatchBuffer repatchBuffer(codeBlock);
repatchBuffer.relink(
exit.codeLocationForRepatch(codeBlock), CodeLocationLabel(exit.m_code.code()));
return exit.m_code.code().executableAddress();
}
} }
#endif // ENABLE(FTL_JIT)