#include "config.h"
#include "DFGPlan.h"
#if ENABLE(DFG_JIT)
#include "DFGArgumentsEliminationPhase.h"
#include "DFGBackwardsPropagationPhase.h"
#include "DFGByteCodeParser.h"
#include "DFGCFAPhase.h"
#include "DFGCFGSimplificationPhase.h"
#include "DFGCPSRethreadingPhase.h"
#include "DFGCSEPhase.h"
#include "DFGCleanUpPhase.h"
#include "DFGConstantFoldingPhase.h"
#include "DFGConstantHoistingPhase.h"
#include "DFGCriticalEdgeBreakingPhase.h"
#include "DFGDCEPhase.h"
#include "DFGFailedFinalizer.h"
#include "DFGFixupPhase.h"
#include "DFGGraphSafepoint.h"
#include "DFGIntegerCheckCombiningPhase.h"
#include "DFGIntegerRangeOptimizationPhase.h"
#include "DFGInvalidationPointInjectionPhase.h"
#include "DFGJITCompiler.h"
#include "DFGLICMPhase.h"
#include "DFGLiveCatchVariablePreservationPhase.h"
#include "DFGLivenessAnalysisPhase.h"
#include "DFGLoopPreHeaderCreationPhase.h"
#include "DFGMaximalFlushInsertionPhase.h"
#include "DFGMovHintRemovalPhase.h"
#include "DFGOSRAvailabilityAnalysisPhase.h"
#include "DFGOSREntrypointCreationPhase.h"
#include "DFGObjectAllocationSinkingPhase.h"
#include "DFGPhantomInsertionPhase.h"
#include "DFGPredictionInjectionPhase.h"
#include "DFGPredictionPropagationPhase.h"
#include "DFGPutStackSinkingPhase.h"
#include "DFGSSAConversionPhase.h"
#include "DFGSSALoweringPhase.h"
#include "DFGStackLayoutPhase.h"
#include "DFGStaticExecutionCountEstimationPhase.h"
#include "DFGStoreBarrierClusteringPhase.h"
#include "DFGStoreBarrierInsertionPhase.h"
#include "DFGStrengthReductionPhase.h"
#include "DFGTierUpCheckInjectionPhase.h"
#include "DFGTypeCheckHoistingPhase.h"
#include "DFGUnificationPhase.h"
#include "DFGValidate.h"
#include "DFGVarargsForwardingPhase.h"
#include "DFGVirtualRegisterAllocationPhase.h"
#include "DFGWatchpointCollectionPhase.h"
#include "JSCInlines.h"
#include "OperandsInlines.h"
#include "ProfilerDatabase.h"
#include "TrackedReferences.h"
#include "VMInlines.h"
#include <wtf/CurrentTime.h>
#if ENABLE(FTL_JIT)
#include "FTLCapabilities.h"
#include "FTLCompile.h"
#include "FTLFail.h"
#include "FTLLink.h"
#include "FTLLowerDFGToB3.h"
#include "FTLState.h"
#endif
namespace JSC {
extern double totalDFGCompileTime;
extern double totalFTLCompileTime;
extern double totalFTLDFGCompileTime;
extern double totalFTLB3CompileTime;
}
namespace JSC { namespace DFG {
namespace {
void dumpAndVerifyGraph(Graph& graph, const char* text, bool forceDump = false)
{
GraphDumpMode modeForFinalValidate = DumpGraph;
if (verboseCompilationEnabled(graph.m_plan.mode) || forceDump) {
dataLog(text, "\n");
graph.dump();
modeForFinalValidate = DontDumpGraph;
}
if (validationEnabled())
validate(graph, modeForFinalValidate);
}
Profiler::CompilationKind profilerCompilationKindForMode(CompilationMode mode)
{
switch (mode) {
case InvalidCompilationMode:
RELEASE_ASSERT_NOT_REACHED();
return Profiler::DFG;
case DFGMode:
return Profiler::DFG;
case FTLMode:
return Profiler::FTL;
case FTLForOSREntryMode:
return Profiler::FTLForOSREntry;
}
RELEASE_ASSERT_NOT_REACHED();
return Profiler::DFG;
}
}
Plan::Plan(CodeBlock* passedCodeBlock, CodeBlock* profiledDFGCodeBlock,
CompilationMode mode, unsigned osrEntryBytecodeIndex,
const Operands<JSValue>& mustHandleValues)
: vm(passedCodeBlock->vm())
, codeBlock(passedCodeBlock)
, profiledDFGCodeBlock(profiledDFGCodeBlock)
, mode(mode)
, osrEntryBytecodeIndex(osrEntryBytecodeIndex)
, mustHandleValues(mustHandleValues)
, compilation(UNLIKELY(vm->m_perBytecodeProfiler) ? adoptRef(new Profiler::Compilation(vm->m_perBytecodeProfiler->ensureBytecodesFor(codeBlock), profilerCompilationKindForMode(mode))) : nullptr)
, inlineCallFrames(adoptRef(new InlineCallFrameSet()))
, identifiers(codeBlock)
, weakReferences(codeBlock)
, stage(Preparing)
{
}
Plan::~Plan()
{
}
bool Plan::computeCompileTimes() const
{
return reportCompileTimes()
|| Options::reportTotalCompileTimes()
|| (vm && vm->m_perBytecodeProfiler);
}
bool Plan::reportCompileTimes() const
{
return Options::reportCompileTimes()
|| Options::reportDFGCompileTimes()
|| (Options::reportFTLCompileTimes() && isFTL(mode));
}
void Plan::compileInThread(ThreadData* threadData)
{
this->threadData = threadData;
double before = 0;
CString codeBlockName;
if (UNLIKELY(computeCompileTimes()))
before = monotonicallyIncreasingTimeMS();
if (UNLIKELY(reportCompileTimes()))
codeBlockName = toCString(*codeBlock);
CompilationScope compilationScope;
if (logCompilationChanges(mode) || Options::reportDFGPhaseTimes())
dataLog("DFG(Plan) compiling ", *codeBlock, " with ", mode, ", number of instructions = ", codeBlock->instructionCount(), "\n");
CompilationPath path = compileInThreadImpl();
RELEASE_ASSERT(path == CancelPath || finalizer);
RELEASE_ASSERT((path == CancelPath) == (stage == Cancelled));
double after = 0;
if (UNLIKELY(computeCompileTimes())) {
after = monotonicallyIncreasingTimeMS();
if (Options::reportTotalCompileTimes()) {
if (isFTL(mode)) {
totalFTLCompileTime += after - before;
totalFTLDFGCompileTime += m_timeBeforeFTL - before;
totalFTLB3CompileTime += after - m_timeBeforeFTL;
} else
totalDFGCompileTime += after - before;
}
}
const char* pathName = nullptr;
switch (path) {
case FailPath:
pathName = "N/A (fail)";
break;
case DFGPath:
pathName = "DFG";
break;
case FTLPath:
pathName = "FTL";
break;
case CancelPath:
pathName = "Cancelled";
break;
default:
RELEASE_ASSERT_NOT_REACHED();
break;
}
if (codeBlock) { if (path == FTLPath)
CODEBLOCK_LOG_EVENT(codeBlock, "ftlCompile", ("took ", after - before, " ms (DFG: ", m_timeBeforeFTL - before, ", B3: ", after - m_timeBeforeFTL, ") with ", pathName));
else
CODEBLOCK_LOG_EVENT(codeBlock, "dfgCompile", ("took ", after - before, " ms with ", pathName));
}
if (UNLIKELY(reportCompileTimes())) {
dataLog("Optimized ", codeBlockName, " using ", mode, " with ", pathName, " into ", finalizer ? finalizer->codeSize() : 0, " bytes in ", after - before, " ms");
if (path == FTLPath)
dataLog(" (DFG: ", m_timeBeforeFTL - before, ", B3: ", after - m_timeBeforeFTL, ")");
dataLog(".\n");
}
}
Plan::CompilationPath Plan::compileInThreadImpl()
{
cleanMustHandleValuesIfNecessary();
if (verboseCompilationEnabled(mode) && osrEntryBytecodeIndex != UINT_MAX) {
dataLog("\n");
dataLog("Compiler must handle OSR entry from bc#", osrEntryBytecodeIndex, " with values: ", mustHandleValues, "\n");
dataLog("\n");
}
Graph dfg(*vm, *this);
parse(dfg);
codeBlock->setCalleeSaveRegisters(RegisterSet::dfgCalleeSaveRegisters());
bool changed = false;
#define RUN_PHASE(phase) \
do { \
if (Options::safepointBeforeEachPhase()) { \
Safepoint::Result safepointResult; \
{ \
GraphSafepoint safepoint(dfg, safepointResult); \
} \
if (safepointResult.didGetCancelled()) \
return CancelPath; \
} \
changed |= phase(dfg); \
} while (false); \
codeBlock->shrinkToFit(CodeBlock::EarlyShrink);
if (validationEnabled())
validate(dfg);
if (Options::dumpGraphAfterParsing()) {
dataLog("Graph after parsing:\n");
dfg.dump();
}
RUN_PHASE(performLiveCatchVariablePreservationPhase);
if (Options::useMaximalFlushInsertionPhase())
RUN_PHASE(performMaximalFlushInsertion);
RUN_PHASE(performCPSRethreading);
RUN_PHASE(performUnification);
RUN_PHASE(performPredictionInjection);
RUN_PHASE(performStaticExecutionCountEstimation);
if (mode == FTLForOSREntryMode) {
bool result = performOSREntrypointCreation(dfg);
if (!result) {
finalizer = std::make_unique<FailedFinalizer>(*this);
return FailPath;
}
RUN_PHASE(performCPSRethreading);
}
if (validationEnabled())
validate(dfg);
RUN_PHASE(performBackwardsPropagation);
RUN_PHASE(performPredictionPropagation);
RUN_PHASE(performFixup);
RUN_PHASE(performInvalidationPointInjection);
RUN_PHASE(performTypeCheckHoisting);
dfg.m_fixpointState = FixpointNotConverged;
if (validationEnabled())
validate(dfg);
RUN_PHASE(performStrengthReduction);
RUN_PHASE(performCPSRethreading);
RUN_PHASE(performCFA);
RUN_PHASE(performConstantFolding);
changed = false;
RUN_PHASE(performCFGSimplification);
RUN_PHASE(performLocalCSE);
if (validationEnabled())
validate(dfg);
RUN_PHASE(performCPSRethreading);
if (!isFTL(mode)) {
RUN_PHASE(performVarargsForwarding); }
if (changed) {
RUN_PHASE(performCFA);
RUN_PHASE(performConstantFolding);
}
if (validationEnabled()) {
dfg.ensureCPSDominators();
dfg.ensureCPSNaturalLoops();
}
switch (mode) {
case DFGMode: {
dfg.m_fixpointState = FixpointConverged;
RUN_PHASE(performTierUpCheckInjection);
RUN_PHASE(performFastStoreBarrierInsertion);
RUN_PHASE(performStoreBarrierClustering);
RUN_PHASE(performCleanUp);
RUN_PHASE(performCPSRethreading);
RUN_PHASE(performDCE);
RUN_PHASE(performPhantomInsertion);
RUN_PHASE(performStackLayout);
RUN_PHASE(performVirtualRegisterAllocation);
RUN_PHASE(performWatchpointCollection);
dumpAndVerifyGraph(dfg, "Graph after optimization:");
JITCompiler dataFlowJIT(dfg);
if (codeBlock->codeType() == FunctionCode)
dataFlowJIT.compileFunction();
else
dataFlowJIT.compile();
return DFGPath;
}
case FTLMode:
case FTLForOSREntryMode: {
#if ENABLE(FTL_JIT)
if (FTL::canCompile(dfg) == FTL::CannotCompile) {
finalizer = std::make_unique<FailedFinalizer>(*this);
return FailPath;
}
RUN_PHASE(performCleanUp); RUN_PHASE(performCriticalEdgeBreaking);
if (Options::createPreHeaders())
RUN_PHASE(performLoopPreHeaderCreation);
RUN_PHASE(performCPSRethreading);
RUN_PHASE(performSSAConversion);
RUN_PHASE(performSSALowering);
RUN_PHASE(performArgumentsElimination);
if (Options::usePutStackSinking())
RUN_PHASE(performPutStackSinking);
RUN_PHASE(performConstantHoisting);
RUN_PHASE(performGlobalCSE);
RUN_PHASE(performLivenessAnalysis);
RUN_PHASE(performCFA);
RUN_PHASE(performConstantFolding);
RUN_PHASE(performCleanUp); changed = false;
RUN_PHASE(performStrengthReduction);
if (Options::useObjectAllocationSinking()) {
RUN_PHASE(performCriticalEdgeBreaking);
RUN_PHASE(performObjectAllocationSinking);
}
if (changed) {
RUN_PHASE(performLivenessAnalysis);
RUN_PHASE(performCFA);
RUN_PHASE(performConstantFolding);
}
RUN_PHASE(performLivenessAnalysis);
RUN_PHASE(performCFA);
RUN_PHASE(performLICM);
RUN_PHASE(performLivenessAnalysis);
RUN_PHASE(performIntegerRangeOptimization);
RUN_PHASE(performCleanUp);
RUN_PHASE(performIntegerCheckCombining);
RUN_PHASE(performGlobalCSE);
dfg.m_fixpointState = FixpointConverged;
RUN_PHASE(performLivenessAnalysis);
RUN_PHASE(performCFA);
RUN_PHASE(performGlobalStoreBarrierInsertion);
RUN_PHASE(performStoreBarrierClustering);
if (Options::useMovHintRemoval())
RUN_PHASE(performMovHintRemoval);
RUN_PHASE(performCleanUp);
RUN_PHASE(performDCE); RUN_PHASE(performStackLayout);
RUN_PHASE(performLivenessAnalysis);
RUN_PHASE(performOSRAvailabilityAnalysis);
RUN_PHASE(performWatchpointCollection);
if (FTL::canCompile(dfg) == FTL::CannotCompile) {
finalizer = std::make_unique<FailedFinalizer>(*this);
return FailPath;
}
dumpAndVerifyGraph(dfg, "Graph just before FTL lowering:", shouldDumpDisassembly(mode));
Safepoint::Result safepointResult;
{
GraphSafepoint safepoint(dfg, safepointResult);
}
if (safepointResult.didGetCancelled())
return CancelPath;
FTL::State state(dfg);
FTL::lowerDFGToB3(state);
if (UNLIKELY(computeCompileTimes()))
m_timeBeforeFTL = monotonicallyIncreasingTimeMS();
if (Options::b3AlwaysFailsBeforeCompile()) {
FTL::fail(state);
return FTLPath;
}
FTL::compile(state, safepointResult);
if (safepointResult.didGetCancelled())
return CancelPath;
if (Options::b3AlwaysFailsBeforeLink()) {
FTL::fail(state);
return FTLPath;
}
if (state.allocationFailed) {
FTL::fail(state);
return FTLPath;
}
FTL::link(state);
if (state.allocationFailed) {
FTL::fail(state);
return FTLPath;
}
return FTLPath;
#else
RELEASE_ASSERT_NOT_REACHED();
return FailPath;
#endif // ENABLE(FTL_JIT)
}
default:
RELEASE_ASSERT_NOT_REACHED();
return FailPath;
}
#undef RUN_PHASE
}
bool Plan::isStillValid()
{
CodeBlock* replacement = codeBlock->replacement();
if (!replacement)
return false;
if (codeBlock->alternative() != replacement->baselineVersion())
return false;
if (!watchpoints.areStillValid())
return false;
return true;
}
void Plan::reallyAdd(CommonData* commonData)
{
watchpoints.reallyAdd(codeBlock, *commonData);
identifiers.reallyAdd(*vm, commonData);
weakReferences.reallyAdd(*vm, commonData);
transitions.reallyAdd(*vm, commonData);
}
void Plan::notifyCompiling()
{
stage = Compiling;
}
void Plan::notifyReady()
{
callback->compilationDidBecomeReadyAsynchronously(codeBlock, profiledDFGCodeBlock);
stage = Ready;
}
CompilationResult Plan::finalizeWithoutNotifyingCallback()
{
vm->heap.writeBarrier(codeBlock);
if (!isStillValid()) {
CODEBLOCK_LOG_EVENT(codeBlock, "dfgFinalize", ("invalidated"));
return CompilationInvalidated;
}
bool result;
if (codeBlock->codeType() == FunctionCode)
result = finalizer->finalizeFunction();
else
result = finalizer->finalize();
if (!result) {
CODEBLOCK_LOG_EVENT(codeBlock, "dfgFinalize", ("failed"));
return CompilationFailed;
}
reallyAdd(codeBlock->jitCode()->dfgCommon());
if (validationEnabled()) {
TrackedReferences trackedReferences;
for (WriteBarrier<JSCell>& reference : codeBlock->jitCode()->dfgCommon()->weakReferences)
trackedReferences.add(reference.get());
for (WriteBarrier<Structure>& reference : codeBlock->jitCode()->dfgCommon()->weakStructureReferences)
trackedReferences.add(reference.get());
for (WriteBarrier<Unknown>& constant : codeBlock->constants())
trackedReferences.add(constant.get());
for (auto* inlineCallFrame : *inlineCallFrames) {
ASSERT(inlineCallFrame->baselineCodeBlock.get());
trackedReferences.add(inlineCallFrame->baselineCodeBlock.get());
}
codeBlock->jitCode()->validateReferences(trackedReferences);
}
CODEBLOCK_LOG_EVENT(codeBlock, "dfgFinalize", ("succeeded"));
return CompilationSuccessful;
}
void Plan::finalizeAndNotifyCallback()
{
callback->compilationDidComplete(codeBlock, profiledDFGCodeBlock, finalizeWithoutNotifyingCallback());
}
CompilationKey Plan::key()
{
return CompilationKey(codeBlock->alternative(), mode);
}
void Plan::checkLivenessAndVisitChildren(SlotVisitor& visitor)
{
if (!isKnownToBeLiveDuringGC())
return;
cleanMustHandleValuesIfNecessary();
for (unsigned i = mustHandleValues.size(); i--;)
visitor.appendUnbarriered(mustHandleValues[i]);
visitor.appendUnbarriered(codeBlock);
visitor.appendUnbarriered(codeBlock->alternative());
visitor.appendUnbarriered(profiledDFGCodeBlock);
if (inlineCallFrames) {
for (auto* inlineCallFrame : *inlineCallFrames) {
ASSERT(inlineCallFrame->baselineCodeBlock.get());
visitor.appendUnbarriered(inlineCallFrame->baselineCodeBlock.get());
}
}
weakReferences.visitChildren(visitor);
transitions.visitChildren(visitor);
}
bool Plan::isKnownToBeLiveDuringGC()
{
if (stage == Cancelled)
return false;
if (!Heap::isMarked(codeBlock->ownerExecutable()))
return false;
if (!Heap::isMarked(codeBlock->alternative()))
return false;
if (!!profiledDFGCodeBlock && !Heap::isMarked(profiledDFGCodeBlock))
return false;
return true;
}
void Plan::cancel()
{
vm = nullptr;
codeBlock = nullptr;
profiledDFGCodeBlock = nullptr;
mustHandleValues.clear();
compilation = nullptr;
finalizer = nullptr;
inlineCallFrames = nullptr;
watchpoints = DesiredWatchpoints();
identifiers = DesiredIdentifiers();
weakReferences = DesiredWeakReferences();
transitions = DesiredTransitions();
callback = nullptr;
stage = Cancelled;
}
void Plan::cleanMustHandleValuesIfNecessary()
{
LockHolder locker(mustHandleValueCleaningLock);
if (!mustHandleValuesMayIncludeGarbage)
return;
mustHandleValuesMayIncludeGarbage = false;
if (!codeBlock)
return;
if (!mustHandleValues.numberOfLocals())
return;
CodeBlock* alternative = codeBlock->alternative();
FastBitVector liveness = alternative->livenessAnalysis().getLivenessInfoAtBytecodeOffset(alternative, osrEntryBytecodeIndex);
for (unsigned local = mustHandleValues.numberOfLocals(); local--;) {
if (!liveness[local])
mustHandleValues.local(local) = jsUndefined();
}
}
} }
#endif // ENABLE(DFG_JIT)