#include "config.h"
#include "DFGLICMPhase.h"
#if ENABLE(DFG_JIT)
#include "DFGAbstractInterpreterInlines.h"
#include "DFGAtTailAbstractState.h"
#include "DFGBasicBlockInlines.h"
#include "DFGClobberSet.h"
#include "DFGClobberize.h"
#include "DFGControlEquivalenceAnalysis.h"
#include "DFGEdgeDominates.h"
#include "DFGGraph.h"
#include "DFGInsertionSet.h"
#include "DFGMayExit.h"
#include "DFGNaturalLoops.h"
#include "DFGPhase.h"
#include "DFGSafeToExecute.h"
#include "JSCInlines.h"
namespace JSC { namespace DFG {
class LICMPhase : public Phase {
static const bool verbose = false;
using NaturalLoop = SSANaturalLoop;
struct LoopData {
ClobberSet writes;
BasicBlock* preHeader { nullptr };
};
public:
LICMPhase(Graph& graph)
: Phase(graph, "LICM")
, m_state(graph)
, m_interpreter(graph, m_state)
{
}
bool run()
{
DFG_ASSERT(m_graph, nullptr, m_graph.m_form == SSA);
m_graph.ensureSSADominators();
m_graph.ensureSSANaturalLoops();
m_graph.ensureControlEquivalenceAnalysis();
if (verbose) {
dataLog("Graph before LICM:\n");
m_graph.dump();
}
m_data.resize(m_graph.m_ssaNaturalLoops->numLoops());
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
if (!block->cfaHasVisited)
continue;
const NaturalLoop* loop = m_graph.m_ssaNaturalLoops->innerMostLoopOf(block);
if (!loop)
continue;
LoopData& data = m_data[loop->index()];
for (auto* node : *block) {
if (node->op() == ForceOSRExit)
break;
addWrites(m_graph, node, data.writes);
}
}
for (unsigned loopIndex = m_graph.m_ssaNaturalLoops->numLoops(); loopIndex--;) {
const NaturalLoop& loop = m_graph.m_ssaNaturalLoops->loop(loopIndex);
LoopData& data = m_data[loop.index()];
for (
const NaturalLoop* outerLoop = m_graph.m_ssaNaturalLoops->innerMostOuterLoop(loop);
outerLoop;
outerLoop = m_graph.m_ssaNaturalLoops->innerMostOuterLoop(*outerLoop))
m_data[outerLoop->index()].writes.addAll(data.writes);
BasicBlock* header = loop.header();
BasicBlock* preHeader = nullptr;
unsigned numberOfPreHeaders = 0;
DFG_ASSERT(m_graph, header->at(0), header->predecessors.size() > 1, header->predecessors.size());
for (unsigned i = header->predecessors.size(); i--;) {
BasicBlock* predecessor = header->predecessors[i];
if (m_graph.m_ssaDominators->dominates(header, predecessor))
continue;
preHeader = predecessor;
++numberOfPreHeaders;
}
if (numberOfPreHeaders != 1)
continue;
DFG_ASSERT(m_graph, preHeader->terminal(), preHeader->terminal()->op() == Jump, preHeader->terminal()->op());
if (!preHeader->terminal()->origin.exitOK)
continue;
data.preHeader = preHeader;
}
m_graph.initializeNodeOwners();
Vector<const NaturalLoop*> loopStack;
bool changed = false;
for (BasicBlock* block : m_graph.blocksInPreOrder()) {
if (!block->cfaHasVisited)
continue;
const NaturalLoop* loop = m_graph.m_ssaNaturalLoops->innerMostLoopOf(block);
if (!loop)
continue;
loopStack.shrink(0);
for (
const NaturalLoop* current = loop;
current;
current = m_graph.m_ssaNaturalLoops->innerMostOuterLoop(*current))
loopStack.append(current);
if (verbose) {
dataLog(
"Attempting to hoist out of block ", *block, " in loops:\n");
for (unsigned stackIndex = loopStack.size(); stackIndex--;) {
dataLog(
" ", *loopStack[stackIndex], ", which writes ",
m_data[loopStack[stackIndex]->index()].writes, "\n");
}
}
for (unsigned nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex) {
Node*& nodeRef = block->at(nodeIndex);
if (nodeRef->op() == ForceOSRExit)
break;
for (unsigned stackIndex = loopStack.size(); stackIndex--;)
changed |= attemptHoist(block, nodeRef, loopStack[stackIndex]);
}
}
return changed;
}
private:
bool attemptHoist(BasicBlock* fromBlock, Node*& nodeRef, const NaturalLoop* loop)
{
Node* node = nodeRef;
LoopData& data = m_data[loop->index()];
if (!data.preHeader) {
if (verbose)
dataLog(" Not hoisting ", node, " because the pre-header is invalid.\n");
return false;
}
if (!data.preHeader->cfaDidFinish) {
if (verbose)
dataLog(" Not hoisting ", node, " because CFA is invalid.\n");
return false;
}
m_state.initializeTo(data.preHeader);
NodeOrigin originalOrigin = node->origin;
bool canSpeculateBlindly = !m_graph.hasGlobalExitSite(originalOrigin.semantic, HoistingFailed);
bool isControlEquivalent = m_graph.m_controlEquivalenceAnalysis->dominatesEquivalently(data.preHeader, fromBlock);
bool addsBlindSpeculation = !isControlEquivalent;
NodeOrigin terminalOrigin = data.preHeader->terminal()->origin;
Vector<Node*, 2> hoistedNodes;
auto insertHoistedNode = [&] (Node* node) {
data.preHeader->insertBeforeTerminal(node);
node->owner = data.preHeader;
node->origin = terminalOrigin.withSemantic(node->origin.semantic);
node->origin.wasHoisted |= addsBlindSpeculation;
hoistedNodes.append(node);
};
auto updateAbstractState = [&] {
m_state.trustEdgeProofs();
for (unsigned i = 0; i < hoistedNodes.size(); ++i)
m_interpreter.execute(hoistedNodes[i]);
m_state.dontTrustEdgeProofs();
for (unsigned bodyIndex = loop->size(); bodyIndex--;) {
BasicBlock* subBlock = loop->at(bodyIndex);
const NaturalLoop* subLoop = m_graph.m_ssaNaturalLoops->headerOf(subBlock);
if (!subLoop)
continue;
BasicBlock* subPreHeader = m_data[subLoop->index()].preHeader;
if (!subPreHeader)
continue;
if (!subPreHeader->cfaDidFinish)
continue;
if (subPreHeader == data.preHeader)
continue;
m_state.initializeTo(subPreHeader);
for (unsigned i = 0; i < hoistedNodes.size(); ++i)
m_interpreter.execute(hoistedNodes[i]);
}
};
auto tryHoistChecks = [&] {
if (addsBlindSpeculation && !canSpeculateBlindly)
return false;
ASSERT(hoistedNodes.isEmpty());
Vector<Edge, 3> checks;
m_graph.doToChildren(node, [&] (Edge edge) {
if (!m_graph.m_ssaDominators->dominates(edge.node()->owner, data.preHeader))
return;
if (!edge.willHaveCheck())
return;
if ((m_state.forNode(edge).m_type & SpecEmpty) && checkMayCrashIfInputIsEmpty(edge.useKind())) {
if (!canSpeculateBlindly)
return;
Node* checkNotEmpty = m_graph.addNode(CheckNotEmpty, originalOrigin, Edge(edge.node(), UntypedUse));
insertHoistedNode(checkNotEmpty);
}
checks.append(edge);
});
if (checks.isEmpty())
return false;
AdjacencyList children;
NodeType checkOp = Check;
if (checks.size() <= AdjacencyList::Size) {
children = AdjacencyList(AdjacencyList::Fixed);
for (unsigned i = 0; i < checks.size(); ++i)
children.setChild(i, checks[i]);
} else {
checkOp = CheckVarargs;
unsigned firstChild = m_graph.m_varArgChildren.size();
for (Edge edge : checks)
m_graph.m_varArgChildren.append(edge);
children = AdjacencyList(AdjacencyList::Variable, firstChild, checks.size());
}
Node* check = m_graph.addNode(checkOp, originalOrigin, children);
insertHoistedNode(check);
updateAbstractState();
if (verbose)
dataLogLn(" Hoisted some checks from ", node, " and created a new Check ", check, ". Hoisted from ", *fromBlock, " to ", *data.preHeader);
return true;
};
if (!edgesDominate(m_graph, node, data.preHeader)) {
if (verbose) {
dataLog(
" Not hoisting ", node, " because it isn't loop invariant.\n");
}
return tryHoistChecks();
}
if (doesWrites(m_graph, node)) {
if (verbose)
dataLog(" Not hoisting ", node, " because it writes things.\n");
return tryHoistChecks();
}
addsBlindSpeculation = mayExit(m_graph, node, m_state) && !isControlEquivalent;
if (readsOverlap(m_graph, node, data.writes)) {
if (verbose) {
dataLog(
" Not hoisting ", node,
" because it reads things that the loop writes.\n");
}
return tryHoistChecks();
}
if (addsBlindSpeculation && !canSpeculateBlindly) {
if (verbose) {
dataLog(
" Not hoisting ", node, " because it may exit and the pre-header (",
*data.preHeader, ") is not control equivalent to the node's original block (",
*fromBlock, ") and hoisting had previously failed.\n");
}
return tryHoistChecks();
}
if (!safeToExecute(m_state, m_graph, node)) {
bool ignoreEmptyChildren = true;
if (canSpeculateBlindly
&& safeToExecute(m_state, m_graph, node, ignoreEmptyChildren)) {
if (verbose) {
dataLog(
" Rescuing hoisting by inserting empty checks.\n");
}
m_graph.doToChildren(
node,
[&] (Edge& edge) {
if (!(m_state.forNode(edge).m_type & SpecEmpty))
return;
Node* check = m_graph.addNode(CheckNotEmpty, originalOrigin, Edge(edge.node(), UntypedUse));
insertHoistedNode(check);
});
} else {
if (verbose) {
dataLog(
" Not hoisting ", node, " because it isn't safe to execute.\n");
}
return tryHoistChecks();
}
}
if (verbose) {
dataLog(
" Hoisting ", node, " from ", *fromBlock, " to ", *data.preHeader,
"\n");
}
insertHoistedNode(node);
updateAbstractState();
if (node->flags() & NodeHasVarArgs)
nodeRef = m_graph.addNode(CheckVarargs, originalOrigin, m_graph.copyVarargChildren(node));
else
nodeRef = m_graph.addNode(Check, originalOrigin, node->children);
return true;
}
AtTailAbstractState m_state;
AbstractInterpreter<AtTailAbstractState> m_interpreter;
Vector<LoopData> m_data;
};
bool performLICM(Graph& graph)
{
return runPhase<LICMPhase>(graph);
}
} }
#endif // ENABLE(DFG_JIT)