#pragma once
#if ENABLE(JIT)
#include "ArithProfile.h"
#include "CCallHelpers.h"
#include "JITAddGenerator.h"
#include "JITMathICInlineResult.h"
#include "JITMulGenerator.h"
#include "JITNegGenerator.h"
#include "JITSubGenerator.h"
#include "LinkBuffer.h"
#include "Repatch.h"
#include "SnippetOperand.h"
namespace JSC {
class LinkBuffer;
struct MathICGenerationState {
MacroAssembler::Label fastPathStart;
MacroAssembler::Label fastPathEnd;
MacroAssembler::Label slowPathStart;
MacroAssembler::Call slowPathCall;
MacroAssembler::JumpList slowPathJumps;
bool shouldSlowPathRepatch;
};
#define ENABLE_MATH_IC_STATS 0
template <typename GeneratorType, bool(*isProfileEmpty)(ArithProfile&)>
class JITMathIC {
public:
JITMathIC(ArithProfile* arithProfile)
: m_arithProfile(arithProfile)
{
}
CodeLocationLabel doneLocation() { return m_inlineStart.labelAtOffset(m_inlineSize); }
CodeLocationLabel slowPathStartLocation() { return m_inlineStart.labelAtOffset(m_deltaFromStartToSlowPathStart); }
CodeLocationCall slowPathCallLocation() { return m_inlineStart.callAtOffset(m_deltaFromStartToSlowPathCallLocation); }
bool generateInline(CCallHelpers& jit, MathICGenerationState& state, bool shouldEmitProfiling = true)
{
#if CPU(ARM_TRADITIONAL)
return false;
#endif
state.fastPathStart = jit.label();
size_t startSize = jit.m_assembler.buffer().codeSize();
if (m_arithProfile) {
if (isProfileEmpty(*m_arithProfile)) {
state.slowPathJumps.append(jit.patchableJump());
size_t inlineSize = jit.m_assembler.buffer().codeSize() - startSize;
ASSERT_UNUSED(inlineSize, static_cast<ptrdiff_t>(inlineSize) <= MacroAssembler::patchableJumpSize());
state.shouldSlowPathRepatch = true;
state.fastPathEnd = jit.label();
ASSERT(!m_generateFastPathOnRepatch); m_generateFastPathOnRepatch = true;
return true;
}
}
JITMathICInlineResult result = m_generator.generateInline(jit, state, m_arithProfile);
switch (result) {
case JITMathICInlineResult::GeneratedFastPath: {
size_t inlineSize = jit.m_assembler.buffer().codeSize() - startSize;
if (static_cast<ptrdiff_t>(inlineSize) < MacroAssembler::patchableJumpSize()) {
size_t nopsToEmitInBytes = MacroAssembler::patchableJumpSize() - inlineSize;
jit.emitNops(nopsToEmitInBytes);
}
state.shouldSlowPathRepatch = true;
state.fastPathEnd = jit.label();
return true;
}
case JITMathICInlineResult::GenerateFullSnippet: {
MacroAssembler::JumpList endJumpList;
bool result = m_generator.generateFastPath(jit, endJumpList, state.slowPathJumps, m_arithProfile, shouldEmitProfiling);
if (result) {
state.fastPathEnd = jit.label();
state.shouldSlowPathRepatch = false;
endJumpList.link(&jit);
return true;
}
return false;
}
case JITMathICInlineResult::DontGenerate: {
return false;
}
default:
ASSERT_NOT_REACHED();
}
return false;
}
void generateOutOfLine(VM& vm, CodeBlock* codeBlock, FunctionPtr callReplacement)
{
auto linkJumpToOutOfLineSnippet = [&] () {
CCallHelpers jit(&vm, codeBlock);
auto jump = jit.jump();
bool needsBranchCompaction = false;
RELEASE_ASSERT(jit.m_assembler.buffer().codeSize() <= static_cast<size_t>(m_inlineSize));
LinkBuffer linkBuffer(jit, m_inlineStart.dataLocation(), jit.m_assembler.buffer().codeSize(), JITCompilationMustSucceed, needsBranchCompaction);
RELEASE_ASSERT(linkBuffer.isValid());
linkBuffer.link(jump, CodeLocationLabel(m_code.code()));
FINALIZE_CODE(linkBuffer, ("JITMathIC: linking constant jump to out of line stub"));
};
auto replaceCall = [&] () {
ftlThunkAwareRepatchCall(codeBlock, slowPathCallLocation(), callReplacement);
};
bool shouldEmitProfiling = !JITCode::isOptimizingJIT(codeBlock->jitType());
if (m_generateFastPathOnRepatch) {
CCallHelpers jit(&vm, codeBlock);
MathICGenerationState generationState;
bool generatedInline = generateInline(jit, generationState, shouldEmitProfiling);
m_generateFastPathOnRepatch = false;
if (generatedInline) {
auto jumpToDone = jit.jump();
LinkBuffer linkBuffer(vm, jit, codeBlock, JITCompilationCanFail);
if (!linkBuffer.didFailToAllocate()) {
linkBuffer.link(generationState.slowPathJumps, slowPathStartLocation());
linkBuffer.link(jumpToDone, doneLocation());
m_code = FINALIZE_CODE_FOR(
codeBlock, linkBuffer, ("JITMathIC: generating out of line fast IC snippet"));
if (!generationState.shouldSlowPathRepatch) {
replaceCall();
}
linkJumpToOutOfLineSnippet();
return;
}
}
}
replaceCall();
{
CCallHelpers jit(&vm, codeBlock);
MacroAssembler::JumpList endJumpList;
MacroAssembler::JumpList slowPathJumpList;
bool emittedFastPath = m_generator.generateFastPath(jit, endJumpList, slowPathJumpList, m_arithProfile, shouldEmitProfiling);
if (!emittedFastPath)
return;
endJumpList.append(jit.jump());
LinkBuffer linkBuffer(vm, jit, codeBlock, JITCompilationCanFail);
if (linkBuffer.didFailToAllocate())
return;
linkBuffer.link(endJumpList, doneLocation());
linkBuffer.link(slowPathJumpList, slowPathStartLocation());
m_code = FINALIZE_CODE_FOR(
codeBlock, linkBuffer, ("JITMathIC: generating out of line IC snippet"));
}
linkJumpToOutOfLineSnippet();
}
void finalizeInlineCode(const MathICGenerationState& state, LinkBuffer& linkBuffer)
{
CodeLocationLabel start = linkBuffer.locationOf(state.fastPathStart);
m_inlineStart = start;
m_inlineSize = MacroAssembler::differenceBetweenCodePtr(
start, linkBuffer.locationOf(state.fastPathEnd));
ASSERT(m_inlineSize > 0);
m_deltaFromStartToSlowPathCallLocation = MacroAssembler::differenceBetweenCodePtr(
start, linkBuffer.locationOf(state.slowPathCall));
m_deltaFromStartToSlowPathStart = MacroAssembler::differenceBetweenCodePtr(
start, linkBuffer.locationOf(state.slowPathStart));
}
ArithProfile* arithProfile() const { return m_arithProfile; }
#if ENABLE(MATH_IC_STATS)
size_t m_generatedCodeSize { 0 };
size_t codeSize() const
{
size_t result = m_generatedCodeSize;
if (m_code)
result += m_code.size();
return result;
}
#endif
ArithProfile* m_arithProfile;
MacroAssemblerCodeRef m_code;
CodeLocationLabel m_inlineStart;
int32_t m_inlineSize;
int32_t m_deltaFromStartToSlowPathCallLocation;
int32_t m_deltaFromStartToSlowPathStart;
bool m_generateFastPathOnRepatch { false };
GeneratorType m_generator;
};
inline bool isBinaryProfileEmpty(ArithProfile& arithProfile)
{
return arithProfile.lhsObservedType().isEmpty() || arithProfile.rhsObservedType().isEmpty();
}
template <typename GeneratorType>
class JITBinaryMathIC : public JITMathIC<GeneratorType, isBinaryProfileEmpty> {
public:
JITBinaryMathIC(ArithProfile* arithProfile)
: JITMathIC<GeneratorType, isBinaryProfileEmpty>(arithProfile)
{
}
};
typedef JITBinaryMathIC<JITAddGenerator> JITAddIC;
typedef JITBinaryMathIC<JITMulGenerator> JITMulIC;
typedef JITBinaryMathIC<JITSubGenerator> JITSubIC;
inline bool isUnaryProfileEmpty(ArithProfile& arithProfile)
{
return arithProfile.lhsObservedType().isEmpty();
}
template <typename GeneratorType>
class JITUnaryMathIC : public JITMathIC<GeneratorType, isUnaryProfileEmpty> {
public:
JITUnaryMathIC(ArithProfile* arithProfile)
: JITMathIC<GeneratorType, isUnaryProfileEmpty>(arithProfile)
{
}
};
typedef JITUnaryMathIC<JITNegGenerator> JITNegIC;
}
#endif // ENABLE(JIT)