DFGJITCompiler.cpp [plain text]
#include "config.h"
#include "DFGJITCompiler.h"
#if ENABLE(DFG_JIT)
#include "CodeBlock.h"
#include "DFGJITCodeGenerator.h"
#include "DFGNonSpeculativeJIT.h"
#include "DFGOperations.h"
#include "DFGRegisterBank.h"
#include "DFGSpeculativeJIT.h"
#include "JSGlobalData.h"
#include "LinkBuffer.h"
namespace JSC { namespace DFG {
void JITCompiler::fillNumericToDouble(NodeIndex nodeIndex, FPRReg fpr, GPRReg temporary)
{
Node& node = graph()[nodeIndex];
if (node.isConstant()) {
ASSERT(isDoubleConstant(nodeIndex));
move(MacroAssembler::ImmPtr(reinterpret_cast<void*>(reinterpretDoubleToIntptr(valueOfDoubleConstant(nodeIndex)))), temporary);
movePtrToDouble(temporary, fpr);
} else {
loadPtr(addressFor(node.virtualRegister()), temporary);
Jump isInteger = branchPtr(MacroAssembler::AboveOrEqual, temporary, GPRInfo::tagTypeNumberRegister);
jitAssertIsJSDouble(temporary);
addPtr(GPRInfo::tagTypeNumberRegister, temporary);
movePtrToDouble(temporary, fpr);
Jump hasUnboxedDouble = jump();
isInteger.link(this);
convertInt32ToDouble(temporary, fpr);
hasUnboxedDouble.link(this);
}
}
void JITCompiler::fillInt32ToInteger(NodeIndex nodeIndex, GPRReg gpr)
{
Node& node = graph()[nodeIndex];
if (node.isConstant()) {
ASSERT(isInt32Constant(nodeIndex));
move(MacroAssembler::Imm32(valueOfInt32Constant(nodeIndex)), gpr);
} else {
#if DFG_JIT_ASSERT
loadPtr(addressFor(node.virtualRegister()), gpr);
jitAssertIsJSInt32(gpr);
#endif
load32(addressFor(node.virtualRegister()), gpr);
}
}
void JITCompiler::fillToJS(NodeIndex nodeIndex, GPRReg gpr)
{
Node& node = graph()[nodeIndex];
if (node.isConstant()) {
if (isInt32Constant(nodeIndex)) {
JSValue jsValue = jsNumber(valueOfInt32Constant(nodeIndex));
move(MacroAssembler::ImmPtr(JSValue::encode(jsValue)), gpr);
} else if (isDoubleConstant(nodeIndex)) {
JSValue jsValue(JSValue::EncodeAsDouble, valueOfDoubleConstant(nodeIndex));
move(MacroAssembler::ImmPtr(JSValue::encode(jsValue)), gpr);
} else {
ASSERT(isJSConstant(nodeIndex));
JSValue jsValue = valueOfJSConstant(nodeIndex);
move(MacroAssembler::ImmPtr(JSValue::encode(jsValue)), gpr);
}
return;
}
loadPtr(addressFor(node.virtualRegister()), gpr);
}
class GeneralizedRegister {
public:
GeneralizedRegister() { }
static GeneralizedRegister createGPR(GPRReg gpr)
{
GeneralizedRegister result;
result.m_isFPR = false;
result.m_register.gpr = gpr;
return result;
}
static GeneralizedRegister createFPR(FPRReg fpr)
{
GeneralizedRegister result;
result.m_isFPR = true;
result.m_register.fpr = fpr;
return result;
}
bool isFPR() const
{
return m_isFPR;
}
GPRReg gpr() const
{
ASSERT(!m_isFPR);
return m_register.gpr;
}
FPRReg fpr() const
{
ASSERT(m_isFPR);
return m_register.fpr;
}
const SpeculationCheck::RegisterInfo& findInSpeculationCheck(const SpeculationCheck& check)
{
if (isFPR())
return check.m_fprInfo[FPRInfo::toIndex(fpr())];
return check.m_gprInfo[GPRInfo::toIndex(gpr())];
}
const EntryLocation::RegisterInfo& findInEntryLocation(const EntryLocation& entry)
{
if (isFPR())
return entry.m_fprInfo[FPRInfo::toIndex(fpr())];
return entry.m_gprInfo[GPRInfo::toIndex(gpr())];
}
DataFormat previousDataFormat(const SpeculationCheck& check)
{
return findInSpeculationCheck(check).format;
}
DataFormat nextDataFormat(const EntryLocation& entry)
{
return findInEntryLocation(entry).format;
}
void convert(DataFormat oldDataFormat, DataFormat newDataFormat, JITCompiler& jit)
{
if (LIKELY(!needDataFormatConversion(oldDataFormat, newDataFormat)))
return;
if (oldDataFormat == DataFormatInteger) {
jit.orPtr(GPRInfo::tagTypeNumberRegister, gpr());
return;
}
ASSERT(newDataFormat == DataFormatInteger);
jit.zeroExtend32ToPtr(gpr(), gpr());
return;
}
void moveTo(GeneralizedRegister& other, DataFormat myDataFormat, DataFormat otherDataFormat, JITCompiler& jit, FPRReg scratchFPR)
{
if (UNLIKELY(isFPR())) {
if (UNLIKELY(other.isFPR())) {
jit.moveDouble(fpr(), other.fpr());
return;
}
JITCompiler::Jump done;
if (scratchFPR != InvalidFPRReg) {
JITCompiler::JumpList notInt;
jit.branchConvertDoubleToInt32(fpr(), other.gpr(), notInt, scratchFPR);
jit.orPtr(GPRInfo::tagTypeNumberRegister, other.gpr());
done = jit.jump();
notInt.link(&jit);
}
jit.moveDoubleToPtr(fpr(), other.gpr());
jit.subPtr(GPRInfo::tagTypeNumberRegister, other.gpr());
if (done.isSet())
done.link(&jit);
return;
}
if (UNLIKELY(other.isFPR())) {
jit.addPtr(GPRInfo::tagTypeNumberRegister, gpr());
jit.movePtrToDouble(gpr(), other.fpr());
return;
}
if (LIKELY(!needDataFormatConversion(myDataFormat, otherDataFormat))) {
jit.move(gpr(), other.gpr());
return;
}
if (myDataFormat == DataFormatInteger) {
jit.orPtr(gpr(), GPRInfo::tagTypeNumberRegister, other.gpr());
return;
}
ASSERT(otherDataFormat == DataFormatInteger);
jit.zeroExtend32ToPtr(gpr(), other.gpr());
}
void swapWith(GeneralizedRegister& other, DataFormat myDataFormat, DataFormat myNewDataFormat, DataFormat otherDataFormat, DataFormat otherNewDataFormat, JITCompiler& jit, GPRReg scratchGPR, FPRReg scratchFPR)
{
if (UNLIKELY(isFPR())) {
if (UNLIKELY(other.isFPR())) {
if (scratchFPR == InvalidFPRReg)
jit.moveDoubleToPtr(fpr(), scratchGPR);
else
jit.moveDouble(fpr(), scratchFPR);
jit.moveDouble(other.fpr(), fpr());
if (scratchFPR == InvalidFPRReg)
jit.movePtrToDouble(scratchGPR, other.fpr());
else
jit.moveDouble(scratchFPR, other.fpr());
return;
}
jit.move(other.gpr(), scratchGPR);
JITCompiler::Jump done;
if (scratchFPR != InvalidFPRReg) {
JITCompiler::JumpList notInt;
jit.branchConvertDoubleToInt32(fpr(), other.gpr(), notInt, scratchFPR);
jit.orPtr(GPRInfo::tagTypeNumberRegister, other.gpr());
done = jit.jump();
notInt.link(&jit);
}
jit.moveDoubleToPtr(fpr(), other.gpr());
jit.subPtr(GPRInfo::tagTypeNumberRegister, other.gpr());
if (done.isSet())
done.link(&jit);
jit.addPtr(GPRInfo::tagTypeNumberRegister, scratchGPR);
jit.movePtrToDouble(scratchGPR, fpr());
return;
}
if (UNLIKELY(other.isFPR())) {
other.swapWith(*this, otherDataFormat, otherNewDataFormat, myDataFormat, myNewDataFormat, jit, scratchGPR, scratchFPR);
return;
}
jit.swap(gpr(), other.gpr());
if (UNLIKELY(needDataFormatConversion(otherDataFormat, myNewDataFormat))) {
if (otherDataFormat == DataFormatInteger)
jit.orPtr(GPRInfo::tagTypeNumberRegister, gpr());
else if (myNewDataFormat == DataFormatInteger)
jit.zeroExtend32ToPtr(gpr(), gpr());
}
if (UNLIKELY(needDataFormatConversion(myDataFormat, otherNewDataFormat))) {
if (myDataFormat == DataFormatInteger)
jit.orPtr(GPRInfo::tagTypeNumberRegister, other.gpr());
else if (otherNewDataFormat == DataFormatInteger)
jit.zeroExtend32ToPtr(other.gpr(), other.gpr());
}
}
private:
bool m_isFPR;
union {
GPRReg gpr;
FPRReg fpr;
} m_register;
};
struct ShuffledRegister {
GeneralizedRegister reg;
ShuffledRegister* previous;
bool hasFrom;
bool hasTo;
bool handled;
ShuffledRegister() { }
ShuffledRegister(GeneralizedRegister reg)
: reg(reg)
, previous(0)
, hasFrom(false)
, hasTo(false)
, handled(false)
{
}
bool isEndOfNonCyclingPermutation()
{
return hasTo && !hasFrom;
}
void handleNonCyclingPermutation(const SpeculationCheck& check, const EntryLocation& entry, JITCompiler& jit, FPRReg& scratchFPR1, FPRReg& scratchFPR2)
{
ShuffledRegister* cur = this;
while (cur->previous) {
cur->previous->reg.moveTo(cur->reg, cur->previous->reg.previousDataFormat(check), cur->reg.nextDataFormat(entry), jit, scratchFPR1);
cur->handled = true;
if (cur->reg.isFPR()) {
if (scratchFPR1 == InvalidFPRReg)
scratchFPR1 = cur->reg.fpr();
else {
ASSERT(scratchFPR1 != cur->reg.fpr());
scratchFPR2 = cur->reg.fpr();
}
}
cur = cur->previous;
}
cur->handled = true;
if (cur->reg.isFPR()) {
if (scratchFPR1 == InvalidFPRReg)
scratchFPR1 = cur->reg.fpr();
else {
ASSERT(scratchFPR1 != cur->reg.fpr());
scratchFPR2 = cur->reg.fpr();
}
}
}
void handleCyclingPermutation(const SpeculationCheck& check, const EntryLocation& entry, JITCompiler& jit, GPRReg scratchGPR, FPRReg scratchFPR1, FPRReg scratchFPR2)
{
unsigned cycleLength = 0;
ShuffledRegister* cur = this;
ShuffledRegister* next = 0;
do {
ASSERT(cur);
cycleLength++;
cur->handled = true;
next = cur;
cur = cur->previous;
} while (cur != this);
ASSERT(cycleLength);
ASSERT(next->previous == cur);
switch (cycleLength) {
case 1:
reg.convert(reg.previousDataFormat(check), reg.nextDataFormat(entry), jit);
break;
case 2:
reg.swapWith(previous->reg, reg.previousDataFormat(check), reg.nextDataFormat(entry), previous->reg.previousDataFormat(check), previous->reg.nextDataFormat(entry), jit, scratchGPR, scratchFPR1);
break;
default:
GeneralizedRegister scratch;
if (UNLIKELY(reg.isFPR() && next->reg.isFPR())) {
if (scratchFPR2 == InvalidFPRReg) {
scratch = GeneralizedRegister::createGPR(scratchGPR);
reg.moveTo(scratch, DataFormatDouble, DataFormatJSDouble, jit, scratchFPR1);
} else {
scratch = GeneralizedRegister::createFPR(scratchFPR2);
reg.moveTo(scratch, DataFormatDouble, DataFormatDouble, jit, scratchFPR1);
}
} else {
scratch = GeneralizedRegister::createGPR(scratchGPR);
reg.moveTo(scratch, reg.previousDataFormat(check), next->reg.nextDataFormat(entry), jit, scratchFPR1);
}
cur = this;
while (cur->previous != this) {
ASSERT(cur);
cur->previous->reg.moveTo(cur->reg, cur->previous->reg.previousDataFormat(check), cur->reg.nextDataFormat(entry), jit, scratchFPR1);
cur = cur->previous;
}
if (UNLIKELY(reg.isFPR() && next->reg.isFPR())) {
if (scratchFPR2 == InvalidFPRReg)
scratch.moveTo(next->reg, DataFormatJSDouble, DataFormatDouble, jit, scratchFPR1);
else
scratch.moveTo(next->reg, DataFormatDouble, DataFormatDouble, jit, scratchFPR1);
} else
scratch.moveTo(next->reg, next->reg.nextDataFormat(entry), next->reg.nextDataFormat(entry), jit, scratchFPR1);
break;
}
}
static ShuffledRegister* lookup(ShuffledRegister* gprs, ShuffledRegister* fprs, GeneralizedRegister& reg)
{
if (reg.isFPR())
return fprs + FPRInfo::toIndex(reg.fpr());
return gprs + GPRInfo::toIndex(reg.gpr());
}
};
template<typename T>
T& lookupForRegister(T* gprs, T* fprs, unsigned index)
{
ASSERT(index < GPRInfo::numberOfRegisters + FPRInfo::numberOfRegisters);
if (index < GPRInfo::numberOfRegisters)
return gprs[index];
return fprs[index - GPRInfo::numberOfRegisters];
}
class NodeToRegisterMap {
public:
struct Tuple {
NodeIndex first;
GeneralizedRegister second;
Tuple()
{
}
};
typedef Tuple* iterator;
NodeToRegisterMap()
: m_occupancy(0)
{
}
void set(NodeIndex first, GeneralizedRegister second)
{
m_payload[m_occupancy].first = first;
m_payload[m_occupancy].second = second;
m_occupancy++;
}
Tuple* end()
{
return 0;
}
Tuple* find(NodeIndex first)
{
for (unsigned i = m_occupancy; i-- > 0;) {
if (m_payload[i].first == first)
return m_payload + i;
}
return 0;
}
void clear()
{
m_occupancy = 0;
}
private:
Tuple m_payload[GPRInfo::numberOfRegisters + FPRInfo::numberOfRegisters];
unsigned m_occupancy;
};
void JITCompiler::jumpFromSpeculativeToNonSpeculative(const SpeculationCheck& check, const EntryLocation& entry, SpeculationRecovery* recovery, NodeToRegisterMap& checkNodeToRegisterMap, NodeToRegisterMap& entryNodeToRegisterMap)
{
ASSERT(check.m_nodeIndex == entry.m_nodeIndex);
check.m_check.link(this);
#if DFG_DEBUG_VERBOSE
fprintf(stderr, "Speculation failure for Node @%d at JIT offset 0x%x\n", (int)check.m_nodeIndex, debugOffset());
#endif
#if DFG_JIT_BREAK_ON_SPECULATION_FAILURE
breakpoint();
#endif
if (recovery) {
ASSERT(recovery->type() == SpeculativeAdd);
ASSERT(check.m_gprInfo[GPRInfo::toIndex(recovery->dest())].nodeIndex != NoNode);
sub32(recovery->src(), recovery->dest());
DataFormat format = check.m_gprInfo[GPRInfo::toIndex(recovery->dest())].format;
ASSERT(format == DataFormatInteger || format == DataFormatJSInteger || format == DataFormatJS);
if (format != DataFormatInteger)
orPtr(GPRInfo::tagTypeNumberRegister, recovery->dest());
}
checkNodeToRegisterMap.clear();
entryNodeToRegisterMap.clear();
GPRReg scratchGPR = InvalidGPRReg;
FPRReg scratchFPR1 = InvalidFPRReg;
FPRReg scratchFPR2 = InvalidFPRReg;
bool needToRestoreTagMaskRegister = false;
for (unsigned index = 0; index < GPRInfo::numberOfRegisters; ++index) {
NodeIndex nodeIndexInCheck = check.m_gprInfo[index].nodeIndex;
if (nodeIndexInCheck != NoNode)
checkNodeToRegisterMap.set(nodeIndexInCheck, GeneralizedRegister::createGPR(GPRInfo::toRegister(index)));
NodeIndex nodeIndexInEntry = entry.m_gprInfo[index].nodeIndex;
if (nodeIndexInEntry != NoNode)
entryNodeToRegisterMap.set(nodeIndexInEntry, GeneralizedRegister::createGPR(GPRInfo::toRegister(index)));
else if (nodeIndexInCheck == NoNode)
scratchGPR = GPRInfo::toRegister(index);
}
for (unsigned index = 0; index < FPRInfo::numberOfRegisters; ++index) {
NodeIndex nodeIndexInCheck = check.m_fprInfo[index].nodeIndex;
if (nodeIndexInCheck != NoNode)
checkNodeToRegisterMap.set(nodeIndexInCheck, GeneralizedRegister::createFPR(FPRInfo::toRegister(index)));
NodeIndex nodeIndexInEntry = entry.m_fprInfo[index].nodeIndex;
if (nodeIndexInEntry != NoNode)
entryNodeToRegisterMap.set(nodeIndexInEntry, GeneralizedRegister::createFPR(FPRInfo::toRegister(index)));
else if (nodeIndexInCheck == NoNode) {
if (scratchFPR1 == InvalidFPRReg)
scratchFPR1 = FPRInfo::toRegister(index);
else
scratchFPR2 = FPRInfo::toRegister(index);
}
}
ASSERT((scratchFPR1 == InvalidFPRReg && scratchFPR2 == InvalidFPRReg) || (scratchFPR1 != scratchFPR2));
ShuffledRegister gprs[GPRInfo::numberOfRegisters];
ShuffledRegister fprs[FPRInfo::numberOfRegisters];
for (unsigned index = 0; index < GPRInfo::numberOfRegisters; ++index)
gprs[index] = ShuffledRegister(GeneralizedRegister::createGPR(GPRInfo::toRegister(index)));
for (unsigned index = 0; index < FPRInfo::numberOfRegisters; ++index)
fprs[index] = ShuffledRegister(GeneralizedRegister::createFPR(FPRInfo::toRegister(index)));
for (unsigned index = 0; index < GPRInfo::numberOfRegisters; ++index) {
NodeIndex nodeIndex = check.m_gprInfo[index].nodeIndex;
if (nodeIndex == NoNode)
continue;
NodeToRegisterMap::iterator mapIterator = entryNodeToRegisterMap.find(nodeIndex);
if (mapIterator != entryNodeToRegisterMap.end()) {
gprs[index].hasFrom = true;
ShuffledRegister* next = ShuffledRegister::lookup(gprs, fprs, mapIterator->second);
next->previous = gprs + index;
next->hasTo = true;
if (!mapIterator->second.findInEntryLocation(entry).isSpilled)
continue;
} else {
if (entry.m_gprInfo[index].nodeIndex == NoNode)
scratchGPR = GPRInfo::toRegister(index);
}
if (check.m_gprInfo[index].isSpilled)
continue;
DataFormat dataFormat = check.m_gprInfo[index].format;
VirtualRegister virtualRegister = graph()[nodeIndex].virtualRegister();
ASSERT(dataFormat == DataFormatInteger || DataFormatCell || dataFormat & DataFormatJS);
if (dataFormat == DataFormatInteger)
orPtr(GPRInfo::tagTypeNumberRegister, GPRInfo::toRegister(index));
storePtr(GPRInfo::toRegister(index), addressFor(virtualRegister));
}
if (scratchGPR == InvalidGPRReg) {
scratchGPR = GPRInfo::tagMaskRegister;
needToRestoreTagMaskRegister = true;
}
for (unsigned index = 0; index < FPRInfo::numberOfRegisters; ++index) {
NodeIndex nodeIndex = check.m_fprInfo[index].nodeIndex;
if (nodeIndex == NoNode)
continue;
NodeToRegisterMap::iterator mapIterator = entryNodeToRegisterMap.find(nodeIndex);
if (mapIterator != entryNodeToRegisterMap.end()) {
fprs[index].hasFrom = true;
ShuffledRegister* next = ShuffledRegister::lookup(gprs, fprs, mapIterator->second);
next->previous = fprs + index;
next->hasTo = true;
if (!mapIterator->second.findInEntryLocation(entry).isSpilled)
continue;
} else {
if (entry.m_fprInfo[index].nodeIndex == NoNode) {
if (scratchFPR1 == InvalidFPRReg)
scratchFPR1 = FPRInfo::toRegister(index);
else if (scratchFPR2)
scratchFPR2 = FPRInfo::toRegister(index);
ASSERT((scratchFPR1 == InvalidFPRReg && scratchFPR2 == InvalidFPRReg) || (scratchFPR1 != scratchFPR2));
}
}
if (check.m_fprInfo[index].isSpilled)
continue;
VirtualRegister virtualRegister = graph()[nodeIndex].virtualRegister();
moveDoubleToPtr(FPRInfo::toRegister(index), scratchGPR);
subPtr(GPRInfo::tagTypeNumberRegister, scratchGPR);
storePtr(scratchGPR, addressFor(virtualRegister));
}
#if !ASSERT_DISABLED
ASSERT(scratchGPR != InvalidGPRReg);
if (scratchGPR != GPRInfo::tagMaskRegister) {
ASSERT(!gprs[GPRInfo::toIndex(scratchGPR)].hasTo);
ASSERT(!gprs[GPRInfo::toIndex(scratchGPR)].hasFrom);
}
if (scratchFPR1 != InvalidFPRReg) {
ASSERT(scratchFPR1 != scratchFPR2);
ASSERT(!fprs[FPRInfo::toIndex(scratchFPR1)].hasTo);
ASSERT(!fprs[FPRInfo::toIndex(scratchFPR1)].hasFrom);
if (scratchFPR2 != InvalidFPRReg) {
ASSERT(!fprs[FPRInfo::toIndex(scratchFPR2)].hasTo);
ASSERT(!fprs[FPRInfo::toIndex(scratchFPR2)].hasFrom);
}
} else
ASSERT(scratchFPR2 == InvalidFPRReg);
#endif
for (unsigned index = 0; index < GPRInfo::numberOfRegisters + FPRInfo::numberOfRegisters; ++index) {
ShuffledRegister& reg = lookupForRegister(gprs, fprs, index);
if (!reg.isEndOfNonCyclingPermutation() || reg.handled || (!reg.hasFrom && !reg.hasTo))
continue;
reg.handleNonCyclingPermutation(check, entry, *this, scratchFPR1, scratchFPR2);
ASSERT((scratchFPR1 == InvalidFPRReg && scratchFPR2 == InvalidFPRReg) || (scratchFPR1 != scratchFPR2));
}
for (unsigned index = 0; index < GPRInfo::numberOfRegisters + FPRInfo::numberOfRegisters; ++index) {
ShuffledRegister& reg = lookupForRegister(gprs, fprs, index);
if (reg.handled || (!reg.hasFrom && !reg.hasTo))
continue;
reg.handleCyclingPermutation(check, entry, *this, scratchGPR, scratchFPR1, scratchFPR2);
ASSERT((scratchFPR1 == InvalidFPRReg && scratchFPR2 == InvalidFPRReg) || (scratchFPR1 != scratchFPR2));
}
#if !ASSERT_DISABLED
for (unsigned index = 0; index < GPRInfo::numberOfRegisters + FPRInfo::numberOfRegisters; ++index) {
ShuffledRegister& reg = lookupForRegister(gprs, fprs, index);
ASSERT(reg.handled || (!reg.hasFrom && !reg.hasTo));
}
#endif
for (unsigned index = 0; index < FPRInfo::numberOfRegisters; ++index) {
NodeIndex nodeIndex = entry.m_fprInfo[index].nodeIndex;
if (nodeIndex == NoNode || entry.m_fprInfo[index].isSpilled)
continue;
NodeToRegisterMap::iterator mapIterator = checkNodeToRegisterMap.find(nodeIndex);
if (mapIterator != checkNodeToRegisterMap.end()
&& !mapIterator->second.findInSpeculationCheck(check).isSpilled)
continue;
fillNumericToDouble(nodeIndex, FPRInfo::toRegister(index), GPRInfo::regT0);
}
for (unsigned index = 0; index < GPRInfo::numberOfRegisters; ++index) {
NodeIndex nodeIndex = entry.m_gprInfo[index].nodeIndex;
if (nodeIndex == NoNode || entry.m_gprInfo[index].isSpilled)
continue;
NodeToRegisterMap::iterator mapIterator = checkNodeToRegisterMap.find(nodeIndex);
if (mapIterator != checkNodeToRegisterMap.end()
&& !mapIterator->second.findInSpeculationCheck(check).isSpilled)
continue;
DataFormat dataFormat = entry.m_gprInfo[index].format;
if (dataFormat == DataFormatInteger)
fillInt32ToInteger(nodeIndex, GPRInfo::toRegister(index));
else {
ASSERT(dataFormat & DataFormatJS || dataFormat == DataFormatCell); fillToJS(nodeIndex, GPRInfo::toRegister(index));
}
}
if (needToRestoreTagMaskRegister)
move(TrustedImmPtr(reinterpret_cast<void*>(TagMask)), GPRInfo::tagMaskRegister);
jump(entry.m_entry);
}
void JITCompiler::linkSpeculationChecks(SpeculativeJIT& speculative, NonSpeculativeJIT& nonSpeculative)
{
SpeculationCheckVector::Iterator checksIter = speculative.speculationChecks().begin();
SpeculationCheckVector::Iterator checksEnd = speculative.speculationChecks().end();
NonSpeculativeJIT::EntryLocationVector::Iterator entriesIter = nonSpeculative.entryLocations().begin();
NonSpeculativeJIT::EntryLocationVector::Iterator entriesEnd = nonSpeculative.entryLocations().end();
NodeToRegisterMap checkNodeToRegisterMap;
NodeToRegisterMap entryNodeToRegisterMap;
while (checksIter != checksEnd) {
ASSERT(checksIter->m_nodeIndex == entriesIter->m_nodeIndex);
do {
ASSERT(checksIter != checksEnd);
ASSERT(entriesIter != entriesEnd);
const SpeculationCheck& check = *checksIter;
const EntryLocation& entry = *entriesIter;
jumpFromSpeculativeToNonSpeculative(check, entry, speculative.speculationRecovery(check.m_recoveryIndex), checkNodeToRegisterMap, entryNodeToRegisterMap);
++checksIter;
} while (checksIter != checksEnd && checksIter->m_nodeIndex == entriesIter->m_nodeIndex);
++entriesIter;
}
ASSERT(!(checksIter != checksEnd));
ASSERT(!(entriesIter != entriesEnd));
}
void JITCompiler::compileEntry()
{
preserveReturnAddressAfterCall(GPRInfo::regT2);
emitPutToCallFrameHeader(GPRInfo::regT2, RegisterFile::ReturnPC);
}
void JITCompiler::compileBody()
{
#if DFG_JIT_BREAK_ON_EVERY_FUNCTION
breakpoint();
#endif
Label speculativePathBegin = label();
SpeculativeJIT speculative(*this);
#if !DFG_DEBUG_LOCAL_DISBALE_SPECULATIVE
bool compiledSpeculative = speculative.compile();
#else
bool compiledSpeculative = false;
#endif
if (compiledSpeculative) {
SpeculationCheckIndexIterator checkIterator(speculative.speculationChecks());
NonSpeculativeJIT nonSpeculative(*this);
nonSpeculative.compile(checkIterator);
linkSpeculationChecks(speculative, nonSpeculative);
} else {
#if DFG_DEBUG_VERBOSE
fprintf(stderr, "SpeculativeJIT was terminated.\n");
#endif
m_calls.clear();
m_propertyAccesses.clear();
m_jsCalls.clear();
m_methodGets.clear();
rewindToLabel(speculativePathBegin);
SpeculationCheckVector noChecks;
SpeculationCheckIndexIterator checkIterator(noChecks);
NonSpeculativeJIT nonSpeculative(*this);
nonSpeculative.compile(checkIterator);
}
for (unsigned i = 0; i < m_calls.size(); ++i) {
Jump& exceptionCheck = m_calls[i].m_exceptionCheck;
if (exceptionCheck.isSet()) {
exceptionCheck.link(this);
++m_exceptionCheckCount;
}
}
if (m_exceptionCheckCount) {
move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0);
peek(GPRInfo::argumentGPR1, -1);
m_calls.append(CallRecord(call(), lookupExceptionHandler));
jump(GPRInfo::returnValueGPR2);
}
}
void JITCompiler::link(LinkBuffer& linkBuffer)
{
#if DFG_DEBUG_VERBOSE
fprintf(stderr, "JIT code start at %p\n", linkBuffer.debugAddress());
#endif
for (unsigned i = 0; i < m_calls.size(); ++i) {
if (m_calls[i].m_function.value())
linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function);
}
if (m_codeBlock->needsCallReturnIndices()) {
m_codeBlock->callReturnIndexVector().reserveCapacity(m_exceptionCheckCount);
for (unsigned i = 0; i < m_calls.size(); ++i) {
if (m_calls[i].m_handlesExceptions) {
unsigned returnAddressOffset = linkBuffer.returnAddressOffset(m_calls[i].m_call);
unsigned exceptionInfo = m_calls[i].m_exceptionInfo;
m_codeBlock->callReturnIndexVector().append(CallReturnOffsetToBytecodeOffset(returnAddressOffset, exceptionInfo));
}
}
}
m_codeBlock->setNumberOfStructureStubInfos(m_propertyAccesses.size());
for (unsigned i = 0; i < m_propertyAccesses.size(); ++i) {
StructureStubInfo& info = m_codeBlock->structureStubInfo(i);
info.callReturnLocation = linkBuffer.locationOf(m_propertyAccesses[i].m_functionCall);
info.u.unset.deltaCheckImmToCall = m_propertyAccesses[i].m_deltaCheckImmToCall;
info.deltaCallToStructCheck = m_propertyAccesses[i].m_deltaCallToStructCheck;
info.u.unset.deltaCallToLoadOrStore = m_propertyAccesses[i].m_deltaCallToLoadOrStore;
info.deltaCallToSlowCase = m_propertyAccesses[i].m_deltaCallToSlowCase;
info.deltaCallToDone = m_propertyAccesses[i].m_deltaCallToDone;
info.baseGPR = m_propertyAccesses[i].m_baseGPR;
info.valueGPR = m_propertyAccesses[i].m_valueGPR;
info.scratchGPR = m_propertyAccesses[i].m_scratchGPR;
}
m_codeBlock->setNumberOfCallLinkInfos(m_jsCalls.size());
for (unsigned i = 0; i < m_jsCalls.size(); ++i) {
CallLinkInfo& info = m_codeBlock->callLinkInfo(i);
info.isCall = m_jsCalls[i].m_isCall;
info.callReturnLocation = CodeLocationLabel(linkBuffer.locationOf(m_jsCalls[i].m_slowCall));
info.hotPathBegin = linkBuffer.locationOf(m_jsCalls[i].m_targetToCheck);
info.hotPathOther = linkBuffer.locationOfNearCall(m_jsCalls[i].m_fastCall);
}
m_codeBlock->addMethodCallLinkInfos(m_methodGets.size());
for (unsigned i = 0; i < m_methodGets.size(); ++i) {
MethodCallLinkInfo& info = m_codeBlock->methodCallLinkInfo(i);
info.cachedStructure.setLocation(linkBuffer.locationOf(m_methodGets[i].m_structToCompare));
info.cachedPrototypeStructure.setLocation(linkBuffer.locationOf(m_methodGets[i].m_protoStructToCompare));
info.cachedFunction.setLocation(linkBuffer.locationOf(m_methodGets[i].m_putFunction));
info.cachedPrototype.setLocation(linkBuffer.locationOf(m_methodGets[i].m_protoObj));
info.callReturnLocation = linkBuffer.locationOf(m_methodGets[i].m_slowCall);
}
}
void JITCompiler::compile(JITCode& entry)
{
compileEntry();
compileBody();
LinkBuffer linkBuffer(*m_globalData, this, m_globalData->executableAllocator);
link(linkBuffer);
entry = JITCode(linkBuffer.finalizeCode(), JITCode::DFGJIT);
}
void JITCompiler::compileFunction(JITCode& entry, MacroAssemblerCodePtr& entryWithArityCheck)
{
compileEntry();
Label fromArityCheck(this);
emitPutImmediateToCallFrameHeader(m_codeBlock, RegisterFile::CodeBlock);
addPtr(Imm32(m_codeBlock->m_numCalleeRegisters * sizeof(Register)), GPRInfo::callFrameRegister, GPRInfo::regT1);
Jump registerFileCheck = branchPtr(Below, AbsoluteAddress(m_globalData->interpreter->registerFile().addressOfEnd()), GPRInfo::regT1);
Label fromRegisterFileCheck = label();
compileBody();
registerFileCheck.link(this);
move(stackPointerRegister, GPRInfo::argumentGPR0);
poke(GPRInfo::callFrameRegister, OBJECT_OFFSETOF(struct JITStackFrame, callFrame) / sizeof(void*));
Call callRegisterFileCheck = call();
jump(fromRegisterFileCheck);
Label arityCheck = label();
preserveReturnAddressAfterCall(GPRInfo::regT2);
emitPutToCallFrameHeader(GPRInfo::regT2, RegisterFile::ReturnPC);
branch32(Equal, GPRInfo::regT1, Imm32(m_codeBlock->m_numParameters)).linkTo(fromArityCheck, this);
move(stackPointerRegister, GPRInfo::argumentGPR0);
poke(GPRInfo::callFrameRegister, OBJECT_OFFSETOF(struct JITStackFrame, callFrame) / sizeof(void*));
Call callArityCheck = call();
move(GPRInfo::regT0, GPRInfo::callFrameRegister);
jump(fromArityCheck);
LinkBuffer linkBuffer(*m_globalData, this, m_globalData->executableAllocator);
link(linkBuffer);
linkBuffer.link(callRegisterFileCheck, cti_register_file_check);
linkBuffer.link(callArityCheck, m_codeBlock->m_isConstructor ? cti_op_construct_arityCheck : cti_op_call_arityCheck);
entryWithArityCheck = linkBuffer.locationOf(arityCheck);
entry = JITCode(linkBuffer.finalizeCode(), JITCode::DFGJIT);
}
#if DFG_JIT_ASSERT
void JITCompiler::jitAssertIsInt32(GPRReg gpr)
{
#if CPU(X86_64)
Jump checkInt32 = branchPtr(BelowOrEqual, gpr, TrustedImmPtr(reinterpret_cast<void*>(static_cast<uintptr_t>(0xFFFFFFFFu))));
breakpoint();
checkInt32.link(this);
#else
UNUSED_PARAM(gpr);
#endif
}
void JITCompiler::jitAssertIsJSInt32(GPRReg gpr)
{
Jump checkJSInt32 = branchPtr(AboveOrEqual, gpr, GPRInfo::tagTypeNumberRegister);
breakpoint();
checkJSInt32.link(this);
}
void JITCompiler::jitAssertIsJSNumber(GPRReg gpr)
{
Jump checkJSNumber = branchTestPtr(MacroAssembler::NonZero, gpr, GPRInfo::tagTypeNumberRegister);
breakpoint();
checkJSNumber.link(this);
}
void JITCompiler::jitAssertIsJSDouble(GPRReg gpr)
{
Jump checkJSInt32 = branchPtr(AboveOrEqual, gpr, GPRInfo::tagTypeNumberRegister);
Jump checkJSNumber = branchTestPtr(MacroAssembler::NonZero, gpr, GPRInfo::tagTypeNumberRegister);
checkJSInt32.link(this);
breakpoint();
checkJSNumber.link(this);
}
void JITCompiler::jitAssertIsCell(GPRReg gpr)
{
Jump checkCell = branchTestPtr(MacroAssembler::Zero, gpr, GPRInfo::tagMaskRegister);
breakpoint();
checkCell.link(this);
}
#endif
#if ENABLE(SAMPLING_COUNTERS) && CPU(X86_64) // Or any other 64-bit platform!
void JITCompiler::emitCount(AbstractSamplingCounter& counter, uint32_t increment)
{
addPtr(TrustedImm32(increment), AbsoluteAddress(counter.addressOfCounter()));
}
#endif
#if ENABLE(SAMPLING_COUNTERS) && CPU(X86) // Or any other little-endian 32-bit platform!
void JITCompiler::emitCount(AbstractSamplingCounter& counter, uint32_t increment)
{
intptr_t hiWord = reinterpret_cast<intptr_t>(counter.addressOfCounter()) + sizeof(int32_t);
add32(TrustedImm32(increment), AbsoluteAddress(counter.addressOfCounter()));
addWithCarry32(TrustedImm32(0), AbsoluteAddress(reinterpret_cast<void*>(hiWord)));
}
#endif
#if ENABLE(SAMPLING_FLAGS)
void JITCompiler::setSamplingFlag(int32_t flag)
{
ASSERT(flag >= 1);
ASSERT(flag <= 32);
or32(TrustedImm32(1u << (flag - 1)), AbsoluteAddress(SamplingFlags::addressOfFlags()));
}
void JITCompiler::clearSamplingFlag(int32_t flag)
{
ASSERT(flag >= 1);
ASSERT(flag <= 32);
and32(TrustedImm32(~(1u << (flag - 1))), AbsoluteAddress(SamplingFlags::addressOfFlags()));
}
#endif
} }
#endif