DFGOSRExitCompiler32_64.cpp [plain text]
#include "config.h"
#include "DFGOSRExitCompiler.h"
#if ENABLE(DFG_JIT) && USE(JSVALUE32_64)
#include "DFGOperations.h"
#include "Operations.h"
#include <wtf/DataLog.h>
namespace JSC { namespace DFG {
void OSRExitCompiler::compileExit(const OSRExit& exit, const Operands<ValueRecovery>& operands, SpeculationRecovery* recovery)
{
#if DFG_ENABLE(DEBUG_VERBOSE)
dataLogF("OSR exit (");
for (CodeOrigin codeOrigin = exit.m_codeOrigin; ; codeOrigin = codeOrigin.inlineCallFrame->caller) {
dataLogF("bc#%u", codeOrigin.bytecodeIndex);
if (!codeOrigin.inlineCallFrame)
break;
dataLogF(" -> %p ", codeOrigin.inlineCallFrame->executable.get());
}
dataLogF(") at JIT offset 0x%x ", m_jit.debugOffset());
dumpOperands(operands, WTF::dataFile());
#endif
if (Options::printEachOSRExit()) {
SpeculationFailureDebugInfo* debugInfo = new SpeculationFailureDebugInfo;
debugInfo->codeBlock = m_jit.codeBlock();
m_jit.debugCall(debugOperationPrintSpeculationFailure, debugInfo);
}
#if DFG_ENABLE(JIT_BREAK_ON_SPECULATION_FAILURE)
m_jit.breakpoint();
#endif
#if DFG_ENABLE(SUCCESS_STATS)
static SamplingCounter counter("SpeculationFailure");
m_jit.emitCount(counter);
#endif
if (recovery) {
switch (recovery->type()) {
case SpeculativeAdd:
m_jit.sub32(recovery->src(), recovery->dest());
break;
case BooleanSpeculationCheck:
break;
default:
break;
}
}
if (!!exit.m_jsValueSource) {
if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) {
CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
if (ArrayProfile* arrayProfile = m_jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) {
GPRReg usedRegister1;
GPRReg usedRegister2;
if (exit.m_jsValueSource.isAddress()) {
usedRegister1 = exit.m_jsValueSource.base();
usedRegister2 = InvalidGPRReg;
} else {
usedRegister1 = exit.m_jsValueSource.payloadGPR();
if (exit.m_jsValueSource.hasKnownTag())
usedRegister2 = InvalidGPRReg;
else
usedRegister2 = exit.m_jsValueSource.tagGPR();
}
GPRReg scratch1;
GPRReg scratch2;
scratch1 = AssemblyHelpers::selectScratchGPR(usedRegister1, usedRegister2);
scratch2 = AssemblyHelpers::selectScratchGPR(usedRegister1, usedRegister2, scratch1);
#if CPU(ARM64)
m_jit.pushToSave(scratch1);
m_jit.pushToSave(scratch2);
#else
m_jit.push(scratch1);
m_jit.push(scratch2);
#endif
GPRReg value;
if (exit.m_jsValueSource.isAddress()) {
value = scratch1;
m_jit.loadPtr(AssemblyHelpers::Address(exit.m_jsValueSource.asAddress()), value);
} else
value = exit.m_jsValueSource.payloadGPR();
m_jit.loadPtr(AssemblyHelpers::Address(value, JSCell::structureOffset()), scratch1);
m_jit.storePtr(scratch1, arrayProfile->addressOfLastSeenStructure());
m_jit.load8(AssemblyHelpers::Address(scratch1, Structure::indexingTypeOffset()), scratch1);
m_jit.move(AssemblyHelpers::TrustedImm32(1), scratch2);
m_jit.lshift32(scratch1, scratch2);
m_jit.or32(scratch2, AssemblyHelpers::AbsoluteAddress(arrayProfile->addressOfArrayModes()));
#if CPU(ARM64)
m_jit.popToRestore(scratch2);
m_jit.popToRestore(scratch1);
#else
m_jit.pop(scratch2);
m_jit.pop(scratch1);
#endif
}
}
if (!!exit.m_valueProfile) {
EncodedJSValue* bucket = exit.m_valueProfile.getSpecFailBucket(0);
if (exit.m_jsValueSource.isAddress()) {
GPRReg scratch = AssemblyHelpers::selectScratchGPR(exit.m_jsValueSource.base());
#if CPU(ARM64)
m_jit.pushToSave(scratch);
#else
m_jit.push(scratch);
#endif
m_jit.load32(exit.m_jsValueSource.asAddress(OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), scratch);
m_jit.store32(scratch, &bitwise_cast<EncodedValueDescriptor*>(bucket)->asBits.tag);
m_jit.load32(exit.m_jsValueSource.asAddress(OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), scratch);
m_jit.store32(scratch, &bitwise_cast<EncodedValueDescriptor*>(bucket)->asBits.payload);
#if CPU(ARM64)
m_jit.popToRestore(scratch);
#else
m_jit.pop(scratch);
#endif
} else if (exit.m_jsValueSource.hasKnownTag()) {
m_jit.store32(AssemblyHelpers::TrustedImm32(exit.m_jsValueSource.tag()), &bitwise_cast<EncodedValueDescriptor*>(bucket)->asBits.tag);
m_jit.store32(exit.m_jsValueSource.payloadGPR(), &bitwise_cast<EncodedValueDescriptor*>(bucket)->asBits.payload);
} else {
m_jit.store32(exit.m_jsValueSource.tagGPR(), &bitwise_cast<EncodedValueDescriptor*>(bucket)->asBits.tag);
m_jit.store32(exit.m_jsValueSource.payloadGPR(), &bitwise_cast<EncodedValueDescriptor*>(bucket)->asBits.payload);
}
}
}
Vector<bool> poisonedVirtualRegisters(operands.numberOfLocals());
for (unsigned i = 0; i < poisonedVirtualRegisters.size(); ++i)
poisonedVirtualRegisters[i] = false;
unsigned numberOfPoisonedVirtualRegisters = 0;
unsigned numberOfDisplacedVirtualRegisters = 0;
bool haveUnboxedInt32InJSStack = false;
bool haveUnboxedCellInJSStack = false;
bool haveUnboxedBooleanInJSStack = false;
bool haveUInt32s = false;
bool haveFPRs = false;
bool haveConstants = false;
bool haveUndefined = false;
bool haveArguments = false;
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
switch (recovery.technique()) {
case DisplacedInJSStack:
case Int32DisplacedInJSStack:
case CellDisplacedInJSStack:
case BooleanDisplacedInJSStack:
numberOfDisplacedVirtualRegisters++;
ASSERT((int)recovery.virtualRegister() >= 0);
if (recovery.virtualRegister() < (int)operands.numberOfLocals()) {
switch (operands.local(recovery.virtualRegister()).technique()) {
case InGPR:
case UnboxedInt32InGPR:
case UnboxedBooleanInGPR:
case UInt32InGPR:
case InPair:
case InFPR:
if (!poisonedVirtualRegisters[recovery.virtualRegister()]) {
poisonedVirtualRegisters[recovery.virtualRegister()] = true;
numberOfPoisonedVirtualRegisters++;
}
break;
default:
break;
}
}
break;
case UInt32InGPR:
haveUInt32s = true;
break;
case AlreadyInJSStackAsUnboxedInt32:
haveUnboxedInt32InJSStack = true;
break;
case AlreadyInJSStackAsUnboxedCell:
haveUnboxedCellInJSStack = true;
break;
case AlreadyInJSStackAsUnboxedBoolean:
haveUnboxedBooleanInJSStack = true;
break;
case InFPR:
haveFPRs = true;
break;
case Constant:
haveConstants = true;
if (recovery.constant().isUndefined())
haveUndefined = true;
break;
case ArgumentsThatWereNotCreated:
haveArguments = true;
break;
default:
break;
}
}
unsigned scratchBufferLengthBeforeUInt32s = numberOfPoisonedVirtualRegisters + ((numberOfDisplacedVirtualRegisters * 2) <= GPRInfo::numberOfRegisters ? 0 : numberOfDisplacedVirtualRegisters);
ScratchBuffer* scratchBuffer = m_jit.vm()->scratchBufferForSize(sizeof(EncodedJSValue) * (scratchBufferLengthBeforeUInt32s + (haveUInt32s ? 2 : 0)));
EncodedJSValue* scratchDataBuffer = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
if (haveUnboxedInt32InJSStack || haveUnboxedCellInJSStack || haveUnboxedBooleanInJSStack) {
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
switch (recovery.technique()) {
case AlreadyInJSStackAsUnboxedInt32:
m_jit.store32(AssemblyHelpers::TrustedImm32(JSValue::Int32Tag), AssemblyHelpers::tagFor(static_cast<VirtualRegister>(operands.operandForIndex(index))));
break;
case AlreadyInJSStackAsUnboxedCell:
m_jit.store32(AssemblyHelpers::TrustedImm32(JSValue::CellTag), AssemblyHelpers::tagFor(static_cast<VirtualRegister>(operands.operandForIndex(index))));
break;
case AlreadyInJSStackAsUnboxedBoolean:
m_jit.store32(AssemblyHelpers::TrustedImm32(JSValue::BooleanTag), AssemblyHelpers::tagFor(static_cast<VirtualRegister>(operands.operandForIndex(index))));
break;
default:
break;
}
}
}
initializePoisoned(operands.numberOfLocals());
unsigned currentPoisonIndex = 0;
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
int operand = operands.operandForIndex(index);
switch (recovery.technique()) {
case InGPR:
case UnboxedInt32InGPR:
case UnboxedBooleanInGPR:
if (operands.isVariable(index) && poisonedVirtualRegisters[operands.variableForIndex(index)]) {
m_jit.store32(recovery.gpr(), reinterpret_cast<char*>(scratchDataBuffer + currentPoisonIndex) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));
m_poisonScratchIndices[operands.variableForIndex(index)] = currentPoisonIndex;
currentPoisonIndex++;
} else {
uint32_t tag = JSValue::EmptyValueTag;
if (recovery.technique() == InGPR)
tag = JSValue::CellTag;
else if (recovery.technique() == UnboxedInt32InGPR)
tag = JSValue::Int32Tag;
else
tag = JSValue::BooleanTag;
m_jit.store32(AssemblyHelpers::TrustedImm32(tag), AssemblyHelpers::tagFor((VirtualRegister)operand));
m_jit.store32(recovery.gpr(), AssemblyHelpers::payloadFor((VirtualRegister)operand));
}
break;
case InPair:
if (operands.isVariable(index) && poisonedVirtualRegisters[operands.variableForIndex(index)]) {
m_jit.store32(recovery.tagGPR(), reinterpret_cast<char*>(scratchDataBuffer + currentPoisonIndex) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag));
m_jit.store32(recovery.payloadGPR(), reinterpret_cast<char*>(scratchDataBuffer + currentPoisonIndex) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));
m_poisonScratchIndices[operands.variableForIndex(index)] = currentPoisonIndex;
currentPoisonIndex++;
} else {
m_jit.store32(recovery.tagGPR(), AssemblyHelpers::tagFor((VirtualRegister)operand));
m_jit.store32(recovery.payloadGPR(), AssemblyHelpers::payloadFor((VirtualRegister)operand));
}
break;
case UInt32InGPR: {
EncodedJSValue* myScratch = scratchDataBuffer + scratchBufferLengthBeforeUInt32s;
GPRReg addressGPR = GPRInfo::regT0;
if (addressGPR == recovery.gpr())
addressGPR = GPRInfo::regT1;
m_jit.storePtr(addressGPR, myScratch);
m_jit.move(AssemblyHelpers::TrustedImmPtr(myScratch + 1), addressGPR);
m_jit.storeDouble(FPRInfo::fpRegT0, addressGPR);
AssemblyHelpers::Jump positive = m_jit.branch32(AssemblyHelpers::GreaterThanOrEqual, recovery.gpr(), AssemblyHelpers::TrustedImm32(0));
m_jit.convertInt32ToDouble(recovery.gpr(), FPRInfo::fpRegT0);
m_jit.addDouble(AssemblyHelpers::AbsoluteAddress(&AssemblyHelpers::twoToThe32), FPRInfo::fpRegT0);
if (operands.isVariable(index) && poisonedVirtualRegisters[operands.variableForIndex(index)]) {
m_jit.move(AssemblyHelpers::TrustedImmPtr(scratchDataBuffer + currentPoisonIndex), addressGPR);
m_jit.storeDouble(FPRInfo::fpRegT0, addressGPR);
} else
m_jit.storeDouble(FPRInfo::fpRegT0, AssemblyHelpers::addressFor((VirtualRegister)operand));
AssemblyHelpers::Jump done = m_jit.jump();
positive.link(&m_jit);
if (operands.isVariable(index) && poisonedVirtualRegisters[operands.variableForIndex(index)]) {
m_jit.store32(recovery.gpr(), reinterpret_cast<char*>(scratchDataBuffer + currentPoisonIndex) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));
m_jit.store32(AssemblyHelpers::TrustedImm32(JSValue::Int32Tag), reinterpret_cast<char*>(scratchDataBuffer + currentPoisonIndex) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag));
} else {
m_jit.store32(recovery.gpr(), AssemblyHelpers::payloadFor((VirtualRegister)operand));
m_jit.store32(AssemblyHelpers::TrustedImm32(JSValue::Int32Tag), AssemblyHelpers::tagFor((VirtualRegister)operand));
}
done.link(&m_jit);
m_jit.move(AssemblyHelpers::TrustedImmPtr(myScratch + 1), addressGPR);
m_jit.loadDouble(addressGPR, FPRInfo::fpRegT0);
m_jit.loadPtr(myScratch, addressGPR);
if (operands.isVariable(index) && poisonedVirtualRegisters[operands.variableForIndex(index)]) {
m_poisonScratchIndices[operands.variableForIndex(index)] = currentPoisonIndex;
currentPoisonIndex++;
}
break;
}
default:
break;
}
}
if (haveFPRs) {
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
if (recovery.technique() != InFPR)
continue;
if (operands.isVariable(index) && poisonedVirtualRegisters[operands.variableForIndex(index)]) {
m_jit.storeDouble(recovery.fpr(), scratchDataBuffer + currentPoisonIndex);
m_poisonScratchIndices[operands.variableForIndex(index)] = currentPoisonIndex;
currentPoisonIndex++;
} else
m_jit.storeDouble(recovery.fpr(), AssemblyHelpers::addressFor((VirtualRegister)operands.operandForIndex(index)));
}
}
ASSERT(currentPoisonIndex == numberOfPoisonedVirtualRegisters);
if (numberOfDisplacedVirtualRegisters) {
if (numberOfDisplacedVirtualRegisters * 2 <= GPRInfo::numberOfRegisters) {
unsigned displacementIndex = 0;
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
switch (recovery.technique()) {
case DisplacedInJSStack:
m_jit.load32(AssemblyHelpers::payloadFor(recovery.virtualRegister()), GPRInfo::toRegister(displacementIndex++));
m_jit.load32(AssemblyHelpers::tagFor(recovery.virtualRegister()), GPRInfo::toRegister(displacementIndex++));
break;
case Int32DisplacedInJSStack:
m_jit.load32(AssemblyHelpers::payloadFor(recovery.virtualRegister()), GPRInfo::toRegister(displacementIndex++));
m_jit.move(AssemblyHelpers::TrustedImm32(JSValue::Int32Tag), GPRInfo::toRegister(displacementIndex++));
break;
case CellDisplacedInJSStack:
m_jit.load32(AssemblyHelpers::payloadFor(recovery.virtualRegister()), GPRInfo::toRegister(displacementIndex++));
m_jit.move(AssemblyHelpers::TrustedImm32(JSValue::CellTag), GPRInfo::toRegister(displacementIndex++));
break;
case BooleanDisplacedInJSStack:
m_jit.load32(AssemblyHelpers::payloadFor(recovery.virtualRegister()), GPRInfo::toRegister(displacementIndex++));
m_jit.move(AssemblyHelpers::TrustedImm32(JSValue::BooleanTag), GPRInfo::toRegister(displacementIndex++));
break;
default:
break;
}
}
displacementIndex = 0;
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
switch (recovery.technique()) {
case DisplacedInJSStack:
case Int32DisplacedInJSStack:
case CellDisplacedInJSStack:
case BooleanDisplacedInJSStack:
m_jit.store32(GPRInfo::toRegister(displacementIndex++), AssemblyHelpers::payloadFor((VirtualRegister)operands.operandForIndex(index)));
m_jit.store32(GPRInfo::toRegister(displacementIndex++), AssemblyHelpers::tagFor((VirtualRegister)operands.operandForIndex(index)));
break;
default:
break;
}
}
} else {
unsigned scratchIndex = numberOfPoisonedVirtualRegisters;
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
switch (recovery.technique()) {
case DisplacedInJSStack:
m_jit.load32(AssemblyHelpers::payloadFor(recovery.virtualRegister()), GPRInfo::regT0);
m_jit.load32(AssemblyHelpers::tagFor(recovery.virtualRegister()), GPRInfo::regT1);
m_jit.store32(GPRInfo::regT0, reinterpret_cast<char*>(scratchDataBuffer + scratchIndex) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));
m_jit.store32(GPRInfo::regT1, reinterpret_cast<char*>(scratchDataBuffer + scratchIndex) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag));
scratchIndex++;
break;
case Int32DisplacedInJSStack:
case CellDisplacedInJSStack:
case BooleanDisplacedInJSStack:
m_jit.load32(AssemblyHelpers::payloadFor(recovery.virtualRegister()), GPRInfo::regT0);
m_jit.store32(GPRInfo::regT0, reinterpret_cast<char*>(scratchDataBuffer + scratchIndex++) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));
break;
default:
break;
}
}
scratchIndex = numberOfPoisonedVirtualRegisters;
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
switch (recovery.technique()) {
case DisplacedInJSStack:
m_jit.load32(reinterpret_cast<char*>(scratchDataBuffer + scratchIndex) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload), GPRInfo::regT0);
m_jit.load32(reinterpret_cast<char*>(scratchDataBuffer + scratchIndex) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag), GPRInfo::regT1);
m_jit.store32(GPRInfo::regT0, AssemblyHelpers::payloadFor((VirtualRegister)operands.operandForIndex(index)));
m_jit.store32(GPRInfo::regT1, AssemblyHelpers::tagFor((VirtualRegister)operands.operandForIndex(index)));
scratchIndex++;
break;
case Int32DisplacedInJSStack:
m_jit.load32(reinterpret_cast<char*>(scratchDataBuffer + scratchIndex++) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload), GPRInfo::regT0);
m_jit.store32(AssemblyHelpers::TrustedImm32(JSValue::Int32Tag), AssemblyHelpers::tagFor((VirtualRegister)operands.operandForIndex(index)));
m_jit.store32(GPRInfo::regT0, AssemblyHelpers::payloadFor((VirtualRegister)operands.operandForIndex(index)));
break;
case CellDisplacedInJSStack:
m_jit.load32(reinterpret_cast<char*>(scratchDataBuffer + scratchIndex++) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload), GPRInfo::regT0);
m_jit.store32(AssemblyHelpers::TrustedImm32(JSValue::CellTag), AssemblyHelpers::tagFor((VirtualRegister)operands.operandForIndex(index)));
m_jit.store32(GPRInfo::regT0, AssemblyHelpers::payloadFor((VirtualRegister)operands.operandForIndex(index)));
break;
case BooleanDisplacedInJSStack:
m_jit.load32(reinterpret_cast<char*>(scratchDataBuffer + scratchIndex++) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload), GPRInfo::regT0);
m_jit.store32(AssemblyHelpers::TrustedImm32(JSValue::BooleanTag), AssemblyHelpers::tagFor((VirtualRegister)operands.operandForIndex(index)));
m_jit.store32(GPRInfo::regT0, AssemblyHelpers::payloadFor((VirtualRegister)operands.operandForIndex(index)));
break;
default:
break;
}
}
ASSERT(scratchIndex == numberOfPoisonedVirtualRegisters + numberOfDisplacedVirtualRegisters);
}
}
if (numberOfPoisonedVirtualRegisters) {
for (int virtualRegister = 0; virtualRegister < (int)operands.numberOfLocals(); ++virtualRegister) {
if (!poisonedVirtualRegisters[virtualRegister])
continue;
const ValueRecovery& recovery = operands.local(virtualRegister);
switch (recovery.technique()) {
case InGPR:
case UnboxedInt32InGPR:
case UnboxedBooleanInGPR: {
m_jit.load32(reinterpret_cast<char*>(scratchDataBuffer + poisonIndex(virtualRegister)) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload), GPRInfo::regT0);
m_jit.store32(GPRInfo::regT0, AssemblyHelpers::payloadFor((VirtualRegister)virtualRegister));
uint32_t tag = JSValue::EmptyValueTag;
if (recovery.technique() == InGPR)
tag = JSValue::CellTag;
else if (recovery.technique() == UnboxedInt32InGPR)
tag = JSValue::Int32Tag;
else
tag = JSValue::BooleanTag;
m_jit.store32(AssemblyHelpers::TrustedImm32(tag), AssemblyHelpers::tagFor((VirtualRegister)virtualRegister));
break;
}
case InFPR:
case InPair:
case UInt32InGPR:
m_jit.load32(reinterpret_cast<char*>(scratchDataBuffer + poisonIndex(virtualRegister)) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload), GPRInfo::regT0);
m_jit.load32(reinterpret_cast<char*>(scratchDataBuffer + poisonIndex(virtualRegister)) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag), GPRInfo::regT1);
m_jit.store32(GPRInfo::regT0, AssemblyHelpers::payloadFor((VirtualRegister)virtualRegister));
m_jit.store32(GPRInfo::regT1, AssemblyHelpers::tagFor((VirtualRegister)virtualRegister));
break;
default:
break;
}
}
}
if (haveConstants) {
if (haveUndefined) {
m_jit.move(AssemblyHelpers::TrustedImm32(jsUndefined().payload()), GPRInfo::regT0);
m_jit.move(AssemblyHelpers::TrustedImm32(jsUndefined().tag()), GPRInfo::regT1);
}
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
if (recovery.technique() != Constant)
continue;
if (recovery.constant().isUndefined()) {
m_jit.store32(GPRInfo::regT0, AssemblyHelpers::payloadFor((VirtualRegister)operands.operandForIndex(index)));
m_jit.store32(GPRInfo::regT1, AssemblyHelpers::tagFor((VirtualRegister)operands.operandForIndex(index)));
} else {
m_jit.store32(AssemblyHelpers::TrustedImm32(recovery.constant().payload()), AssemblyHelpers::payloadFor((VirtualRegister)operands.operandForIndex(index)));
m_jit.store32(AssemblyHelpers::TrustedImm32(recovery.constant().tag()), AssemblyHelpers::tagFor((VirtualRegister)operands.operandForIndex(index)));
}
}
}
handleExitCounts(exit);
ASSERT(m_jit.baselineCodeBlock()->getJITType() == JITCode::BaselineJIT);
m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(m_jit.baselineCodeBlock()), AssemblyHelpers::addressFor((VirtualRegister)JSStack::CodeBlock));
for (CodeOrigin codeOrigin = exit.m_codeOrigin; codeOrigin.inlineCallFrame; codeOrigin = codeOrigin.inlineCallFrame->caller) {
InlineCallFrame* inlineCallFrame = codeOrigin.inlineCallFrame;
CodeBlock* baselineCodeBlock = m_jit.baselineCodeBlockFor(codeOrigin);
CodeBlock* baselineCodeBlockForCaller = m_jit.baselineCodeBlockFor(inlineCallFrame->caller);
Vector<BytecodeAndMachineOffset>& decodedCodeMap = m_jit.decodedCodeMapFor(baselineCodeBlockForCaller);
unsigned returnBytecodeIndex = inlineCallFrame->caller.bytecodeIndex + OPCODE_LENGTH(op_call);
BytecodeAndMachineOffset* mapping = binarySearch<BytecodeAndMachineOffset, unsigned>(decodedCodeMap, decodedCodeMap.size(), returnBytecodeIndex, BytecodeAndMachineOffset::getBytecodeIndex);
ASSERT(mapping);
ASSERT(mapping->m_bytecodeIndex == returnBytecodeIndex);
void* jumpTarget = baselineCodeBlockForCaller->getJITCode().executableAddressAtOffset(mapping->m_machineCodeOffset);
GPRReg callerFrameGPR;
if (inlineCallFrame->caller.inlineCallFrame) {
m_jit.add32(AssemblyHelpers::TrustedImm32(inlineCallFrame->caller.inlineCallFrame->stackOffset * sizeof(EncodedJSValue)), GPRInfo::callFrameRegister, GPRInfo::regT3);
callerFrameGPR = GPRInfo::regT3;
} else
callerFrameGPR = GPRInfo::callFrameRegister;
m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(baselineCodeBlock), AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::CodeBlock)));
m_jit.store32(AssemblyHelpers::TrustedImm32(JSValue::CellTag), AssemblyHelpers::tagFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::ScopeChain)));
if (!inlineCallFrame->isClosureCall())
m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(inlineCallFrame->callee->scope()), AssemblyHelpers::payloadFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::ScopeChain)));
m_jit.store32(AssemblyHelpers::TrustedImm32(JSValue::CellTag), AssemblyHelpers::tagFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::CallerFrame)));
m_jit.storePtr(callerFrameGPR, AssemblyHelpers::payloadFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::CallerFrame)));
m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(jumpTarget), AssemblyHelpers::payloadFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::ReturnPC)));
m_jit.store32(AssemblyHelpers::TrustedImm32(inlineCallFrame->arguments.size()), AssemblyHelpers::payloadFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::ArgumentCount)));
m_jit.store32(AssemblyHelpers::TrustedImm32(JSValue::CellTag), AssemblyHelpers::tagFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::Callee)));
if (!inlineCallFrame->isClosureCall())
m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(inlineCallFrame->callee.get()), AssemblyHelpers::payloadFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::Callee)));
}
if (haveArguments) {
HashSet<InlineCallFrame*, DefaultHash<InlineCallFrame*>::Hash,
NullableHashTraits<InlineCallFrame*> > didCreateArgumentsObject;
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
if (recovery.technique() != ArgumentsThatWereNotCreated)
continue;
int operand = operands.operandForIndex(index);
InlineCallFrame* inlineCallFrame = 0;
for (InlineCallFrame* current = exit.m_codeOrigin.inlineCallFrame;
current;
current = current->caller.inlineCallFrame) {
if (current->stackOffset <= operand) {
inlineCallFrame = current;
break;
}
}
if (!m_jit.baselineCodeBlockFor(inlineCallFrame)->usesArguments())
continue;
int argumentsRegister = m_jit.argumentsRegisterFor(inlineCallFrame);
if (didCreateArgumentsObject.add(inlineCallFrame).isNewEntry) {
if (inlineCallFrame) {
m_jit.setupArgumentsWithExecState(
AssemblyHelpers::TrustedImmPtr(inlineCallFrame));
m_jit.move(
AssemblyHelpers::TrustedImmPtr(
bitwise_cast<void*>(operationCreateInlinedArguments)),
GPRInfo::nonArgGPR0);
} else {
m_jit.setupArgumentsExecState();
m_jit.move(
AssemblyHelpers::TrustedImmPtr(
bitwise_cast<void*>(operationCreateArguments)),
GPRInfo::nonArgGPR0);
}
m_jit.call(GPRInfo::nonArgGPR0);
m_jit.store32(
AssemblyHelpers::TrustedImm32(JSValue::CellTag),
AssemblyHelpers::tagFor(argumentsRegister));
m_jit.store32(
GPRInfo::returnValueGPR,
AssemblyHelpers::payloadFor(argumentsRegister));
m_jit.store32(
AssemblyHelpers::TrustedImm32(JSValue::CellTag),
AssemblyHelpers::tagFor(unmodifiedArgumentsRegister(argumentsRegister)));
m_jit.store32(
GPRInfo::returnValueGPR,
AssemblyHelpers::payloadFor(unmodifiedArgumentsRegister(argumentsRegister)));
m_jit.move(GPRInfo::returnValueGPR, GPRInfo::regT0); }
m_jit.load32(AssemblyHelpers::payloadFor(argumentsRegister), GPRInfo::regT0);
m_jit.store32(
AssemblyHelpers::TrustedImm32(JSValue::CellTag),
AssemblyHelpers::tagFor(operand));
m_jit.store32(GPRInfo::regT0, AssemblyHelpers::payloadFor(operand));
}
}
if (exit.m_lastSetOperand != std::numeric_limits<int>::max()) {
m_jit.load32(AssemblyHelpers::payloadFor((VirtualRegister)exit.m_lastSetOperand), GPRInfo::cachedResultRegister);
m_jit.load32(AssemblyHelpers::tagFor((VirtualRegister)exit.m_lastSetOperand), GPRInfo::cachedResultRegister2);
}
if (exit.m_codeOrigin.inlineCallFrame)
m_jit.addPtr(AssemblyHelpers::TrustedImm32(exit.m_codeOrigin.inlineCallFrame->stackOffset * sizeof(EncodedJSValue)), GPRInfo::callFrameRegister);
CodeBlock* baselineCodeBlock = m_jit.baselineCodeBlockFor(exit.m_codeOrigin);
Vector<BytecodeAndMachineOffset>& decodedCodeMap = m_jit.decodedCodeMapFor(baselineCodeBlock);
BytecodeAndMachineOffset* mapping = binarySearch<BytecodeAndMachineOffset, unsigned>(decodedCodeMap, decodedCodeMap.size(), exit.m_codeOrigin.bytecodeIndex, BytecodeAndMachineOffset::getBytecodeIndex);
ASSERT(mapping);
ASSERT(mapping->m_bytecodeIndex == exit.m_codeOrigin.bytecodeIndex);
void* jumpTarget = baselineCodeBlock->getJITCode().executableAddressAtOffset(mapping->m_machineCodeOffset);
ASSERT(GPRInfo::regT2 != GPRInfo::cachedResultRegister && GPRInfo::regT2 != GPRInfo::cachedResultRegister2);
m_jit.move(AssemblyHelpers::TrustedImmPtr(jumpTarget), GPRInfo::regT2);
m_jit.jump(GPRInfo::regT2);
#if DFG_ENABLE(DEBUG_VERBOSE)
dataLogF(" -> %p\n", jumpTarget);
#endif
}
} }
#endif // ENABLE(DFG_JIT) && USE(JSVALUE32_64)