StatepointLowering.cpp [plain text]
#include "StatepointLowering.h"
#include "SelectionDAGBuilder.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/GCMetadata.h"
#include "llvm/CodeGen/GCStrategy.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/StackMaps.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Statepoint.h"
#include "llvm/Target/TargetLowering.h"
#include <algorithm>
using namespace llvm;
#define DEBUG_TYPE "statepoint-lowering"
STATISTIC(NumSlotsAllocatedForStatepoints,
"Number of stack slots allocated for statepoints");
STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
STATISTIC(StatepointMaxSlotsRequired,
"Maximum number of stack slots required for a singe statepoint");
static void pushStackMapConstant(SmallVectorImpl<SDValue>& Ops,
SelectionDAGBuilder &Builder, uint64_t Value) {
SDLoc L = Builder.getCurSDLoc();
Ops.push_back(Builder.DAG.getTargetConstant(StackMaps::ConstantOp, L,
MVT::i64));
Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64));
}
void StatepointLoweringState::startNewStatepoint(SelectionDAGBuilder &Builder) {
assert(PendingGCRelocateCalls.empty() &&
"Trying to visit statepoint before finished processing previous one");
Locations.clear();
NextSlotToAllocate = 0;
AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
for (size_t i = 0; i < AllocatedStackSlots.size(); i++) {
AllocatedStackSlots[i] = false;
}
}
void StatepointLoweringState::clear() {
Locations.clear();
AllocatedStackSlots.clear();
assert(PendingGCRelocateCalls.empty() &&
"cleared before statepoint sequence completed");
}
SDValue
StatepointLoweringState::allocateStackSlot(EVT ValueType,
SelectionDAGBuilder &Builder) {
NumSlotsAllocatedForStatepoints++;
for (int i = 0; i < 40000; i++) {
assert(Builder.FuncInfo.StatepointStackSlots.size() ==
AllocatedStackSlots.size() &&
"broken invariant");
const size_t NumSlots = AllocatedStackSlots.size();
assert(NextSlotToAllocate <= NumSlots && "broken invariant");
if (NextSlotToAllocate >= NumSlots) {
assert(NextSlotToAllocate == NumSlots);
if (NumSlots + 1 > StatepointMaxSlotsRequired) {
StatepointMaxSlotsRequired = NumSlots + 1;
}
SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);
const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
auto *MFI = Builder.DAG.getMachineFunction().getFrameInfo();
MFI->markAsStatepointSpillSlotObjectIndex(FI);
Builder.FuncInfo.StatepointStackSlots.push_back(FI);
AllocatedStackSlots.push_back(true);
return SpillSlot;
}
if (!AllocatedStackSlots[NextSlotToAllocate]) {
const int FI = Builder.FuncInfo.StatepointStackSlots[NextSlotToAllocate];
AllocatedStackSlots[NextSlotToAllocate] = true;
return Builder.DAG.getFrameIndex(FI, ValueType);
}
NextSlotToAllocate++;
}
llvm_unreachable("infinite loop?");
}
static Optional<int> findPreviousSpillSlot(const Value *Val,
SelectionDAGBuilder &Builder,
int LookUpDepth) {
if (LookUpDepth <= 0)
return Optional<int>();
if (const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
FunctionLoweringInfo::StatepointSpilledValueMapTy &SpillMap =
Builder.FuncInfo.StatepointRelocatedValues[Relocate->getStatepoint()];
auto It = SpillMap.find(Relocate->getDerivedPtr());
if (It == SpillMap.end())
return Optional<int>();
return It->second;
}
if (const BitCastInst *Cast = dyn_cast<BitCastInst>(Val)) {
return findPreviousSpillSlot(Cast->getOperand(0), Builder, LookUpDepth - 1);
}
if (const PHINode *Phi = dyn_cast<PHINode>(Val)) {
Optional<int> MergedResult = None;
for (auto &IncomingValue : Phi->incoming_values()) {
Optional<int> SpillSlot =
findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth - 1);
if (!SpillSlot.hasValue())
return Optional<int>();
if (MergedResult.hasValue() && *MergedResult != *SpillSlot)
return Optional<int>();
MergedResult = SpillSlot;
}
return MergedResult;
}
return Optional<int>();
}
static void reservePreviousStackSlotForValue(const Value *IncomingValue,
SelectionDAGBuilder &Builder) {
SDValue Incoming = Builder.getValue(IncomingValue);
if (isa<ConstantSDNode>(Incoming) || isa<FrameIndexSDNode>(Incoming)) {
return;
}
SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
if (OldLocation.getNode())
return;
const int LookUpDepth = 6;
Optional<int> Index =
findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth);
if (!Index.hasValue())
return;
auto Itr = std::find(Builder.FuncInfo.StatepointStackSlots.begin(),
Builder.FuncInfo.StatepointStackSlots.end(), *Index);
assert(Itr != Builder.FuncInfo.StatepointStackSlots.end() &&
"value spilled to the unknown stack slot");
const int Offset =
std::distance(Builder.FuncInfo.StatepointStackSlots.begin(), Itr);
if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {
return;
}
Builder.StatepointLowering.reserveStackSlot(Offset);
SDValue Loc = Builder.DAG.getTargetFrameIndex(*Index, Incoming.getValueType());
Builder.StatepointLowering.setLocation(Incoming, Loc);
}
static void removeDuplicatesGCPtrs(SmallVectorImpl<const Value *> &Bases,
SmallVectorImpl<const Value *> &Ptrs,
SmallVectorImpl<const Value *> &Relocs,
SelectionDAGBuilder &Builder) {
SmallSet<SDValue, 32> Seen;
SmallVector<const Value *, 64> NewBases, NewPtrs, NewRelocs;
for (size_t i = 0; i < Ptrs.size(); i++) {
SDValue SD = Builder.getValue(Ptrs[i]);
if (Seen.count(SD) == 0) {
NewBases.push_back(Bases[i]);
NewPtrs.push_back(Ptrs[i]);
NewRelocs.push_back(Relocs[i]);
}
Seen.insert(SD);
}
assert(Bases.size() >= NewBases.size());
assert(Ptrs.size() >= NewPtrs.size());
assert(Relocs.size() >= NewRelocs.size());
Bases = NewBases;
Ptrs = NewPtrs;
Relocs = NewRelocs;
assert(Ptrs.size() == Bases.size());
assert(Ptrs.size() == Relocs.size());
}
static SDNode *
lowerCallFromStatepoint(ImmutableStatepoint ISP, const BasicBlock *EHPadBB,
SelectionDAGBuilder &Builder,
SmallVectorImpl<SDValue> &PendingExports) {
ImmutableCallSite CS(ISP.getCallSite());
SDValue ActualCallee;
if (ISP.getNumPatchBytes() > 0) {
const auto &TLI = Builder.DAG.getTargetLoweringInfo();
const auto &DL = Builder.DAG.getDataLayout();
unsigned AS = ISP.getCalledValue()->getType()->getPointerAddressSpace();
ActualCallee = Builder.DAG.getConstant(0, Builder.getCurSDLoc(),
TLI.getPointerTy(DL, AS));
} else
ActualCallee = Builder.getValue(ISP.getCalledValue());
assert(CS.getCallingConv() != CallingConv::AnyReg &&
"anyregcc is not supported on statepoints!");
Type *DefTy = ISP.getActualReturnType();
bool HasDef = !DefTy->isVoidTy();
SDValue ReturnValue, CallEndVal;
std::tie(ReturnValue, CallEndVal) = Builder.lowerCallOperands(
ISP.getCallSite(), ImmutableStatepoint::CallArgsBeginPos,
ISP.getNumCallArgs(), ActualCallee, DefTy, EHPadBB,
false );
SDNode *CallEnd = CallEndVal.getNode();
if (HasDef) {
if (CallEnd->getOpcode() == ISD::LOAD)
CallEnd = CallEnd->getOperand(0).getNode();
else
while (CallEnd->getOpcode() == ISD::CopyFromReg)
CallEnd = CallEnd->getOperand(0).getNode();
}
assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");
const Instruction *GCResult = ISP.getGCResult();
if (HasDef && GCResult) {
if (GCResult->getParent() != CS.getParent()) {
unsigned Reg = Builder.FuncInfo.CreateRegs(ISP.getActualReturnType());
RegsForValue RFV(
*Builder.DAG.getContext(), Builder.DAG.getTargetLoweringInfo(),
Builder.DAG.getDataLayout(), Reg, ISP.getActualReturnType());
SDValue Chain = Builder.DAG.getEntryNode();
RFV.getCopyToRegs(ReturnValue, Builder.DAG, Builder.getCurSDLoc(), Chain,
nullptr);
PendingExports.push_back(Chain);
Builder.FuncInfo.ValueMap[CS.getInstruction()] = Reg;
} else {
Builder.setValue(CS.getInstruction(), ReturnValue);
}
} else {
Builder.setValue(CS.getInstruction(),
Builder.DAG.getIntPtrConstant(-1, Builder.getCurSDLoc()));
}
return CallEnd->getOperand(0).getNode();
}
static void getIncomingStatepointGCValues(
SmallVectorImpl<const Value *> &Bases, SmallVectorImpl<const Value *> &Ptrs,
SmallVectorImpl<const Value *> &Relocs, ImmutableStatepoint StatepointSite,
SelectionDAGBuilder &Builder) {
for (const GCRelocateInst *Relocate : StatepointSite.getRelocates()) {
Relocs.push_back(Relocate);
Bases.push_back(Relocate->getBasePtr());
Ptrs.push_back(Relocate->getDerivedPtr());
}
removeDuplicatesGCPtrs(Bases, Ptrs, Relocs, Builder);
assert(Bases.size() == Ptrs.size() && Ptrs.size() == Relocs.size());
}
static std::pair<SDValue, SDValue>
spillIncomingStatepointValue(SDValue Incoming, SDValue Chain,
SelectionDAGBuilder &Builder) {
SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
if (!Loc.getNode()) {
Loc = Builder.StatepointLowering.allocateStackSlot(Incoming.getValueType(),
Builder);
assert(isa<FrameIndexSDNode>(Loc));
int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
Loc = Builder.DAG.getTargetFrameIndex(Index, Incoming.getValueType());
Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
MachinePointerInfo::getFixedStack(
Builder.DAG.getMachineFunction(), Index),
false, false, 0);
Builder.StatepointLowering.setLocation(Incoming, Loc);
}
assert(Loc.getNode());
return std::make_pair(Loc, Chain);
}
static void lowerIncomingStatepointValue(SDValue Incoming,
SmallVectorImpl<SDValue> &Ops,
SelectionDAGBuilder &Builder) {
SDValue Chain = Builder.getRoot();
if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
pushStackMapConstant(Ops, Builder, C->getSExtValue());
} else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
Incoming.getValueType()));
} else {
std::pair<SDValue, SDValue> Res =
spillIncomingStatepointValue(Incoming, Chain, Builder);
Ops.push_back(Res.first);
Chain = Res.second;
}
Builder.DAG.setRoot(Chain);
}
static void lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
ImmutableStatepoint StatepointSite,
SelectionDAGBuilder &Builder) {
SmallVector<const Value *, 64> Bases, Ptrs, Relocations;
getIncomingStatepointGCValues(Bases, Ptrs, Relocations, StatepointSite,
Builder);
#ifndef NDEBUG
GCStrategy &S = Builder.GFI->getStrategy();
for (const Value *V : Bases) {
auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
if (Opt.hasValue()) {
assert(Opt.getValue() &&
"non gc managed base pointer found in statepoint");
}
}
for (const Value *V : Ptrs) {
auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
if (Opt.hasValue()) {
assert(Opt.getValue() &&
"non gc managed derived pointer found in statepoint");
}
}
for (const Value *V : Relocations) {
auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
if (Opt.hasValue()) {
assert(Opt.getValue() && "non gc managed pointer relocated");
}
}
#endif
for (const Value *V : StatepointSite.vm_state_args()) {
reservePreviousStackSlotForValue(V, Builder);
}
for (unsigned i = 0; i < Bases.size(); ++i) {
reservePreviousStackSlotForValue(Bases[i], Builder);
reservePreviousStackSlotForValue(Ptrs[i], Builder);
}
const int NumVMSArgs = StatepointSite.getNumTotalVMSArgs();
pushStackMapConstant(Ops, Builder, NumVMSArgs);
assert(NumVMSArgs == std::distance(StatepointSite.vm_state_begin(),
StatepointSite.vm_state_end()));
for (const Value *V : StatepointSite.vm_state_args()) {
SDValue Incoming = Builder.getValue(V);
lowerIncomingStatepointValue(Incoming, Ops, Builder);
}
for (unsigned i = 0; i < Bases.size(); ++i) {
const Value *Base = Bases[i];
lowerIncomingStatepointValue(Builder.getValue(Base), Ops, Builder);
const Value *Ptr = Ptrs[i];
lowerIncomingStatepointValue(Builder.getValue(Ptr), Ops, Builder);
}
for (Value *V : StatepointSite.gc_args()) {
SDValue Incoming = Builder.getValue(V);
if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
Incoming.getValueType()));
}
}
const Instruction *StatepointInstr =
StatepointSite.getCallSite().getInstruction();
FunctionLoweringInfo::StatepointSpilledValueMapTy &SpillMap =
Builder.FuncInfo.StatepointRelocatedValues[StatepointInstr];
for (const GCRelocateInst *Relocate : StatepointSite.getRelocates()) {
const Value *V = Relocate->getDerivedPtr();
SDValue SDV = Builder.getValue(V);
SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
if (Loc.getNode()) {
SpillMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
} else {
SpillMap[V] = None;
if (Relocate->getParent() != StatepointInstr->getParent())
Builder.ExportFromCurrentBlock(V);
}
}
}
void SelectionDAGBuilder::visitStatepoint(const CallInst &CI) {
assert(isStatepoint(&CI) &&
"function called must be the statepoint function");
LowerStatepoint(ImmutableStatepoint(&CI));
}
void SelectionDAGBuilder::LowerStatepoint(
ImmutableStatepoint ISP, const BasicBlock *EHPadBB ) {
NumOfStatepoints++;
StatepointLowering.startNewStatepoint(*this);
ImmutableCallSite CS(ISP.getCallSite());
#ifndef NDEBUG
for (const User *U : CS->users()) {
const CallInst *Call = cast<CallInst>(U);
if (isa<GCRelocateInst>(Call) && Call->getParent() == CS.getParent())
StatepointLowering.scheduleRelocCall(*Call);
}
#endif
#ifndef NDEBUG
ISP.verify();
assert(GFI->getStrategy().useStatepoints() &&
"GCStrategy does not expect to encounter statepoints");
#endif
SmallVector<SDValue, 10> LoweredMetaArgs;
lowerStatepointMetaArgs(LoweredMetaArgs, ISP, *this);
SDNode *CallNode =
lowerCallFromStatepoint(ISP, EHPadBB, *this, PendingExports);
SDValue Chain = CallNode->getOperand(0);
SDValue Glue;
bool CallHasIncomingGlue = CallNode->getGluedNode();
if (CallHasIncomingGlue) {
Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
}
const bool IsGCTransition =
(ISP.getFlags() & (uint64_t)StatepointFlags::GCTransition) ==
(uint64_t)StatepointFlags::GCTransition;
if (IsGCTransition) {
SmallVector<SDValue, 8> TSOps;
TSOps.push_back(Chain);
for (const Value *V : ISP.gc_transition_args()) {
TSOps.push_back(getValue(V));
if (V->getType()->isPointerTy())
TSOps.push_back(DAG.getSrcValue(V));
}
if (CallHasIncomingGlue)
TSOps.push_back(Glue);
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
SDValue GCTransitionStart =
DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);
Chain = GCTransitionStart.getValue(0);
Glue = GCTransitionStart.getValue(1);
}
SmallVector<SDValue, 40> Ops;
Ops.push_back(DAG.getTargetConstant(ISP.getID(), getCurSDLoc(), MVT::i64));
Ops.push_back(
DAG.getTargetConstant(ISP.getNumPatchBytes(), getCurSDLoc(), MVT::i32));
unsigned NumCallRegArgs =
CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
Ops.push_back(CallTarget);
SDNode::op_iterator RegMaskIt;
if (CallHasIncomingGlue)
RegMaskIt = CallNode->op_end() - 2;
else
RegMaskIt = CallNode->op_end() - 1;
Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
pushStackMapConstant(Ops, *this, CS.getCallingConv());
uint64_t Flags = ISP.getFlags();
assert(
((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0)
&& "unknown flag used");
pushStackMapConstant(Ops, *this, Flags);
Ops.insert(Ops.end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());
Ops.push_back(*RegMaskIt);
Ops.push_back(Chain);
if (Glue.getNode())
Ops.push_back(Glue);
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
SDNode *StatepointMCNode =
DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
SDNode *SinkNode = StatepointMCNode;
if (IsGCTransition) {
SmallVector<SDValue, 8> TEOps;
TEOps.push_back(SDValue(StatepointMCNode, 0));
for (const Value *V : ISP.gc_transition_args()) {
TEOps.push_back(getValue(V));
if (V->getType()->isPointerTy())
TEOps.push_back(DAG.getSrcValue(V));
}
TEOps.push_back(SDValue(StatepointMCNode, 1));
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
SDValue GCTransitionStart =
DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
SinkNode = GCTransitionStart.getNode();
}
DAG.ReplaceAllUsesWith(CallNode, SinkNode); DAG.DeleteNode(CallNode);
}
void SelectionDAGBuilder::visitGCResult(const CallInst &CI) {
Instruction *I = cast<Instruction>(CI.getArgOperand(0));
assert(isStatepoint(I) && "first argument must be a statepoint token");
if (I->getParent() != CI.getParent()) {
PointerType *CalleeType = cast<PointerType>(
ImmutableStatepoint(I).getCalledValue()->getType());
Type *RetTy =
cast<FunctionType>(CalleeType->getElementType())->getReturnType();
SDValue CopyFromReg = getCopyFromRegs(I, RetTy);
assert(CopyFromReg.getNode());
setValue(&CI, CopyFromReg);
} else {
setValue(&CI, getValue(I));
}
}
void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
#ifndef NDEBUG
if (Relocate.getStatepoint()->getParent() == Relocate.getParent()) {
StatepointLowering.relocCallVisited(Relocate);
}
#endif
const Value *DerivedPtr = Relocate.getDerivedPtr();
SDValue SD = getValue(DerivedPtr);
FunctionLoweringInfo::StatepointSpilledValueMapTy &SpillMap =
FuncInfo.StatepointRelocatedValues[Relocate.getStatepoint()];
assert(SpillMap.count(DerivedPtr) && "Relocating not lowered gc value");
Optional<int> DerivedPtrLocation = SpillMap[DerivedPtr];
if (!DerivedPtrLocation) {
setValue(&Relocate, SD);
return;
}
SDValue SpillSlot = DAG.getTargetFrameIndex(*DerivedPtrLocation,
SD.getValueType());
SDValue Chain = getRoot();
SDValue SpillLoad =
DAG.getLoad(SpillSlot.getValueType(), getCurSDLoc(), Chain, SpillSlot,
MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
*DerivedPtrLocation),
false, false, false, 0);
DAG.setRoot(SpillLoad.getValue(1));
assert(SpillLoad.getNode());
setValue(&Relocate, SpillLoad);
}