StatepointLowering.cpp [plain text]
#include "StatepointLowering.h"
#include "SelectionDAGBuilder.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/GCMetadata.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/GCStrategy.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/StackMaps.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Statepoint.h"
#include "llvm/Target/TargetLowering.h"
#include <algorithm>
using namespace llvm;
#define DEBUG_TYPE "statepoint-lowering"
STATISTIC(NumSlotsAllocatedForStatepoints,
"Number of stack slots allocated for statepoints");
STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
STATISTIC(StatepointMaxSlotsRequired,
"Maximum number of stack slots required for a singe statepoint");
void
StatepointLoweringState::startNewStatepoint(SelectionDAGBuilder &Builder) {
assert(PendingGCRelocateCalls.empty() &&
"Trying to visit statepoint before finished processing previous one");
Locations.clear();
RelocLocations.clear();
NextSlotToAllocate = 0;
AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
for (size_t i = 0; i < AllocatedStackSlots.size(); i++) {
AllocatedStackSlots[i] = false;
}
}
void StatepointLoweringState::clear() {
Locations.clear();
RelocLocations.clear();
AllocatedStackSlots.clear();
assert(PendingGCRelocateCalls.empty() &&
"cleared before statepoint sequence completed");
}
SDValue
StatepointLoweringState::allocateStackSlot(EVT ValueType,
SelectionDAGBuilder &Builder) {
NumSlotsAllocatedForStatepoints++;
for (int i = 0; i < 40000; i++) {
assert(Builder.FuncInfo.StatepointStackSlots.size() ==
AllocatedStackSlots.size() &&
"broken invariant");
const size_t NumSlots = AllocatedStackSlots.size();
assert(NextSlotToAllocate <= NumSlots && "broken invariant");
if (NextSlotToAllocate >= NumSlots) {
assert(NextSlotToAllocate == NumSlots);
if (NumSlots + 1 > StatepointMaxSlotsRequired) {
StatepointMaxSlotsRequired = NumSlots + 1;
}
SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);
const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
Builder.FuncInfo.StatepointStackSlots.push_back(FI);
AllocatedStackSlots.push_back(true);
return SpillSlot;
}
if (!AllocatedStackSlots[NextSlotToAllocate]) {
const int FI = Builder.FuncInfo.StatepointStackSlots[NextSlotToAllocate];
AllocatedStackSlots[NextSlotToAllocate] = true;
return Builder.DAG.getFrameIndex(FI, ValueType);
}
NextSlotToAllocate++;
}
llvm_unreachable("infinite loop?");
}
static void reservePreviousStackSlotForValue(SDValue Incoming,
SelectionDAGBuilder &Builder) {
if (isa<ConstantSDNode>(Incoming) || isa<FrameIndexSDNode>(Incoming)) {
return;
}
SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
if (Loc.getNode()) {
return;
}
if (LoadSDNode *Load = dyn_cast<LoadSDNode>(Incoming)) {
if (auto *FI = dyn_cast<FrameIndexSDNode>(Load->getBasePtr())) {
const int Index = FI->getIndex();
auto Itr = std::find(Builder.FuncInfo.StatepointStackSlots.begin(),
Builder.FuncInfo.StatepointStackSlots.end(), Index);
if (Itr == Builder.FuncInfo.StatepointStackSlots.end()) {
return;
} else {
const int Offset =
std::distance(Builder.FuncInfo.StatepointStackSlots.begin(), Itr);
if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {
return;
}
Builder.StatepointLowering.reserveStackSlot(Offset);
}
SDValue Loc =
Builder.DAG.getTargetFrameIndex(Index, Incoming.getValueType());
Builder.StatepointLowering.setLocation(Incoming, Loc);
}
}
}
static void removeDuplicatesGCPtrs(SmallVectorImpl<const Value *> &Bases,
SmallVectorImpl<const Value *> &Ptrs,
SmallVectorImpl<const Value *> &Relocs,
SelectionDAGBuilder &Builder) {
SmallSet<SDValue, 64> Seen;
SmallVector<const Value *, 64> NewBases, NewPtrs, NewRelocs;
for (size_t i = 0; i < Ptrs.size(); i++) {
SDValue SD = Builder.getValue(Ptrs[i]);
if (Seen.count(SD) == 0) {
NewBases.push_back(Bases[i]);
NewPtrs.push_back(Ptrs[i]);
NewRelocs.push_back(Relocs[i]);
}
Seen.insert(SD);
}
assert(Bases.size() >= NewBases.size());
assert(Ptrs.size() >= NewPtrs.size());
assert(Relocs.size() >= NewRelocs.size());
Bases = NewBases;
Ptrs = NewPtrs;
Relocs = NewRelocs;
assert(Ptrs.size() == Bases.size());
assert(Ptrs.size() == Relocs.size());
}
static SDNode *lowerCallFromStatepoint(const CallInst &CI,
SelectionDAGBuilder &Builder) {
assert(Intrinsic::experimental_gc_statepoint ==
dyn_cast<IntrinsicInst>(&CI)->getIntrinsicID() &&
"function called must be the statepoint function");
ImmutableStatepoint StatepointOperands(&CI);
Value *ActualCallee = const_cast<Value *>(StatepointOperands.actualCallee());
std::vector<Value *> Args;
CallInst::const_op_iterator arg_begin = StatepointOperands.call_args_begin();
CallInst::const_op_iterator arg_end = StatepointOperands.call_args_end();
Args.insert(Args.end(), arg_begin, arg_end);
CallInst *Tmp = CallInst::Create(ActualCallee, Args);
Tmp->setTailCall(CI.isTailCall());
Tmp->setCallingConv(CI.getCallingConv());
Tmp->setAttributes(CI.getAttributes());
Builder.LowerCallTo(Tmp, Builder.getValue(ActualCallee), false);
const bool HasDef = !Tmp->getType()->isVoidTy();
if (HasDef) {
Builder.setValue(&CI, Builder.getValue(Tmp));
} else {
Builder.setValue(&CI, Builder.DAG.getIntPtrConstant(-1));
}
Builder.removeValue(Tmp);
delete Tmp;
Tmp = nullptr;
SDNode *CallNode = nullptr;
SDValue Chain = Builder.DAG.getRoot();
SDNode *CallEnd = Chain.getNode();
int Sanity = 0;
while (CallEnd->getOpcode() != ISD::CALLSEQ_END) {
CallEnd = CallEnd->getGluedNode();
assert(CallEnd && "Can not find call node");
assert(Sanity < 20 && "should have found call end already");
Sanity++;
}
assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
"Expected a callseq node.");
assert(CallEnd->getGluedNode());
CallNode = CallEnd->getGluedNode();
return CallNode;
}
static void
getIncomingStatepointGCValues(SmallVectorImpl<const Value *> &Bases,
SmallVectorImpl<const Value *> &Ptrs,
SmallVectorImpl<const Value *> &Relocs,
ImmutableCallSite Statepoint,
SelectionDAGBuilder &Builder) {
for (const User *U : cast<CallInst>(Statepoint.getInstruction())->users()) {
if (!isGCRelocate(U)) {
continue;
}
GCRelocateOperands relocateOpers(U);
Relocs.push_back(cast<Value>(U));
Bases.push_back(relocateOpers.basePtr());
Ptrs.push_back(relocateOpers.derivedPtr());
}
removeDuplicatesGCPtrs(Bases, Ptrs, Relocs, Builder);
assert(Bases.size() == Ptrs.size() && Ptrs.size() == Relocs.size());
}
static std::pair<SDValue, SDValue>
spillIncomingStatepointValue(SDValue Incoming, SDValue Chain,
SelectionDAGBuilder &Builder) {
SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
if (!Loc.getNode()) {
Loc = Builder.StatepointLowering.allocateStackSlot(Incoming.getValueType(),
Builder);
assert(isa<FrameIndexSDNode>(Loc));
int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
Loc = Builder.DAG.getTargetFrameIndex(Index, Incoming.getValueType());
Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
MachinePointerInfo::getFixedStack(Index),
false, false, 0);
Builder.StatepointLowering.setLocation(Incoming, Loc);
}
assert(Loc.getNode());
return std::make_pair(Loc, Chain);
}
static void lowerIncomingStatepointValue(SDValue Incoming,
SmallVectorImpl<SDValue> &Ops,
SelectionDAGBuilder &Builder) {
SDValue Chain = Builder.getRoot();
if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
Ops.push_back(
Builder.DAG.getTargetConstant(StackMaps::ConstantOp, MVT::i64));
Ops.push_back(Builder.DAG.getTargetConstant(C->getSExtValue(), MVT::i64));
} else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
const TargetLowering &TLI = Builder.DAG.getTargetLoweringInfo();
Ops.push_back(
Builder.DAG.getTargetFrameIndex(FI->getIndex(), TLI.getPointerTy()));
} else {
std::pair<SDValue, SDValue> Res =
spillIncomingStatepointValue(Incoming, Chain, Builder);
Ops.push_back(Res.first);
Chain = Res.second;
}
Builder.DAG.setRoot(Chain);
}
static void lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
ImmutableStatepoint Statepoint,
SelectionDAGBuilder &Builder) {
SmallVector<const Value *, 64> Bases, Ptrs, Relocations;
getIncomingStatepointGCValues(Bases, Ptrs, Relocations,
Statepoint.getCallSite(), Builder);
#ifndef NDEBUG
if (Builder.GFI) {
GCStrategy &S = Builder.GFI->getStrategy();
for (const Value *V : Bases) {
auto Opt = S.isGCManagedPointer(V);
if (Opt.hasValue()) {
assert(Opt.getValue() &&
"non gc managed base pointer found in statepoint");
}
}
for (const Value *V : Ptrs) {
auto Opt = S.isGCManagedPointer(V);
if (Opt.hasValue()) {
assert(Opt.getValue() &&
"non gc managed derived pointer found in statepoint");
}
}
for (const Value *V : Relocations) {
auto Opt = S.isGCManagedPointer(V);
if (Opt.hasValue()) {
assert(Opt.getValue() && "non gc managed pointer relocated");
}
}
}
#endif
for (auto I = Statepoint.vm_state_begin() + 1, E = Statepoint.vm_state_end();
I != E; ++I) {
Value *V = *I;
SDValue Incoming = Builder.getValue(V);
reservePreviousStackSlotForValue(Incoming, Builder);
}
for (unsigned i = 0; i < Bases.size() * 2; ++i) {
const Value *V = i % 2 ? Bases[i / 2] : Ptrs[i / 2];
SDValue Incoming = Builder.getValue(V);
reservePreviousStackSlotForValue(Incoming, Builder);
}
const int NumVMSArgs = Statepoint.numTotalVMSArgs();
Ops.push_back(
Builder.DAG.getTargetConstant(StackMaps::ConstantOp, MVT::i64));
Ops.push_back(Builder.DAG.getTargetConstant(NumVMSArgs, MVT::i64));
assert(NumVMSArgs + 1 == std::distance(Statepoint.vm_state_begin(),
Statepoint.vm_state_end()));
for (auto I = Statepoint.vm_state_begin() + 1, E = Statepoint.vm_state_end();
I != E; ++I) {
const Value *V = *I;
SDValue Incoming = Builder.getValue(V);
lowerIncomingStatepointValue(Incoming, Ops, Builder);
}
for (unsigned i = 0; i < Bases.size() * 2; ++i) {
const Value *V = i % 2 ? Bases[i / 2] : Ptrs[i / 2];
SDValue Incoming = Builder.getValue(V);
lowerIncomingStatepointValue(Incoming, Ops, Builder);
}
}
void SelectionDAGBuilder::visitStatepoint(const CallInst &CI) {
assert(isStatepoint(&CI) &&
"function called must be the statepoint function");
NumOfStatepoints++;
StatepointLowering.startNewStatepoint(*this);
#ifndef NDEBUG
for (const User *U : CI.users()) {
const CallInst *Call = cast<CallInst>(U);
if (isGCRelocate(Call))
StatepointLowering.scheduleRelocCall(*Call);
}
#endif
ImmutableStatepoint ISP(&CI);
#ifndef NDEBUG
ISP.verify();
if (GFI) {
assert(GFI->getStrategy().useStatepoints() &&
"GCStrategy does not expect to encounter statepoints");
}
#endif
SmallVector<SDValue, 10> LoweredArgs;
lowerStatepointMetaArgs(LoweredArgs, ISP, *this);
SDNode *CallNode = lowerCallFromStatepoint(CI, *this);
SmallVector<SDValue, 40> Ops;
SDValue Glue;
if (CallNode->getGluedNode()) {
Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
}
unsigned NumCallRegArgs =
CallNode->getNumOperands() - (Glue.getNode() ? 4 : 3);
Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, MVT::i32));
SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
Ops.push_back(CallTarget);
SDNode::op_iterator RegMaskIt;
if (Glue.getNode())
RegMaskIt = CallNode->op_end() - 2;
else
RegMaskIt = CallNode->op_end() - 1;
Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
CallingConv::ID CallConv = CI.getCallingConv();
int Flags = dyn_cast<ConstantInt>(CI.getArgOperand(2))->getZExtValue();
assert(Flags == 0 && "not expected to be used");
Ops.push_back(DAG.getTargetConstant(StackMaps::ConstantOp, MVT::i64));
Ops.push_back(
DAG.getTargetConstant(Flags | ((unsigned)CallConv << 1), MVT::i64));
Ops.insert(Ops.end(), LoweredArgs.begin(), LoweredArgs.end());
Ops.push_back(*RegMaskIt);
Ops.push_back(CallNode->getOperand(0));
if (Glue.getNode())
Ops.push_back(Glue);
SmallVector<EVT, 21> ValueVTs;
ValueVTs.push_back(MVT::Other);
ValueVTs.push_back(MVT::Glue); SDVTList NodeTys = DAG.getVTList(ValueVTs);
SDNode *StatepointMCNode = DAG.getMachineNode(TargetOpcode::STATEPOINT,
getCurSDLoc(), NodeTys, Ops);
DAG.ReplaceAllUsesWith(CallNode, StatepointMCNode); DAG.DeleteNode(CallNode);
}
void SelectionDAGBuilder::visitGCResult(const CallInst &CI) {
Instruction *I = cast<Instruction>(CI.getArgOperand(0));
assert(isStatepoint(I) &&
"first argument must be a statepoint token");
setValue(&CI, getValue(I));
}
void SelectionDAGBuilder::visitGCRelocate(const CallInst &CI) {
#ifndef NDEBUG
StatepointLowering.relocCallVisited(CI);
#endif
GCRelocateOperands relocateOpers(&CI);
SDValue SD = getValue(relocateOpers.derivedPtr());
if (isa<ConstantSDNode>(SD) || isa<FrameIndexSDNode>(SD)) {
setValue(&CI, SD);
return;
}
SDValue Loc = StatepointLowering.getRelocLocation(SD);
if (!Loc.getNode()) {
SDValue SpillSlot = StatepointLowering.getLocation(SD);
int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
SDValue Chain = getRoot();
Loc = DAG.getLoad(SpillSlot.getValueType(), getCurSDLoc(), Chain,
SpillSlot, MachinePointerInfo::getFixedStack(FI), false,
false, false, 0);
StatepointLowering.setRelocLocation(SD, Loc);
DAG.setRoot(Loc.getValue(1));
}
assert(Loc.getNode());
setValue(&CI, Loc);
}