#include "llvm/CodeGen/Passes.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SparseSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/LiveInterval.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/PseudoSourceValue.h"
#include "llvm/CodeGen/SlotIndexes.h"
#include "llvm/CodeGen/StackProtector.h"
#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Module.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetInstrInfo.h"
#include "llvm/Target/TargetRegisterInfo.h"
using namespace llvm;
#define DEBUG_TYPE "stackcoloring"
static cl::opt<bool>
DisableColoring("no-stack-coloring",
cl::init(false), cl::Hidden,
cl::desc("Disable stack coloring"));
static cl::opt<bool>
ProtectFromEscapedAllocas("protect-from-escaped-allocas",
cl::init(false), cl::Hidden,
cl::desc("Do not optimize lifetime zones that "
"are broken"));
STATISTIC(NumMarkerSeen, "Number of lifetime markers found.");
STATISTIC(StackSpaceSaved, "Number of bytes saved due to merging slots.");
STATISTIC(StackSlotMerged, "Number of stack slot merged.");
STATISTIC(EscapedAllocas, "Number of allocas that escaped the lifetime region");
namespace {
class StackColoring : public MachineFunctionPass {
MachineFrameInfo *MFI;
MachineFunction *MF;
struct BlockLifetimeInfo {
BitVector Begin;
BitVector End;
BitVector LiveIn;
BitVector LiveOut;
};
typedef DenseMap<const MachineBasicBlock*, BlockLifetimeInfo> LivenessMap;
LivenessMap BlockLiveness;
DenseMap<const MachineBasicBlock*, int> BasicBlocks;
SmallVector<const MachineBasicBlock*, 8> BasicBlockNumbering;
SmallVector<std::unique_ptr<LiveInterval>, 16> Intervals;
VNInfo::Allocator VNInfoAllocator;
SlotIndexes *Indexes;
StackProtector *SP;
SmallVector<MachineInstr*, 8> Markers;
public:
static char ID;
StackColoring() : MachineFunctionPass(ID) {
initializeStackColoringPass(*PassRegistry::getPassRegistry());
}
void getAnalysisUsage(AnalysisUsage &AU) const override;
bool runOnMachineFunction(MachineFunction &MF) override;
private:
void dump() const;
bool removeAllMarkers();
unsigned collectMarkers(unsigned NumSlot);
void calculateLocalLiveness();
void calculateLiveIntervals(unsigned NumSlots);
void remapInstructions(DenseMap<int, int> &SlotRemap);
void removeInvalidSlotRanges();
void expungeSlotMap(DenseMap<int, int> &SlotRemap, unsigned NumSlots);
};
}
char StackColoring::ID = 0;
char &llvm::StackColoringID = StackColoring::ID;
INITIALIZE_PASS_BEGIN(StackColoring,
"stack-coloring", "Merge disjoint stack slots", false, false)
INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
INITIALIZE_PASS_DEPENDENCY(SlotIndexes)
INITIALIZE_PASS_DEPENDENCY(StackProtector)
INITIALIZE_PASS_END(StackColoring,
"stack-coloring", "Merge disjoint stack slots", false, false)
void StackColoring::getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<MachineDominatorTree>();
AU.addPreserved<MachineDominatorTree>();
AU.addRequired<SlotIndexes>();
AU.addRequired<StackProtector>();
MachineFunctionPass::getAnalysisUsage(AU);
}
void StackColoring::dump() const {
for (MachineBasicBlock *MBB : depth_first(MF)) {
DEBUG(dbgs() << "Inspecting block #" << BasicBlocks.lookup(MBB) << " ["
<< MBB->getName() << "]\n");
LivenessMap::const_iterator BI = BlockLiveness.find(MBB);
assert(BI != BlockLiveness.end() && "Block not found");
const BlockLifetimeInfo &BlockInfo = BI->second;
DEBUG(dbgs()<<"BEGIN : {");
for (unsigned i=0; i < BlockInfo.Begin.size(); ++i)
DEBUG(dbgs()<<BlockInfo.Begin.test(i)<<" ");
DEBUG(dbgs()<<"}\n");
DEBUG(dbgs()<<"END : {");
for (unsigned i=0; i < BlockInfo.End.size(); ++i)
DEBUG(dbgs()<<BlockInfo.End.test(i)<<" ");
DEBUG(dbgs()<<"}\n");
DEBUG(dbgs()<<"LIVE_IN: {");
for (unsigned i=0; i < BlockInfo.LiveIn.size(); ++i)
DEBUG(dbgs()<<BlockInfo.LiveIn.test(i)<<" ");
DEBUG(dbgs()<<"}\n");
DEBUG(dbgs()<<"LIVEOUT: {");
for (unsigned i=0; i < BlockInfo.LiveOut.size(); ++i)
DEBUG(dbgs()<<BlockInfo.LiveOut.test(i)<<" ");
DEBUG(dbgs()<<"}\n");
}
}
unsigned StackColoring::collectMarkers(unsigned NumSlot) {
unsigned MarkersFound = 0;
for (MachineBasicBlock *MBB : depth_first(MF)) {
BasicBlocks[MBB] = BasicBlockNumbering.size();
BasicBlockNumbering.push_back(MBB);
BlockLifetimeInfo &BlockInfo = BlockLiveness[MBB];
BlockInfo.Begin.resize(NumSlot);
BlockInfo.End.resize(NumSlot);
for (MachineInstr &MI : *MBB) {
if (MI.getOpcode() != TargetOpcode::LIFETIME_START &&
MI.getOpcode() != TargetOpcode::LIFETIME_END)
continue;
Markers.push_back(&MI);
bool IsStart = MI.getOpcode() == TargetOpcode::LIFETIME_START;
const MachineOperand &MO = MI.getOperand(0);
unsigned Slot = MO.getIndex();
MarkersFound++;
const AllocaInst *Allocation = MFI->getObjectAllocation(Slot);
if (Allocation) {
DEBUG(dbgs()<<"Found a lifetime marker for slot #"<<Slot<<
" with allocation: "<< Allocation->getName()<<"\n");
}
if (IsStart) {
BlockInfo.Begin.set(Slot);
} else {
if (BlockInfo.Begin.test(Slot)) {
BlockInfo.Begin.reset(Slot);
} else {
BlockInfo.End.set(Slot);
}
}
}
}
NumMarkerSeen += MarkersFound;
return MarkersFound;
}
void StackColoring::calculateLocalLiveness() {
SmallPtrSet<const MachineBasicBlock*, 8> BBSet(BasicBlockNumbering.begin(),
BasicBlockNumbering.end());
unsigned NumSSMIters = 0;
bool changed = true;
while (changed) {
changed = false;
++NumSSMIters;
SmallPtrSet<const MachineBasicBlock*, 8> NextBBSet;
for (const MachineBasicBlock *BB : BasicBlockNumbering) {
if (!BBSet.count(BB)) continue;
LivenessMap::iterator BI = BlockLiveness.find(BB);
assert(BI != BlockLiveness.end() && "Block not found");
BlockLifetimeInfo &BlockInfo = BI->second;
BitVector LocalLiveIn;
BitVector LocalLiveOut;
for (MachineBasicBlock::const_pred_iterator PI = BB->pred_begin(),
PE = BB->pred_end(); PI != PE; ++PI) {
LivenessMap::const_iterator I = BlockLiveness.find(*PI);
assert(I != BlockLiveness.end() && "Predecessor not found");
LocalLiveIn |= I->second.LiveOut;
}
LocalLiveIn |= BlockInfo.End;
LocalLiveIn.reset(BlockInfo.Begin);
for (MachineBasicBlock::const_succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end(); SI != SE; ++SI) {
LivenessMap::const_iterator I = BlockLiveness.find(*SI);
assert(I != BlockLiveness.end() && "Successor not found");
LocalLiveOut |= I->second.LiveIn;
}
LocalLiveOut |= BlockInfo.Begin;
LocalLiveOut.reset(BlockInfo.End);
LocalLiveIn |= LocalLiveOut;
LocalLiveOut |= LocalLiveIn;
LocalLiveOut.reset(BlockInfo.End);
LocalLiveIn.reset(BlockInfo.Begin);
BitVector LocalEndBegin = BlockInfo.End;
LocalEndBegin &= BlockInfo.Begin;
LocalLiveIn |= LocalEndBegin;
LocalLiveOut |= LocalEndBegin;
if (LocalLiveIn.test(BlockInfo.LiveIn)) {
changed = true;
BlockInfo.LiveIn |= LocalLiveIn;
NextBBSet.insert(BB->pred_begin(), BB->pred_end());
}
if (LocalLiveOut.test(BlockInfo.LiveOut)) {
changed = true;
BlockInfo.LiveOut |= LocalLiveOut;
NextBBSet.insert(BB->succ_begin(), BB->succ_end());
}
}
BBSet = std::move(NextBBSet);
}}
void StackColoring::calculateLiveIntervals(unsigned NumSlots) {
SmallVector<SlotIndex, 16> Starts;
SmallVector<SlotIndex, 16> Finishes;
for (const MachineBasicBlock &MBB : *MF) {
Starts.clear();
Starts.resize(NumSlots);
Finishes.clear();
Finishes.resize(NumSlots);
for (const MachineInstr *MI : Markers) {
if (MI->getParent() != &MBB)
continue;
assert((MI->getOpcode() == TargetOpcode::LIFETIME_START ||
MI->getOpcode() == TargetOpcode::LIFETIME_END) &&
"Invalid Lifetime marker");
bool IsStart = MI->getOpcode() == TargetOpcode::LIFETIME_START;
const MachineOperand &Mo = MI->getOperand(0);
int Slot = Mo.getIndex();
assert(Slot >= 0 && "Invalid slot");
SlotIndex ThisIndex = Indexes->getInstructionIndex(MI);
if (IsStart) {
if (!Starts[Slot].isValid() || Starts[Slot] > ThisIndex)
Starts[Slot] = ThisIndex;
} else {
if (!Finishes[Slot].isValid() || Finishes[Slot] < ThisIndex)
Finishes[Slot] = ThisIndex;
}
}
BlockLifetimeInfo &MBBLiveness = BlockLiveness[&MBB];
for (int pos = MBBLiveness.LiveIn.find_first(); pos != -1;
pos = MBBLiveness.LiveIn.find_next(pos)) {
Starts[pos] = Indexes->getMBBStartIdx(&MBB);
}
for (int pos = MBBLiveness.LiveOut.find_first(); pos != -1;
pos = MBBLiveness.LiveOut.find_next(pos)) {
Finishes[pos] = Indexes->getMBBEndIdx(&MBB);
}
for (unsigned i = 0; i < NumSlots; ++i) {
assert(Starts[i].isValid() == Finishes[i].isValid() && "Unmatched range");
if (!Starts[i].isValid())
continue;
assert(Starts[i] && Finishes[i] && "Invalid interval");
VNInfo *ValNum = Intervals[i]->getValNumInfo(0);
SlotIndex S = Starts[i];
SlotIndex F = Finishes[i];
if (S < F) {
Intervals[i]->addSegment(LiveInterval::Segment(S, F, ValNum));
} else {
SlotIndex NewStart = Indexes->getMBBStartIdx(&MBB);
SlotIndex NewFin = Indexes->getMBBEndIdx(&MBB);
Intervals[i]->addSegment(LiveInterval::Segment(NewStart, F, ValNum));
Intervals[i]->addSegment(LiveInterval::Segment(S, NewFin, ValNum));
}
}
}
}
bool StackColoring::removeAllMarkers() {
unsigned Count = 0;
for (MachineInstr *MI : Markers) {
MI->eraseFromParent();
Count++;
}
Markers.clear();
DEBUG(dbgs()<<"Removed "<<Count<<" markers.\n");
return Count;
}
void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
unsigned FixedInstr = 0;
unsigned FixedMemOp = 0;
unsigned FixedDbg = 0;
MachineModuleInfo *MMI = &MF->getMMI();
for (auto &VI : MMI->getVariableDbgInfo()) {
if (!VI.Var)
continue;
if (SlotRemap.count(VI.Slot)) {
DEBUG(dbgs() << "Remapping debug info for ["
<< cast<DILocalVariable>(VI.Var)->getName() << "].\n");
VI.Slot = SlotRemap[VI.Slot];
FixedDbg++;
}
}
DenseMap<const AllocaInst*, const AllocaInst*> Allocas;
for (const std::pair<int, int> &SI : SlotRemap) {
const AllocaInst *From = MFI->getObjectAllocation(SI.first);
const AllocaInst *To = MFI->getObjectAllocation(SI.second);
assert(To && From && "Invalid allocation object");
Allocas[From] = To;
Instruction *Inst = const_cast<AllocaInst *>(To);
if (From->getType() != To->getType()) {
BitCastInst *Cast = new BitCastInst(Inst, From->getType());
Cast->insertAfter(Inst);
Inst = Cast;
}
SP->adjustForColoring(From, To);
const_cast<AllocaInst *>(From)->replaceAllUsesWith(Inst);
}
for (MachineBasicBlock &BB : *MF)
for (MachineInstr &I : BB) {
if (I.getOpcode() == TargetOpcode::LIFETIME_START ||
I.getOpcode() == TargetOpcode::LIFETIME_END)
continue;
for (MachineMemOperand *MMO : I.memoperands()) {
const AllocaInst *AI = dyn_cast_or_null<AllocaInst>(MMO->getValue());
if (!AI)
continue;
if (!Allocas.count(AI))
continue;
MMO->setValue(Allocas[AI]);
FixedMemOp++;
}
for (MachineOperand &MO : I.operands()) {
if (!MO.isFI())
continue;
int FromSlot = MO.getIndex();
if (FromSlot<0)
continue;
if (!SlotRemap.count(FromSlot))
continue;
#ifndef NDEBUG
bool TouchesMemory = I.mayLoad() || I.mayStore();
if (!I.isDebugValue() && TouchesMemory && ProtectFromEscapedAllocas) {
SlotIndex Index = Indexes->getInstructionIndex(&I);
const LiveInterval *Interval = &*Intervals[FromSlot];
assert(Interval->find(Index) != Interval->end() &&
"Found instruction usage outside of live range.");
}
#endif
int ToSlot = SlotRemap[FromSlot];
MO.setIndex(ToSlot);
FixedInstr++;
}
}
DEBUG(dbgs()<<"Fixed "<<FixedMemOp<<" machine memory operands.\n");
DEBUG(dbgs()<<"Fixed "<<FixedDbg<<" debug locations.\n");
DEBUG(dbgs()<<"Fixed "<<FixedInstr<<" machine instructions.\n");
}
void StackColoring::removeInvalidSlotRanges() {
for (MachineBasicBlock &BB : *MF)
for (MachineInstr &I : BB) {
if (I.getOpcode() == TargetOpcode::LIFETIME_START ||
I.getOpcode() == TargetOpcode::LIFETIME_END || I.isDebugValue())
continue;
if (!I.mayLoad() && !I.mayStore())
continue;
for (const MachineOperand &MO : I.operands()) {
if (!MO.isFI())
continue;
int Slot = MO.getIndex();
if (Slot<0)
continue;
if (Intervals[Slot]->empty())
continue;
LiveInterval *Interval = &*Intervals[Slot];
SlotIndex Index = Indexes->getInstructionIndex(&I);
if (Interval->find(Index) == Interval->end()) {
Interval->clear();
DEBUG(dbgs()<<"Invalidating range #"<<Slot<<"\n");
EscapedAllocas++;
}
}
}
}
void StackColoring::expungeSlotMap(DenseMap<int, int> &SlotRemap,
unsigned NumSlots) {
for (unsigned i=0; i < NumSlots; ++i) {
if (SlotRemap.count(i)) {
int Target = SlotRemap[i];
while (SlotRemap.count(Target)) {
Target = SlotRemap[Target];
SlotRemap[i] = Target;
}
}
}
}
bool StackColoring::runOnMachineFunction(MachineFunction &Func) {
if (skipOptnoneFunction(*Func.getFunction()))
return false;
DEBUG(dbgs() << "********** Stack Coloring **********\n"
<< "********** Function: "
<< ((const Value*)Func.getFunction())->getName() << '\n');
MF = &Func;
MFI = MF->getFrameInfo();
Indexes = &getAnalysis<SlotIndexes>();
SP = &getAnalysis<StackProtector>();
BlockLiveness.clear();
BasicBlocks.clear();
BasicBlockNumbering.clear();
Markers.clear();
Intervals.clear();
VNInfoAllocator.Reset();
unsigned NumSlots = MFI->getObjectIndexEnd();
if (!NumSlots)
return false;
SmallVector<int, 8> SortedSlots;
SortedSlots.reserve(NumSlots);
Intervals.reserve(NumSlots);
unsigned NumMarkers = collectMarkers(NumSlots);
unsigned TotalSize = 0;
DEBUG(dbgs()<<"Found "<<NumMarkers<<" markers and "<<NumSlots<<" slots\n");
DEBUG(dbgs()<<"Slot structure:\n");
for (int i=0; i < MFI->getObjectIndexEnd(); ++i) {
DEBUG(dbgs()<<"Slot #"<<i<<" - "<<MFI->getObjectSize(i)<<" bytes.\n");
TotalSize += MFI->getObjectSize(i);
}
DEBUG(dbgs()<<"Total Stack size: "<<TotalSize<<" bytes\n\n");
if (NumMarkers < 2 || TotalSize < 16 || DisableColoring) {
DEBUG(dbgs()<<"Will not try to merge slots.\n");
return removeAllMarkers();
}
for (unsigned i=0; i < NumSlots; ++i) {
std::unique_ptr<LiveInterval> LI(new LiveInterval(i, 0));
LI->getNextValue(Indexes->getZeroIndex(), VNInfoAllocator);
Intervals.push_back(std::move(LI));
SortedSlots.push_back(i);
}
calculateLocalLiveness();
calculateLiveIntervals(NumSlots);
if (ProtectFromEscapedAllocas)
removeInvalidSlotRanges();
DenseMap<int, int> SlotRemap;
unsigned RemovedSlots = 0;
unsigned ReducedSize = 0;
for (unsigned I = 0; I < NumSlots; ++I) {
if (Intervals[SortedSlots[I]]->empty())
SortedSlots[I] = -1;
}
std::stable_sort(SortedSlots.begin(), SortedSlots.end(),
[this](int LHS, int RHS) {
if (LHS == -1) return false;
if (RHS == -1) return true;
return MFI->getObjectSize(LHS) > MFI->getObjectSize(RHS);
});
bool Changed = true;
while (Changed) {
Changed = false;
for (unsigned I = 0; I < NumSlots; ++I) {
if (SortedSlots[I] == -1)
continue;
for (unsigned J=I+1; J < NumSlots; ++J) {
if (SortedSlots[J] == -1)
continue;
int FirstSlot = SortedSlots[I];
int SecondSlot = SortedSlots[J];
LiveInterval *First = &*Intervals[FirstSlot];
LiveInterval *Second = &*Intervals[SecondSlot];
assert (!First->empty() && !Second->empty() && "Found an empty range");
if (!First->overlaps(*Second)) {
Changed = true;
First->MergeSegmentsInAsValue(*Second, First->getValNumInfo(0));
SlotRemap[SecondSlot] = FirstSlot;
SortedSlots[J] = -1;
DEBUG(dbgs()<<"Merging #"<<FirstSlot<<" and slots #"<<
SecondSlot<<" together.\n");
unsigned MaxAlignment = std::max(MFI->getObjectAlignment(FirstSlot),
MFI->getObjectAlignment(SecondSlot));
assert(MFI->getObjectSize(FirstSlot) >=
MFI->getObjectSize(SecondSlot) &&
"Merging a small object into a larger one");
RemovedSlots+=1;
ReducedSize += MFI->getObjectSize(SecondSlot);
MFI->setObjectAlignment(FirstSlot, MaxAlignment);
MFI->RemoveStackObject(SecondSlot);
}
}
}
}
StackSpaceSaved += ReducedSize;
StackSlotMerged += RemovedSlots;
DEBUG(dbgs()<<"Merge "<<RemovedSlots<<" slots. Saved "<<
ReducedSize<<" bytes\n");
expungeSlotMap(SlotRemap, NumSlots);
remapInstructions(SlotRemap);
return removeAllMarkers();
}