MachineBlockPlacement.cpp [plain text]
#define DEBUG_TYPE "block-placement2"
#include "llvm/CodeGen/Passes.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Debug.h"
#include "llvm/Target/TargetInstrInfo.h"
#include "llvm/Target/TargetLowering.h"
#include <algorithm>
using namespace llvm;
STATISTIC(NumCondBranches, "Number of conditional branches");
STATISTIC(NumUncondBranches, "Number of uncondittional branches");
STATISTIC(CondBranchTakenFreq,
"Potential frequency of taking conditional branches");
STATISTIC(UncondBranchTakenFreq,
"Potential frequency of taking unconditional branches");
namespace {
class BlockChain;
typedef DenseMap<MachineBasicBlock *, BlockChain *> BlockToChainMapType;
}
namespace {
class BlockChain {
SmallVector<MachineBasicBlock *, 4> Blocks;
BlockToChainMapType &BlockToChain;
public:
BlockChain(BlockToChainMapType &BlockToChain, MachineBasicBlock *BB)
: Blocks(1, BB), BlockToChain(BlockToChain), LoopPredecessors(0) {
assert(BB && "Cannot create a chain with a null basic block");
BlockToChain[BB] = this;
}
typedef SmallVectorImpl<MachineBasicBlock *>::iterator iterator;
iterator begin() { return Blocks.begin(); }
iterator end() { return Blocks.end(); }
void merge(MachineBasicBlock *BB, BlockChain *Chain) {
assert(BB);
assert(!Blocks.empty());
if (!Chain) {
assert(!BlockToChain[BB]);
Blocks.push_back(BB);
BlockToChain[BB] = this;
return;
}
assert(BB == *Chain->begin());
assert(Chain->begin() != Chain->end());
for (BlockChain::iterator BI = Chain->begin(), BE = Chain->end();
BI != BE; ++BI) {
Blocks.push_back(*BI);
assert(BlockToChain[*BI] == Chain && "Incoming blocks not in chain");
BlockToChain[*BI] = this;
}
}
#ifndef NDEBUG
void dump() LLVM_ATTRIBUTE_USED {
for (iterator I = begin(), E = end(); I != E; ++I)
(*I)->dump();
}
#endif // NDEBUG
unsigned LoopPredecessors;
};
}
namespace {
class MachineBlockPlacement : public MachineFunctionPass {
typedef SmallPtrSet<MachineBasicBlock *, 16> BlockFilterSet;
const MachineBranchProbabilityInfo *MBPI;
const MachineBlockFrequencyInfo *MBFI;
const MachineLoopInfo *MLI;
const TargetInstrInfo *TII;
const TargetLoweringBase *TLI;
SpecificBumpPtrAllocator<BlockChain> ChainAllocator;
DenseMap<MachineBasicBlock *, BlockChain *> BlockToChain;
void markChainSuccessors(BlockChain &Chain,
MachineBasicBlock *LoopHeaderBB,
SmallVectorImpl<MachineBasicBlock *> &BlockWorkList,
const BlockFilterSet *BlockFilter = 0);
MachineBasicBlock *selectBestSuccessor(MachineBasicBlock *BB,
BlockChain &Chain,
const BlockFilterSet *BlockFilter);
MachineBasicBlock *selectBestCandidateBlock(
BlockChain &Chain, SmallVectorImpl<MachineBasicBlock *> &WorkList,
const BlockFilterSet *BlockFilter);
MachineBasicBlock *getFirstUnplacedBlock(
MachineFunction &F,
const BlockChain &PlacedChain,
MachineFunction::iterator &PrevUnplacedBlockIt,
const BlockFilterSet *BlockFilter);
void buildChain(MachineBasicBlock *BB, BlockChain &Chain,
SmallVectorImpl<MachineBasicBlock *> &BlockWorkList,
const BlockFilterSet *BlockFilter = 0);
MachineBasicBlock *findBestLoopTop(MachineLoop &L,
const BlockFilterSet &LoopBlockSet);
MachineBasicBlock *findBestLoopExit(MachineFunction &F,
MachineLoop &L,
const BlockFilterSet &LoopBlockSet);
void buildLoopChains(MachineFunction &F, MachineLoop &L);
void rotateLoop(BlockChain &LoopChain, MachineBasicBlock *ExitingBB,
const BlockFilterSet &LoopBlockSet);
void buildCFGChains(MachineFunction &F);
public:
static char ID; MachineBlockPlacement() : MachineFunctionPass(ID) {
initializeMachineBlockPlacementPass(*PassRegistry::getPassRegistry());
}
bool runOnMachineFunction(MachineFunction &F);
void getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<MachineBranchProbabilityInfo>();
AU.addRequired<MachineBlockFrequencyInfo>();
AU.addRequired<MachineLoopInfo>();
MachineFunctionPass::getAnalysisUsage(AU);
}
};
}
char MachineBlockPlacement::ID = 0;
char &llvm::MachineBlockPlacementID = MachineBlockPlacement::ID;
INITIALIZE_PASS_BEGIN(MachineBlockPlacement, "block-placement2",
"Branch Probability Basic Block Placement", false, false)
INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo)
INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfo)
INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
INITIALIZE_PASS_END(MachineBlockPlacement, "block-placement2",
"Branch Probability Basic Block Placement", false, false)
#ifndef NDEBUG
static std::string getBlockName(MachineBasicBlock *BB) {
std::string Result;
raw_string_ostream OS(Result);
OS << "BB#" << BB->getNumber()
<< " (derived from LLVM BB '" << BB->getName() << "')";
OS.flush();
return Result;
}
static std::string getBlockNum(MachineBasicBlock *BB) {
std::string Result;
raw_string_ostream OS(Result);
OS << "BB#" << BB->getNumber();
OS.flush();
return Result;
}
#endif
void MachineBlockPlacement::markChainSuccessors(
BlockChain &Chain,
MachineBasicBlock *LoopHeaderBB,
SmallVectorImpl<MachineBasicBlock *> &BlockWorkList,
const BlockFilterSet *BlockFilter) {
for (BlockChain::iterator CBI = Chain.begin(), CBE = Chain.end();
CBI != CBE; ++CBI) {
for (MachineBasicBlock::succ_iterator SI = (*CBI)->succ_begin(),
SE = (*CBI)->succ_end();
SI != SE; ++SI) {
if (BlockFilter && !BlockFilter->count(*SI))
continue;
BlockChain &SuccChain = *BlockToChain[*SI];
if (&Chain == &SuccChain || *SI == LoopHeaderBB)
continue;
if (SuccChain.LoopPredecessors > 0 && --SuccChain.LoopPredecessors == 0)
BlockWorkList.push_back(*SuccChain.begin());
}
}
}
MachineBasicBlock *MachineBlockPlacement::selectBestSuccessor(
MachineBasicBlock *BB, BlockChain &Chain,
const BlockFilterSet *BlockFilter) {
const BranchProbability HotProb(4, 5);
MachineBasicBlock *BestSucc = 0;
uint32_t BestWeight = 0;
uint32_t WeightScale = 0;
uint32_t SumWeight = MBPI->getSumForBlock(BB, WeightScale);
DEBUG(dbgs() << "Attempting merge from: " << getBlockName(BB) << "\n");
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end();
SI != SE; ++SI) {
if (BlockFilter && !BlockFilter->count(*SI))
continue;
BlockChain &SuccChain = *BlockToChain[*SI];
if (&SuccChain == &Chain) {
DEBUG(dbgs() << " " << getBlockName(*SI) << " -> Already merged!\n");
continue;
}
if (*SI != *SuccChain.begin()) {
DEBUG(dbgs() << " " << getBlockName(*SI) << " -> Mid chain!\n");
continue;
}
uint32_t SuccWeight = MBPI->getEdgeWeight(BB, *SI);
BranchProbability SuccProb(SuccWeight / WeightScale, SumWeight);
if (SuccChain.LoopPredecessors != 0) {
if (SuccProb < HotProb) {
DEBUG(dbgs() << " " << getBlockName(*SI) << " -> CFG conflict\n");
continue;
}
BlockFrequency CandidateEdgeFreq
= MBFI->getBlockFreq(BB) * SuccProb * HotProb.getCompl();
bool BadCFGConflict = false;
for (MachineBasicBlock::pred_iterator PI = (*SI)->pred_begin(),
PE = (*SI)->pred_end();
PI != PE; ++PI) {
if (*PI == *SI || (BlockFilter && !BlockFilter->count(*PI)) ||
BlockToChain[*PI] == &Chain)
continue;
BlockFrequency PredEdgeFreq
= MBFI->getBlockFreq(*PI) * MBPI->getEdgeProbability(*PI, *SI);
if (PredEdgeFreq >= CandidateEdgeFreq) {
BadCFGConflict = true;
break;
}
}
if (BadCFGConflict) {
DEBUG(dbgs() << " " << getBlockName(*SI)
<< " -> non-cold CFG conflict\n");
continue;
}
}
DEBUG(dbgs() << " " << getBlockName(*SI) << " -> " << SuccProb
<< " (prob)"
<< (SuccChain.LoopPredecessors != 0 ? " (CFG break)" : "")
<< "\n");
if (BestSucc && BestWeight >= SuccWeight)
continue;
BestSucc = *SI;
BestWeight = SuccWeight;
}
return BestSucc;
}
namespace {
class IsBlockPlaced {
const BlockChain &PlacedChain;
const BlockToChainMapType &BlockToChain;
public:
IsBlockPlaced(const BlockChain &PlacedChain,
const BlockToChainMapType &BlockToChain)
: PlacedChain(PlacedChain), BlockToChain(BlockToChain) {}
bool operator()(MachineBasicBlock *BB) const {
return BlockToChain.lookup(BB) == &PlacedChain;
}
};
}
MachineBasicBlock *MachineBlockPlacement::selectBestCandidateBlock(
BlockChain &Chain, SmallVectorImpl<MachineBasicBlock *> &WorkList,
const BlockFilterSet *BlockFilter) {
WorkList.erase(std::remove_if(WorkList.begin(), WorkList.end(),
IsBlockPlaced(Chain, BlockToChain)),
WorkList.end());
MachineBasicBlock *BestBlock = 0;
BlockFrequency BestFreq;
for (SmallVectorImpl<MachineBasicBlock *>::iterator WBI = WorkList.begin(),
WBE = WorkList.end();
WBI != WBE; ++WBI) {
BlockChain &SuccChain = *BlockToChain[*WBI];
if (&SuccChain == &Chain) {
DEBUG(dbgs() << " " << getBlockName(*WBI)
<< " -> Already merged!\n");
continue;
}
assert(SuccChain.LoopPredecessors == 0 && "Found CFG-violating block");
BlockFrequency CandidateFreq = MBFI->getBlockFreq(*WBI);
DEBUG(dbgs() << " " << getBlockName(*WBI) << " -> " << CandidateFreq
<< " (freq)\n");
if (BestBlock && BestFreq >= CandidateFreq)
continue;
BestBlock = *WBI;
BestFreq = CandidateFreq;
}
return BestBlock;
}
MachineBasicBlock *MachineBlockPlacement::getFirstUnplacedBlock(
MachineFunction &F, const BlockChain &PlacedChain,
MachineFunction::iterator &PrevUnplacedBlockIt,
const BlockFilterSet *BlockFilter) {
for (MachineFunction::iterator I = PrevUnplacedBlockIt, E = F.end(); I != E;
++I) {
if (BlockFilter && !BlockFilter->count(I))
continue;
if (BlockToChain[I] != &PlacedChain) {
PrevUnplacedBlockIt = I;
return *BlockToChain[I]->begin();
}
}
return 0;
}
void MachineBlockPlacement::buildChain(
MachineBasicBlock *BB,
BlockChain &Chain,
SmallVectorImpl<MachineBasicBlock *> &BlockWorkList,
const BlockFilterSet *BlockFilter) {
assert(BB);
assert(BlockToChain[BB] == &Chain);
MachineFunction &F = *BB->getParent();
MachineFunction::iterator PrevUnplacedBlockIt = F.begin();
MachineBasicBlock *LoopHeaderBB = BB;
markChainSuccessors(Chain, LoopHeaderBB, BlockWorkList, BlockFilter);
BB = *llvm::prior(Chain.end());
for (;;) {
assert(BB);
assert(BlockToChain[BB] == &Chain);
assert(*llvm::prior(Chain.end()) == BB);
MachineBasicBlock *BestSucc = selectBestSuccessor(BB, Chain, BlockFilter);
if (!BestSucc)
BestSucc = selectBestCandidateBlock(Chain, BlockWorkList, BlockFilter);
if (!BestSucc) {
BestSucc = getFirstUnplacedBlock(F, Chain, PrevUnplacedBlockIt,
BlockFilter);
if (!BestSucc)
break;
DEBUG(dbgs() << "Unnatural loop CFG detected, forcibly merging the "
"layout successor until the CFG reduces\n");
}
BlockChain &SuccChain = *BlockToChain[BestSucc];
SuccChain.LoopPredecessors = 0;
DEBUG(dbgs() << "Merging from " << getBlockNum(BB)
<< " to " << getBlockNum(BestSucc) << "\n");
markChainSuccessors(SuccChain, LoopHeaderBB, BlockWorkList, BlockFilter);
Chain.merge(BestSucc, &SuccChain);
BB = *llvm::prior(Chain.end());
}
DEBUG(dbgs() << "Finished forming chain for header block "
<< getBlockNum(*Chain.begin()) << "\n");
}
MachineBasicBlock *
MachineBlockPlacement::findBestLoopTop(MachineLoop &L,
const BlockFilterSet &LoopBlockSet) {
BlockChain &HeaderChain = *BlockToChain[L.getHeader()];
if (!LoopBlockSet.count(*HeaderChain.begin()))
return L.getHeader();
DEBUG(dbgs() << "Finding best loop top for: "
<< getBlockName(L.getHeader()) << "\n");
BlockFrequency BestPredFreq;
MachineBasicBlock *BestPred = 0;
for (MachineBasicBlock::pred_iterator PI = L.getHeader()->pred_begin(),
PE = L.getHeader()->pred_end();
PI != PE; ++PI) {
MachineBasicBlock *Pred = *PI;
if (!LoopBlockSet.count(Pred))
continue;
DEBUG(dbgs() << " header pred: " << getBlockName(Pred) << ", "
<< Pred->succ_size() << " successors, "
<< MBFI->getBlockFreq(Pred) << " freq\n");
if (Pred->succ_size() > 1)
continue;
BlockFrequency PredFreq = MBFI->getBlockFreq(Pred);
if (!BestPred || PredFreq > BestPredFreq ||
(!(PredFreq < BestPredFreq) &&
Pred->isLayoutSuccessor(L.getHeader()))) {
BestPred = Pred;
BestPredFreq = PredFreq;
}
}
if (!BestPred)
return L.getHeader();
while (BestPred->pred_size() == 1 &&
(*BestPred->pred_begin())->succ_size() == 1 &&
*BestPred->pred_begin() != L.getHeader())
BestPred = *BestPred->pred_begin();
DEBUG(dbgs() << " final top: " << getBlockName(BestPred) << "\n");
return BestPred;
}
MachineBasicBlock *
MachineBlockPlacement::findBestLoopExit(MachineFunction &F,
MachineLoop &L,
const BlockFilterSet &LoopBlockSet) {
BlockChain &HeaderChain = *BlockToChain[L.getHeader()];
if (!LoopBlockSet.count(*HeaderChain.begin()))
return 0;
BlockFrequency BestExitEdgeFreq;
unsigned BestExitLoopDepth = 0;
MachineBasicBlock *ExitingBB = 0;
SmallPtrSet<MachineBasicBlock *, 4> BlocksExitingToOuterLoop;
DEBUG(dbgs() << "Finding best loop exit for: "
<< getBlockName(L.getHeader()) << "\n");
for (MachineLoop::block_iterator I = L.block_begin(),
E = L.block_end();
I != E; ++I) {
BlockChain &Chain = *BlockToChain[*I];
if (*I != *llvm::prior(Chain.end()))
continue;
MachineBasicBlock *OldExitingBB = ExitingBB;
BlockFrequency OldBestExitEdgeFreq = BestExitEdgeFreq;
bool HasLoopingSucc = false;
uint32_t WeightScale = 0;
uint32_t SumWeight = MBPI->getSumForBlock(*I, WeightScale);
for (MachineBasicBlock::succ_iterator SI = (*I)->succ_begin(),
SE = (*I)->succ_end();
SI != SE; ++SI) {
if ((*SI)->isLandingPad())
continue;
if (*SI == *I)
continue;
BlockChain &SuccChain = *BlockToChain[*SI];
if (&Chain == &SuccChain) {
DEBUG(dbgs() << " exiting: " << getBlockName(*I) << " -> "
<< getBlockName(*SI) << " (chain conflict)\n");
continue;
}
uint32_t SuccWeight = MBPI->getEdgeWeight(*I, *SI);
if (LoopBlockSet.count(*SI)) {
DEBUG(dbgs() << " looping: " << getBlockName(*I) << " -> "
<< getBlockName(*SI) << " (" << SuccWeight << ")\n");
HasLoopingSucc = true;
continue;
}
unsigned SuccLoopDepth = 0;
if (MachineLoop *ExitLoop = MLI->getLoopFor(*SI)) {
SuccLoopDepth = ExitLoop->getLoopDepth();
if (ExitLoop->contains(&L))
BlocksExitingToOuterLoop.insert(*I);
}
BranchProbability SuccProb(SuccWeight / WeightScale, SumWeight);
BlockFrequency ExitEdgeFreq = MBFI->getBlockFreq(*I) * SuccProb;
DEBUG(dbgs() << " exiting: " << getBlockName(*I) << " -> "
<< getBlockName(*SI) << " [L:" << SuccLoopDepth
<< "] (" << ExitEdgeFreq << ")\n");
if (!ExitingBB || BestExitLoopDepth < SuccLoopDepth ||
ExitEdgeFreq > BestExitEdgeFreq ||
((*I)->isLayoutSuccessor(*SI) &&
!(ExitEdgeFreq < BestExitEdgeFreq))) {
BestExitEdgeFreq = ExitEdgeFreq;
ExitingBB = *I;
}
}
if (!HasLoopingSucc) {
ExitingBB = OldExitingBB;
BestExitEdgeFreq = OldBestExitEdgeFreq;
continue;
}
}
if (!ExitingBB || L.getNumBlocks() == 1)
return 0;
if (!BlocksExitingToOuterLoop.empty() &&
!BlocksExitingToOuterLoop.count(ExitingBB))
return 0;
DEBUG(dbgs() << " Best exiting block: " << getBlockName(ExitingBB) << "\n");
return ExitingBB;
}
void MachineBlockPlacement::rotateLoop(BlockChain &LoopChain,
MachineBasicBlock *ExitingBB,
const BlockFilterSet &LoopBlockSet) {
if (!ExitingBB)
return;
MachineBasicBlock *Top = *LoopChain.begin();
bool ViableTopFallthrough = false;
for (MachineBasicBlock::pred_iterator PI = Top->pred_begin(),
PE = Top->pred_end();
PI != PE; ++PI) {
BlockChain *PredChain = BlockToChain[*PI];
if (!LoopBlockSet.count(*PI) &&
(!PredChain || *PI == *llvm::prior(PredChain->end()))) {
ViableTopFallthrough = true;
break;
}
}
if (ViableTopFallthrough) {
MachineBasicBlock *Bottom = *llvm::prior(LoopChain.end());
for (MachineBasicBlock::succ_iterator SI = Bottom->succ_begin(),
SE = Bottom->succ_end();
SI != SE; ++SI) {
BlockChain *SuccChain = BlockToChain[*SI];
if (!LoopBlockSet.count(*SI) &&
(!SuccChain || *SI == *SuccChain->begin()))
return;
}
}
BlockChain::iterator ExitIt = std::find(LoopChain.begin(), LoopChain.end(),
ExitingBB);
if (ExitIt == LoopChain.end())
return;
std::rotate(LoopChain.begin(), llvm::next(ExitIt), LoopChain.end());
}
void MachineBlockPlacement::buildLoopChains(MachineFunction &F,
MachineLoop &L) {
for (MachineLoop::iterator LI = L.begin(), LE = L.end(); LI != LE; ++LI)
buildLoopChains(F, **LI);
SmallVector<MachineBasicBlock *, 16> BlockWorkList;
BlockFilterSet LoopBlockSet(L.block_begin(), L.block_end());
MachineBasicBlock *LoopTop = findBestLoopTop(L, LoopBlockSet);
MachineBasicBlock *ExitingBB = 0;
if (LoopTop == L.getHeader())
ExitingBB = findBestLoopExit(F, L, LoopBlockSet);
BlockChain &LoopChain = *BlockToChain[LoopTop];
SmallPtrSet<BlockChain *, 4> UpdatedPreds;
assert(LoopChain.LoopPredecessors == 0);
UpdatedPreds.insert(&LoopChain);
for (MachineLoop::block_iterator BI = L.block_begin(),
BE = L.block_end();
BI != BE; ++BI) {
BlockChain &Chain = *BlockToChain[*BI];
if (!UpdatedPreds.insert(&Chain))
continue;
assert(Chain.LoopPredecessors == 0);
for (BlockChain::iterator BCI = Chain.begin(), BCE = Chain.end();
BCI != BCE; ++BCI) {
assert(BlockToChain[*BCI] == &Chain);
for (MachineBasicBlock::pred_iterator PI = (*BCI)->pred_begin(),
PE = (*BCI)->pred_end();
PI != PE; ++PI) {
if (BlockToChain[*PI] == &Chain || !LoopBlockSet.count(*PI))
continue;
++Chain.LoopPredecessors;
}
}
if (Chain.LoopPredecessors == 0)
BlockWorkList.push_back(*Chain.begin());
}
buildChain(LoopTop, LoopChain, BlockWorkList, &LoopBlockSet);
rotateLoop(LoopChain, ExitingBB, LoopBlockSet);
DEBUG({
bool BadLoop = false;
if (LoopChain.LoopPredecessors) {
BadLoop = true;
dbgs() << "Loop chain contains a block without its preds placed!\n"
<< " Loop header: " << getBlockName(*L.block_begin()) << "\n"
<< " Chain header: " << getBlockName(*LoopChain.begin()) << "\n";
}
for (BlockChain::iterator BCI = LoopChain.begin(), BCE = LoopChain.end();
BCI != BCE; ++BCI) {
dbgs() << " ... " << getBlockName(*BCI) << "\n";
if (!LoopBlockSet.erase(*BCI)) {
dbgs() << "Loop chain contains a block not contained by the loop!\n"
<< " Loop header: " << getBlockName(*L.block_begin()) << "\n"
<< " Chain header: " << getBlockName(*LoopChain.begin()) << "\n"
<< " Bad block: " << getBlockName(*BCI) << "\n";
}
}
if (!LoopBlockSet.empty()) {
BadLoop = true;
for (BlockFilterSet::iterator LBI = LoopBlockSet.begin(),
LBE = LoopBlockSet.end();
LBI != LBE; ++LBI)
dbgs() << "Loop contains blocks never placed into a chain!\n"
<< " Loop header: " << getBlockName(*L.block_begin()) << "\n"
<< " Chain header: " << getBlockName(*LoopChain.begin()) << "\n"
<< " Bad block: " << getBlockName(*LBI) << "\n";
}
assert(!BadLoop && "Detected problems with the placement of this loop.");
});
}
void MachineBlockPlacement::buildCFGChains(MachineFunction &F) {
SmallVector<MachineOperand, 4> Cond; for (MachineFunction::iterator FI = F.begin(), FE = F.end(); FI != FE; ++FI) {
MachineBasicBlock *BB = FI;
BlockChain *Chain
= new (ChainAllocator.Allocate()) BlockChain(BlockToChain, BB);
for (;;) {
Cond.clear();
MachineBasicBlock *TBB = 0, *FBB = 0; if (!TII->AnalyzeBranch(*BB, TBB, FBB, Cond) || !FI->canFallThrough())
break;
MachineFunction::iterator NextFI(llvm::next(FI));
MachineBasicBlock *NextBB = NextFI;
assert(NextFI != FE && "Can't fallthrough past the last block.");
DEBUG(dbgs() << "Pre-merging due to unanalyzable fallthrough: "
<< getBlockName(BB) << " -> " << getBlockName(NextBB)
<< "\n");
Chain->merge(NextBB, 0);
FI = NextFI;
BB = NextBB;
}
}
for (MachineLoopInfo::iterator LI = MLI->begin(), LE = MLI->end(); LI != LE;
++LI)
buildLoopChains(F, **LI);
SmallVector<MachineBasicBlock *, 16> BlockWorkList;
SmallPtrSet<BlockChain *, 4> UpdatedPreds;
for (MachineFunction::iterator FI = F.begin(), FE = F.end(); FI != FE; ++FI) {
MachineBasicBlock *BB = &*FI;
BlockChain &Chain = *BlockToChain[BB];
if (!UpdatedPreds.insert(&Chain))
continue;
assert(Chain.LoopPredecessors == 0);
for (BlockChain::iterator BCI = Chain.begin(), BCE = Chain.end();
BCI != BCE; ++BCI) {
assert(BlockToChain[*BCI] == &Chain);
for (MachineBasicBlock::pred_iterator PI = (*BCI)->pred_begin(),
PE = (*BCI)->pred_end();
PI != PE; ++PI) {
if (BlockToChain[*PI] == &Chain)
continue;
++Chain.LoopPredecessors;
}
}
if (Chain.LoopPredecessors == 0)
BlockWorkList.push_back(*Chain.begin());
}
BlockChain &FunctionChain = *BlockToChain[&F.front()];
buildChain(&F.front(), FunctionChain, BlockWorkList);
typedef SmallPtrSet<MachineBasicBlock *, 16> FunctionBlockSetType;
DEBUG({
bool BadFunc = false;
FunctionBlockSetType FunctionBlockSet;
for (MachineFunction::iterator FI = F.begin(), FE = F.end(); FI != FE; ++FI)
FunctionBlockSet.insert(FI);
for (BlockChain::iterator BCI = FunctionChain.begin(),
BCE = FunctionChain.end();
BCI != BCE; ++BCI)
if (!FunctionBlockSet.erase(*BCI)) {
BadFunc = true;
dbgs() << "Function chain contains a block not in the function!\n"
<< " Bad block: " << getBlockName(*BCI) << "\n";
}
if (!FunctionBlockSet.empty()) {
BadFunc = true;
for (FunctionBlockSetType::iterator FBI = FunctionBlockSet.begin(),
FBE = FunctionBlockSet.end();
FBI != FBE; ++FBI)
dbgs() << "Function contains blocks never placed into a chain!\n"
<< " Bad block: " << getBlockName(*FBI) << "\n";
}
assert(!BadFunc && "Detected problems with the block placement.");
});
MachineFunction::iterator InsertPos = F.begin();
for (BlockChain::iterator BI = FunctionChain.begin(),
BE = FunctionChain.end();
BI != BE; ++BI) {
DEBUG(dbgs() << (BI == FunctionChain.begin() ? "Placing chain "
: " ... ")
<< getBlockName(*BI) << "\n");
if (InsertPos != MachineFunction::iterator(*BI))
F.splice(InsertPos, *BI);
else
++InsertPos;
if (BI == FunctionChain.begin())
continue;
MachineBasicBlock *PrevBB = llvm::prior(MachineFunction::iterator(*BI));
Cond.clear();
MachineBasicBlock *TBB = 0, *FBB = 0; if (!TII->AnalyzeBranch(*PrevBB, TBB, FBB, Cond)) {
if (TBB && !Cond.empty() && FBB &&
MBPI->getEdgeWeight(PrevBB, FBB) > MBPI->getEdgeWeight(PrevBB, TBB) &&
!TII->ReverseBranchCondition(Cond)) {
DEBUG(dbgs() << "Reverse order of the two branches: "
<< getBlockName(PrevBB) << "\n");
DEBUG(dbgs() << " Edge weight: " << MBPI->getEdgeWeight(PrevBB, FBB)
<< " vs " << MBPI->getEdgeWeight(PrevBB, TBB) << "\n");
DebugLoc dl; TII->RemoveBranch(*PrevBB);
TII->InsertBranch(*PrevBB, FBB, TBB, Cond, dl);
}
PrevBB->updateTerminator();
}
}
Cond.clear();
MachineBasicBlock *TBB = 0, *FBB = 0; if (!TII->AnalyzeBranch(F.back(), TBB, FBB, Cond))
F.back().updateTerminator();
if (F.getFunction()->getAttributes().
hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize))
return;
unsigned Align = TLI->getPrefLoopAlignment();
if (!Align)
return; if (FunctionChain.begin() == FunctionChain.end())
return;
const BranchProbability ColdProb(1, 5); BlockFrequency EntryFreq = MBFI->getBlockFreq(F.begin());
BlockFrequency WeightedEntryFreq = EntryFreq * ColdProb;
for (BlockChain::iterator BI = llvm::next(FunctionChain.begin()),
BE = FunctionChain.end();
BI != BE; ++BI) {
MachineLoop *L = MLI->getLoopFor(*BI);
if (!L)
continue;
BlockFrequency Freq = MBFI->getBlockFreq(*BI);
if (Freq < WeightedEntryFreq)
continue;
MachineBasicBlock *LoopHeader = L->getHeader();
BlockFrequency LoopHeaderFreq = MBFI->getBlockFreq(LoopHeader);
if (Freq < (LoopHeaderFreq * ColdProb))
continue;
MachineBasicBlock *LayoutPred = *llvm::prior(BI);
if (!LayoutPred->isSuccessor(*BI)) {
(*BI)->setAlignment(Align);
continue;
}
BranchProbability LayoutProb = MBPI->getEdgeProbability(LayoutPred, *BI);
BlockFrequency LayoutEdgeFreq = MBFI->getBlockFreq(LayoutPred) * LayoutProb;
if (LayoutEdgeFreq <= (Freq * ColdProb))
(*BI)->setAlignment(Align);
}
}
bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &F) {
if (llvm::next(F.begin()) == F.end())
return false;
MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
MLI = &getAnalysis<MachineLoopInfo>();
TII = F.getTarget().getInstrInfo();
TLI = F.getTarget().getTargetLowering();
assert(BlockToChain.empty());
buildCFGChains(F);
BlockToChain.clear();
ChainAllocator.DestroyAll();
return true;
}
namespace {
class MachineBlockPlacementStats : public MachineFunctionPass {
const MachineBranchProbabilityInfo *MBPI;
const MachineBlockFrequencyInfo *MBFI;
public:
static char ID; MachineBlockPlacementStats() : MachineFunctionPass(ID) {
initializeMachineBlockPlacementStatsPass(*PassRegistry::getPassRegistry());
}
bool runOnMachineFunction(MachineFunction &F);
void getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<MachineBranchProbabilityInfo>();
AU.addRequired<MachineBlockFrequencyInfo>();
AU.setPreservesAll();
MachineFunctionPass::getAnalysisUsage(AU);
}
};
}
char MachineBlockPlacementStats::ID = 0;
char &llvm::MachineBlockPlacementStatsID = MachineBlockPlacementStats::ID;
INITIALIZE_PASS_BEGIN(MachineBlockPlacementStats, "block-placement-stats",
"Basic Block Placement Stats", false, false)
INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo)
INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfo)
INITIALIZE_PASS_END(MachineBlockPlacementStats, "block-placement-stats",
"Basic Block Placement Stats", false, false)
bool MachineBlockPlacementStats::runOnMachineFunction(MachineFunction &F) {
if (llvm::next(F.begin()) == F.end())
return false;
MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
BlockFrequency BlockFreq = MBFI->getBlockFreq(I);
Statistic &NumBranches = (I->succ_size() > 1) ? NumCondBranches
: NumUncondBranches;
Statistic &BranchTakenFreq = (I->succ_size() > 1) ? CondBranchTakenFreq
: UncondBranchTakenFreq;
for (MachineBasicBlock::succ_iterator SI = I->succ_begin(),
SE = I->succ_end();
SI != SE; ++SI) {
if (I->isLayoutSuccessor(*SI))
continue;
BlockFrequency EdgeFreq = BlockFreq * MBPI->getEdgeProbability(I, *SI);
++NumBranches;
BranchTakenFreq += EdgeFreq.getFrequency();
}
}
return false;
}