MachineBlockPlacement.cpp [plain text]
#include "llvm/CodeGen/Passes.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetInstrInfo.h"
#include "llvm/Target/TargetLowering.h"
#include "llvm/Target/TargetSubtargetInfo.h"
#include <algorithm>
using namespace llvm;
#define DEBUG_TYPE "block-placement"
STATISTIC(NumCondBranches, "Number of conditional branches");
STATISTIC(NumUncondBranches, "Number of unconditional branches");
STATISTIC(CondBranchTakenFreq,
"Potential frequency of taking conditional branches");
STATISTIC(UncondBranchTakenFreq,
"Potential frequency of taking unconditional branches");
static cl::opt<unsigned> AlignAllBlock("align-all-blocks",
cl::desc("Force the alignment of all "
"blocks in the function."),
cl::init(0), cl::Hidden);
static cl::opt<unsigned> ExitBlockBias(
"block-placement-exit-block-bias",
cl::desc("Block frequency percentage a loop exit block needs "
"over the original exit to be considered the new exit."),
cl::init(0), cl::Hidden);
static cl::opt<bool> OutlineOptionalBranches(
"outline-optional-branches",
cl::desc("Put completely optional branches, i.e. branches with a common "
"post dominator, out of line."),
cl::init(false), cl::Hidden);
static cl::opt<unsigned> OutlineOptionalThreshold(
"outline-optional-threshold",
cl::desc("Don't outline optional branches that are a single block with an "
"instruction count below this threshold"),
cl::init(4), cl::Hidden);
static cl::opt<unsigned> LoopToColdBlockRatio(
"loop-to-cold-block-ratio",
cl::desc("Outline loop blocks from loop chain if (frequency of loop) / "
"(frequency of block) is greater than this ratio"),
cl::init(5), cl::Hidden);
static cl::opt<bool>
PreciseRotationCost("precise-rotation-cost",
cl::desc("Model the cost of loop rotation more "
"precisely by using profile data."),
cl::init(false), cl::Hidden);
static cl::opt<unsigned> MisfetchCost(
"misfetch-cost",
cl::desc("Cost that models the probablistic risk of an instruction "
"misfetch due to a jump comparing to falling through, whose cost "
"is zero."),
cl::init(1), cl::Hidden);
static cl::opt<unsigned> JumpInstCost("jump-inst-cost",
cl::desc("Cost of jump instructions."),
cl::init(1), cl::Hidden);
namespace {
class BlockChain;
typedef DenseMap<MachineBasicBlock *, BlockChain *> BlockToChainMapType;
}
namespace {
class BlockChain {
SmallVector<MachineBasicBlock *, 4> Blocks;
BlockToChainMapType &BlockToChain;
public:
BlockChain(BlockToChainMapType &BlockToChain, MachineBasicBlock *BB)
: Blocks(1, BB), BlockToChain(BlockToChain), LoopPredecessors(0) {
assert(BB && "Cannot create a chain with a null basic block");
BlockToChain[BB] = this;
}
typedef SmallVectorImpl<MachineBasicBlock *>::iterator iterator;
iterator begin() { return Blocks.begin(); }
iterator end() { return Blocks.end(); }
void merge(MachineBasicBlock *BB, BlockChain *Chain) {
assert(BB);
assert(!Blocks.empty());
if (!Chain) {
assert(!BlockToChain[BB]);
Blocks.push_back(BB);
BlockToChain[BB] = this;
return;
}
assert(BB == *Chain->begin());
assert(Chain->begin() != Chain->end());
for (MachineBasicBlock *ChainBB : *Chain) {
Blocks.push_back(ChainBB);
assert(BlockToChain[ChainBB] == Chain && "Incoming blocks not in chain");
BlockToChain[ChainBB] = this;
}
}
#ifndef NDEBUG
LLVM_DUMP_METHOD void dump() {
for (MachineBasicBlock *MBB : *this)
MBB->dump();
}
#endif // NDEBUG
unsigned LoopPredecessors;
};
}
namespace {
class MachineBlockPlacement : public MachineFunctionPass {
typedef SmallPtrSet<MachineBasicBlock *, 16> BlockFilterSet;
const MachineBranchProbabilityInfo *MBPI;
const MachineBlockFrequencyInfo *MBFI;
const MachineLoopInfo *MLI;
const TargetInstrInfo *TII;
const TargetLoweringBase *TLI;
MachineDominatorTree *MDT;
SmallPtrSet<MachineBasicBlock *, 4> UnavoidableBlocks;
SpecificBumpPtrAllocator<BlockChain> ChainAllocator;
DenseMap<MachineBasicBlock *, BlockChain *> BlockToChain;
void markChainSuccessors(BlockChain &Chain, MachineBasicBlock *LoopHeaderBB,
SmallVectorImpl<MachineBasicBlock *> &BlockWorkList,
const BlockFilterSet *BlockFilter = nullptr);
MachineBasicBlock *selectBestSuccessor(MachineBasicBlock *BB,
BlockChain &Chain,
const BlockFilterSet *BlockFilter);
MachineBasicBlock *
selectBestCandidateBlock(BlockChain &Chain,
SmallVectorImpl<MachineBasicBlock *> &WorkList,
const BlockFilterSet *BlockFilter);
MachineBasicBlock *
getFirstUnplacedBlock(MachineFunction &F, const BlockChain &PlacedChain,
MachineFunction::iterator &PrevUnplacedBlockIt,
const BlockFilterSet *BlockFilter);
void buildChain(MachineBasicBlock *BB, BlockChain &Chain,
SmallVectorImpl<MachineBasicBlock *> &BlockWorkList,
const BlockFilterSet *BlockFilter = nullptr);
MachineBasicBlock *findBestLoopTop(MachineLoop &L,
const BlockFilterSet &LoopBlockSet);
MachineBasicBlock *findBestLoopExit(MachineFunction &F, MachineLoop &L,
const BlockFilterSet &LoopBlockSet);
BlockFilterSet collectLoopBlockSet(MachineFunction &F, MachineLoop &L);
void buildLoopChains(MachineFunction &F, MachineLoop &L);
void rotateLoop(BlockChain &LoopChain, MachineBasicBlock *ExitingBB,
const BlockFilterSet &LoopBlockSet);
void rotateLoopWithProfile(BlockChain &LoopChain, MachineLoop &L,
const BlockFilterSet &LoopBlockSet);
void buildCFGChains(MachineFunction &F);
public:
static char ID; MachineBlockPlacement() : MachineFunctionPass(ID) {
initializeMachineBlockPlacementPass(*PassRegistry::getPassRegistry());
}
bool runOnMachineFunction(MachineFunction &F) override;
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<MachineBranchProbabilityInfo>();
AU.addRequired<MachineBlockFrequencyInfo>();
AU.addRequired<MachineDominatorTree>();
AU.addRequired<MachineLoopInfo>();
MachineFunctionPass::getAnalysisUsage(AU);
}
};
}
char MachineBlockPlacement::ID = 0;
char &llvm::MachineBlockPlacementID = MachineBlockPlacement::ID;
INITIALIZE_PASS_BEGIN(MachineBlockPlacement, "block-placement",
"Branch Probability Basic Block Placement", false, false)
INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo)
INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfo)
INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
INITIALIZE_PASS_END(MachineBlockPlacement, "block-placement",
"Branch Probability Basic Block Placement", false, false)
#ifndef NDEBUG
static std::string getBlockName(MachineBasicBlock *BB) {
std::string Result;
raw_string_ostream OS(Result);
OS << "BB#" << BB->getNumber();
OS << " (derived from LLVM BB '" << BB->getName() << "')";
OS.flush();
return Result;
}
static std::string getBlockNum(MachineBasicBlock *BB) {
std::string Result;
raw_string_ostream OS(Result);
OS << "BB#" << BB->getNumber();
OS.flush();
return Result;
}
#endif
void MachineBlockPlacement::markChainSuccessors(
BlockChain &Chain, MachineBasicBlock *LoopHeaderBB,
SmallVectorImpl<MachineBasicBlock *> &BlockWorkList,
const BlockFilterSet *BlockFilter) {
for (MachineBasicBlock *MBB : Chain) {
for (MachineBasicBlock *Succ : MBB->successors()) {
if (BlockFilter && !BlockFilter->count(Succ))
continue;
BlockChain &SuccChain = *BlockToChain[Succ];
if (&Chain == &SuccChain || Succ == LoopHeaderBB)
continue;
if (SuccChain.LoopPredecessors > 0 && --SuccChain.LoopPredecessors == 0)
BlockWorkList.push_back(*SuccChain.begin());
}
}
}
MachineBasicBlock *
MachineBlockPlacement::selectBestSuccessor(MachineBasicBlock *BB,
BlockChain &Chain,
const BlockFilterSet *BlockFilter) {
const BranchProbability HotProb(4, 5);
MachineBasicBlock *BestSucc = nullptr;
uint32_t BestWeight = 0;
uint32_t WeightScale = 0;
uint32_t SumWeight = MBPI->getSumForBlock(BB, WeightScale);
DEBUG(dbgs() << "Attempting merge from: " << getBlockName(BB) << "\n");
for (MachineBasicBlock *Succ : BB->successors()) {
if (BlockFilter && !BlockFilter->count(Succ))
continue;
BlockChain &SuccChain = *BlockToChain[Succ];
if (&SuccChain == &Chain) {
DEBUG(dbgs() << " " << getBlockName(Succ) << " -> Already merged!\n");
continue;
}
if (Succ != *SuccChain.begin()) {
DEBUG(dbgs() << " " << getBlockName(Succ) << " -> Mid chain!\n");
continue;
}
uint32_t SuccWeight = MBPI->getEdgeWeight(BB, Succ);
BranchProbability SuccProb(SuccWeight / WeightScale, SumWeight);
if (OutlineOptionalBranches && SuccProb > HotProb.getCompl() &&
UnavoidableBlocks.count(Succ) > 0) {
auto HasShortOptionalBranch = [&]() {
for (MachineBasicBlock *Pred : Succ->predecessors()) {
if (Pred == Succ || (BlockFilter && !BlockFilter->count(Pred)) ||
BlockToChain[Pred] == &Chain)
continue;
if (Pred->pred_size() > 1 || *Pred->pred_begin() != BB)
continue;
if (Pred->size() < OutlineOptionalThreshold)
return true;
}
return false;
};
if (!HasShortOptionalBranch())
return Succ;
}
if (SuccChain.LoopPredecessors != 0) {
if (SuccProb < HotProb) {
DEBUG(dbgs() << " " << getBlockName(Succ) << " -> " << SuccProb
<< " (prob) (CFG conflict)\n");
continue;
}
BlockFrequency CandidateEdgeFreq =
MBFI->getBlockFreq(BB) * SuccProb * HotProb.getCompl();
bool BadCFGConflict = false;
for (MachineBasicBlock *Pred : Succ->predecessors()) {
if (Pred == Succ || (BlockFilter && !BlockFilter->count(Pred)) ||
BlockToChain[Pred] == &Chain)
continue;
BlockFrequency PredEdgeFreq =
MBFI->getBlockFreq(Pred) * MBPI->getEdgeProbability(Pred, Succ);
if (PredEdgeFreq >= CandidateEdgeFreq) {
BadCFGConflict = true;
break;
}
}
if (BadCFGConflict) {
DEBUG(dbgs() << " " << getBlockName(Succ) << " -> " << SuccProb
<< " (prob) (non-cold CFG conflict)\n");
continue;
}
}
DEBUG(dbgs() << " " << getBlockName(Succ) << " -> " << SuccProb
<< " (prob)"
<< (SuccChain.LoopPredecessors != 0 ? " (CFG break)" : "")
<< "\n");
if (BestSucc && BestWeight >= SuccWeight)
continue;
BestSucc = Succ;
BestWeight = SuccWeight;
}
return BestSucc;
}
MachineBasicBlock *MachineBlockPlacement::selectBestCandidateBlock(
BlockChain &Chain, SmallVectorImpl<MachineBasicBlock *> &WorkList,
const BlockFilterSet *BlockFilter) {
WorkList.erase(std::remove_if(WorkList.begin(), WorkList.end(),
[&](MachineBasicBlock *BB) {
return BlockToChain.lookup(BB) == &Chain;
}),
WorkList.end());
MachineBasicBlock *BestBlock = nullptr;
BlockFrequency BestFreq;
for (MachineBasicBlock *MBB : WorkList) {
BlockChain &SuccChain = *BlockToChain[MBB];
if (&SuccChain == &Chain) {
DEBUG(dbgs() << " " << getBlockName(MBB) << " -> Already merged!\n");
continue;
}
assert(SuccChain.LoopPredecessors == 0 && "Found CFG-violating block");
BlockFrequency CandidateFreq = MBFI->getBlockFreq(MBB);
DEBUG(dbgs() << " " << getBlockName(MBB) << " -> ";
MBFI->printBlockFreq(dbgs(), CandidateFreq) << " (freq)\n");
if (BestBlock && BestFreq >= CandidateFreq)
continue;
BestBlock = MBB;
BestFreq = CandidateFreq;
}
return BestBlock;
}
MachineBasicBlock *MachineBlockPlacement::getFirstUnplacedBlock(
MachineFunction &F, const BlockChain &PlacedChain,
MachineFunction::iterator &PrevUnplacedBlockIt,
const BlockFilterSet *BlockFilter) {
for (MachineFunction::iterator I = PrevUnplacedBlockIt, E = F.end(); I != E;
++I) {
if (BlockFilter && !BlockFilter->count(&*I))
continue;
if (BlockToChain[&*I] != &PlacedChain) {
PrevUnplacedBlockIt = I;
return *BlockToChain[&*I]->begin();
}
}
return nullptr;
}
void MachineBlockPlacement::buildChain(
MachineBasicBlock *BB, BlockChain &Chain,
SmallVectorImpl<MachineBasicBlock *> &BlockWorkList,
const BlockFilterSet *BlockFilter) {
assert(BB);
assert(BlockToChain[BB] == &Chain);
MachineFunction &F = *BB->getParent();
MachineFunction::iterator PrevUnplacedBlockIt = F.begin();
MachineBasicBlock *LoopHeaderBB = BB;
markChainSuccessors(Chain, LoopHeaderBB, BlockWorkList, BlockFilter);
BB = *std::prev(Chain.end());
for (;;) {
assert(BB);
assert(BlockToChain[BB] == &Chain);
assert(*std::prev(Chain.end()) == BB);
MachineBasicBlock *BestSucc = selectBestSuccessor(BB, Chain, BlockFilter);
if (!BestSucc)
BestSucc = selectBestCandidateBlock(Chain, BlockWorkList, BlockFilter);
if (!BestSucc) {
BestSucc =
getFirstUnplacedBlock(F, Chain, PrevUnplacedBlockIt, BlockFilter);
if (!BestSucc)
break;
DEBUG(dbgs() << "Unnatural loop CFG detected, forcibly merging the "
"layout successor until the CFG reduces\n");
}
BlockChain &SuccChain = *BlockToChain[BestSucc];
SuccChain.LoopPredecessors = 0;
DEBUG(dbgs() << "Merging from " << getBlockNum(BB) << " to "
<< getBlockNum(BestSucc) << "\n");
markChainSuccessors(SuccChain, LoopHeaderBB, BlockWorkList, BlockFilter);
Chain.merge(BestSucc, &SuccChain);
BB = *std::prev(Chain.end());
}
DEBUG(dbgs() << "Finished forming chain for header block "
<< getBlockNum(*Chain.begin()) << "\n");
}
MachineBasicBlock *
MachineBlockPlacement::findBestLoopTop(MachineLoop &L,
const BlockFilterSet &LoopBlockSet) {
BlockChain &HeaderChain = *BlockToChain[L.getHeader()];
if (!LoopBlockSet.count(*HeaderChain.begin()))
return L.getHeader();
DEBUG(dbgs() << "Finding best loop top for: " << getBlockName(L.getHeader())
<< "\n");
BlockFrequency BestPredFreq;
MachineBasicBlock *BestPred = nullptr;
for (MachineBasicBlock *Pred : L.getHeader()->predecessors()) {
if (!LoopBlockSet.count(Pred))
continue;
DEBUG(dbgs() << " header pred: " << getBlockName(Pred) << ", "
<< Pred->succ_size() << " successors, ";
MBFI->printBlockFreq(dbgs(), Pred) << " freq\n");
if (Pred->succ_size() > 1)
continue;
BlockFrequency PredFreq = MBFI->getBlockFreq(Pred);
if (!BestPred || PredFreq > BestPredFreq ||
(!(PredFreq < BestPredFreq) &&
Pred->isLayoutSuccessor(L.getHeader()))) {
BestPred = Pred;
BestPredFreq = PredFreq;
}
}
if (!BestPred)
return L.getHeader();
while (BestPred->pred_size() == 1 &&
(*BestPred->pred_begin())->succ_size() == 1 &&
*BestPred->pred_begin() != L.getHeader())
BestPred = *BestPred->pred_begin();
DEBUG(dbgs() << " final top: " << getBlockName(BestPred) << "\n");
return BestPred;
}
MachineBasicBlock *
MachineBlockPlacement::findBestLoopExit(MachineFunction &F, MachineLoop &L,
const BlockFilterSet &LoopBlockSet) {
BlockChain &HeaderChain = *BlockToChain[L.getHeader()];
if (!LoopBlockSet.count(*HeaderChain.begin()))
return nullptr;
BlockFrequency BestExitEdgeFreq;
unsigned BestExitLoopDepth = 0;
MachineBasicBlock *ExitingBB = nullptr;
SmallPtrSet<MachineBasicBlock *, 4> BlocksExitingToOuterLoop;
DEBUG(dbgs() << "Finding best loop exit for: " << getBlockName(L.getHeader())
<< "\n");
for (MachineBasicBlock *MBB : L.getBlocks()) {
BlockChain &Chain = *BlockToChain[MBB];
if (MBB != *std::prev(Chain.end()))
continue;
MachineBasicBlock *OldExitingBB = ExitingBB;
BlockFrequency OldBestExitEdgeFreq = BestExitEdgeFreq;
bool HasLoopingSucc = false;
uint32_t WeightScale = 0;
uint32_t SumWeight = MBPI->getSumForBlock(MBB, WeightScale);
for (MachineBasicBlock *Succ : MBB->successors()) {
if (Succ->isEHPad())
continue;
if (Succ == MBB)
continue;
BlockChain &SuccChain = *BlockToChain[Succ];
if (&Chain == &SuccChain) {
DEBUG(dbgs() << " exiting: " << getBlockName(MBB) << " -> "
<< getBlockName(Succ) << " (chain conflict)\n");
continue;
}
uint32_t SuccWeight = MBPI->getEdgeWeight(MBB, Succ);
if (LoopBlockSet.count(Succ)) {
DEBUG(dbgs() << " looping: " << getBlockName(MBB) << " -> "
<< getBlockName(Succ) << " (" << SuccWeight << ")\n");
HasLoopingSucc = true;
continue;
}
unsigned SuccLoopDepth = 0;
if (MachineLoop *ExitLoop = MLI->getLoopFor(Succ)) {
SuccLoopDepth = ExitLoop->getLoopDepth();
if (ExitLoop->contains(&L))
BlocksExitingToOuterLoop.insert(MBB);
}
BranchProbability SuccProb(SuccWeight / WeightScale, SumWeight);
BlockFrequency ExitEdgeFreq = MBFI->getBlockFreq(MBB) * SuccProb;
DEBUG(dbgs() << " exiting: " << getBlockName(MBB) << " -> "
<< getBlockName(Succ) << " [L:" << SuccLoopDepth << "] (";
MBFI->printBlockFreq(dbgs(), ExitEdgeFreq) << ")\n");
BranchProbability Bias(100 - ExitBlockBias, 100);
if (!ExitingBB || SuccLoopDepth > BestExitLoopDepth ||
ExitEdgeFreq > BestExitEdgeFreq ||
(MBB->isLayoutSuccessor(Succ) &&
!(ExitEdgeFreq < BestExitEdgeFreq * Bias))) {
BestExitEdgeFreq = ExitEdgeFreq;
ExitingBB = MBB;
}
}
if (!HasLoopingSucc) {
ExitingBB = OldExitingBB;
BestExitEdgeFreq = OldBestExitEdgeFreq;
continue;
}
}
if (!ExitingBB || L.getNumBlocks() == 1)
return nullptr;
if (!BlocksExitingToOuterLoop.empty() &&
!BlocksExitingToOuterLoop.count(ExitingBB))
return nullptr;
DEBUG(dbgs() << " Best exiting block: " << getBlockName(ExitingBB) << "\n");
return ExitingBB;
}
void MachineBlockPlacement::rotateLoop(BlockChain &LoopChain,
MachineBasicBlock *ExitingBB,
const BlockFilterSet &LoopBlockSet) {
if (!ExitingBB)
return;
MachineBasicBlock *Top = *LoopChain.begin();
bool ViableTopFallthrough = false;
for (MachineBasicBlock *Pred : Top->predecessors()) {
BlockChain *PredChain = BlockToChain[Pred];
if (!LoopBlockSet.count(Pred) &&
(!PredChain || Pred == *std::prev(PredChain->end()))) {
ViableTopFallthrough = true;
break;
}
}
if (ViableTopFallthrough) {
MachineBasicBlock *Bottom = *std::prev(LoopChain.end());
for (MachineBasicBlock *Succ : Bottom->successors()) {
BlockChain *SuccChain = BlockToChain[Succ];
if (!LoopBlockSet.count(Succ) &&
(!SuccChain || Succ == *SuccChain->begin()))
return;
}
}
BlockChain::iterator ExitIt =
std::find(LoopChain.begin(), LoopChain.end(), ExitingBB);
if (ExitIt == LoopChain.end())
return;
std::rotate(LoopChain.begin(), std::next(ExitIt), LoopChain.end());
}
void MachineBlockPlacement::rotateLoopWithProfile(
BlockChain &LoopChain, MachineLoop &L, const BlockFilterSet &LoopBlockSet) {
auto HeaderBB = L.getHeader();
auto HeaderIter = std::find(LoopChain.begin(), LoopChain.end(), HeaderBB);
auto RotationPos = LoopChain.end();
BlockFrequency SmallestRotationCost = BlockFrequency::getMaxFrequency();
auto ScaleBlockFrequency = [](BlockFrequency Freq,
unsigned Scale) -> BlockFrequency {
if (Scale == 0)
return 0;
return Freq / BranchProbability(1, Scale);
};
BlockFrequency HeaderFallThroughCost(0);
for (auto *Pred : HeaderBB->predecessors()) {
BlockChain *PredChain = BlockToChain[Pred];
if (!LoopBlockSet.count(Pred) &&
(!PredChain || Pred == *std::prev(PredChain->end()))) {
auto EdgeFreq =
MBFI->getBlockFreq(Pred) * MBPI->getEdgeProbability(Pred, HeaderBB);
auto FallThruCost = ScaleBlockFrequency(EdgeFreq, MisfetchCost);
if (Pred->succ_size() == 1)
FallThruCost += ScaleBlockFrequency(EdgeFreq, JumpInstCost);
HeaderFallThroughCost = std::max(HeaderFallThroughCost, FallThruCost);
}
}
SmallVector<std::pair<MachineBasicBlock *, BlockFrequency>, 4> ExitsWithFreq;
for (auto BB : LoopChain) {
uint32_t LargestExitEdgeWeight = 0;
for (auto *Succ : BB->successors()) {
BlockChain *SuccChain = BlockToChain[Succ];
if (!LoopBlockSet.count(Succ) &&
(!SuccChain || Succ == *SuccChain->begin())) {
uint32_t SuccWeight = MBPI->getEdgeWeight(BB, Succ);
LargestExitEdgeWeight = std::max(LargestExitEdgeWeight, SuccWeight);
}
}
if (LargestExitEdgeWeight > 0) {
uint32_t WeightScale = 0;
uint32_t SumWeight = MBPI->getSumForBlock(BB, WeightScale);
auto ExitFreq =
MBFI->getBlockFreq(BB) *
BranchProbability(LargestExitEdgeWeight / WeightScale, SumWeight);
ExitsWithFreq.emplace_back(BB, ExitFreq);
}
}
for (auto Iter = LoopChain.begin(), TailIter = std::prev(LoopChain.end()),
EndIter = LoopChain.end();
Iter != EndIter; Iter++, TailIter++) {
if (TailIter == LoopChain.end())
TailIter = LoopChain.begin();
auto TailBB = *TailIter;
BlockFrequency Cost = 0;
if (Iter != HeaderIter)
Cost += HeaderFallThroughCost;
for (auto &ExitWithFreq : ExitsWithFreq)
if (TailBB != ExitWithFreq.first)
Cost += ExitWithFreq.second;
if (TailBB->isSuccessor(*Iter)) {
auto TailBBFreq = MBFI->getBlockFreq(TailBB);
if (TailBB->succ_size() == 1)
Cost += ScaleBlockFrequency(TailBBFreq.getFrequency(),
MisfetchCost + JumpInstCost);
else if (TailBB->succ_size() == 2) {
auto TailToHeadProb = MBPI->getEdgeProbability(TailBB, *Iter);
auto TailToHeadFreq = TailBBFreq * TailToHeadProb;
auto ColderEdgeFreq = TailToHeadProb > BranchProbability(1, 2)
? TailBBFreq * TailToHeadProb.getCompl()
: TailToHeadFreq;
Cost += ScaleBlockFrequency(TailToHeadFreq, MisfetchCost) +
ScaleBlockFrequency(ColderEdgeFreq, JumpInstCost);
}
}
DEBUG(dbgs() << "The cost of loop rotation by making " << getBlockNum(*Iter)
<< " to the top: " << Cost.getFrequency() << "\n");
if (Cost < SmallestRotationCost) {
SmallestRotationCost = Cost;
RotationPos = Iter;
}
}
if (RotationPos != LoopChain.end()) {
DEBUG(dbgs() << "Rotate loop by making " << getBlockNum(*RotationPos)
<< " to the top\n");
std::rotate(LoopChain.begin(), RotationPos, LoopChain.end());
}
}
MachineBlockPlacement::BlockFilterSet
MachineBlockPlacement::collectLoopBlockSet(MachineFunction &F, MachineLoop &L) {
BlockFilterSet LoopBlockSet;
if (F.getFunction()->getEntryCount()) {
BlockFrequency LoopFreq(0);
for (auto LoopPred : L.getHeader()->predecessors())
if (!L.contains(LoopPred))
LoopFreq += MBFI->getBlockFreq(LoopPred) *
MBPI->getEdgeProbability(LoopPred, L.getHeader());
for (MachineBasicBlock *LoopBB : L.getBlocks()) {
auto Freq = MBFI->getBlockFreq(LoopBB).getFrequency();
if (Freq == 0 || LoopFreq.getFrequency() / Freq > LoopToColdBlockRatio)
continue;
LoopBlockSet.insert(LoopBB);
}
} else
LoopBlockSet.insert(L.block_begin(), L.block_end());
return LoopBlockSet;
}
void MachineBlockPlacement::buildLoopChains(MachineFunction &F,
MachineLoop &L) {
for (MachineLoop *InnerLoop : L)
buildLoopChains(F, *InnerLoop);
SmallVector<MachineBasicBlock *, 16> BlockWorkList;
BlockFilterSet LoopBlockSet = collectLoopBlockSet(F, L);
bool RotateLoopWithProfile =
PreciseRotationCost && F.getFunction()->getEntryCount();
MachineBasicBlock *LoopTop =
RotateLoopWithProfile ? L.getHeader() : findBestLoopTop(L, LoopBlockSet);
MachineBasicBlock *ExitingBB = nullptr;
if (!RotateLoopWithProfile && LoopTop == L.getHeader())
ExitingBB = findBestLoopExit(F, L, LoopBlockSet);
BlockChain &LoopChain = *BlockToChain[LoopTop];
SmallPtrSet<BlockChain *, 4> UpdatedPreds;
assert(LoopChain.LoopPredecessors == 0);
UpdatedPreds.insert(&LoopChain);
for (MachineBasicBlock *LoopBB : LoopBlockSet) {
BlockChain &Chain = *BlockToChain[LoopBB];
if (!UpdatedPreds.insert(&Chain).second)
continue;
assert(Chain.LoopPredecessors == 0);
for (MachineBasicBlock *ChainBB : Chain) {
assert(BlockToChain[ChainBB] == &Chain);
for (MachineBasicBlock *Pred : ChainBB->predecessors()) {
if (BlockToChain[Pred] == &Chain || !LoopBlockSet.count(Pred))
continue;
++Chain.LoopPredecessors;
}
}
if (Chain.LoopPredecessors == 0)
BlockWorkList.push_back(*Chain.begin());
}
buildChain(LoopTop, LoopChain, BlockWorkList, &LoopBlockSet);
if (RotateLoopWithProfile)
rotateLoopWithProfile(LoopChain, L, LoopBlockSet);
else
rotateLoop(LoopChain, ExitingBB, LoopBlockSet);
DEBUG({
bool BadLoop = false;
if (LoopChain.LoopPredecessors) {
BadLoop = true;
dbgs() << "Loop chain contains a block without its preds placed!\n"
<< " Loop header: " << getBlockName(*L.block_begin()) << "\n"
<< " Chain header: " << getBlockName(*LoopChain.begin()) << "\n";
}
for (MachineBasicBlock *ChainBB : LoopChain) {
dbgs() << " ... " << getBlockName(ChainBB) << "\n";
if (!LoopBlockSet.erase(ChainBB)) {
dbgs() << "Loop chain contains a block not contained by the loop!\n"
<< " Loop header: " << getBlockName(*L.block_begin()) << "\n"
<< " Chain header: " << getBlockName(*LoopChain.begin()) << "\n"
<< " Bad block: " << getBlockName(ChainBB) << "\n";
}
}
if (!LoopBlockSet.empty()) {
BadLoop = true;
for (MachineBasicBlock *LoopBB : LoopBlockSet)
dbgs() << "Loop contains blocks never placed into a chain!\n"
<< " Loop header: " << getBlockName(*L.block_begin()) << "\n"
<< " Chain header: " << getBlockName(*LoopChain.begin()) << "\n"
<< " Bad block: " << getBlockName(LoopBB) << "\n";
}
assert(!BadLoop && "Detected problems with the placement of this loop.");
});
}
void MachineBlockPlacement::buildCFGChains(MachineFunction &F) {
SmallVector<MachineOperand, 4> Cond; for (MachineFunction::iterator FI = F.begin(), FE = F.end(); FI != FE; ++FI) {
MachineBasicBlock *BB = &*FI;
BlockChain *Chain =
new (ChainAllocator.Allocate()) BlockChain(BlockToChain, BB);
for (;;) {
Cond.clear();
MachineBasicBlock *TBB = nullptr, *FBB = nullptr; if (!TII->AnalyzeBranch(*BB, TBB, FBB, Cond) || !FI->canFallThrough())
break;
MachineFunction::iterator NextFI = std::next(FI);
MachineBasicBlock *NextBB = &*NextFI;
assert(NextFI != FE && "Can't fallthrough past the last block.");
DEBUG(dbgs() << "Pre-merging due to unanalyzable fallthrough: "
<< getBlockName(BB) << " -> " << getBlockName(NextBB)
<< "\n");
Chain->merge(NextBB, nullptr);
FI = NextFI;
BB = NextBB;
}
}
if (OutlineOptionalBranches) {
MachineBasicBlock *Terminator = nullptr;
for (MachineBasicBlock &MBB : F) {
if (MBB.succ_size() == 0) {
if (Terminator == nullptr)
Terminator = &MBB;
else
Terminator = MDT->findNearestCommonDominator(Terminator, &MBB);
}
}
UnavoidableBlocks.clear();
for (MachineBasicBlock &MBB : F) {
if (MDT->dominates(&MBB, Terminator)) {
UnavoidableBlocks.insert(&MBB);
}
}
}
for (MachineLoop *L : *MLI)
buildLoopChains(F, *L);
SmallVector<MachineBasicBlock *, 16> BlockWorkList;
SmallPtrSet<BlockChain *, 4> UpdatedPreds;
for (MachineBasicBlock &MBB : F) {
BlockChain &Chain = *BlockToChain[&MBB];
if (!UpdatedPreds.insert(&Chain).second)
continue;
assert(Chain.LoopPredecessors == 0);
for (MachineBasicBlock *ChainBB : Chain) {
assert(BlockToChain[ChainBB] == &Chain);
for (MachineBasicBlock *Pred : ChainBB->predecessors()) {
if (BlockToChain[Pred] == &Chain)
continue;
++Chain.LoopPredecessors;
}
}
if (Chain.LoopPredecessors == 0)
BlockWorkList.push_back(*Chain.begin());
}
BlockChain &FunctionChain = *BlockToChain[&F.front()];
buildChain(&F.front(), FunctionChain, BlockWorkList);
#ifndef NDEBUG
typedef SmallPtrSet<MachineBasicBlock *, 16> FunctionBlockSetType;
#endif
DEBUG({
bool BadFunc = false;
FunctionBlockSetType FunctionBlockSet;
for (MachineBasicBlock &MBB : F)
FunctionBlockSet.insert(&MBB);
for (MachineBasicBlock *ChainBB : FunctionChain)
if (!FunctionBlockSet.erase(ChainBB)) {
BadFunc = true;
dbgs() << "Function chain contains a block not in the function!\n"
<< " Bad block: " << getBlockName(ChainBB) << "\n";
}
if (!FunctionBlockSet.empty()) {
BadFunc = true;
for (MachineBasicBlock *RemainingBB : FunctionBlockSet)
dbgs() << "Function contains blocks never placed into a chain!\n"
<< " Bad block: " << getBlockName(RemainingBB) << "\n";
}
assert(!BadFunc && "Detected problems with the block placement.");
});
MachineFunction::iterator InsertPos = F.begin();
for (MachineBasicBlock *ChainBB : FunctionChain) {
DEBUG(dbgs() << (ChainBB == *FunctionChain.begin() ? "Placing chain "
: " ... ")
<< getBlockName(ChainBB) << "\n");
if (InsertPos != MachineFunction::iterator(ChainBB))
F.splice(InsertPos, ChainBB);
else
++InsertPos;
if (ChainBB == *FunctionChain.begin())
continue;
MachineBasicBlock *PrevBB = &*std::prev(MachineFunction::iterator(ChainBB));
Cond.clear();
MachineBasicBlock *TBB = nullptr, *FBB = nullptr; if (!TII->AnalyzeBranch(*PrevBB, TBB, FBB, Cond)) {
bool needUpdateBr = true;
if (!Cond.empty() && (!FBB || FBB == ChainBB)) {
PrevBB->updateTerminator();
needUpdateBr = false;
Cond.clear();
TBB = FBB = nullptr;
if (TII->AnalyzeBranch(*PrevBB, TBB, FBB, Cond)) {
TBB = FBB = nullptr;
}
}
if (TBB && !Cond.empty() && FBB &&
MBPI->getEdgeWeight(PrevBB, FBB) > MBPI->getEdgeWeight(PrevBB, TBB) &&
!TII->ReverseBranchCondition(Cond)) {
DEBUG(dbgs() << "Reverse order of the two branches: "
<< getBlockName(PrevBB) << "\n");
DEBUG(dbgs() << " Edge weight: " << MBPI->getEdgeWeight(PrevBB, FBB)
<< " vs " << MBPI->getEdgeWeight(PrevBB, TBB) << "\n");
DebugLoc dl; TII->RemoveBranch(*PrevBB);
TII->InsertBranch(*PrevBB, FBB, TBB, Cond, dl);
needUpdateBr = true;
}
if (needUpdateBr)
PrevBB->updateTerminator();
}
}
Cond.clear();
MachineBasicBlock *TBB = nullptr, *FBB = nullptr; if (!TII->AnalyzeBranch(F.back(), TBB, FBB, Cond))
F.back().updateTerminator();
if (F.getFunction()->hasFnAttribute(Attribute::OptimizeForSize))
return;
if (FunctionChain.begin() == FunctionChain.end())
return;
const BranchProbability ColdProb(1, 5); BlockFrequency EntryFreq = MBFI->getBlockFreq(&F.front());
BlockFrequency WeightedEntryFreq = EntryFreq * ColdProb;
for (MachineBasicBlock *ChainBB : FunctionChain) {
if (ChainBB == *FunctionChain.begin())
continue;
MachineLoop *L = MLI->getLoopFor(ChainBB);
if (!L)
continue;
unsigned Align = TLI->getPrefLoopAlignment(L);
if (!Align)
continue;
BlockFrequency Freq = MBFI->getBlockFreq(ChainBB);
if (Freq < WeightedEntryFreq)
continue;
MachineBasicBlock *LoopHeader = L->getHeader();
BlockFrequency LoopHeaderFreq = MBFI->getBlockFreq(LoopHeader);
if (Freq < (LoopHeaderFreq * ColdProb))
continue;
MachineBasicBlock *LayoutPred =
&*std::prev(MachineFunction::iterator(ChainBB));
if (!LayoutPred->isSuccessor(ChainBB)) {
ChainBB->setAlignment(Align);
continue;
}
BranchProbability LayoutProb =
MBPI->getEdgeProbability(LayoutPred, ChainBB);
BlockFrequency LayoutEdgeFreq = MBFI->getBlockFreq(LayoutPred) * LayoutProb;
if (LayoutEdgeFreq <= (Freq * ColdProb))
ChainBB->setAlignment(Align);
}
}
bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &F) {
if (std::next(F.begin()) == F.end())
return false;
if (skipOptnoneFunction(*F.getFunction()))
return false;
MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
MLI = &getAnalysis<MachineLoopInfo>();
TII = F.getSubtarget().getInstrInfo();
TLI = F.getSubtarget().getTargetLowering();
MDT = &getAnalysis<MachineDominatorTree>();
assert(BlockToChain.empty());
buildCFGChains(F);
BlockToChain.clear();
ChainAllocator.DestroyAll();
if (AlignAllBlock)
for (MachineBasicBlock &MBB : F)
MBB.setAlignment(AlignAllBlock);
return true;
}
namespace {
class MachineBlockPlacementStats : public MachineFunctionPass {
const MachineBranchProbabilityInfo *MBPI;
const MachineBlockFrequencyInfo *MBFI;
public:
static char ID; MachineBlockPlacementStats() : MachineFunctionPass(ID) {
initializeMachineBlockPlacementStatsPass(*PassRegistry::getPassRegistry());
}
bool runOnMachineFunction(MachineFunction &F) override;
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<MachineBranchProbabilityInfo>();
AU.addRequired<MachineBlockFrequencyInfo>();
AU.setPreservesAll();
MachineFunctionPass::getAnalysisUsage(AU);
}
};
}
char MachineBlockPlacementStats::ID = 0;
char &llvm::MachineBlockPlacementStatsID = MachineBlockPlacementStats::ID;
INITIALIZE_PASS_BEGIN(MachineBlockPlacementStats, "block-placement-stats",
"Basic Block Placement Stats", false, false)
INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo)
INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfo)
INITIALIZE_PASS_END(MachineBlockPlacementStats, "block-placement-stats",
"Basic Block Placement Stats", false, false)
bool MachineBlockPlacementStats::runOnMachineFunction(MachineFunction &F) {
if (std::next(F.begin()) == F.end())
return false;
MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
for (MachineBasicBlock &MBB : F) {
BlockFrequency BlockFreq = MBFI->getBlockFreq(&MBB);
Statistic &NumBranches =
(MBB.succ_size() > 1) ? NumCondBranches : NumUncondBranches;
Statistic &BranchTakenFreq =
(MBB.succ_size() > 1) ? CondBranchTakenFreq : UncondBranchTakenFreq;
for (MachineBasicBlock *Succ : MBB.successors()) {
if (MBB.isLayoutSuccessor(Succ))
continue;
BlockFrequency EdgeFreq =
BlockFreq * MBPI->getEdgeProbability(&MBB, Succ);
++NumBranches;
BranchTakenFreq += EdgeFreq.getFrequency();
}
}
return false;
}