LoopUnrollRuntime.cpp [plain text]
#define DEBUG_TYPE "loop-unroll"
#include "llvm/Transforms/Utils/UnrollLoop.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/LoopIterator.h"
#include "llvm/Analysis/LoopPass.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpander.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include <algorithm>
using namespace llvm;
STATISTIC(NumRuntimeUnrolled,
"Number of loops unrolled with run-time trip counts");
static void ConnectProlog(Loop *L, Value *TripCount, unsigned Count,
BasicBlock *LastPrologBB, BasicBlock *PrologEnd,
BasicBlock *OrigPH, BasicBlock *NewPH,
ValueToValueMapTy &LVMap, Pass *P) {
BasicBlock *Latch = L->getLoopLatch();
assert(Latch != 0 && "Loop must have a latch");
for (succ_iterator SBI = succ_begin(Latch), SBE = succ_end(Latch);
SBI != SBE; ++SBI) {
for (BasicBlock::iterator BBI = (*SBI)->begin();
PHINode *PN = dyn_cast<PHINode>(BBI); ++BBI) {
PHINode *NewPN = PHINode::Create(PN->getType(), 2, PN->getName()+".unr",
PrologEnd->getTerminator());
if (L->contains(PN)) {
NewPN->addIncoming(PN->getIncomingValueForBlock(NewPH), OrigPH);
} else {
NewPN->addIncoming(Constant::getNullValue(PN->getType()), OrigPH);
}
Value *V = PN->getIncomingValueForBlock(Latch);
if (Instruction *I = dyn_cast<Instruction>(V)) {
if (L->contains(I)) {
V = LVMap[I];
}
}
NewPN->addIncoming(V, LastPrologBB);
if (L->contains(PN)) {
PN->setIncomingValue(PN->getBasicBlockIndex(NewPH), NewPN);
} else {
PN->addIncoming(NewPN, PrologEnd);
}
}
}
Instruction *InsertPt = PrologEnd->getTerminator();
Instruction *BrLoopExit =
new ICmpInst(InsertPt, ICmpInst::ICMP_ULT, TripCount,
ConstantInt::get(TripCount->getType(), Count));
BasicBlock *Exit = L->getUniqueExitBlock();
assert(Exit != 0 && "Loop must have a single exit block only");
SmallVector<BasicBlock*, 4> Preds(pred_begin(Exit), pred_end(Exit));
if (!Exit->isLandingPad()) {
SplitBlockPredecessors(Exit, Preds, ".unr-lcssa", P);
} else {
SmallVector<BasicBlock*, 2> NewBBs;
SplitLandingPadPredecessors(Exit, Preds, ".unr1-lcssa", ".unr2-lcssa",
P, NewBBs);
}
BranchInst::Create(Exit, NewPH, BrLoopExit, InsertPt);
InsertPt->eraseFromParent();
}
static void CloneLoopBlocks(Loop *L,
bool FirstCopy,
BasicBlock *InsertTop,
BasicBlock *InsertBot,
std::vector<BasicBlock *> &NewBlocks,
LoopBlocksDFS &LoopBlocks,
ValueToValueMapTy &VMap,
ValueToValueMapTy &LVMap,
LoopInfo *LI) {
BasicBlock *Preheader = L->getLoopPreheader();
BasicBlock *Header = L->getHeader();
BasicBlock *Latch = L->getLoopLatch();
Function *F = Header->getParent();
LoopBlocksDFS::RPOIterator BlockBegin = LoopBlocks.beginRPO();
LoopBlocksDFS::RPOIterator BlockEnd = LoopBlocks.endRPO();
for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) {
BasicBlock *NewBB = CloneBasicBlock(*BB, VMap, ".unr", F);
NewBlocks.push_back(NewBB);
if (Loop *ParentLoop = L->getParentLoop())
ParentLoop->addBasicBlockToLoop(NewBB, LI->getBase());
VMap[*BB] = NewBB;
if (Header == *BB) {
InsertTop->getTerminator()->setSuccessor(0, NewBB);
for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) {
PHINode *NewPHI = cast<PHINode>(VMap[I]);
if (FirstCopy) {
VMap[I] = NewPHI->getIncomingValueForBlock(Preheader);
NewBB->getInstList().erase(NewPHI);
} else {
unsigned idx = NewPHI->getBasicBlockIndex(Latch);
Value *InVal = NewPHI->getIncomingValue(idx);
if (Instruction *I = dyn_cast<Instruction>(InVal))
if (L->contains(I))
InVal = LVMap[InVal];
NewPHI->setIncomingValue(idx, InVal);
NewPHI->setIncomingBlock(idx, InsertTop);
}
}
}
if (Latch == *BB) {
VMap.erase((*BB)->getTerminator());
NewBB->getTerminator()->eraseFromParent();
BranchInst::Create(InsertBot, NewBB);
}
}
for (ValueToValueMapTy::iterator VI = VMap.begin(), VE = VMap.end();
VI != VE; ++VI) {
LVMap[VI->first] = VI->second;
}
}
bool llvm::UnrollRuntimeLoopProlog(Loop *L, unsigned Count, LoopInfo *LI,
LPPassManager *LPM) {
if (!L->getExitingBlock())
return false;
if (!L->isLoopSimplifyForm() || L->getUniqueExitBlock() == 0)
return false;
if (!LPM)
return false;
ScalarEvolution *SE = LPM->getAnalysisIfAvailable<ScalarEvolution>();
if (SE == 0)
return false;
const SCEV *BECount = SE->getBackedgeTakenCount(L);
if (isa<SCEVCouldNotCompute>(BECount) || !BECount->getType()->isIntegerTy())
return false;
const SCEV *TripCountSC =
SE->getAddExpr(BECount, SE->getConstant(BECount->getType(), 1));
if (isa<SCEVCouldNotCompute>(TripCountSC))
return false;
if ((Count & (Count-1)) != 0)
return false;
if (Loop *ParentLoop = L->getParentLoop())
SE->forgetLoop(ParentLoop);
BasicBlock *PH = L->getLoopPreheader();
BasicBlock *Header = L->getHeader();
BasicBlock *Latch = L->getLoopLatch();
BasicBlock *PEnd = SplitEdge(PH, Header, LPM->getAsPass());
BasicBlock *NewPH = SplitBlock(PEnd, PEnd->getTerminator(), LPM->getAsPass());
BranchInst *PreHeaderBR = cast<BranchInst>(PH->getTerminator());
SCEVExpander Expander(*SE, "loop-unroll");
Value *TripCount = Expander.expandCodeFor(TripCountSC, TripCountSC->getType(),
PreHeaderBR);
Type *CountTy = TripCount->getType();
BinaryOperator *ModVal =
BinaryOperator::CreateURem(TripCount,
ConstantInt::get(CountTy, Count),
"xtraiter");
ModVal->insertBefore(PreHeaderBR);
Value *BranchVal = new ICmpInst(PreHeaderBR,
ICmpInst::ICMP_NE, ModVal,
ConstantInt::get(CountTy, 0), "lcmp");
BranchInst::Create(PEnd, PEnd, BranchVal, PreHeaderBR);
assert(PreHeaderBR->isUnconditional() &&
PreHeaderBR->getSuccessor(0) == PEnd &&
"CFG edges in Preheader are not correct");
PreHeaderBR->eraseFromParent();
ValueToValueMapTy LVMap;
Function *F = Header->getParent();
BasicBlock *CompareBB = 0;
BasicBlock *LastLoopBB = PH;
LoopBlocksDFS LoopBlocks(L);
LoopBlocks.perform(LI);
for (unsigned leftOverIters = Count-1; leftOverIters > 0; --leftOverIters) {
std::vector<BasicBlock*> NewBlocks;
ValueToValueMapTy VMap;
CloneLoopBlocks(L, (leftOverIters == Count-1), LastLoopBB, PEnd, NewBlocks,
LoopBlocks, VMap, LVMap, LI);
LastLoopBB = cast<BasicBlock>(VMap[Latch]);
F->getBasicBlockList().splice(PEnd, F->getBasicBlockList(),
NewBlocks[0], F->end());
if (leftOverIters == Count-1) {
CompareBB = NewBlocks[0];
} else {
BasicBlock *NewBB = BasicBlock::Create(CompareBB->getContext(), "unr.cmp",
F, CompareBB);
if (Loop *ParentLoop = L->getParentLoop()) {
ParentLoop->addBasicBlockToLoop(NewBB, LI->getBase());
}
Value *BranchVal = new ICmpInst(*NewBB, ICmpInst::ICMP_EQ, ModVal,
ConstantInt::get(CountTy, leftOverIters),
"un.tmp");
BranchInst::Create(NewBlocks[0], CompareBB,
BranchVal, NewBB);
CompareBB = NewBB;
PH->getTerminator()->setSuccessor(0, NewBB);
VMap[NewPH] = CompareBB;
}
for (unsigned i = 0, e = NewBlocks.size(); i != e; ++i) {
for (BasicBlock::iterator I = NewBlocks[i]->begin(),
E = NewBlocks[i]->end(); I != E; ++I) {
RemapInstruction(I, VMap,
RF_NoModuleLevelChanges|RF_IgnoreMissingEntries);
}
}
}
ConnectProlog(L, TripCount, Count, LastLoopBB, PEnd, PH, NewPH, LVMap,
LPM->getAsPass());
NumRuntimeUnrolled++;
return true;
}