LoopDistribute.cpp [plain text]
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/EquivalenceClasses.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include <list>
#define LDIST_NAME "loop-distribute"
#define DEBUG_TYPE LDIST_NAME
using namespace llvm;
static cl::opt<bool>
LDistVerify("loop-distribute-verify", cl::Hidden,
cl::desc("Turn on DominatorTree and LoopInfo verification "
"after Loop Distribution"),
cl::init(false));
static cl::opt<bool> DistributeNonIfConvertible(
"loop-distribute-non-if-convertible", cl::Hidden,
cl::desc("Whether to distribute into a loop that may not be "
"if-convertible by the loop vectorizer"),
cl::init(false));
static cl::opt<bool> AddMemcheckForStoreToLoadElimination(
"loop-distribute-add-memchecks-for-store-to-load-elimination", cl::Hidden,
cl::desc("For a distribute loop which contains loop-carried store-to-load "
"forwarding, add additional memchecks to allow load elimination"),
cl::init(true));
STATISTIC(NumLoopsDistributed, "Number of loops distributed");
namespace {
void remapInstructionsInLoop(const SmallVectorImpl<BasicBlock *> &Blocks,
ValueToValueMapTy &VMap) {
for (auto *BB : Blocks)
for (auto &Inst : *BB)
RemapInstruction(&Inst, VMap,
RF_NoModuleLevelChanges | RF_IgnoreMissingEntries);
}
static Loop *cloneLoopWithPreheader(BasicBlock *Before, BasicBlock *LoopDomBB,
Loop *OrigLoop, ValueToValueMapTy &VMap,
const Twine &NameSuffix, LoopInfo *LI,
DominatorTree *DT,
SmallVectorImpl<BasicBlock *> &Blocks) {
Function *F = OrigLoop->getHeader()->getParent();
Loop *ParentLoop = OrigLoop->getParentLoop();
Loop *NewLoop = new Loop();
if (ParentLoop)
ParentLoop->addChildLoop(NewLoop);
else
LI->addTopLevelLoop(NewLoop);
BasicBlock *OrigPH = OrigLoop->getLoopPreheader();
BasicBlock *NewPH = CloneBasicBlock(OrigPH, VMap, NameSuffix, F);
VMap[OrigPH] = NewPH;
Blocks.push_back(NewPH);
if (ParentLoop)
ParentLoop->addBasicBlockToLoop(NewPH, *LI);
DT->addNewBlock(NewPH, LoopDomBB);
for (BasicBlock *BB : OrigLoop->getBlocks()) {
BasicBlock *NewBB = CloneBasicBlock(BB, VMap, NameSuffix, F);
VMap[BB] = NewBB;
NewLoop->addBasicBlockToLoop(NewBB, *LI);
BasicBlock *IDomBB = DT->getNode(BB)->getIDom()->getBlock();
DT->addNewBlock(NewBB, cast<BasicBlock>(VMap[IDomBB]));
Blocks.push_back(NewBB);
}
F->getBasicBlockList().splice(Before, F->getBasicBlockList(), NewPH);
F->getBasicBlockList().splice(Before, F->getBasicBlockList(),
NewLoop->getHeader(), F->end());
return NewLoop;
}
class InstPartition {
typedef SmallPtrSet<Instruction *, 8> InstructionSet;
public:
InstPartition(Instruction *I, Loop *L, bool DepCycle = false)
: DepCycle(DepCycle), OrigLoop(L), ClonedLoop(nullptr) {
Set.insert(I);
}
bool hasDepCycle() const { return DepCycle; }
void add(Instruction *I) { Set.insert(I); }
InstructionSet::iterator begin() { return Set.begin(); }
InstructionSet::iterator end() { return Set.end(); }
InstructionSet::const_iterator begin() const { return Set.begin(); }
InstructionSet::const_iterator end() const { return Set.end(); }
bool empty() const { return Set.empty(); }
void moveTo(InstPartition &Other) {
Other.Set.insert(Set.begin(), Set.end());
Set.clear();
Other.DepCycle |= DepCycle;
}
void populateUsedSet() {
for (auto *B : OrigLoop->getBlocks())
Set.insert(B->getTerminator());
SmallVector<Instruction *, 8> Worklist(Set.begin(), Set.end());
while (!Worklist.empty()) {
Instruction *I = Worklist.pop_back_val();
for (Value *V : I->operand_values()) {
auto *I = dyn_cast<Instruction>(V);
if (I && OrigLoop->contains(I->getParent()) && Set.insert(I).second)
Worklist.push_back(I);
}
}
}
Loop *cloneLoopWithPreheader(BasicBlock *InsertBefore, BasicBlock *LoopDomBB,
unsigned Index, LoopInfo *LI,
DominatorTree *DT) {
ClonedLoop = ::cloneLoopWithPreheader(InsertBefore, LoopDomBB, OrigLoop,
VMap, Twine(".ldist") + Twine(Index),
LI, DT, ClonedLoopBlocks);
return ClonedLoop;
}
const Loop *getClonedLoop() const { return ClonedLoop; }
const Loop *getDistributedLoop() const {
return ClonedLoop ? ClonedLoop : OrigLoop;
}
ValueToValueMapTy &getVMap() { return VMap; }
Instruction *getNewInst(Instruction *I) {
return ClonedLoop ? cast<Instruction>(VMap[I]) : I;
}
void remapInstructions() { remapInstructionsInLoop(ClonedLoopBlocks, VMap); }
void removeUnusedInsts() {
SmallVector<Instruction *, 8> Unused;
for (auto *Block : OrigLoop->getBlocks())
for (auto &Inst : *Block)
if (!Set.count(&Inst)) {
Instruction *NewInst = getNewInst(&Inst);
assert(!isa<BranchInst>(NewInst) &&
"Branches are marked used early on");
Unused.push_back(NewInst);
}
for (auto I = Unused.rbegin(), E = Unused.rend(); I != E; ++I) {
auto *Inst = *I;
if (!Inst->use_empty())
Inst->replaceAllUsesWith(UndefValue::get(Inst->getType()));
Inst->eraseFromParent();
}
}
void annotateNoAlias(
MDNode *Scope,
const SmallSet<Instruction *, 8> &InstsInStoreToLoadForwarding) {
for (auto *OrigInst : Set) {
auto *I = getNewInst(OrigInst);
if ((isa<LoadInst>(I) && InstsInStoreToLoadForwarding.count(I)) ||
(isa<StoreInst>(I) && !InstsInStoreToLoadForwarding.count(I))) {
I->setMetadata(LLVMContext::MD_noalias,
MDNode::concatenate(
I->getMetadata(LLVMContext::MD_noalias), Scope));
I->setMetadata(LLVMContext::MD_alias_scope,
MDNode::concatenate(
I->getMetadata(LLVMContext::MD_alias_scope), Scope));
}
}
}
void print() {
if (DepCycle)
dbgs() << " (cycle)\n";
for (auto *I : Set)
dbgs() << " " << I->getParent()->getName() << ":" << *I << "\n";
}
void printBlocks() const {
for (auto *BB : getDistributedLoop()->getBlocks())
dbgs() << *BB;
}
private:
InstructionSet Set;
bool DepCycle;
Loop *OrigLoop;
Loop *ClonedLoop;
SmallVector<BasicBlock *, 8> ClonedLoopBlocks;
ValueToValueMapTy VMap;
};
class InstPartitionContainer {
typedef std::list<std::unique_ptr<InstPartition>> PartitionContainerT;
typedef DenseMap<Instruction *, int> InstToPartitionIdT;
public:
InstPartitionContainer(Loop *L, LoopInfo *LI, DominatorTree *DT)
: L(L), LI(LI), DT(DT) {}
unsigned getSize() const { return PartitionContainer.size(); }
void addToCyclicPartition(Instruction *Inst) {
if (PartitionContainer.empty() || !PartitionContainer.back()->hasDepCycle())
PartitionContainer.push_back(
llvm::make_unique<InstPartition>(Inst, L, true));
else
PartitionContainer.back()->add(Inst);
}
void addToNewNonCyclicPartition(Instruction *Inst) {
PartitionContainer.push_back(llvm::make_unique<InstPartition>(Inst, L));
}
void mergeAdjacentNonCyclic() {
mergeAdjacentPartitionsIf(
[](const InstPartition *P) { return !P->hasDepCycle(); });
}
void mergeNonIfConvertible() {
mergeAdjacentPartitionsIf([&](const InstPartition *Partition) {
if (Partition->hasDepCycle())
return true;
bool seenStore = false;
for (auto *Inst : *Partition)
if (isa<StoreInst>(Inst)) {
seenStore = true;
if (!LoopAccessInfo::blockNeedsPredication(Inst->getParent(), L, DT))
return false;
}
return seenStore;
});
}
void mergeBeforePopulating() {
mergeAdjacentNonCyclic();
if (!DistributeNonIfConvertible)
mergeNonIfConvertible();
}
bool mergeToAvoidDuplicatedLoads() {
typedef DenseMap<Instruction *, InstPartition *> LoadToPartitionT;
typedef EquivalenceClasses<InstPartition *> ToBeMergedT;
LoadToPartitionT LoadToPartition;
ToBeMergedT ToBeMerged;
for (PartitionContainerT::iterator I = PartitionContainer.begin(),
E = PartitionContainer.end();
I != E; ++I) {
auto *PartI = I->get();
for (Instruction *Inst : *PartI)
if (isa<LoadInst>(Inst)) {
bool NewElt;
LoadToPartitionT::iterator LoadToPart;
std::tie(LoadToPart, NewElt) =
LoadToPartition.insert(std::make_pair(Inst, PartI));
if (!NewElt) {
DEBUG(dbgs() << "Merging partitions due to this load in multiple "
<< "partitions: " << PartI << ", "
<< LoadToPart->second << "\n" << *Inst << "\n");
auto PartJ = I;
do {
--PartJ;
ToBeMerged.unionSets(PartI, PartJ->get());
} while (PartJ->get() != LoadToPart->second);
}
}
}
if (ToBeMerged.empty())
return false;
for (ToBeMergedT::iterator I = ToBeMerged.begin(), E = ToBeMerged.end();
I != E; ++I) {
if (!I->isLeader())
continue;
auto PartI = I->getData();
for (auto PartJ : make_range(std::next(ToBeMerged.member_begin(I)),
ToBeMerged.member_end())) {
PartJ->moveTo(*PartI);
}
}
for (PartitionContainerT::iterator PartI = PartitionContainer.begin(),
E = PartitionContainer.end();
PartI != E;)
if ((*PartI)->empty())
PartI = PartitionContainer.erase(PartI);
else
++PartI;
return true;
}
void setupPartitionIdOnInstructions() {
int PartitionID = 0;
for (auto &PartitionPtr : PartitionContainer) {
for (Instruction *Inst : *PartitionPtr) {
bool NewElt;
InstToPartitionIdT::iterator Iter;
std::tie(Iter, NewElt) =
InstToPartitionId.insert(std::make_pair(Inst, PartitionID));
if (!NewElt)
Iter->second = -1;
}
++PartitionID;
}
}
void populateUsedSet() {
for (auto &P : PartitionContainer)
P->populateUsedSet();
}
void cloneLoops(Pass *P) {
BasicBlock *OrigPH = L->getLoopPreheader();
BasicBlock *Pred = OrigPH->getSinglePredecessor();
assert(Pred && "Preheader does not have a single predecessor");
BasicBlock *ExitBlock = L->getExitBlock();
assert(ExitBlock && "No single exit block");
Loop *NewLoop;
assert(!PartitionContainer.empty() && "at least two partitions expected");
assert(&*OrigPH->begin() == OrigPH->getTerminator() &&
"preheader not empty");
BasicBlock *TopPH = OrigPH;
unsigned Index = getSize() - 1;
for (auto I = std::next(PartitionContainer.crbegin()),
E = PartitionContainer.crend();
I != E; ++I, --Index, TopPH = NewLoop->getLoopPreheader()) {
auto &Part = *I;
NewLoop = Part->cloneLoopWithPreheader(TopPH, Pred, Index, LI, DT);
Part->getVMap()[ExitBlock] = TopPH;
Part->remapInstructions();
}
Pred->getTerminator()->replaceUsesOfWith(OrigPH, TopPH);
for (auto Curr = PartitionContainer.cbegin(),
Next = std::next(PartitionContainer.cbegin()),
E = PartitionContainer.cend();
Next != E; ++Curr, ++Next)
DT->changeImmediateDominator(
(*Next)->getDistributedLoop()->getLoopPreheader(),
(*Curr)->getDistributedLoop()->getExitingBlock());
}
void removeUnusedInsts() {
for (auto &PartitionPtr : PartitionContainer)
PartitionPtr->removeUnusedInsts();
}
SmallVector<int, 8>
computePartitionSetForPointers(const LoopAccessInfo &LAI) {
const LoopAccessInfo::RuntimePointerCheck *RtPtrCheck =
LAI.getRuntimePointerCheck();
unsigned N = RtPtrCheck->Pointers.size();
SmallVector<int, 8> PtrToPartitions(N);
for (unsigned I = 0; I < N; ++I) {
Value *Ptr = RtPtrCheck->Pointers[I];
auto Instructions =
LAI.getInstructionsForAccess(Ptr, RtPtrCheck->IsWritePtr[I]);
int &Partition = PtrToPartitions[I];
Partition = -2;
for (Instruction *Inst : Instructions) {
int ThisPartition = this->InstToPartitionId[Inst];
if (Partition == -2)
Partition = ThisPartition;
else if (Partition == -1)
break;
else if (Partition != (int)ThisPartition)
Partition = -1;
}
assert(Partition != -2 && "Pointer not belonging to any partition");
}
return PtrToPartitions;
}
void annotateNoAlias(
const SmallSet<Instruction *, 8> &InstsInStoreToLoadForwarding) {
MDBuilder MDB(L->getHeader()->getContext());
MDNode *Domain = MDB.createAnonymousAliasScopeDomain("MemCheckDomain");
MDNode *Scope = MDB.createAnonymousAliasScope(Domain, "MemCheckScope");
for (auto &P : PartitionContainer)
if (P->hasDepCycle())
P->annotateNoAlias(Scope, InstsInStoreToLoadForwarding);
}
void print(raw_ostream &OS) const {
unsigned Index = 0;
for (auto &P : PartitionContainer) {
OS << "Partition " << Index++ << " (" << P.get() << "):\n";
P->print();
}
}
void dump() const { print(dbgs()); }
#ifndef NDEBUG
friend raw_ostream &operator<<(raw_ostream &OS,
const InstPartitionContainer &Partitions) {
Partitions.print(OS);
return OS;
}
#endif
void printBlocks() const {
unsigned Index = 0;
for (auto &P : PartitionContainer) {
dbgs() << "\nPartition " << Index++ << " (" << P.get() << "):\n";
P->printBlocks();
}
}
PartitionContainerT::iterator begin() { return PartitionContainer.begin(); }
PartitionContainerT::iterator end() { return PartitionContainer.end(); }
private:
PartitionContainerT PartitionContainer;
InstToPartitionIdT InstToPartitionId;
Loop *L;
LoopInfo *LI;
DominatorTree *DT;
template <class UnaryPredicate>
void mergeAdjacentPartitionsIf(UnaryPredicate Predicate) {
for (auto I = PartitionContainer.begin(); I != PartitionContainer.end();) {
auto &FirstPartition = *I;
if (!Predicate(FirstPartition.get()))
++I;
else
for (I = std::next(I);
I != PartitionContainer.end() && Predicate(I->get());) {
(*I)->moveTo(*FirstPartition);
I = PartitionContainer.erase(I);
}
}
}
};
class MemoryInstructionDependences {
typedef MemoryDepChecker::Dependence Dependence;
public:
struct Entry {
Instruction *Inst;
unsigned NumUnsafeDependencesStartOrEnd;
Entry(Instruction *Inst) : Inst(Inst), NumUnsafeDependencesStartOrEnd(0) {}
};
typedef SmallVector<Entry, 8> AccessesType;
AccessesType::const_iterator begin() const { return Accesses.begin(); }
AccessesType::const_iterator end() const { return Accesses.end(); }
MemoryInstructionDependences(
const SmallVectorImpl<Instruction *> &Instructions,
const SmallVectorImpl<Dependence> &InterestingDependences) {
std::transform(Instructions.begin(), Instructions.end(),
std::back_inserter(Accesses),
[](Instruction *Inst) { return Entry(Inst); });
DEBUG(dbgs() << "Backward dependences:\n");
for (auto &Dep : InterestingDependences)
if (Dep.isPossiblyBackward()) {
++Accesses[Dep.Source].NumUnsafeDependencesStartOrEnd;
--Accesses[Dep.Destination].NumUnsafeDependencesStartOrEnd;
DEBUG(Dep.print(dbgs(), 2, Instructions));
}
}
private:
AccessesType Accesses;
};
class RuntimeCheckEmitter {
public:
RuntimeCheckEmitter(const LoopAccessInfo &LAI, Loop *L, LoopInfo *LI,
DominatorTree *DT,
const SmallSet<Value *, 8> &PtrsInStoreToLoadForwarding)
: OrigLoop(L), NonDistributedLoop(nullptr),
PtrsInStoreToLoadForwarding(PtrsInStoreToLoadForwarding), LAI(LAI),
LI(LI), DT(DT) {}
void partitionPointers(InstPartitionContainer &Partitions) {
PtrToPartition = Partitions.computePartitionSetForPointers(LAI);
DEBUG(dbgs() << "\nPointers:\n");
DEBUG(LAI.getRuntimePointerCheck()->print(dbgs(), 0, &PtrToPartition,
&PtrsInStoreToLoadForwarding));
}
bool needsRuntimeChecks() const {
return LAI.getRuntimePointerCheck()->needsAnyChecking(
&PtrToPartition, &PtrsInStoreToLoadForwarding);
}
void versionLoop(Pass *P) {
Instruction *FirstCheckInst;
Instruction *MemRuntimeCheck;
BasicBlock *MemCheckBB = OrigLoop->getLoopPreheader();
std::tie(FirstCheckInst, MemRuntimeCheck) =
LAI.addRuntimeCheck(MemCheckBB->getTerminator(), &PtrToPartition,
&PtrsInStoreToLoadForwarding);
assert(MemRuntimeCheck && "called even though needsAnyChecking = false");
MemCheckBB->setName(OrigLoop->getHeader()->getName() + ".ldist.memcheck");
BasicBlock *PH =
SplitBlock(MemCheckBB, MemCheckBB->getTerminator(), DT, LI);
PH->setName(OrigLoop->getHeader()->getName() + ".ph");
SmallVector<BasicBlock *, 8> NonDistributedLoopBlocks;
NonDistributedLoop =
cloneLoopWithPreheader(PH, MemCheckBB, OrigLoop, VMap, ".ldist.nondist",
LI, DT, NonDistributedLoopBlocks);
remapInstructionsInLoop(NonDistributedLoopBlocks, VMap);
Instruction *OrigTerm = MemCheckBB->getTerminator();
BranchInst::Create(NonDistributedLoop->getLoopPreheader(),
OrigLoop->getLoopPreheader(), MemRuntimeCheck, OrigTerm);
OrigTerm->eraseFromParent();
DT->changeImmediateDominator(OrigLoop->getExitBlock(), MemCheckBB);
}
void addPHINodes(const SmallVectorImpl<Instruction *> &DefsUsedOutside) {
BasicBlock *PHIBlock = OrigLoop->getExitBlock();
assert(PHIBlock && "No single successor to loop exit block");
for (auto *Inst : DefsUsedOutside) {
auto *NonDistInst = cast<Instruction>(VMap[Inst]);
PHINode *PN;
BasicBlock::iterator I;
for (I = PHIBlock->begin(); (PN = dyn_cast<PHINode>(I)); ++I) {
assert(PN->getNumOperands() == 1 &&
"Exit block should only have on predecessor");
if (PN->getIncomingValue(0) == Inst)
break;
}
if (!PN) {
PN = PHINode::Create(Inst->getType(), 2, Inst->getName() + ".ldist",
PHIBlock->begin());
for (auto *User : Inst->users())
if (!OrigLoop->contains(cast<Instruction>(User)->getParent()))
User->replaceUsesOfWith(Inst, PN);
PN->addIncoming(Inst, OrigLoop->getExitingBlock());
}
PN->addIncoming(NonDistInst, NonDistributedLoop->getExitingBlock());
}
}
private:
Loop *OrigLoop;
Loop *NonDistributedLoop;
SmallVector<int, 8> PtrToPartition;
SmallSet<Value *, 8> PtrsInStoreToLoadForwarding;
ValueToValueMapTy VMap;
const LoopAccessInfo &LAI;
LoopInfo *LI;
DominatorTree *DT;
};
static SmallVector<Instruction *, 8> findDefsUsedOutsideOfLoop(Loop *L) {
SmallVector<Instruction *, 8> UsedOutside;
for (auto *Block : L->getBlocks())
for (auto &Inst : *Block) {
auto Users = Inst.users();
if (std::any_of(Users.begin(), Users.end(), [&](User *U) {
auto *Use = cast<Instruction>(U);
return !L->contains(Use->getParent());
}))
UsedOutside.push_back(&Inst);
}
return UsedOutside;
}
class LoopDistribute : public FunctionPass {
public:
LoopDistribute() : FunctionPass(ID) {
initializeLoopDistributePass(*PassRegistry::getPassRegistry());
}
bool runOnFunction(Function &F) override {
LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
LAA = &getAnalysis<LoopAccessAnalysis>();
DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
SmallVector<Loop *, 8> Worklist;
for (Loop *TopLevelLoop : *LI)
for (Loop *L : depth_first(TopLevelLoop))
if (L->empty())
Worklist.push_back(L);
bool Changed = false;
for (Loop *L : Worklist)
Changed |= processLoop(L);
return Changed;
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<LoopInfoWrapperPass>();
AU.addPreserved<LoopInfoWrapperPass>();
AU.addRequired<LoopAccessAnalysis>();
AU.addRequired<DominatorTreeWrapperPass>();
AU.addPreserved<DominatorTreeWrapperPass>();
}
static char ID;
private:
bool processLoop(Loop *L) {
assert(L->empty() && "Only process inner loops.");
DEBUG(dbgs() << "\nLDist: In \"" << L->getHeader()->getParent()->getName()
<< "\" checking " << *L << "\n");
BasicBlock *PH = L->getLoopPreheader();
if (!PH) {
DEBUG(dbgs() << "Skipping; no preheader");
return false;
}
if (!L->getExitBlock()) {
DEBUG(dbgs() << "Skipping; multiple exit blocks");
return false;
}
const LoopAccessInfo &LAI = LAA->getInfo(L, ValueToValueMap());
if (LAI.canVectorizeMemory()) {
DEBUG(dbgs() << "Skipping; memory operations are safe for vectorization");
return false;
}
auto *InterestingDependences =
LAI.getDepChecker().getInterestingDependences();
if (!InterestingDependences || InterestingDependences->empty()) {
DEBUG(dbgs() << "Skipping; No unsafe dependences to isolate");
return false;
}
InstPartitionContainer Partitions(L, LI, DT);
const MemoryDepChecker &DepChecker = LAI.getDepChecker();
const auto &MemoryInstructions = DepChecker.getMemoryInstructions();
MemoryInstructionDependences MID(MemoryInstructions,
*InterestingDependences);
int NumUnsafeDependencesActive = 0;
for (auto &InstDep : MID) {
Instruction *I = InstDep.Inst;
if (NumUnsafeDependencesActive ||
InstDep.NumUnsafeDependencesStartOrEnd > 0)
Partitions.addToCyclicPartition(I);
else
Partitions.addToNewNonCyclicPartition(I);
NumUnsafeDependencesActive += InstDep.NumUnsafeDependencesStartOrEnd;
assert(NumUnsafeDependencesActive >= 0 &&
"Negative number of dependences active");
}
auto DefsUsedOutside = findDefsUsedOutsideOfLoop(L);
for (auto *Inst : DefsUsedOutside)
Partitions.addToNewNonCyclicPartition(Inst);
DEBUG(dbgs() << "Seeded partitions:\n" << Partitions);
if (Partitions.getSize() < 2)
return false;
Partitions.mergeBeforePopulating();
DEBUG(dbgs() << "\nMerged partitions:\n" << Partitions);
if (Partitions.getSize() < 2)
return false;
Partitions.populateUsedSet();
DEBUG(dbgs() << "\nPopulated partitions:\n" << Partitions);
if (Partitions.mergeToAvoidDuplicatedLoads()) {
DEBUG(dbgs() << "\nPartitions merged to ensure unique loads:\n"
<< Partitions);
if (Partitions.getSize() < 2)
return false;
}
DEBUG(dbgs() << "\nDistributing loop: " << *L << "\n");
Partitions.setupPartitionIdOnInstructions();
if (!PH->getSinglePredecessor() || &*PH->begin() != PH->getTerminator())
SplitBlock(PH, PH->getTerminator(), DT, LI);
SmallSet<Instruction *, 8> InstsInStoreToLoadForwarding;
SmallSet<Value *, 8> PtrsInStoreToLoadForwarding;
if (AddMemcheckForStoreToLoadElimination) {
for (auto &Dep : *InterestingDependences)
if (Dep.isPossiblyBackward()) {
Instruction *Source = MemoryInstructions[Dep.Source];
Instruction *Destination = MemoryInstructions[Dep.Destination];
if (auto *LD = dyn_cast<LoadInst>(Source))
if (auto *ST = dyn_cast<StoreInst>(Destination)) {
InstsInStoreToLoadForwarding.insert(LD);
PtrsInStoreToLoadForwarding.insert(LD->getPointerOperand());
InstsInStoreToLoadForwarding.insert(ST);
PtrsInStoreToLoadForwarding.insert(ST->getPointerOperand());
}
}
}
RuntimeCheckEmitter RtCheckEmitter(LAI, L, LI, DT,
PtrsInStoreToLoadForwarding);
RtCheckEmitter.partitionPointers(Partitions);
if (RtCheckEmitter.needsRuntimeChecks()) {
RtCheckEmitter.versionLoop(this);
RtCheckEmitter.addPHINodes(DefsUsedOutside);
if (!InstsInStoreToLoadForwarding.empty())
Partitions.annotateNoAlias(InstsInStoreToLoadForwarding);
}
Partitions.cloneLoops(this);
Partitions.removeUnusedInsts();
DEBUG(dbgs() << "\nAfter removing unused Instrs:\n");
DEBUG(Partitions.printBlocks());
if (LDistVerify) {
LI->verify();
DT->verifyDomTree();
}
++NumLoopsDistributed;
return true;
}
LoopInfo *LI;
LoopAccessAnalysis *LAA;
DominatorTree *DT;
};
}
char LoopDistribute::ID;
static const char ldist_name[] = "Loop Distribition";
INITIALIZE_PASS_BEGIN(LoopDistribute, LDIST_NAME, ldist_name, false, false)
INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(LoopAccessAnalysis)
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
INITIALIZE_PASS_END(LoopDistribute, LDIST_NAME, ldist_name, false, false)
namespace llvm {
FunctionPass *createLoopDistributePass() { return new LoopDistribute(); }
}