PostRASchedulerList.cpp [plain text]
#define DEBUG_TYPE "post-RA-sched"
#include "ScheduleDAGInstrs.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/CodeGen/LatencyPriorityQueue.h"
#include "llvm/CodeGen/SchedulerRegistry.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/ScheduleHazardRecognizer.h"
#include "llvm/Target/TargetLowering.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetInstrInfo.h"
#include "llvm/Target/TargetRegisterInfo.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/ADT/Statistic.h"
#include <map>
using namespace llvm;
STATISTIC(NumNoops, "Number of noops inserted");
STATISTIC(NumStalls, "Number of pipeline stalls");
static cl::opt<bool>
EnableAntiDepBreaking("break-anti-dependencies",
cl::desc("Break post-RA scheduling anti-dependencies"),
cl::init(true), cl::Hidden);
static cl::opt<bool>
EnablePostRAHazardAvoidance("avoid-hazards",
cl::desc("Enable simple hazard-avoidance"),
cl::init(true), cl::Hidden);
namespace {
class VISIBILITY_HIDDEN PostRAScheduler : public MachineFunctionPass {
public:
static char ID;
PostRAScheduler() : MachineFunctionPass(&ID) {}
void getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<MachineDominatorTree>();
AU.addPreserved<MachineDominatorTree>();
AU.addRequired<MachineLoopInfo>();
AU.addPreserved<MachineLoopInfo>();
MachineFunctionPass::getAnalysisUsage(AU);
}
const char *getPassName() const {
return "Post RA top-down list latency scheduler";
}
bool runOnMachineFunction(MachineFunction &Fn);
};
char PostRAScheduler::ID = 0;
class VISIBILITY_HIDDEN SchedulePostRATDList : public ScheduleDAGInstrs {
LatencyPriorityQueue AvailableQueue;
std::vector<SUnit*> PendingQueue;
ScheduleDAGTopologicalSort Topo;
const BitVector AllocatableSet;
ScheduleHazardRecognizer *HazardRec;
const TargetRegisterClass *
Classes[TargetRegisterInfo::FirstVirtualRegister];
std::multimap<unsigned, MachineOperand *> RegRefs;
unsigned KillIndices[TargetRegisterInfo::FirstVirtualRegister];
unsigned DefIndices[TargetRegisterInfo::FirstVirtualRegister];
public:
SchedulePostRATDList(MachineFunction &MF,
const MachineLoopInfo &MLI,
const MachineDominatorTree &MDT,
ScheduleHazardRecognizer *HR)
: ScheduleDAGInstrs(MF, MLI, MDT), Topo(SUnits),
AllocatableSet(TRI->getAllocatableSet(MF)),
HazardRec(HR) {}
~SchedulePostRATDList() {
delete HazardRec;
}
void StartBlock(MachineBasicBlock *BB);
void Schedule();
void Observe(MachineInstr *MI, unsigned Count);
void FinishBlock();
private:
void PrescanInstruction(MachineInstr *MI);
void ScanInstruction(MachineInstr *MI, unsigned Count);
void ReleaseSucc(SUnit *SU, SDep *SuccEdge);
void ReleaseSuccessors(SUnit *SU);
void ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle);
void ListScheduleTopDown();
bool BreakAntiDependencies();
};
class SimpleHazardRecognizer : public ScheduleHazardRecognizer {
enum Class {
Other, Load, Store
};
Class Window[8];
Class getClass(const SUnit *SU) {
const MachineInstr *MI = SU->getInstr();
const TargetInstrDesc &TID = MI->getDesc();
if (TID.mayLoad())
return Load;
if (TID.mayStore())
return Store;
return Other;
}
void Step(Class C) {
std::copy(Window+1, array_endof(Window), Window);
Window[array_lengthof(Window)-1] = C;
}
public:
SimpleHazardRecognizer() : Window() {}
virtual HazardType getHazardType(SUnit *SU) {
Class C = getClass(SU);
if (C == Other)
return NoHazard;
unsigned Score = 0;
for (unsigned i = 0; i != array_lengthof(Window); ++i)
if (Window[i] == C)
Score += i + 1;
if (Score > array_lengthof(Window) * 2)
return Hazard;
return NoHazard;
}
virtual void EmitInstruction(SUnit *SU) {
Step(getClass(SU));
}
virtual void AdvanceCycle() {
Step(Other);
}
};
}
static bool isSchedulingBoundary(const MachineInstr *MI,
const MachineFunction &MF) {
if (MI->getDesc().isTerminator() || MI->isLabel())
return true;
const TargetLowering &TLI = *MF.getTarget().getTargetLowering();
if (MI->modifiesRegister(TLI.getStackPointerRegisterToSaveRestore()))
return true;
return false;
}
bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) {
DOUT << "PostRAScheduler\n";
const MachineLoopInfo &MLI = getAnalysis<MachineLoopInfo>();
const MachineDominatorTree &MDT = getAnalysis<MachineDominatorTree>();
ScheduleHazardRecognizer *HR = EnablePostRAHazardAvoidance ?
new SimpleHazardRecognizer :
new ScheduleHazardRecognizer();
SchedulePostRATDList Scheduler(Fn, MLI, MDT, HR);
for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end();
MBB != MBBe; ++MBB) {
Scheduler.StartBlock(MBB);
MachineBasicBlock::iterator Current = MBB->end();
unsigned Count = MBB->size(), CurrentCount = Count;
for (MachineBasicBlock::iterator I = Current; I != MBB->begin(); ) {
MachineInstr *MI = prior(I);
if (isSchedulingBoundary(MI, Fn)) {
Scheduler.Run(MBB, I, Current, CurrentCount);
Scheduler.EmitSchedule();
Current = MI;
CurrentCount = Count - 1;
Scheduler.Observe(MI, CurrentCount);
}
I = MI;
--Count;
}
assert(Count == 0 && "Instruction count mismatch!");
assert((MBB->begin() == Current || CurrentCount != 0) &&
"Instruction count mismatch!");
Scheduler.Run(MBB, MBB->begin(), Current, CurrentCount);
Scheduler.EmitSchedule();
Scheduler.FinishBlock();
}
return true;
}
void SchedulePostRATDList::StartBlock(MachineBasicBlock *BB) {
ScheduleDAGInstrs::StartBlock(BB);
std::fill(Classes, array_endof(Classes),
static_cast<const TargetRegisterClass *>(0));
std::fill(KillIndices, array_endof(KillIndices), ~0u);
std::fill(DefIndices, array_endof(DefIndices), BB->size());
if (!BB->empty() && BB->back().getDesc().isReturn())
for (MachineRegisterInfo::liveout_iterator I = MRI.liveout_begin(),
E = MRI.liveout_end(); I != E; ++I) {
unsigned Reg = *I;
Classes[Reg] = reinterpret_cast<TargetRegisterClass *>(-1);
KillIndices[Reg] = BB->size();
DefIndices[Reg] = ~0u;
for (const unsigned *Alias = TRI->getAliasSet(Reg); *Alias; ++Alias) {
unsigned AliasReg = *Alias;
Classes[AliasReg] = reinterpret_cast<TargetRegisterClass *>(-1);
KillIndices[AliasReg] = BB->size();
DefIndices[AliasReg] = ~0u;
}
}
else
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end(); SI != SE; ++SI)
for (MachineBasicBlock::livein_iterator I = (*SI)->livein_begin(),
E = (*SI)->livein_end(); I != E; ++I) {
unsigned Reg = *I;
Classes[Reg] = reinterpret_cast<TargetRegisterClass *>(-1);
KillIndices[Reg] = BB->size();
DefIndices[Reg] = ~0u;
for (const unsigned *Alias = TRI->getAliasSet(Reg); *Alias; ++Alias) {
unsigned AliasReg = *Alias;
Classes[AliasReg] = reinterpret_cast<TargetRegisterClass *>(-1);
KillIndices[AliasReg] = BB->size();
DefIndices[AliasReg] = ~0u;
}
}
for (const unsigned *I = TRI->getCalleeSavedRegs(); *I; ++I) {
unsigned Reg = *I;
Classes[Reg] = reinterpret_cast<TargetRegisterClass *>(-1);
KillIndices[Reg] = BB->size();
DefIndices[Reg] = ~0u;
for (const unsigned *Alias = TRI->getAliasSet(Reg); *Alias; ++Alias) {
unsigned AliasReg = *Alias;
Classes[AliasReg] = reinterpret_cast<TargetRegisterClass *>(-1);
KillIndices[AliasReg] = BB->size();
DefIndices[AliasReg] = ~0u;
}
}
}
void SchedulePostRATDList::Schedule() {
DOUT << "********** List Scheduling **********\n";
BuildSchedGraph();
if (EnableAntiDepBreaking) {
if (BreakAntiDependencies()) {
SUnits.clear();
EntrySU = SUnit();
ExitSU = SUnit();
BuildSchedGraph();
}
}
AvailableQueue.initNodes(SUnits);
ListScheduleTopDown();
AvailableQueue.releaseState();
}
void SchedulePostRATDList::Observe(MachineInstr *MI, unsigned Count) {
assert(Count < InsertPosIndex && "Instruction index out of expected range!");
for (unsigned Reg = 0; Reg != TargetRegisterInfo::FirstVirtualRegister; ++Reg)
if (DefIndices[Reg] < InsertPosIndex && DefIndices[Reg] >= Count) {
assert(KillIndices[Reg] == ~0u && "Clobbered register is live!");
Classes[Reg] = reinterpret_cast<TargetRegisterClass *>(-1);
DefIndices[Reg] = InsertPosIndex;
}
PrescanInstruction(MI);
ScanInstruction(MI, Count);
}
void SchedulePostRATDList::FinishBlock() {
RegRefs.clear();
ScheduleDAGInstrs::FinishBlock();
}
static SDep *CriticalPathStep(SUnit *SU) {
SDep *Next = 0;
unsigned NextDepth = 0;
for (SUnit::pred_iterator P = SU->Preds.begin(), PE = SU->Preds.end();
P != PE; ++P) {
SUnit *PredSU = P->getSUnit();
unsigned PredLatency = P->getLatency();
unsigned PredTotalLatency = PredSU->getDepth() + PredLatency;
if (NextDepth < PredTotalLatency ||
(NextDepth == PredTotalLatency && P->getKind() == SDep::Anti)) {
NextDepth = PredTotalLatency;
Next = &*P;
}
}
return Next;
}
void SchedulePostRATDList::PrescanInstruction(MachineInstr *MI) {
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
MachineOperand &MO = MI->getOperand(i);
if (!MO.isReg()) continue;
unsigned Reg = MO.getReg();
if (Reg == 0) continue;
const TargetRegisterClass *NewRC =
getInstrOperandRegClass(TRI, MI->getDesc(), i);
if (!Classes[Reg] && NewRC)
Classes[Reg] = NewRC;
else if (!NewRC || Classes[Reg] != NewRC)
Classes[Reg] = reinterpret_cast<TargetRegisterClass *>(-1);
for (const unsigned *Alias = TRI->getAliasSet(Reg); *Alias; ++Alias) {
unsigned AliasReg = *Alias;
if (Classes[AliasReg]) {
Classes[AliasReg] = reinterpret_cast<TargetRegisterClass *>(-1);
Classes[Reg] = reinterpret_cast<TargetRegisterClass *>(-1);
}
}
if (Classes[Reg] != reinterpret_cast<TargetRegisterClass *>(-1))
RegRefs.insert(std::make_pair(Reg, &MO));
}
}
void SchedulePostRATDList::ScanInstruction(MachineInstr *MI,
unsigned Count) {
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
MachineOperand &MO = MI->getOperand(i);
if (!MO.isReg()) continue;
unsigned Reg = MO.getReg();
if (Reg == 0) continue;
if (!MO.isDef()) continue;
if (MI->isRegTiedToUseOperand(i)) continue;
DefIndices[Reg] = Count;
KillIndices[Reg] = ~0u;
assert(((KillIndices[Reg] == ~0u) !=
(DefIndices[Reg] == ~0u)) &&
"Kill and Def maps aren't consistent for Reg!");
Classes[Reg] = 0;
RegRefs.erase(Reg);
for (const unsigned *Subreg = TRI->getSubRegisters(Reg);
*Subreg; ++Subreg) {
unsigned SubregReg = *Subreg;
DefIndices[SubregReg] = Count;
KillIndices[SubregReg] = ~0u;
Classes[SubregReg] = 0;
RegRefs.erase(SubregReg);
}
for (const unsigned *Super = TRI->getSuperRegisters(Reg);
*Super; ++Super) {
unsigned SuperReg = *Super;
Classes[SuperReg] = reinterpret_cast<TargetRegisterClass *>(-1);
}
}
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
MachineOperand &MO = MI->getOperand(i);
if (!MO.isReg()) continue;
unsigned Reg = MO.getReg();
if (Reg == 0) continue;
if (!MO.isUse()) continue;
const TargetRegisterClass *NewRC =
getInstrOperandRegClass(TRI, MI->getDesc(), i);
if (!Classes[Reg] && NewRC)
Classes[Reg] = NewRC;
else if (!NewRC || Classes[Reg] != NewRC)
Classes[Reg] = reinterpret_cast<TargetRegisterClass *>(-1);
RegRefs.insert(std::make_pair(Reg, &MO));
if (KillIndices[Reg] == ~0u) {
KillIndices[Reg] = Count;
DefIndices[Reg] = ~0u;
assert(((KillIndices[Reg] == ~0u) !=
(DefIndices[Reg] == ~0u)) &&
"Kill and Def maps aren't consistent for Reg!");
}
for (const unsigned *Alias = TRI->getAliasSet(Reg); *Alias; ++Alias) {
unsigned AliasReg = *Alias;
if (KillIndices[AliasReg] == ~0u) {
KillIndices[AliasReg] = Count;
DefIndices[AliasReg] = ~0u;
}
}
}
}
bool SchedulePostRATDList::BreakAntiDependencies() {
if (SUnits.empty()) return false;
SUnit *Max = 0;
for (unsigned i = 0, e = SUnits.size(); i != e; ++i) {
SUnit *SU = &SUnits[i];
if (!Max || SU->getDepth() + SU->Latency > Max->getDepth() + Max->Latency)
Max = SU;
}
DOUT << "Critical path has total latency "
<< (Max->getDepth() + Max->Latency) << "\n";
SUnit *CriticalPathSU = Max;
MachineInstr *CriticalPathMI = CriticalPathSU->getInstr();
unsigned LastNewReg[TargetRegisterInfo::FirstVirtualRegister] = {};
bool Changed = false;
unsigned Count = InsertPosIndex - 1;
for (MachineBasicBlock::iterator I = InsertPos, E = Begin;
I != E; --Count) {
MachineInstr *MI = --I;
if (MI->getOpcode() == TargetInstrInfo::IMPLICIT_DEF)
continue;
unsigned AntiDepReg = 0;
if (MI == CriticalPathMI) {
if (SDep *Edge = CriticalPathStep(CriticalPathSU)) {
SUnit *NextSU = Edge->getSUnit();
if (Edge->getKind() == SDep::Anti) {
AntiDepReg = Edge->getReg();
assert(AntiDepReg != 0 && "Anti-dependence on reg0?");
if (!AllocatableSet.test(AntiDepReg))
AntiDepReg = 0;
else {
for (SUnit::pred_iterator P = CriticalPathSU->Preds.begin(),
PE = CriticalPathSU->Preds.end(); P != PE; ++P)
if (P->getSUnit() == NextSU ?
(P->getKind() != SDep::Anti || P->getReg() != AntiDepReg) :
(P->getKind() == SDep::Data && P->getReg() == AntiDepReg)) {
AntiDepReg = 0;
break;
}
}
}
CriticalPathSU = NextSU;
CriticalPathMI = CriticalPathSU->getInstr();
} else {
CriticalPathSU = 0;
CriticalPathMI = 0;
}
}
PrescanInstruction(MI);
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
MachineOperand &MO = MI->getOperand(i);
if (!MO.isReg()) continue;
unsigned Reg = MO.getReg();
if (Reg == 0) continue;
if (MO.isUse() && AntiDepReg == Reg) {
AntiDepReg = 0;
break;
}
}
const TargetRegisterClass *RC = AntiDepReg != 0 ? Classes[AntiDepReg] : 0;
assert((AntiDepReg == 0 || RC != NULL) &&
"Register should be live if it's causing an anti-dependence!");
if (RC == reinterpret_cast<TargetRegisterClass *>(-1))
AntiDepReg = 0;
if (AntiDepReg != 0) {
for (TargetRegisterClass::iterator R = RC->allocation_order_begin(MF),
RE = RC->allocation_order_end(MF); R != RE; ++R) {
unsigned NewReg = *R;
if (NewReg == AntiDepReg) continue;
if (NewReg == LastNewReg[AntiDepReg]) continue;
assert(((KillIndices[AntiDepReg] == ~0u) != (DefIndices[AntiDepReg] == ~0u)) &&
"Kill and Def maps aren't consistent for AntiDepReg!");
assert(((KillIndices[NewReg] == ~0u) != (DefIndices[NewReg] == ~0u)) &&
"Kill and Def maps aren't consistent for NewReg!");
if (KillIndices[NewReg] == ~0u &&
Classes[NewReg] != reinterpret_cast<TargetRegisterClass *>(-1) &&
KillIndices[AntiDepReg] <= DefIndices[NewReg]) {
DOUT << "Breaking anti-dependence edge on "
<< TRI->getName(AntiDepReg)
<< " with " << RegRefs.count(AntiDepReg) << " references"
<< " using " << TRI->getName(NewReg) << "!\n";
std::pair<std::multimap<unsigned, MachineOperand *>::iterator,
std::multimap<unsigned, MachineOperand *>::iterator>
Range = RegRefs.equal_range(AntiDepReg);
for (std::multimap<unsigned, MachineOperand *>::iterator
Q = Range.first, QE = Range.second; Q != QE; ++Q)
Q->second->setReg(NewReg);
Classes[NewReg] = Classes[AntiDepReg];
DefIndices[NewReg] = DefIndices[AntiDepReg];
KillIndices[NewReg] = KillIndices[AntiDepReg];
assert(((KillIndices[NewReg] == ~0u) !=
(DefIndices[NewReg] == ~0u)) &&
"Kill and Def maps aren't consistent for NewReg!");
Classes[AntiDepReg] = 0;
DefIndices[AntiDepReg] = KillIndices[AntiDepReg];
KillIndices[AntiDepReg] = ~0u;
assert(((KillIndices[AntiDepReg] == ~0u) !=
(DefIndices[AntiDepReg] == ~0u)) &&
"Kill and Def maps aren't consistent for AntiDepReg!");
RegRefs.erase(AntiDepReg);
Changed = true;
LastNewReg[AntiDepReg] = NewReg;
break;
}
}
}
ScanInstruction(MI, Count);
}
return Changed;
}
void SchedulePostRATDList::ReleaseSucc(SUnit *SU, SDep *SuccEdge) {
SUnit *SuccSU = SuccEdge->getSUnit();
--SuccSU->NumPredsLeft;
#ifndef NDEBUG
if (SuccSU->NumPredsLeft < 0) {
cerr << "*** Scheduling failed! ***\n";
SuccSU->dump(this);
cerr << " has been released too many times!\n";
assert(0);
}
#endif
SuccSU->setDepthToAtLeast(SU->getDepth() + SuccEdge->getLatency());
if (SuccSU->NumPredsLeft == 0 && SuccSU != &ExitSU)
PendingQueue.push_back(SuccSU);
}
void SchedulePostRATDList::ReleaseSuccessors(SUnit *SU) {
for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
I != E; ++I)
ReleaseSucc(SU, &*I);
}
void SchedulePostRATDList::ScheduleNodeTopDown(SUnit *SU, unsigned CurCycle) {
DOUT << "*** Scheduling [" << CurCycle << "]: ";
DEBUG(SU->dump(this));
Sequence.push_back(SU);
assert(CurCycle >= SU->getDepth() && "Node scheduled above its depth!");
SU->setDepthToAtLeast(CurCycle);
ReleaseSuccessors(SU);
SU->isScheduled = true;
AvailableQueue.ScheduledNode(SU);
}
void SchedulePostRATDList::ListScheduleTopDown() {
unsigned CurCycle = 0;
ReleaseSuccessors(&EntrySU);
for (unsigned i = 0, e = SUnits.size(); i != e; ++i) {
if (SUnits[i].Preds.empty()) {
AvailableQueue.push(&SUnits[i]);
SUnits[i].isAvailable = true;
}
}
std::vector<SUnit*> NotReady;
Sequence.reserve(SUnits.size());
while (!AvailableQueue.empty() || !PendingQueue.empty()) {
unsigned MinDepth = ~0u;
for (unsigned i = 0, e = PendingQueue.size(); i != e; ++i) {
if (PendingQueue[i]->getDepth() <= CurCycle) {
AvailableQueue.push(PendingQueue[i]);
PendingQueue[i]->isAvailable = true;
PendingQueue[i] = PendingQueue.back();
PendingQueue.pop_back();
--i; --e;
} else if (PendingQueue[i]->getDepth() < MinDepth)
MinDepth = PendingQueue[i]->getDepth();
}
if (AvailableQueue.empty()) {
CurCycle = MinDepth != ~0u ? MinDepth : CurCycle + 1;
continue;
}
SUnit *FoundSUnit = 0;
bool HasNoopHazards = false;
while (!AvailableQueue.empty()) {
SUnit *CurSUnit = AvailableQueue.pop();
ScheduleHazardRecognizer::HazardType HT =
HazardRec->getHazardType(CurSUnit);
if (HT == ScheduleHazardRecognizer::NoHazard) {
FoundSUnit = CurSUnit;
break;
}
HasNoopHazards |= HT == ScheduleHazardRecognizer::NoopHazard;
NotReady.push_back(CurSUnit);
}
if (!NotReady.empty()) {
AvailableQueue.push_all(NotReady);
NotReady.clear();
}
if (FoundSUnit) {
ScheduleNodeTopDown(FoundSUnit, CurCycle);
HazardRec->EmitInstruction(FoundSUnit);
if (FoundSUnit->Latency) ++CurCycle;
} else if (!HasNoopHazards) {
DOUT << "*** Advancing cycle, no work to do\n";
HazardRec->AdvanceCycle();
++NumStalls;
++CurCycle;
} else {
DOUT << "*** Emitting noop\n";
HazardRec->EmitNoop();
Sequence.push_back(0); ++NumNoops;
++CurCycle;
}
}
#ifndef NDEBUG
VerifySchedule(false);
#endif
}
FunctionPass *llvm::createPostRAScheduler() {
return new PostRAScheduler();
}