#define DEBUG_TYPE "x86-vzeroupper"
#include "X86.h"
#include "X86InstrInfo.h"
#include "X86Subtarget.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetInstrInfo.h"
using namespace llvm;
STATISTIC(NumVZU, "Number of vzeroupper instructions inserted");
namespace {
class VZeroUpperInserter : public MachineFunctionPass {
public:
VZeroUpperInserter() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override;
const char *getPassName() const override {return "X86 vzeroupper inserter";}
private:
void processBasicBlock(MachineBasicBlock &MBB);
void insertVZeroUpper(MachineBasicBlock::iterator I,
MachineBasicBlock &MBB);
void addDirtySuccessor(MachineBasicBlock &MBB);
typedef enum { PASS_THROUGH, EXITS_CLEAN, EXITS_DIRTY } BlockExitState;
static const char* getBlockExitStateName(BlockExitState ST);
struct BlockState {
BlockState() : ExitState(PASS_THROUGH), AddedToDirtySuccessors(false) {}
BlockExitState ExitState;
bool AddedToDirtySuccessors;
MachineBasicBlock::iterator FirstUnguardedCall;
};
typedef SmallVector<BlockState, 8> BlockStateMap;
typedef SmallVector<MachineBasicBlock*, 8> DirtySuccessorsWorkList;
BlockStateMap BlockStates;
DirtySuccessorsWorkList DirtySuccessors;
bool EverMadeChange;
const TargetInstrInfo *TII;
static char ID;
};
char VZeroUpperInserter::ID = 0;
}
FunctionPass *llvm::createX86IssueVZeroUpperPass() {
return new VZeroUpperInserter();
}
const char* VZeroUpperInserter::getBlockExitStateName(BlockExitState ST) {
switch (ST) {
case PASS_THROUGH: return "Pass-through";
case EXITS_DIRTY: return "Exits-dirty";
case EXITS_CLEAN: return "Exits-clean";
}
llvm_unreachable("Invalid block exit state.");
}
static bool isYmmReg(unsigned Reg) {
return (Reg >= X86::YMM0 && Reg <= X86::YMM15);
}
static bool checkFnHasLiveInYmm(MachineRegisterInfo &MRI) {
for (MachineRegisterInfo::livein_iterator I = MRI.livein_begin(),
E = MRI.livein_end(); I != E; ++I)
if (isYmmReg(I->first))
return true;
return false;
}
static bool clobbersAllYmmRegs(const MachineOperand &MO) {
for (unsigned reg = X86::YMM0; reg <= X86::YMM15; ++reg) {
if (!MO.clobbersPhysReg(reg))
return false;
}
return true;
}
static bool hasYmmReg(MachineInstr *MI) {
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
const MachineOperand &MO = MI->getOperand(i);
if (MI->isCall() && MO.isRegMask() && !clobbersAllYmmRegs(MO))
return true;
if (!MO.isReg())
continue;
if (MO.isDebug())
continue;
if (isYmmReg(MO.getReg()))
return true;
}
return false;
}
static bool callClobbersAnyYmmReg(MachineInstr *MI) {
assert(MI->isCall() && "Can only be called on call instructions.");
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
const MachineOperand &MO = MI->getOperand(i);
if (!MO.isRegMask())
continue;
for (unsigned reg = X86::YMM0; reg <= X86::YMM15; ++reg) {
if (MO.clobbersPhysReg(reg))
return true;
}
}
return false;
}
void VZeroUpperInserter::insertVZeroUpper(MachineBasicBlock::iterator I,
MachineBasicBlock &MBB) {
DebugLoc dl = I->getDebugLoc();
BuildMI(MBB, I, dl, TII->get(X86::VZEROUPPER));
++NumVZU;
EverMadeChange = true;
}
void VZeroUpperInserter::addDirtySuccessor(MachineBasicBlock &MBB) {
if (!BlockStates[MBB.getNumber()].AddedToDirtySuccessors) {
DirtySuccessors.push_back(&MBB);
BlockStates[MBB.getNumber()].AddedToDirtySuccessors = true;
}
}
void VZeroUpperInserter::processBasicBlock(MachineBasicBlock &MBB) {
BlockExitState CurState = PASS_THROUGH;
BlockStates[MBB.getNumber()].FirstUnguardedCall = MBB.end();
for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ++I) {
MachineInstr *MI = I;
bool isControlFlow = MI->isCall() || MI->isReturn();
if (!isControlFlow && CurState == EXITS_DIRTY)
continue;
if (hasYmmReg(MI)) {
CurState = EXITS_DIRTY;
continue;
}
if (!isControlFlow)
continue;
if (MI->isCall() && !callClobbersAnyYmmReg(MI))
continue;
if (CurState == EXITS_DIRTY) {
insertVZeroUpper(I, MBB);
CurState = EXITS_CLEAN;
} else if (CurState == PASS_THROUGH) {
BlockStates[MBB.getNumber()].FirstUnguardedCall = I;
CurState = EXITS_CLEAN;
}
}
DEBUG(dbgs() << "MBB #" << MBB.getNumber() << " exit state: "
<< getBlockExitStateName(CurState) << '\n');
if (CurState == EXITS_DIRTY)
for (MachineBasicBlock::succ_iterator SI = MBB.succ_begin(),
SE = MBB.succ_end();
SI != SE; ++SI)
addDirtySuccessor(**SI);
BlockStates[MBB.getNumber()].ExitState = CurState;
}
bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
if (MF.getTarget().getSubtarget<X86Subtarget>().hasAVX512())
return false;
TII = MF.getTarget().getInstrInfo();
MachineRegisterInfo &MRI = MF.getRegInfo();
EverMadeChange = false;
bool YMMUsed = false;
const TargetRegisterClass *RC = &X86::VR256RegClass;
for (TargetRegisterClass::iterator i = RC->begin(), e = RC->end();
i != e; i++) {
if (!MRI.reg_nodbg_empty(*i)) {
YMMUsed = true;
break;
}
}
if (!YMMUsed) {
return false;
}
assert(BlockStates.empty() && DirtySuccessors.empty() &&
"X86VZeroUpper state should be clear");
BlockStates.resize(MF.getNumBlockIDs());
for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I)
processBasicBlock(*I);
if (checkFnHasLiveInYmm(MRI))
addDirtySuccessor(MF.front());
while (!DirtySuccessors.empty()) {
MachineBasicBlock &MBB = *DirtySuccessors.back();
DirtySuccessors.pop_back();
BlockState &BBState = BlockStates[MBB.getNumber()];
if (BBState.FirstUnguardedCall != MBB.end())
insertVZeroUpper(BBState.FirstUnguardedCall, MBB);
if (BBState.ExitState == PASS_THROUGH) {
DEBUG(dbgs() << "MBB #" << MBB.getNumber()
<< " was Pass-through, is now Dirty-out.\n");
for (MachineBasicBlock::succ_iterator SI = MBB.succ_begin(),
SE = MBB.succ_end();
SI != SE; ++SI)
addDirtySuccessor(**SI);
}
}
BlockStates.clear();
return EverMadeChange;
}