#define CM_NAME "cost-model"
#define DEBUG_TYPE CM_NAME
#include "llvm/ADT/STLExtras.h"
#include "llvm/Analysis/Passes.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Value.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
using namespace llvm;
static cl::opt<bool> EnableReduxCost("costmodel-reduxcost", cl::init(false),
cl::Hidden,
cl::desc("Recognize reduction patterns."));
namespace {
class CostModelAnalysis : public FunctionPass {
public:
static char ID; CostModelAnalysis() : FunctionPass(ID), F(0), TTI(0) {
initializeCostModelAnalysisPass(
*PassRegistry::getPassRegistry());
}
unsigned getInstructionCost(const Instruction *I) const;
private:
virtual void getAnalysisUsage(AnalysisUsage &AU) const;
virtual bool runOnFunction(Function &F);
virtual void print(raw_ostream &OS, const Module*) const;
Function *F;
const TargetTransformInfo *TTI;
};
}
char CostModelAnalysis::ID = 0;
static const char cm_name[] = "Cost Model Analysis";
INITIALIZE_PASS_BEGIN(CostModelAnalysis, CM_NAME, cm_name, false, true)
INITIALIZE_PASS_END (CostModelAnalysis, CM_NAME, cm_name, false, true)
FunctionPass *llvm::createCostModelAnalysisPass() {
return new CostModelAnalysis();
}
void
CostModelAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesAll();
}
bool
CostModelAnalysis::runOnFunction(Function &F) {
this->F = &F;
TTI = getAnalysisIfAvailable<TargetTransformInfo>();
return false;
}
static bool isReverseVectorMask(SmallVectorImpl<int> &Mask) {
for (unsigned i = 0, MaskSize = Mask.size(); i < MaskSize; ++i)
if (Mask[i] > 0 && Mask[i] != (int)(MaskSize - 1 - i))
return false;
return true;
}
static TargetTransformInfo::OperandValueKind getOperandInfo(Value *V) {
TargetTransformInfo::OperandValueKind OpInfo =
TargetTransformInfo::OK_AnyValue;
ConstantDataVector *CDV = 0;
if ((CDV = dyn_cast<ConstantDataVector>(V))) {
OpInfo = TargetTransformInfo::OK_NonUniformConstantValue;
if (CDV->getSplatValue() != NULL)
OpInfo = TargetTransformInfo::OK_UniformConstantValue;
}
ConstantVector *CV = 0;
if ((CV = dyn_cast<ConstantVector>(V))) {
OpInfo = TargetTransformInfo::OK_NonUniformConstantValue;
if (CV->getSplatValue() != NULL)
OpInfo = TargetTransformInfo::OK_UniformConstantValue;
}
return OpInfo;
}
static bool matchMask(SmallVectorImpl<int> &M1, SmallVectorImpl<int> &M2) {
if (M1.size() != M2.size())
return false;
for (unsigned i = 0, e = M1.size(); i != e; ++i)
if (M1[i] != M2[i])
return false;
return true;
}
static bool matchPairwiseShuffleMask(ShuffleVectorInst *SI, bool IsLeft,
unsigned Level) {
if (!SI && Level == 0 && IsLeft)
return true;
else if (!SI)
return false;
SmallVector<int, 32> Mask(SI->getType()->getVectorNumElements(), -1);
for (unsigned i = 0, e = (1 << Level), val = !IsLeft; i != e; ++i, val += 2)
Mask[i] = val;
SmallVector<int, 16> ActualMask = SI->getShuffleMask();
if (!matchMask(Mask, ActualMask))
return false;
return true;
}
static bool matchPairwiseReductionAtLevel(const BinaryOperator *BinOp,
unsigned Level, unsigned NumLevels) {
if (BinOp == 0)
return false;
assert(BinOp->getType()->isVectorTy() && "Expecting a vector type");
unsigned Opcode = BinOp->getOpcode();
Value *L = BinOp->getOperand(0);
Value *R = BinOp->getOperand(1);
ShuffleVectorInst *LS = dyn_cast<ShuffleVectorInst>(L);
if (!LS && Level)
return false;
ShuffleVectorInst *RS = dyn_cast<ShuffleVectorInst>(R);
if (!RS && Level)
return false;
if (!Level && !RS && !LS)
return false;
Value *NextLevelOpL = LS ? LS->getOperand(0) : 0;
Value *NextLevelOpR = RS ? RS->getOperand(0) : 0;
Value *NextLevelOp = 0;
if (NextLevelOpR && NextLevelOpL) {
if (NextLevelOpL != NextLevelOpR)
return false;
NextLevelOp = NextLevelOpL;
} else if (Level == 0 && (NextLevelOpR || NextLevelOpL)) {
if (NextLevelOpL && NextLevelOpL != R)
return false;
else if (NextLevelOpR && NextLevelOpR != L)
return false;
NextLevelOp = NextLevelOpL ? R : L;
} else
return false;
BinaryOperator *NextLevelBinOp = 0;
if (Level + 1 != NumLevels) {
if (!(NextLevelBinOp = dyn_cast<BinaryOperator>(NextLevelOp)))
return false;
else if (NextLevelBinOp->getOpcode() != Opcode)
return false;
}
if (matchPairwiseShuffleMask(LS, true, Level)) {
if (!matchPairwiseShuffleMask(RS, false, Level))
return false;
} else if (matchPairwiseShuffleMask(RS, true, Level)) {
if (!matchPairwiseShuffleMask(LS, false, Level))
return false;
} else
return false;
if (++Level == NumLevels)
return true;
return matchPairwiseReductionAtLevel(NextLevelBinOp, Level, NumLevels);
}
static bool matchPairwiseReduction(const ExtractElementInst *ReduxRoot,
unsigned &Opcode, Type *&Ty) {
if (!EnableReduxCost)
return false;
ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1));
unsigned Idx = ~0u;
if (CI)
Idx = CI->getZExtValue();
if (Idx != 0)
return false;
BinaryOperator *RdxStart = dyn_cast<BinaryOperator>(ReduxRoot->getOperand(0));
if (!RdxStart)
return false;
Type *VecTy = ReduxRoot->getOperand(0)->getType();
unsigned NumVecElems = VecTy->getVectorNumElements();
if (!isPowerOf2_32(NumVecElems))
return false;
if (!matchPairwiseReductionAtLevel(RdxStart, 0, Log2_32(NumVecElems)))
return false;
Opcode = RdxStart->getOpcode();
Ty = VecTy;
return true;
}
static std::pair<Value *, ShuffleVectorInst *>
getShuffleAndOtherOprd(BinaryOperator *B) {
Value *L = B->getOperand(0);
Value *R = B->getOperand(1);
ShuffleVectorInst *S = 0;
if ((S = dyn_cast<ShuffleVectorInst>(L)))
return std::make_pair(R, S);
S = dyn_cast<ShuffleVectorInst>(R);
return std::make_pair(L, S);
}
static bool matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot,
unsigned &Opcode, Type *&Ty) {
if (!EnableReduxCost)
return false;
ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1));
unsigned Idx = ~0u;
if (CI)
Idx = CI->getZExtValue();
if (Idx != 0)
return false;
BinaryOperator *RdxStart = dyn_cast<BinaryOperator>(ReduxRoot->getOperand(0));
if (!RdxStart)
return false;
unsigned RdxOpcode = RdxStart->getOpcode();
Type *VecTy = ReduxRoot->getOperand(0)->getType();
unsigned NumVecElems = VecTy->getVectorNumElements();
if (!isPowerOf2_32(NumVecElems))
return false;
unsigned MaskStart = 1;
Value *RdxOp = RdxStart;
SmallVector<int, 32> ShuffleMask(NumVecElems, 0);
unsigned NumVecElemsRemain = NumVecElems;
while (NumVecElemsRemain - 1) {
BinaryOperator *BinOp;
if (!(BinOp = dyn_cast<BinaryOperator>(RdxOp)))
return false;
if (BinOp->getOpcode() != RdxOpcode)
return false;
Value *NextRdxOp;
ShuffleVectorInst *Shuffle;
tie(NextRdxOp, Shuffle) = getShuffleAndOtherOprd(BinOp);
if (Shuffle == 0)
return false;
if (Shuffle->getOperand(0) != NextRdxOp)
return false;
for (unsigned j = 0; j != MaskStart; ++j)
ShuffleMask[j] = MaskStart + j;
std::fill(&ShuffleMask[MaskStart], ShuffleMask.end(), -1);
SmallVector<int, 16> Mask = Shuffle->getShuffleMask();
if (!matchMask(ShuffleMask, Mask))
return false;
RdxOp = NextRdxOp;
NumVecElemsRemain /= 2;
MaskStart *= 2;
}
Opcode = RdxOpcode;
Ty = VecTy;
return true;
}
unsigned CostModelAnalysis::getInstructionCost(const Instruction *I) const {
if (!TTI)
return -1;
switch (I->getOpcode()) {
case Instruction::GetElementPtr:{
Type *ValTy = I->getOperand(0)->getType()->getPointerElementType();
return TTI->getAddressComputationCost(ValTy);
}
case Instruction::Ret:
case Instruction::PHI:
case Instruction::Br: {
return TTI->getCFInstrCost(I->getOpcode());
}
case Instruction::Add:
case Instruction::FAdd:
case Instruction::Sub:
case Instruction::FSub:
case Instruction::Mul:
case Instruction::FMul:
case Instruction::UDiv:
case Instruction::SDiv:
case Instruction::FDiv:
case Instruction::URem:
case Instruction::SRem:
case Instruction::FRem:
case Instruction::Shl:
case Instruction::LShr:
case Instruction::AShr:
case Instruction::And:
case Instruction::Or:
case Instruction::Xor: {
TargetTransformInfo::OperandValueKind Op1VK =
getOperandInfo(I->getOperand(0));
TargetTransformInfo::OperandValueKind Op2VK =
getOperandInfo(I->getOperand(1));
return TTI->getArithmeticInstrCost(I->getOpcode(), I->getType(), Op1VK,
Op2VK);
}
case Instruction::Select: {
const SelectInst *SI = cast<SelectInst>(I);
Type *CondTy = SI->getCondition()->getType();
return TTI->getCmpSelInstrCost(I->getOpcode(), I->getType(), CondTy);
}
case Instruction::ICmp:
case Instruction::FCmp: {
Type *ValTy = I->getOperand(0)->getType();
return TTI->getCmpSelInstrCost(I->getOpcode(), ValTy);
}
case Instruction::Store: {
const StoreInst *SI = cast<StoreInst>(I);
Type *ValTy = SI->getValueOperand()->getType();
return TTI->getMemoryOpCost(I->getOpcode(), ValTy,
SI->getAlignment(),
SI->getPointerAddressSpace());
}
case Instruction::Load: {
const LoadInst *LI = cast<LoadInst>(I);
return TTI->getMemoryOpCost(I->getOpcode(), I->getType(),
LI->getAlignment(),
LI->getPointerAddressSpace());
}
case Instruction::ZExt:
case Instruction::SExt:
case Instruction::FPToUI:
case Instruction::FPToSI:
case Instruction::FPExt:
case Instruction::PtrToInt:
case Instruction::IntToPtr:
case Instruction::SIToFP:
case Instruction::UIToFP:
case Instruction::Trunc:
case Instruction::FPTrunc:
case Instruction::BitCast: {
Type *SrcTy = I->getOperand(0)->getType();
return TTI->getCastInstrCost(I->getOpcode(), I->getType(), SrcTy);
}
case Instruction::ExtractElement: {
const ExtractElementInst * EEI = cast<ExtractElementInst>(I);
ConstantInt *CI = dyn_cast<ConstantInt>(I->getOperand(1));
unsigned Idx = -1;
if (CI)
Idx = CI->getZExtValue();
unsigned ReduxOpCode;
Type *ReduxType;
if (matchVectorSplittingReduction(EEI, ReduxOpCode, ReduxType))
return TTI->getReductionCost(ReduxOpCode, ReduxType, false);
else if (matchPairwiseReduction(EEI, ReduxOpCode, ReduxType))
return TTI->getReductionCost(ReduxOpCode, ReduxType, true);
return TTI->getVectorInstrCost(I->getOpcode(),
EEI->getOperand(0)->getType(), Idx);
}
case Instruction::InsertElement: {
const InsertElementInst * IE = cast<InsertElementInst>(I);
ConstantInt *CI = dyn_cast<ConstantInt>(IE->getOperand(2));
unsigned Idx = -1;
if (CI)
Idx = CI->getZExtValue();
return TTI->getVectorInstrCost(I->getOpcode(),
IE->getType(), Idx);
}
case Instruction::ShuffleVector: {
const ShuffleVectorInst *Shuffle = cast<ShuffleVectorInst>(I);
Type *VecTypOp0 = Shuffle->getOperand(0)->getType();
unsigned NumVecElems = VecTypOp0->getVectorNumElements();
SmallVector<int, 16> Mask = Shuffle->getShuffleMask();
if (NumVecElems == Mask.size() && isReverseVectorMask(Mask))
return TTI->getShuffleCost(TargetTransformInfo::SK_Reverse, VecTypOp0, 0,
0);
return -1;
}
case Instruction::Call:
if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
SmallVector<Type*, 4> Tys;
for (unsigned J = 0, JE = II->getNumArgOperands(); J != JE; ++J)
Tys.push_back(II->getArgOperand(J)->getType());
return TTI->getIntrinsicInstrCost(II->getIntrinsicID(), II->getType(),
Tys);
}
return -1;
default:
return -1;
}
}
void CostModelAnalysis::print(raw_ostream &OS, const Module*) const {
if (!F)
return;
for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) {
for (BasicBlock::iterator it = B->begin(), e = B->end(); it != e; ++it) {
Instruction *Inst = it;
unsigned Cost = getInstructionCost(Inst);
if (Cost != (unsigned)-1)
OS << "Cost Model: Found an estimated cost of " << Cost;
else
OS << "Cost Model: Unknown cost";
OS << " for instruction: "<< *Inst << "\n";
}
}
}