#include "llvm/Analysis/InlineCost.h"
#include "llvm/Support/CallSite.h"
#include "llvm/CallingConv.h"
#include "llvm/IntrinsicInst.h"
#include "llvm/Target/TargetData.h"
#include "llvm/ADT/SmallPtrSet.h"
using namespace llvm;
bool llvm::callIsSmall(const Function *F) {
if (!F) return false;
if (F->hasLocalLinkage()) return false;
if (!F->hasName()) return false;
StringRef Name = F->getName();
if (Name == "copysign" || Name == "copysignf" || Name == "copysignl" ||
Name == "fabs" || Name == "fabsf" || Name == "fabsl" ||
Name == "sin" || Name == "sinf" || Name == "sinl" ||
Name == "cos" || Name == "cosf" || Name == "cosl" ||
Name == "sqrt" || Name == "sqrtf" || Name == "sqrtl" )
return true;
if (Name == "pow" || Name == "powf" || Name == "powl" ||
Name == "exp2" || Name == "exp2l" || Name == "exp2f" ||
Name == "floor" || Name == "floorf" || Name == "ceil" ||
Name == "round" || Name == "ffs" || Name == "ffsl" ||
Name == "abs" || Name == "labs" || Name == "llabs")
return true;
return false;
}
void CodeMetrics::analyzeBasicBlock(const BasicBlock *BB,
const TargetData *TD) {
++NumBlocks;
unsigned NumInstsBeforeThisBB = NumInsts;
for (BasicBlock::const_iterator II = BB->begin(), E = BB->end();
II != E; ++II) {
if (isa<PHINode>(II)) continue;
if (isa<CallInst>(II) || isa<InvokeInst>(II)) {
if (const IntrinsicInst *IntrinsicI = dyn_cast<IntrinsicInst>(II)) {
switch (IntrinsicI->getIntrinsicID()) {
default: break;
case Intrinsic::dbg_declare:
case Intrinsic::dbg_value:
case Intrinsic::invariant_start:
case Intrinsic::invariant_end:
case Intrinsic::lifetime_start:
case Intrinsic::lifetime_end:
case Intrinsic::objectsize:
case Intrinsic::ptr_annotation:
case Intrinsic::var_annotation:
continue;
}
}
ImmutableCallSite CS(cast<Instruction>(II));
if (const Function *F = CS.getCalledFunction()) {
if (!CS.isNoInline() && F->hasInternalLinkage() && F->hasOneUse())
++NumInlineCandidates;
if (F == BB->getParent())
isRecursive = true;
}
if (!isa<IntrinsicInst>(II) && !callIsSmall(CS.getCalledFunction())) {
NumInsts += CS.arg_size();
if (!isa<InlineAsm>(CS.getCalledValue()))
++NumCalls;
}
}
if (const AllocaInst *AI = dyn_cast<AllocaInst>(II)) {
if (!AI->isStaticAlloca())
this->usesDynamicAlloca = true;
}
if (isa<ExtractElementInst>(II) || II->getType()->isVectorTy())
++NumVectorInsts;
if (const CastInst *CI = dyn_cast<CastInst>(II)) {
if (CI->isLosslessCast() || isa<IntToPtrInst>(CI) ||
isa<PtrToIntInst>(CI))
continue;
if (isa<TruncInst>(CI) && TD &&
TD->isLegalInteger(TD->getTypeSizeInBits(CI->getType())))
continue;
if (isa<CmpInst>(CI->getOperand(0)))
continue;
} else if (const GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(II)){
if (GEPI->hasAllConstantIndices())
continue;
}
++NumInsts;
}
if (isa<ReturnInst>(BB->getTerminator()))
++NumRets;
if (isa<IndirectBrInst>(BB->getTerminator()))
containsIndirectBr = true;
NumBBInsts[BB] = NumInsts - NumInstsBeforeThisBB;
}
unsigned CodeMetrics::CountCodeReductionForConstant(Value *V) {
unsigned Reduction = 0;
for (Value::use_iterator UI = V->use_begin(), E = V->use_end(); UI != E;++UI){
User *U = *UI;
if (isa<BranchInst>(U) || isa<SwitchInst>(U)) {
const TerminatorInst &TI = cast<TerminatorInst>(*U);
const unsigned NumSucc = TI.getNumSuccessors();
unsigned Instrs = 0;
for (unsigned I = 0; I != NumSucc; ++I)
Instrs += NumBBInsts[TI.getSuccessor(I)];
Reduction += InlineConstants::InstrCost*Instrs*(NumSucc-1)/NumSucc;
} else {
Instruction &Inst = cast<Instruction>(*U);
if (Inst.mayReadFromMemory() || Inst.mayHaveSideEffects() ||
isa<AllocaInst>(Inst))
continue;
bool AllOperandsConstant = true;
for (unsigned i = 0, e = Inst.getNumOperands(); i != e; ++i)
if (!isa<Constant>(Inst.getOperand(i)) && Inst.getOperand(i) != V) {
AllOperandsConstant = false;
break;
}
if (AllOperandsConstant) {
Reduction += InlineConstants::InstrCost;
Reduction += CountCodeReductionForConstant(&Inst);
}
}
}
return Reduction;
}
unsigned CodeMetrics::CountCodeReductionForAlloca(Value *V) {
if (!V->getType()->isPointerTy()) return 0; unsigned Reduction = 0;
SmallVector<ICmpInst *, 4> ICmpInsts;
SmallVector<Value *, 4> Worklist;
Worklist.push_back(V);
do {
Value *V = Worklist.pop_back_val();
for (Value::use_iterator UI = V->use_begin(), E = V->use_end();
UI != E; ++UI){
Instruction *I = cast<Instruction>(*UI);
if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
if (!LI->isSimple())
return 0;
Reduction += InlineConstants::InstrCost;
} else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
if (!SI->isSimple())
return 0;
Reduction += InlineConstants::InstrCost;
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(I)) {
if (!GEP->hasAllConstantIndices())
return 0;
if (GEP->hasAllZeroIndices())
Reduction += InlineConstants::InstrCost;
Worklist.push_back(GEP);
} else if (BitCastInst *BCI = dyn_cast<BitCastInst>(I)) {
Worklist.push_back(BCI);
Reduction += InlineConstants::InstrCost;
} else if (SelectInst *SI = dyn_cast<SelectInst>(I)) {
for (Value::use_iterator UI = SI->use_begin(), UE = SI->use_end();
UI != UE; ++UI) {
LoadInst *LI = dyn_cast<LoadInst>(*UI);
if (LI == 0 || !LI->isSimple()) return 0;
}
continue;
} else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
switch (II->getIntrinsicID()) {
default:
return 0;
case Intrinsic::memset:
case Intrinsic::memcpy:
case Intrinsic::memmove:
case Intrinsic::lifetime_start:
case Intrinsic::lifetime_end:
Reduction += InlineConstants::InstrCost;
break;
}
} else if (ICmpInst *ICI = dyn_cast<ICmpInst>(I)) {
if (!isa<Constant>(ICI->getOperand(1)))
return 0;
ICmpInsts.push_back(ICI);
} else {
return 0;
}
}
} while (!Worklist.empty());
while (!ICmpInsts.empty()) {
ICmpInst *ICI = ICmpInsts.pop_back_val();
bool Result = ICI->isTrueWhenEqual();
SmallVector<Instruction *, 4> Worklist;
Worklist.push_back(ICI);
do {
Instruction *U = Worklist.pop_back_val();
Reduction += InlineConstants::InstrCost;
for (Value::use_iterator UI = U->use_begin(), UE = U->use_end();
UI != UE; ++UI) {
Instruction *I = dyn_cast<Instruction>(*UI);
if (!I || I->mayHaveSideEffects()) continue;
if (I->getNumOperands() == 1)
Worklist.push_back(I);
if (BinaryOperator *BO = dyn_cast<BinaryOperator>(I)) {
if (Result == true) {
if (BO->getOpcode() == Instruction::Or)
Worklist.push_back(I);
if (BO->getOpcode() == Instruction::And)
Reduction += InlineConstants::InstrCost;
} else {
if (BO->getOpcode() == Instruction::Or ||
BO->getOpcode() == Instruction::Xor)
Reduction += InlineConstants::InstrCost;
if (BO->getOpcode() == Instruction::And)
Worklist.push_back(I);
}
}
if (BranchInst *BI = dyn_cast<BranchInst>(I)) {
BasicBlock *BB = BI->getSuccessor(Result ? 0 : 1);
if (BB->getSinglePredecessor())
Reduction += InlineConstants::InstrCost * NumBBInsts[BB];
}
}
} while (!Worklist.empty());
}
return Reduction;
}
void CodeMetrics::analyzeFunction(Function *F, const TargetData *TD) {
exposesReturnsTwice = F->callsFunctionThatReturnsTwice() &&
!F->hasFnAttr(Attribute::ReturnsTwice);
for (Function::const_iterator BB = F->begin(), E = F->end(); BB != E; ++BB)
analyzeBasicBlock(&*BB, TD);
}
void InlineCostAnalyzer::FunctionInfo::analyzeFunction(Function *F,
const TargetData *TD) {
Metrics.analyzeFunction(F, TD);
if (Metrics.NumRets==1)
--Metrics.NumInsts;
ArgumentWeights.reserve(F->arg_size());
for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E; ++I)
ArgumentWeights.push_back(ArgInfo(Metrics.CountCodeReductionForConstant(I),
Metrics.CountCodeReductionForAlloca(I)));
}
bool InlineCostAnalyzer::FunctionInfo::NeverInline() {
return (Metrics.exposesReturnsTwice || Metrics.isRecursive ||
Metrics.containsIndirectBr);
}
int InlineCostAnalyzer::getSpecializationBonus(Function *Callee,
SmallVectorImpl<unsigned> &SpecializedArgNos)
{
if (Callee->mayBeOverridden())
return 0;
int Bonus = 0;
if (Callee->getCallingConv() == CallingConv::Cold)
Bonus -= InlineConstants::ColdccPenalty;
FunctionInfo *CalleeFI = &CachedFunctionInfo[Callee];
if (CalleeFI->Metrics.NumBlocks == 0)
CalleeFI->analyzeFunction(Callee, TD);
unsigned ArgNo = 0;
unsigned i = 0;
for (Function::arg_iterator I = Callee->arg_begin(), E = Callee->arg_end();
I != E; ++I, ++ArgNo)
if (ArgNo == SpecializedArgNos[i]) {
++i;
Bonus += CountBonusForConstant(I);
}
Bonus -= CalleeFI->Metrics.NumCalls * InlineConstants::CallPenalty;
return Bonus;
}
int InlineCostAnalyzer::ConstantFunctionBonus(CallSite CS, Constant *C) {
if (!C) return 0;
Function *F = dyn_cast<Function>(C);
if (!F) return 0;
int Bonus = InlineConstants::IndirectCallBonus + getInlineSize(CS, F);
return (Bonus > 0) ? 0 : Bonus;
}
int InlineCostAnalyzer::CountBonusForConstant(Value *V, Constant *C) {
unsigned Bonus = 0;
for (Value::use_iterator UI = V->use_begin(), E = V->use_end(); UI != E;++UI){
User *U = *UI;
if (CallInst *CI = dyn_cast<CallInst>(U)) {
if (CI->getCalledValue() == V)
Bonus += ConstantFunctionBonus(CallSite(CI), C);
} else if (InvokeInst *II = dyn_cast<InvokeInst>(U)) {
if (II->getCalledValue() == V)
Bonus += ConstantFunctionBonus(CallSite(II), C);
}
else {
Instruction &Inst = cast<Instruction>(*U);
if (Inst.mayReadFromMemory() || Inst.mayHaveSideEffects() ||
isa<AllocaInst>(Inst))
continue;
bool AllOperandsConstant = true;
for (unsigned i = 0, e = Inst.getNumOperands(); i != e; ++i)
if (!isa<Constant>(Inst.getOperand(i)) && Inst.getOperand(i) != V) {
AllOperandsConstant = false;
break;
}
if (AllOperandsConstant)
Bonus += CountBonusForConstant(&Inst);
}
}
return Bonus;
}
int InlineCostAnalyzer::getInlineSize(CallSite CS, Function *Callee) {
FunctionInfo *CalleeFI = &CachedFunctionInfo[Callee];
if (CalleeFI->Metrics.NumBlocks == 0)
CalleeFI->analyzeFunction(Callee, TD);
int InlineCost = 0;
unsigned ArgNo = 0;
CallSite::arg_iterator I = CS.arg_begin();
for (Function::arg_iterator FI = Callee->arg_begin(), FE = Callee->arg_end();
FI != FE; ++I, ++FI, ++ArgNo) {
if (isa<AllocaInst>(I))
InlineCost -= CalleeFI->ArgumentWeights[ArgNo].AllocaWeight;
else if (isa<Constant>(I))
InlineCost -= CalleeFI->ArgumentWeights[ArgNo].ConstantWeight;
}
InlineCost -= (CS.arg_size() * InlineConstants::InstrCost);
InlineCost += CalleeFI->Metrics.NumCalls * InlineConstants::CallPenalty;
InlineCost += CalleeFI->Metrics.NumInsts * InlineConstants::InstrCost;
return InlineCost;
}
int InlineCostAnalyzer::getInlineBonuses(CallSite CS, Function *Callee) {
FunctionInfo *CalleeFI = &CachedFunctionInfo[Callee];
if (CalleeFI->Metrics.NumBlocks == 0)
CalleeFI->analyzeFunction(Callee, TD);
bool isDirectCall = CS.getCalledFunction() == Callee;
Instruction *TheCall = CS.getInstruction();
int Bonus = 0;
if (Callee->hasLocalLinkage() && Callee->hasOneUse() && isDirectCall)
Bonus += InlineConstants::LastCallToStaticBonus;
if (InvokeInst *II = dyn_cast<InvokeInst>(TheCall)) {
if (isa<UnreachableInst>(II->getNormalDest()->begin()))
Bonus += InlineConstants::NoreturnPenalty;
} else if (isa<UnreachableInst>(++BasicBlock::iterator(TheCall)))
Bonus += InlineConstants::NoreturnPenalty;
if (Callee->getCallingConv() == CallingConv::Cold)
Bonus += InlineConstants::ColdccPenalty;
CallSite::arg_iterator I = CS.arg_begin();
for (Function::arg_iterator FI = Callee->arg_begin(), FE = Callee->arg_end();
FI != FE; ++I, ++FI)
if (isa<Constant>(I))
Bonus += CountBonusForConstant(FI, cast<Constant>(I));
return Bonus;
}
InlineCost InlineCostAnalyzer::getInlineCost(CallSite CS,
SmallPtrSet<const Function*, 16> &NeverInline) {
return getInlineCost(CS, CS.getCalledFunction(), NeverInline);
}
InlineCost InlineCostAnalyzer::getInlineCost(CallSite CS,
Function *Callee,
SmallPtrSet<const Function*, 16> &NeverInline) {
Instruction *TheCall = CS.getInstruction();
Function *Caller = TheCall->getParent()->getParent();
if (Callee->mayBeOverridden() ||
Callee->hasFnAttr(Attribute::NoInline) || NeverInline.count(Callee) ||
CS.isNoInline())
return llvm::InlineCost::getNever();
FunctionInfo *CalleeFI = &CachedFunctionInfo[Callee];
if (CalleeFI->Metrics.NumBlocks == 0)
CalleeFI->analyzeFunction(Callee, TD);
if (CalleeFI->NeverInline())
return InlineCost::getNever();
if (!Callee->isDeclaration() && Callee->hasFnAttr(Attribute::AlwaysInline))
return InlineCost::getAlways();
if (CalleeFI->Metrics.usesDynamicAlloca) {
FunctionInfo &CallerFI = CachedFunctionInfo[Caller];
if (CallerFI.Metrics.NumBlocks == 0) {
CallerFI.analyzeFunction(Caller, TD);
CalleeFI = &CachedFunctionInfo[Callee];
}
if (!CallerFI.Metrics.usesDynamicAlloca)
return InlineCost::getNever();
}
int InlineCost = getInlineSize(CS, Callee) + getInlineBonuses(CS, Callee);
return llvm::InlineCost::get(InlineCost);
}
InlineCost InlineCostAnalyzer::getSpecializationCost(Function *Callee,
SmallVectorImpl<unsigned> &SpecializedArgNos)
{
if (Callee->mayBeOverridden())
return llvm::InlineCost::getNever();
FunctionInfo *CalleeFI = &CachedFunctionInfo[Callee];
if (CalleeFI->Metrics.NumBlocks == 0)
CalleeFI->analyzeFunction(Callee, TD);
int Cost = 0;
Cost += CalleeFI->Metrics.NumInsts * InlineConstants::InstrCost;
for (SmallVectorImpl<unsigned>::iterator an = SpecializedArgNos.begin(),
ae = SpecializedArgNos.end(); an != ae; ++an)
Cost -= CalleeFI->ArgumentWeights[*an].ConstantWeight;
return llvm::InlineCost::get(Cost);
}
float InlineCostAnalyzer::getInlineFudgeFactor(CallSite CS) {
Function *Callee = CS.getCalledFunction();
FunctionInfo &CalleeFI = CachedFunctionInfo[Callee];
if (CalleeFI.Metrics.NumBlocks == 0)
CalleeFI.analyzeFunction(Callee, TD);
float Factor = 1.0f;
if (CalleeFI.Metrics.NumBlocks == 1)
Factor += 0.5f;
if (CalleeFI.Metrics.NumVectorInsts > CalleeFI.Metrics.NumInsts/2)
Factor += 2.0f;
else if (CalleeFI.Metrics.NumVectorInsts > CalleeFI.Metrics.NumInsts/10)
Factor += 1.5f;
return Factor;
}
void
InlineCostAnalyzer::growCachedCostInfo(Function *Caller, Function *Callee) {
CodeMetrics &CallerMetrics = CachedFunctionInfo[Caller].Metrics;
if (CallerMetrics.NumBlocks < 10 && CallerMetrics.NumInsts < 1000) {
resetCachedCostInfo(Caller);
return;
}
if (CallerMetrics.NumCalls > 0)
--CallerMetrics.NumCalls;
if (Callee == 0) return;
CodeMetrics &CalleeMetrics = CachedFunctionInfo[Callee].Metrics;
if (CalleeMetrics.NumBlocks == 0) {
resetCachedCostInfo(Caller);
return;
}
CallerMetrics.usesDynamicAlloca |= CalleeMetrics.usesDynamicAlloca;
CallerMetrics.exposesReturnsTwice |= CalleeMetrics.exposesReturnsTwice;
CallerMetrics.isRecursive |= CalleeMetrics.isRecursive;
CallerMetrics.containsIndirectBr |= CalleeMetrics.containsIndirectBr;
CallerMetrics.NumInsts += CalleeMetrics.NumInsts;
CallerMetrics.NumBlocks += CalleeMetrics.NumBlocks;
CallerMetrics.NumCalls += CalleeMetrics.NumCalls;
CallerMetrics.NumVectorInsts += CalleeMetrics.NumVectorInsts;
CallerMetrics.NumRets += CalleeMetrics.NumRets;
if (CallerMetrics.NumInsts >= Callee->arg_size())
CallerMetrics.NumInsts -= Callee->arg_size();
else
CallerMetrics.NumInsts = 0;
}
void InlineCostAnalyzer::clear() {
CachedFunctionInfo.clear();
}