InstructionCombining.cpp [plain text]
#include "llvm/Transforms/InstCombine/InstCombine.h"
#include "InstCombineInternal.h"
#include "llvm-c/Initialization.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/LibCallSemantics.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/GetElementPtrTypeIterator.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/Local.h"
#include <algorithm>
#include <climits>
using namespace llvm;
using namespace llvm::PatternMatch;
#define DEBUG_TYPE "instcombine"
STATISTIC(NumCombined , "Number of insts combined");
STATISTIC(NumConstProp, "Number of constant folds");
STATISTIC(NumDeadInst , "Number of dead inst eliminated");
STATISTIC(NumSunkInst , "Number of instructions sunk");
STATISTIC(NumExpand, "Number of expansions");
STATISTIC(NumFactor , "Number of factorizations");
STATISTIC(NumReassoc , "Number of reassociations");
Value *InstCombiner::EmitGEPOffset(User *GEP) {
return llvm::EmitGEPOffset(Builder, DL, GEP);
}
bool InstCombiner::ShouldChangeType(unsigned FromWidth,
unsigned ToWidth) const {
bool FromLegal = DL.isLegalInteger(FromWidth);
bool ToLegal = DL.isLegalInteger(ToWidth);
if (FromLegal && !ToLegal)
return false;
if (!FromLegal && !ToLegal && ToWidth > FromWidth)
return false;
return true;
}
bool InstCombiner::ShouldChangeType(Type *From, Type *To) const {
assert(From->isIntegerTy() && To->isIntegerTy());
unsigned FromWidth = From->getPrimitiveSizeInBits();
unsigned ToWidth = To->getPrimitiveSizeInBits();
return ShouldChangeType(FromWidth, ToWidth);
}
static bool MaintainNoSignedWrap(BinaryOperator &I, Value *B, Value *C) {
OverflowingBinaryOperator *OBO = dyn_cast<OverflowingBinaryOperator>(&I);
if (!OBO || !OBO->hasNoSignedWrap()) {
return false;
}
Instruction::BinaryOps Opcode = I.getOpcode();
if (Opcode != Instruction::Add &&
Opcode != Instruction::Sub) {
return false;
}
ConstantInt *CB = dyn_cast<ConstantInt>(B);
ConstantInt *CC = dyn_cast<ConstantInt>(C);
if (!CB || !CC) {
return false;
}
const APInt &BVal = CB->getValue();
const APInt &CVal = CC->getValue();
bool Overflow = false;
if (Opcode == Instruction::Add) {
BVal.sadd_ov(CVal, Overflow);
} else {
BVal.ssub_ov(CVal, Overflow);
}
return !Overflow;
}
static void ClearSubclassDataAfterReassociation(BinaryOperator &I) {
FPMathOperator *FPMO = dyn_cast<FPMathOperator>(&I);
if (!FPMO) {
I.clearSubclassOptionalData();
return;
}
FastMathFlags FMF = I.getFastMathFlags();
I.clearSubclassOptionalData();
I.setFastMathFlags(FMF);
}
bool InstCombiner::SimplifyAssociativeOrCommutative(BinaryOperator &I) {
Instruction::BinaryOps Opcode = I.getOpcode();
bool Changed = false;
do {
if (I.isCommutative() && getComplexity(I.getOperand(0)) <
getComplexity(I.getOperand(1)))
Changed = !I.swapOperands();
BinaryOperator *Op0 = dyn_cast<BinaryOperator>(I.getOperand(0));
BinaryOperator *Op1 = dyn_cast<BinaryOperator>(I.getOperand(1));
if (I.isAssociative()) {
if (Op0 && Op0->getOpcode() == Opcode) {
Value *A = Op0->getOperand(0);
Value *B = Op0->getOperand(1);
Value *C = I.getOperand(1);
if (Value *V = SimplifyBinOp(Opcode, B, C, DL)) {
I.setOperand(0, A);
I.setOperand(1, V);
if (MaintainNoSignedWrap(I, B, C) &&
(!Op0 || (isa<BinaryOperator>(Op0) && Op0->hasNoSignedWrap()))) {
I.clearSubclassOptionalData();
I.setHasNoSignedWrap(true);
} else {
ClearSubclassDataAfterReassociation(I);
}
Changed = true;
++NumReassoc;
continue;
}
}
if (Op1 && Op1->getOpcode() == Opcode) {
Value *A = I.getOperand(0);
Value *B = Op1->getOperand(0);
Value *C = Op1->getOperand(1);
if (Value *V = SimplifyBinOp(Opcode, A, B, DL)) {
I.setOperand(0, V);
I.setOperand(1, C);
ClearSubclassDataAfterReassociation(I);
Changed = true;
++NumReassoc;
continue;
}
}
}
if (I.isAssociative() && I.isCommutative()) {
if (Op0 && Op0->getOpcode() == Opcode) {
Value *A = Op0->getOperand(0);
Value *B = Op0->getOperand(1);
Value *C = I.getOperand(1);
if (Value *V = SimplifyBinOp(Opcode, C, A, DL)) {
I.setOperand(0, V);
I.setOperand(1, B);
ClearSubclassDataAfterReassociation(I);
Changed = true;
++NumReassoc;
continue;
}
}
if (Op1 && Op1->getOpcode() == Opcode) {
Value *A = I.getOperand(0);
Value *B = Op1->getOperand(0);
Value *C = Op1->getOperand(1);
if (Value *V = SimplifyBinOp(Opcode, C, A, DL)) {
I.setOperand(0, B);
I.setOperand(1, V);
ClearSubclassDataAfterReassociation(I);
Changed = true;
++NumReassoc;
continue;
}
}
if (Op0 && Op1 &&
Op0->getOpcode() == Opcode && Op1->getOpcode() == Opcode &&
isa<Constant>(Op0->getOperand(1)) &&
isa<Constant>(Op1->getOperand(1)) &&
Op0->hasOneUse() && Op1->hasOneUse()) {
Value *A = Op0->getOperand(0);
Constant *C1 = cast<Constant>(Op0->getOperand(1));
Value *B = Op1->getOperand(0);
Constant *C2 = cast<Constant>(Op1->getOperand(1));
Constant *Folded = ConstantExpr::get(Opcode, C1, C2);
BinaryOperator *New = BinaryOperator::Create(Opcode, A, B);
if (isa<FPMathOperator>(New)) {
FastMathFlags Flags = I.getFastMathFlags();
Flags &= Op0->getFastMathFlags();
Flags &= Op1->getFastMathFlags();
New->setFastMathFlags(Flags);
}
InsertNewInstWith(New, I);
New->takeName(Op1);
I.setOperand(0, New);
I.setOperand(1, Folded);
ClearSubclassDataAfterReassociation(I);
Changed = true;
continue;
}
}
return Changed;
} while (1);
}
static bool LeftDistributesOverRight(Instruction::BinaryOps LOp,
Instruction::BinaryOps ROp) {
switch (LOp) {
default:
return false;
case Instruction::And:
switch (ROp) {
default:
return false;
case Instruction::Or:
case Instruction::Xor:
return true;
}
case Instruction::Mul:
switch (ROp) {
default:
return false;
case Instruction::Add:
case Instruction::Sub:
return true;
}
case Instruction::Or:
switch (ROp) {
default:
return false;
case Instruction::And:
return true;
}
}
}
static bool RightDistributesOverLeft(Instruction::BinaryOps LOp,
Instruction::BinaryOps ROp) {
if (Instruction::isCommutative(ROp))
return LeftDistributesOverRight(ROp, LOp);
switch (LOp) {
default:
return false;
case Instruction::And:
case Instruction::Or:
case Instruction::Xor:
switch (ROp) {
default:
return false;
case Instruction::Shl:
case Instruction::LShr:
case Instruction::AShr:
return true;
}
}
return false;
}
static Value *getIdentityValue(Instruction::BinaryOps OpCode, Value *V) {
if (isa<Constant>(V))
return nullptr;
if (OpCode == Instruction::Mul)
return ConstantInt::get(V->getType(), 1);
return nullptr;
}
static Instruction::BinaryOps
getBinOpsForFactorization(Instruction::BinaryOps TopLevelOpcode,
BinaryOperator *Op, Value *&LHS, Value *&RHS) {
if (!Op)
return Instruction::BinaryOpsEnd;
LHS = Op->getOperand(0);
RHS = Op->getOperand(1);
switch (TopLevelOpcode) {
default:
return Op->getOpcode();
case Instruction::Add:
case Instruction::Sub:
if (Op->getOpcode() == Instruction::Shl) {
if (Constant *CST = dyn_cast<Constant>(Op->getOperand(1))) {
RHS = ConstantExpr::getShl(ConstantInt::get(Op->getType(), 1), CST);
return Instruction::Mul;
}
}
return Op->getOpcode();
}
}
static Value *tryFactorization(InstCombiner::BuilderTy *Builder,
const DataLayout &DL, BinaryOperator &I,
Instruction::BinaryOps InnerOpcode, Value *A,
Value *B, Value *C, Value *D) {
if (!A || !C || !B || !D)
return nullptr;
Value *V = nullptr;
Value *SimplifiedInst = nullptr;
Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
Instruction::BinaryOps TopLevelOpcode = I.getOpcode();
bool InnerCommutative = Instruction::isCommutative(InnerOpcode);
if (LeftDistributesOverRight(InnerOpcode, TopLevelOpcode))
if (A == C || (InnerCommutative && A == D)) {
if (A != C)
std::swap(C, D);
V = SimplifyBinOp(TopLevelOpcode, B, D, DL);
if (!V && LHS->hasOneUse() && RHS->hasOneUse())
V = Builder->CreateBinOp(TopLevelOpcode, B, D, RHS->getName());
if (V) {
SimplifiedInst = Builder->CreateBinOp(InnerOpcode, A, V);
}
}
if (!SimplifiedInst && RightDistributesOverLeft(TopLevelOpcode, InnerOpcode))
if (B == D || (InnerCommutative && B == C)) {
if (B != D)
std::swap(C, D);
V = SimplifyBinOp(TopLevelOpcode, A, C, DL);
if (!V && LHS->hasOneUse() && RHS->hasOneUse())
V = Builder->CreateBinOp(TopLevelOpcode, A, C, LHS->getName());
if (V) {
SimplifiedInst = Builder->CreateBinOp(InnerOpcode, V, B);
}
}
if (SimplifiedInst) {
++NumFactor;
SimplifiedInst->takeName(&I);
if (BinaryOperator *BO = dyn_cast<BinaryOperator>(SimplifiedInst)) {
if (isa<OverflowingBinaryOperator>(SimplifiedInst)) {
bool HasNSW = false;
if (isa<OverflowingBinaryOperator>(&I))
HasNSW = I.hasNoSignedWrap();
if (BinaryOperator *Op0 = dyn_cast<BinaryOperator>(LHS))
if (isa<OverflowingBinaryOperator>(Op0))
HasNSW &= Op0->hasNoSignedWrap();
if (BinaryOperator *Op1 = dyn_cast<BinaryOperator>(RHS))
if (isa<OverflowingBinaryOperator>(Op1))
HasNSW &= Op1->hasNoSignedWrap();
const APInt *CInt;
if (TopLevelOpcode == Instruction::Add &&
InnerOpcode == Instruction::Mul)
if (match(V, m_APInt(CInt)) && !CInt->isMinSignedValue())
BO->setHasNoSignedWrap(HasNSW);
}
}
}
return SimplifiedInst;
}
Value *InstCombiner::SimplifyUsingDistributiveLaws(BinaryOperator &I) {
Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
BinaryOperator *Op0 = dyn_cast<BinaryOperator>(LHS);
BinaryOperator *Op1 = dyn_cast<BinaryOperator>(RHS);
Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr;
auto TopLevelOpcode = I.getOpcode();
auto LHSOpcode = getBinOpsForFactorization(TopLevelOpcode, Op0, A, B);
auto RHSOpcode = getBinOpsForFactorization(TopLevelOpcode, Op1, C, D);
if (LHSOpcode == RHSOpcode) {
if (Value *V = tryFactorization(Builder, DL, I, LHSOpcode, A, B, C, D))
return V;
}
if (Value *V = tryFactorization(Builder, DL, I, LHSOpcode, A, B, RHS,
getIdentityValue(LHSOpcode, RHS)))
return V;
if (Value *V = tryFactorization(Builder, DL, I, RHSOpcode, LHS,
getIdentityValue(RHSOpcode, LHS), C, D))
return V;
if (Op0 && RightDistributesOverLeft(Op0->getOpcode(), TopLevelOpcode)) {
Value *A = Op0->getOperand(0), *B = Op0->getOperand(1), *C = RHS;
Instruction::BinaryOps InnerOpcode = Op0->getOpcode();
if (Value *L = SimplifyBinOp(TopLevelOpcode, A, C, DL))
if (Value *R = SimplifyBinOp(TopLevelOpcode, B, C, DL)) {
++NumExpand;
if ((L == A && R == B) ||
(Instruction::isCommutative(InnerOpcode) && L == B && R == A))
return Op0;
if (Value *V = SimplifyBinOp(InnerOpcode, L, R, DL))
return V;
C = Builder->CreateBinOp(InnerOpcode, L, R);
C->takeName(&I);
return C;
}
}
if (Op1 && LeftDistributesOverRight(TopLevelOpcode, Op1->getOpcode())) {
Value *A = LHS, *B = Op1->getOperand(0), *C = Op1->getOperand(1);
Instruction::BinaryOps InnerOpcode = Op1->getOpcode();
if (Value *L = SimplifyBinOp(TopLevelOpcode, A, B, DL))
if (Value *R = SimplifyBinOp(TopLevelOpcode, A, C, DL)) {
++NumExpand;
if ((L == B && R == C) ||
(Instruction::isCommutative(InnerOpcode) && L == C && R == B))
return Op1;
if (Value *V = SimplifyBinOp(InnerOpcode, L, R, DL))
return V;
A = Builder->CreateBinOp(InnerOpcode, L, R);
A->takeName(&I);
return A;
}
}
if (auto *SI0 = dyn_cast<SelectInst>(LHS)) {
if (auto *SI1 = dyn_cast<SelectInst>(RHS)) {
if (SI0->getCondition() == SI1->getCondition()) {
Value *SI = nullptr;
if (Value *V = SimplifyBinOp(TopLevelOpcode, SI0->getFalseValue(),
SI1->getFalseValue(), DL, TLI, DT, AC))
SI = Builder->CreateSelect(SI0->getCondition(),
Builder->CreateBinOp(TopLevelOpcode,
SI0->getTrueValue(),
SI1->getTrueValue()),
V);
if (Value *V = SimplifyBinOp(TopLevelOpcode, SI0->getTrueValue(),
SI1->getTrueValue(), DL, TLI, DT, AC))
SI = Builder->CreateSelect(
SI0->getCondition(), V,
Builder->CreateBinOp(TopLevelOpcode, SI0->getFalseValue(),
SI1->getFalseValue()));
if (SI) {
SI->takeName(&I);
return SI;
}
}
}
}
return nullptr;
}
Value *InstCombiner::dyn_castNegVal(Value *V) const {
if (BinaryOperator::isNeg(V))
return BinaryOperator::getNegArgument(V);
if (ConstantInt *C = dyn_cast<ConstantInt>(V))
return ConstantExpr::getNeg(C);
if (ConstantDataVector *C = dyn_cast<ConstantDataVector>(V))
if (C->getType()->getElementType()->isIntegerTy())
return ConstantExpr::getNeg(C);
return nullptr;
}
Value *InstCombiner::dyn_castFNegVal(Value *V, bool IgnoreZeroSign) const {
if (BinaryOperator::isFNeg(V, IgnoreZeroSign))
return BinaryOperator::getFNegArgument(V);
if (ConstantFP *C = dyn_cast<ConstantFP>(V))
return ConstantExpr::getFNeg(C);
if (ConstantDataVector *C = dyn_cast<ConstantDataVector>(V))
if (C->getType()->getElementType()->isFloatingPointTy())
return ConstantExpr::getFNeg(C);
return nullptr;
}
static Value *FoldOperationIntoSelectOperand(Instruction &I, Value *SO,
InstCombiner *IC) {
if (CastInst *CI = dyn_cast<CastInst>(&I)) {
return IC->Builder->CreateCast(CI->getOpcode(), SO, I.getType());
}
bool ConstIsRHS = isa<Constant>(I.getOperand(1));
Constant *ConstOperand = cast<Constant>(I.getOperand(ConstIsRHS));
if (Constant *SOC = dyn_cast<Constant>(SO)) {
if (ConstIsRHS)
return ConstantExpr::get(I.getOpcode(), SOC, ConstOperand);
return ConstantExpr::get(I.getOpcode(), ConstOperand, SOC);
}
Value *Op0 = SO, *Op1 = ConstOperand;
if (!ConstIsRHS)
std::swap(Op0, Op1);
if (BinaryOperator *BO = dyn_cast<BinaryOperator>(&I)) {
Value *RI = IC->Builder->CreateBinOp(BO->getOpcode(), Op0, Op1,
SO->getName()+".op");
Instruction *FPInst = dyn_cast<Instruction>(RI);
if (FPInst && isa<FPMathOperator>(FPInst))
FPInst->copyFastMathFlags(BO);
return RI;
}
if (ICmpInst *CI = dyn_cast<ICmpInst>(&I))
return IC->Builder->CreateICmp(CI->getPredicate(), Op0, Op1,
SO->getName()+".cmp");
if (FCmpInst *CI = dyn_cast<FCmpInst>(&I))
return IC->Builder->CreateICmp(CI->getPredicate(), Op0, Op1,
SO->getName()+".cmp");
llvm_unreachable("Unknown binary instruction type!");
}
Instruction *InstCombiner::FoldOpIntoSelect(Instruction &Op, SelectInst *SI) {
if (!SI->hasOneUse()) return nullptr;
Value *TV = SI->getOperand(1);
Value *FV = SI->getOperand(2);
if (isa<Constant>(TV) || isa<Constant>(FV)) {
if (SI->getType()->isIntegerTy(1)) return nullptr;
if (BitCastInst *BC = dyn_cast<BitCastInst>(&Op)) {
VectorType *DestTy = dyn_cast<VectorType>(BC->getDestTy());
VectorType *SrcTy = dyn_cast<VectorType>(BC->getSrcTy());
if ((SrcTy == nullptr) != (DestTy == nullptr)) return nullptr;
if (SrcTy && SrcTy->getNumElements() != DestTy->getNumElements())
return nullptr;
}
if (auto *CI = dyn_cast<CmpInst>(SI->getCondition())) {
if (CI->hasOneUse()) {
Value *Op0 = CI->getOperand(0), *Op1 = CI->getOperand(1);
if ((SI->getOperand(1) == Op0 && SI->getOperand(2) == Op1) ||
(SI->getOperand(2) == Op0 && SI->getOperand(1) == Op1))
return nullptr;
}
}
Value *SelectTrueVal = FoldOperationIntoSelectOperand(Op, TV, this);
Value *SelectFalseVal = FoldOperationIntoSelectOperand(Op, FV, this);
return SelectInst::Create(SI->getCondition(),
SelectTrueVal, SelectFalseVal);
}
return nullptr;
}
Instruction *InstCombiner::FoldOpIntoPhi(Instruction &I) {
PHINode *PN = cast<PHINode>(I.getOperand(0));
unsigned NumPHIValues = PN->getNumIncomingValues();
if (NumPHIValues == 0)
return nullptr;
if (!PN->hasOneUse()) {
for (User *U : PN->users()) {
Instruction *UI = cast<Instruction>(U);
if (UI != &I && !I.isIdenticalTo(UI))
return nullptr;
}
}
BasicBlock *NonConstBB = nullptr;
for (unsigned i = 0; i != NumPHIValues; ++i) {
Value *InVal = PN->getIncomingValue(i);
if (isa<Constant>(InVal) && !isa<ConstantExpr>(InVal))
continue;
if (isa<PHINode>(InVal)) return nullptr; if (NonConstBB) return nullptr;
NonConstBB = PN->getIncomingBlock(i);
if (InvokeInst *II = dyn_cast<InvokeInst>(InVal))
if (II->getParent() == NonConstBB)
return nullptr;
if (isPotentiallyReachable(I.getParent(), NonConstBB, DT, LI))
return nullptr;
}
if (NonConstBB != nullptr) {
BranchInst *BI = dyn_cast<BranchInst>(NonConstBB->getTerminator());
if (!BI || !BI->isUnconditional()) return nullptr;
}
PHINode *NewPN = PHINode::Create(I.getType(), PN->getNumIncomingValues());
InsertNewInstBefore(NewPN, *PN);
NewPN->takeName(PN);
if (NonConstBB)
Builder->SetInsertPoint(NonConstBB->getTerminator());
if (SelectInst *SI = dyn_cast<SelectInst>(&I)) {
Value *TrueV = SI->getTrueValue();
Value *FalseV = SI->getFalseValue();
BasicBlock *PhiTransBB = PN->getParent();
for (unsigned i = 0; i != NumPHIValues; ++i) {
BasicBlock *ThisBB = PN->getIncomingBlock(i);
Value *TrueVInPred = TrueV->DoPHITranslation(PhiTransBB, ThisBB);
Value *FalseVInPred = FalseV->DoPHITranslation(PhiTransBB, ThisBB);
Value *InV = nullptr;
Constant *InC = dyn_cast<Constant>(PN->getIncomingValue(i));
if (InC && !isa<ConstantExpr>(InC))
InV = InC->isNullValue() ? FalseVInPred : TrueVInPred;
else
InV = Builder->CreateSelect(PN->getIncomingValue(i),
TrueVInPred, FalseVInPred, "phitmp");
NewPN->addIncoming(InV, ThisBB);
}
} else if (CmpInst *CI = dyn_cast<CmpInst>(&I)) {
Constant *C = cast<Constant>(I.getOperand(1));
for (unsigned i = 0; i != NumPHIValues; ++i) {
Value *InV = nullptr;
if (Constant *InC = dyn_cast<Constant>(PN->getIncomingValue(i)))
InV = ConstantExpr::getCompare(CI->getPredicate(), InC, C);
else if (isa<ICmpInst>(CI))
InV = Builder->CreateICmp(CI->getPredicate(), PN->getIncomingValue(i),
C, "phitmp");
else
InV = Builder->CreateFCmp(CI->getPredicate(), PN->getIncomingValue(i),
C, "phitmp");
NewPN->addIncoming(InV, PN->getIncomingBlock(i));
}
} else if (I.getNumOperands() == 2) {
Constant *C = cast<Constant>(I.getOperand(1));
for (unsigned i = 0; i != NumPHIValues; ++i) {
Value *InV = nullptr;
if (Constant *InC = dyn_cast<Constant>(PN->getIncomingValue(i)))
InV = ConstantExpr::get(I.getOpcode(), InC, C);
else
InV = Builder->CreateBinOp(cast<BinaryOperator>(I).getOpcode(),
PN->getIncomingValue(i), C, "phitmp");
NewPN->addIncoming(InV, PN->getIncomingBlock(i));
}
} else {
CastInst *CI = cast<CastInst>(&I);
Type *RetTy = CI->getType();
for (unsigned i = 0; i != NumPHIValues; ++i) {
Value *InV;
if (Constant *InC = dyn_cast<Constant>(PN->getIncomingValue(i)))
InV = ConstantExpr::getCast(CI->getOpcode(), InC, RetTy);
else
InV = Builder->CreateCast(CI->getOpcode(),
PN->getIncomingValue(i), I.getType(), "phitmp");
NewPN->addIncoming(InV, PN->getIncomingBlock(i));
}
}
for (auto UI = PN->user_begin(), E = PN->user_end(); UI != E;) {
Instruction *User = cast<Instruction>(*UI++);
if (User == &I) continue;
ReplaceInstUsesWith(*User, NewPN);
EraseInstFromFunction(*User);
}
return ReplaceInstUsesWith(I, NewPN);
}
Type *InstCombiner::FindElementAtOffset(PointerType *PtrTy, int64_t Offset,
SmallVectorImpl<Value *> &NewIndices) {
Type *Ty = PtrTy->getElementType();
if (!Ty->isSized())
return nullptr;
Type *IntPtrTy = DL.getIntPtrType(PtrTy);
int64_t FirstIdx = 0;
if (int64_t TySize = DL.getTypeAllocSize(Ty)) {
FirstIdx = Offset/TySize;
Offset -= FirstIdx*TySize;
if (Offset < 0) {
--FirstIdx;
Offset += TySize;
assert(Offset >= 0);
}
assert((uint64_t)Offset < (uint64_t)TySize && "Out of range offset");
}
NewIndices.push_back(ConstantInt::get(IntPtrTy, FirstIdx));
while (Offset) {
if (uint64_t(Offset * 8) >= DL.getTypeSizeInBits(Ty))
return nullptr;
if (StructType *STy = dyn_cast<StructType>(Ty)) {
const StructLayout *SL = DL.getStructLayout(STy);
assert(Offset < (int64_t)SL->getSizeInBytes() &&
"Offset must stay within the indexed type");
unsigned Elt = SL->getElementContainingOffset(Offset);
NewIndices.push_back(ConstantInt::get(Type::getInt32Ty(Ty->getContext()),
Elt));
Offset -= SL->getElementOffset(Elt);
Ty = STy->getElementType(Elt);
} else if (ArrayType *AT = dyn_cast<ArrayType>(Ty)) {
uint64_t EltSize = DL.getTypeAllocSize(AT->getElementType());
assert(EltSize && "Cannot index into a zero-sized array");
NewIndices.push_back(ConstantInt::get(IntPtrTy,Offset/EltSize));
Offset %= EltSize;
Ty = AT->getElementType();
} else {
return nullptr;
}
}
return Ty;
}
static bool shouldMergeGEPs(GEPOperator &GEP, GEPOperator &Src) {
if (GEP.hasAllZeroIndices() && !Src.hasAllZeroIndices() &&
!Src.hasOneUse())
return false;
return true;
}
Value *InstCombiner::Descale(Value *Val, APInt Scale, bool &NoSignedWrap) {
assert(isa<IntegerType>(Val->getType()) && "Can only descale integers!");
assert(cast<IntegerType>(Val->getType())->getBitWidth() ==
Scale.getBitWidth() && "Scale not compatible with value!");
if (match(Val, m_Zero()) || Scale == 1) {
NoSignedWrap = true;
return Val;
}
if (Scale.isMinValue())
return nullptr;
Value *Op = Val;
std::pair<Instruction*, unsigned> Parent;
bool RequireNoSignedWrap = false;
int32_t logScale = Scale.exactLogBase2();
for (;; Op = Parent.first->getOperand(Parent.second)) {
if (ConstantInt *CI = dyn_cast<ConstantInt>(Op)) {
APInt Quotient(Scale), Remainder(Scale); APInt::sdivrem(CI->getValue(), Scale, Quotient, Remainder);
if (!Remainder.isMinValue())
return nullptr;
Op = ConstantInt::get(CI->getType(), Quotient);
NoSignedWrap = true;
break;
}
if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Op)) {
if (BO->getOpcode() == Instruction::Mul) {
NoSignedWrap = BO->hasNoSignedWrap();
if (RequireNoSignedWrap && !NoSignedWrap)
return nullptr;
Value *LHS = BO->getOperand(0);
Value *RHS = BO->getOperand(1);
if (ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
if (CI->getValue() == Scale) {
Op = LHS;
break;
}
if (!Op->hasOneUse())
return nullptr;
Parent = std::make_pair(BO, 1);
continue;
}
if (!Op->hasOneUse())
return nullptr;
Parent = std::make_pair(BO, 0);
continue;
}
if (logScale > 0 && BO->getOpcode() == Instruction::Shl &&
isa<ConstantInt>(BO->getOperand(1))) {
NoSignedWrap = BO->hasNoSignedWrap();
if (RequireNoSignedWrap && !NoSignedWrap)
return nullptr;
Value *LHS = BO->getOperand(0);
int32_t Amt = cast<ConstantInt>(BO->getOperand(1))->
getLimitedValue(Scale.getBitWidth());
if (Amt == logScale) {
Op = LHS;
break;
}
if (Amt < logScale || !Op->hasOneUse())
return nullptr;
Parent = std::make_pair(BO, 1);
Op = ConstantInt::get(BO->getType(), Amt - logScale);
break;
}
}
if (!Op->hasOneUse())
return nullptr;
if (CastInst *Cast = dyn_cast<CastInst>(Op)) {
if (Cast->getOpcode() == Instruction::SExt) {
unsigned SmallSize = Cast->getSrcTy()->getPrimitiveSizeInBits();
APInt SmallScale = Scale.trunc(SmallSize);
if (SmallScale.sext(Scale.getBitWidth()) != Scale)
return nullptr;
assert(SmallScale.exactLogBase2() == logScale);
RequireNoSignedWrap = true;
Parent = std::make_pair(Cast, 0);
Scale = SmallScale;
continue;
}
if (Cast->getOpcode() == Instruction::Trunc) {
if (RequireNoSignedWrap)
return nullptr;
unsigned LargeSize = Cast->getSrcTy()->getPrimitiveSizeInBits();
Parent = std::make_pair(Cast, 0);
Scale = Scale.sext(LargeSize);
if (logScale + 1 == (int32_t)Cast->getType()->getPrimitiveSizeInBits())
logScale = -1;
assert(Scale.exactLogBase2() == logScale);
continue;
}
}
return nullptr;
}
if (match(Op, m_Zero())) {
NoSignedWrap = true;
return Op;
}
if (!Parent.first)
return Op;
assert(Parent.first->hasOneUse() && "Drilled down when more than one use!");
assert(Op != Parent.first->getOperand(Parent.second) &&
"Descaling was a no-op?");
Parent.first->setOperand(Parent.second, Op);
Worklist.Add(Parent.first);
Instruction *Ancestor = Parent.first;
do {
if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Ancestor)) {
bool OpNoSignedWrap = BO->hasNoSignedWrap();
NoSignedWrap &= OpNoSignedWrap;
if (NoSignedWrap != OpNoSignedWrap) {
BO->setHasNoSignedWrap(NoSignedWrap);
Worklist.Add(Ancestor);
}
} else if (Ancestor->getOpcode() == Instruction::Trunc) {
NoSignedWrap = false;
}
assert((Ancestor->getOpcode() != Instruction::SExt || NoSignedWrap) &&
"Failed to keep proper track of nsw flags while drilling down?");
if (Ancestor == Val)
return Val;
assert(Ancestor->hasOneUse() && "Drilled down when more than one use!");
Ancestor = Ancestor->user_back();
} while (1);
}
static Value *CreateBinOpAsGiven(BinaryOperator &Inst, Value *LHS, Value *RHS,
InstCombiner::BuilderTy *B) {
Value *BORes = B->CreateBinOp(Inst.getOpcode(), LHS, RHS);
if (BinaryOperator *NewBO = dyn_cast<BinaryOperator>(BORes)) {
if (isa<OverflowingBinaryOperator>(NewBO)) {
NewBO->setHasNoSignedWrap(Inst.hasNoSignedWrap());
NewBO->setHasNoUnsignedWrap(Inst.hasNoUnsignedWrap());
}
if (isa<PossiblyExactOperator>(NewBO))
NewBO->setIsExact(Inst.isExact());
}
return BORes;
}
Value *InstCombiner::SimplifyVectorOp(BinaryOperator &Inst) {
if (!Inst.getType()->isVectorTy()) return nullptr;
if (!isSafeToSpeculativelyExecute(&Inst))
return nullptr;
unsigned VWidth = cast<VectorType>(Inst.getType())->getNumElements();
Value *LHS = Inst.getOperand(0), *RHS = Inst.getOperand(1);
assert(cast<VectorType>(LHS->getType())->getNumElements() == VWidth);
assert(cast<VectorType>(RHS->getType())->getNumElements() == VWidth);
if (isa<ShuffleVectorInst>(LHS) && isa<ShuffleVectorInst>(RHS)) {
ShuffleVectorInst *LShuf = cast<ShuffleVectorInst>(LHS);
ShuffleVectorInst *RShuf = cast<ShuffleVectorInst>(RHS);
if (isa<UndefValue>(LShuf->getOperand(1)) &&
isa<UndefValue>(RShuf->getOperand(1)) &&
LShuf->getOperand(0)->getType() == RShuf->getOperand(0)->getType() &&
LShuf->getMask() == RShuf->getMask()) {
Value *NewBO = CreateBinOpAsGiven(Inst, LShuf->getOperand(0),
RShuf->getOperand(0), Builder);
Value *Res = Builder->CreateShuffleVector(NewBO,
UndefValue::get(NewBO->getType()), LShuf->getMask());
return Res;
}
}
ShuffleVectorInst *Shuffle = nullptr;
Constant *C1 = nullptr;
if (isa<ShuffleVectorInst>(LHS)) Shuffle = cast<ShuffleVectorInst>(LHS);
if (isa<ShuffleVectorInst>(RHS)) Shuffle = cast<ShuffleVectorInst>(RHS);
if (isa<Constant>(LHS)) C1 = cast<Constant>(LHS);
if (isa<Constant>(RHS)) C1 = cast<Constant>(RHS);
if (Shuffle && C1 &&
(isa<ConstantVector>(C1) || isa<ConstantDataVector>(C1)) &&
isa<UndefValue>(Shuffle->getOperand(1)) &&
Shuffle->getType() == Shuffle->getOperand(0)->getType()) {
SmallVector<int, 16> ShMask = Shuffle->getShuffleMask();
SmallVector<Constant*, 16> C2M(VWidth,
UndefValue::get(C1->getType()->getScalarType()));
bool MayChange = true;
for (unsigned I = 0; I < VWidth; ++I) {
if (ShMask[I] >= 0) {
assert(ShMask[I] < (int)VWidth);
if (!isa<UndefValue>(C2M[ShMask[I]])) {
MayChange = false;
break;
}
C2M[ShMask[I]] = C1->getAggregateElement(I);
}
}
if (MayChange) {
Constant *C2 = ConstantVector::get(C2M);
Value *NewLHS, *NewRHS;
if (isa<Constant>(LHS)) {
NewLHS = C2;
NewRHS = Shuffle->getOperand(0);
} else {
NewLHS = Shuffle->getOperand(0);
NewRHS = C2;
}
Value *NewBO = CreateBinOpAsGiven(Inst, NewLHS, NewRHS, Builder);
Value *Res = Builder->CreateShuffleVector(NewBO,
UndefValue::get(Inst.getType()), Shuffle->getMask());
return Res;
}
}
return nullptr;
}
Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
SmallVector<Value*, 8> Ops(GEP.op_begin(), GEP.op_end());
if (Value *V = SimplifyGEPInst(Ops, DL, TLI, DT, AC))
return ReplaceInstUsesWith(GEP, V);
Value *PtrOp = GEP.getOperand(0);
bool MadeChange = false;
Type *IntPtrTy =
DL.getIntPtrType(GEP.getPointerOperandType()->getScalarType());
gep_type_iterator GTI = gep_type_begin(GEP);
for (User::op_iterator I = GEP.op_begin() + 1, E = GEP.op_end(); I != E;
++I, ++GTI) {
SequentialType *SeqTy = dyn_cast<SequentialType>(*GTI);
if (!SeqTy)
continue;
Type *IndexTy = (*I)->getType();
Type *NewIndexType = IndexTy->isVectorTy() ?
VectorType::get(IntPtrTy, IndexTy->getVectorNumElements()) : IntPtrTy;
if (SeqTy->getElementType()->isSized() &&
DL.getTypeAllocSize(SeqTy->getElementType()) == 0)
if (!isa<Constant>(*I) || !cast<Constant>(*I)->isNullValue()) {
*I = Constant::getNullValue(NewIndexType);
MadeChange = true;
}
if (IndexTy != NewIndexType) {
*I = Builder->CreateIntCast(*I, NewIndexType, true);
MadeChange = true;
}
}
if (MadeChange)
return &GEP;
if (PHINode *PN = dyn_cast<PHINode>(PtrOp)) {
GetElementPtrInst *Op1 = dyn_cast<GetElementPtrInst>(PN->getOperand(0));
if (!Op1)
return nullptr;
if (Op1 == &GEP)
return nullptr;
signed DI = -1;
for (auto I = PN->op_begin()+1, E = PN->op_end(); I !=E; ++I) {
GetElementPtrInst *Op2 = dyn_cast<GetElementPtrInst>(*I);
if (!Op2 || Op1->getNumOperands() != Op2->getNumOperands())
return nullptr;
if (Op2 == &GEP)
return nullptr;
Type *CurTy = Op1->getOperand(0)->getType()->getScalarType();
for (unsigned J = 0, F = Op1->getNumOperands(); J != F; ++J) {
if (Op1->getOperand(J)->getType() != Op2->getOperand(J)->getType())
return nullptr;
if (Op1->getOperand(J) != Op2->getOperand(J)) {
if (DI == -1) {
if (J > 1 && CurTy->isStructTy())
return nullptr;
DI = J;
} else {
return nullptr;
}
}
if (J > 0) {
if (CompositeType *CT = dyn_cast<CompositeType>(CurTy)) {
CurTy = CT->getTypeAtIndex(Op1->getOperand(J));
} else {
CurTy = nullptr;
}
}
}
}
if (DI != -1 && !PN->hasOneUse())
return nullptr;
GetElementPtrInst *NewGEP = cast<GetElementPtrInst>(Op1->clone());
if (DI == -1) {
GEP.getParent()->getInstList().insert(
GEP.getParent()->getFirstInsertionPt(), NewGEP);
} else {
PHINode *NewPN;
{
IRBuilderBase::InsertPointGuard Guard(*Builder);
Builder->SetInsertPoint(PN);
NewPN = Builder->CreatePHI(Op1->getOperand(DI)->getType(),
PN->getNumOperands());
}
for (auto &I : PN->operands())
NewPN->addIncoming(cast<GEPOperator>(I)->getOperand(DI),
PN->getIncomingBlock(I));
NewGEP->setOperand(DI, NewPN);
GEP.getParent()->getInstList().insert(
GEP.getParent()->getFirstInsertionPt(), NewGEP);
NewGEP->setOperand(DI, NewPN);
}
GEP.setOperand(0, NewGEP);
PtrOp = NewGEP;
}
if (GEPOperator *Src = dyn_cast<GEPOperator>(PtrOp)) {
if (!shouldMergeGEPs(*cast<GEPOperator>(&GEP), *Src))
return nullptr;
if (GEPOperator *SrcGEP =
dyn_cast<GEPOperator>(Src->getOperand(0)))
if (SrcGEP->getNumOperands() == 2 && shouldMergeGEPs(*Src, *SrcGEP))
return nullptr;
SmallVector<Value*, 8> Indices;
bool EndsWithSequential = false;
for (gep_type_iterator I = gep_type_begin(*Src), E = gep_type_end(*Src);
I != E; ++I)
EndsWithSequential = !(*I)->isStructTy();
if (EndsWithSequential) {
Value *Sum;
Value *SO1 = Src->getOperand(Src->getNumOperands()-1);
Value *GO1 = GEP.getOperand(1);
if (SO1 == Constant::getNullValue(SO1->getType())) {
Sum = GO1;
} else if (GO1 == Constant::getNullValue(GO1->getType())) {
Sum = SO1;
} else {
if (SO1->getType() != GO1->getType())
return nullptr;
if (!isa<Constant>(GO1) || !isa<Constant>(SO1))
return nullptr;
Sum = Builder->CreateAdd(SO1, GO1, PtrOp->getName()+".sum");
}
if (Src->getNumOperands() == 2) {
GEP.setOperand(0, Src->getOperand(0));
GEP.setOperand(1, Sum);
return &GEP;
}
Indices.append(Src->op_begin()+1, Src->op_end()-1);
Indices.push_back(Sum);
Indices.append(GEP.op_begin()+2, GEP.op_end());
} else if (isa<Constant>(*GEP.idx_begin()) &&
cast<Constant>(*GEP.idx_begin())->isNullValue() &&
Src->getNumOperands() != 1) {
Indices.append(Src->op_begin()+1, Src->op_end());
Indices.append(GEP.idx_begin()+1, GEP.idx_end());
}
if (!Indices.empty())
return GEP.isInBounds() && Src->isInBounds()
? GetElementPtrInst::CreateInBounds(
Src->getSourceElementType(), Src->getOperand(0), Indices,
GEP.getName())
: GetElementPtrInst::Create(Src->getSourceElementType(),
Src->getOperand(0), Indices,
GEP.getName());
}
if (GEP.getNumIndices() == 1) {
unsigned AS = GEP.getPointerAddressSpace();
if (GEP.getOperand(1)->getType()->getScalarSizeInBits() ==
DL.getPointerSizeInBits(AS)) {
Type *PtrTy = GEP.getPointerOperandType();
Type *Ty = PtrTy->getPointerElementType();
uint64_t TyAllocSize = DL.getTypeAllocSize(Ty);
bool Matched = false;
uint64_t C;
Value *V = nullptr;
if (TyAllocSize == 1) {
V = GEP.getOperand(1);
Matched = true;
} else if (match(GEP.getOperand(1),
m_AShr(m_Value(V), m_ConstantInt(C)))) {
if (TyAllocSize == 1ULL << C)
Matched = true;
} else if (match(GEP.getOperand(1),
m_SDiv(m_Value(V), m_ConstantInt(C)))) {
if (TyAllocSize == C)
Matched = true;
}
if (Matched) {
if (match(V, m_Neg(m_PtrToInt(m_Value())))) {
Operator *Index = cast<Operator>(V);
Value *PtrToInt = Builder->CreatePtrToInt(PtrOp, Index->getType());
Value *NewSub = Builder->CreateSub(PtrToInt, Index->getOperand(1));
return CastInst::Create(Instruction::IntToPtr, NewSub, GEP.getType());
}
Value *Y;
if (match(V, m_Sub(m_PtrToInt(m_Value(Y)),
m_PtrToInt(m_Specific(GEP.getOperand(0)))))) {
return CastInst::CreatePointerBitCastOrAddrSpaceCast(Y,
GEP.getType());
}
}
}
}
Value *StrippedPtr = PtrOp->stripPointerCasts();
PointerType *StrippedPtrTy = dyn_cast<PointerType>(StrippedPtr->getType());
if (!StrippedPtrTy)
return nullptr;
if (StrippedPtr != PtrOp) {
bool HasZeroPointerIndex = false;
if (ConstantInt *C = dyn_cast<ConstantInt>(GEP.getOperand(1)))
HasZeroPointerIndex = C->isZero();
if (HasZeroPointerIndex) {
PointerType *CPTy = cast<PointerType>(PtrOp->getType());
if (ArrayType *CATy =
dyn_cast<ArrayType>(CPTy->getElementType())) {
if (CATy->getElementType() == StrippedPtrTy->getElementType()) {
SmallVector<Value*, 8> Idx(GEP.idx_begin()+1, GEP.idx_end());
GetElementPtrInst *Res = GetElementPtrInst::Create(
StrippedPtrTy->getElementType(), StrippedPtr, Idx, GEP.getName());
Res->setIsInBounds(GEP.isInBounds());
if (StrippedPtrTy->getAddressSpace() == GEP.getAddressSpace())
return Res;
return new AddrSpaceCastInst(Builder->Insert(Res), GEP.getType());
}
if (ArrayType *XATy =
dyn_cast<ArrayType>(StrippedPtrTy->getElementType())){
if (CATy->getElementType() == XATy->getElementType()) {
if (StrippedPtrTy->getAddressSpace() == GEP.getAddressSpace()) {
GEP.setOperand(0, StrippedPtr);
GEP.setSourceElementType(XATy);
return &GEP;
}
SmallVector<Value*, 8> Idx(GEP.idx_begin(), GEP.idx_end());
Value *NewGEP = GEP.isInBounds()
? Builder->CreateInBoundsGEP(
nullptr, StrippedPtr, Idx, GEP.getName())
: Builder->CreateGEP(nullptr, StrippedPtr, Idx,
GEP.getName());
return new AddrSpaceCastInst(NewGEP, GEP.getType());
}
}
}
} else if (GEP.getNumOperands() == 2) {
Type *SrcElTy = StrippedPtrTy->getElementType();
Type *ResElTy = PtrOp->getType()->getPointerElementType();
if (SrcElTy->isArrayTy() &&
DL.getTypeAllocSize(SrcElTy->getArrayElementType()) ==
DL.getTypeAllocSize(ResElTy)) {
Type *IdxType = DL.getIntPtrType(GEP.getType());
Value *Idx[2] = { Constant::getNullValue(IdxType), GEP.getOperand(1) };
Value *NewGEP =
GEP.isInBounds()
? Builder->CreateInBoundsGEP(nullptr, StrippedPtr, Idx,
GEP.getName())
: Builder->CreateGEP(nullptr, StrippedPtr, Idx, GEP.getName());
return CastInst::CreatePointerBitCastOrAddrSpaceCast(NewGEP,
GEP.getType());
}
if (ResElTy->isSized() && SrcElTy->isSized()) {
uint64_t ResSize = DL.getTypeAllocSize(ResElTy);
uint64_t SrcSize = DL.getTypeAllocSize(SrcElTy);
if (ResSize && SrcSize % ResSize == 0) {
Value *Idx = GEP.getOperand(1);
unsigned BitWidth = Idx->getType()->getPrimitiveSizeInBits();
uint64_t Scale = SrcSize / ResSize;
assert(Idx->getType() == DL.getIntPtrType(GEP.getType()) &&
"Index not cast to pointer width?");
bool NSW;
if (Value *NewIdx = Descale(Idx, APInt(BitWidth, Scale), NSW)) {
Value *NewGEP =
GEP.isInBounds() && NSW
? Builder->CreateInBoundsGEP(nullptr, StrippedPtr, NewIdx,
GEP.getName())
: Builder->CreateGEP(nullptr, StrippedPtr, NewIdx,
GEP.getName());
return CastInst::CreatePointerBitCastOrAddrSpaceCast(NewGEP,
GEP.getType());
}
}
}
if (ResElTy->isSized() && SrcElTy->isSized() && SrcElTy->isArrayTy()) {
uint64_t ResSize = DL.getTypeAllocSize(ResElTy);
uint64_t ArrayEltSize =
DL.getTypeAllocSize(SrcElTy->getArrayElementType());
if (ResSize && ArrayEltSize % ResSize == 0) {
Value *Idx = GEP.getOperand(1);
unsigned BitWidth = Idx->getType()->getPrimitiveSizeInBits();
uint64_t Scale = ArrayEltSize / ResSize;
assert(Idx->getType() == DL.getIntPtrType(GEP.getType()) &&
"Index not cast to pointer width?");
bool NSW;
if (Value *NewIdx = Descale(Idx, APInt(BitWidth, Scale), NSW)) {
Value *Off[2] = {
Constant::getNullValue(DL.getIntPtrType(GEP.getType())),
NewIdx};
Value *NewGEP = GEP.isInBounds() && NSW
? Builder->CreateInBoundsGEP(
SrcElTy, StrippedPtr, Off, GEP.getName())
: Builder->CreateGEP(SrcElTy, StrippedPtr, Off,
GEP.getName());
return CastInst::CreatePointerBitCastOrAddrSpaceCast(NewGEP,
GEP.getType());
}
}
}
}
}
if (AddrSpaceCastInst *ASC = dyn_cast<AddrSpaceCastInst>(PtrOp)) {
if (BitCastInst *BC = dyn_cast<BitCastInst>(ASC->getOperand(0)))
PtrOp = BC;
}
if (BitCastInst *BCI = dyn_cast<BitCastInst>(PtrOp)) {
Value *Operand = BCI->getOperand(0);
PointerType *OpType = cast<PointerType>(Operand->getType());
unsigned OffsetBits = DL.getPointerTypeSizeInBits(GEP.getType());
APInt Offset(OffsetBits, 0);
if (!isa<BitCastInst>(Operand) &&
GEP.accumulateConstantOffset(DL, Offset)) {
if (!Offset) {
if (isa<AllocaInst>(Operand) || isAllocationFn(Operand, TLI)) {
if (Instruction *I = visitBitCast(*BCI)) {
if (I != BCI) {
I->takeName(BCI);
BCI->getParent()->getInstList().insert(BCI->getIterator(), I);
ReplaceInstUsesWith(*BCI, I);
}
return &GEP;
}
}
if (Operand->getType()->getPointerAddressSpace() != GEP.getAddressSpace())
return new AddrSpaceCastInst(Operand, GEP.getType());
return new BitCastInst(Operand, GEP.getType());
}
SmallVector<Value*, 8> NewIndices;
if (FindElementAtOffset(OpType, Offset.getSExtValue(), NewIndices)) {
Value *NGEP =
GEP.isInBounds()
? Builder->CreateInBoundsGEP(nullptr, Operand, NewIndices)
: Builder->CreateGEP(nullptr, Operand, NewIndices);
if (NGEP->getType() == GEP.getType())
return ReplaceInstUsesWith(GEP, NGEP);
NGEP->takeName(&GEP);
if (NGEP->getType()->getPointerAddressSpace() != GEP.getAddressSpace())
return new AddrSpaceCastInst(NGEP, GEP.getType());
return new BitCastInst(NGEP, GEP.getType());
}
}
}
return nullptr;
}
static bool
isAllocSiteRemovable(Instruction *AI, SmallVectorImpl<WeakVH> &Users,
const TargetLibraryInfo *TLI) {
SmallVector<Instruction*, 4> Worklist;
Worklist.push_back(AI);
do {
Instruction *PI = Worklist.pop_back_val();
for (User *U : PI->users()) {
Instruction *I = cast<Instruction>(U);
switch (I->getOpcode()) {
default:
return false;
case Instruction::BitCast:
case Instruction::GetElementPtr:
Users.emplace_back(I);
Worklist.push_back(I);
continue;
case Instruction::ICmp: {
ICmpInst *ICI = cast<ICmpInst>(I);
if (!ICI->isEquality() || !isa<ConstantPointerNull>(ICI->getOperand(1)))
return false;
Users.emplace_back(I);
continue;
}
case Instruction::Call:
if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
switch (II->getIntrinsicID()) {
default:
return false;
case Intrinsic::memmove:
case Intrinsic::memcpy:
case Intrinsic::memset: {
MemIntrinsic *MI = cast<MemIntrinsic>(II);
if (MI->isVolatile() || MI->getRawDest() != PI)
return false;
}
case Intrinsic::dbg_declare:
case Intrinsic::dbg_value:
case Intrinsic::invariant_start:
case Intrinsic::invariant_end:
case Intrinsic::lifetime_start:
case Intrinsic::lifetime_end:
case Intrinsic::objectsize:
Users.emplace_back(I);
continue;
}
}
if (isFreeCall(I, TLI)) {
Users.emplace_back(I);
continue;
}
return false;
case Instruction::Store: {
StoreInst *SI = cast<StoreInst>(I);
if (SI->isVolatile() || SI->getPointerOperand() != PI)
return false;
Users.emplace_back(I);
continue;
}
}
llvm_unreachable("missing a return?");
}
} while (!Worklist.empty());
return true;
}
Instruction *InstCombiner::visitAllocSite(Instruction &MI) {
SmallVector<WeakVH, 64> Users;
if (isAllocSiteRemovable(&MI, Users, TLI)) {
for (unsigned i = 0, e = Users.size(); i != e; ++i) {
Instruction *I = cast_or_null<Instruction>(&*Users[i]);
if (!I) continue;
if (ICmpInst *C = dyn_cast<ICmpInst>(I)) {
ReplaceInstUsesWith(*C,
ConstantInt::get(Type::getInt1Ty(C->getContext()),
C->isFalseWhenEqual()));
} else if (isa<BitCastInst>(I) || isa<GetElementPtrInst>(I)) {
ReplaceInstUsesWith(*I, UndefValue::get(I->getType()));
} else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
if (II->getIntrinsicID() == Intrinsic::objectsize) {
ConstantInt *CI = cast<ConstantInt>(II->getArgOperand(1));
uint64_t DontKnow = CI->isZero() ? -1ULL : 0;
ReplaceInstUsesWith(*I, ConstantInt::get(I->getType(), DontKnow));
}
}
EraseInstFromFunction(*I);
}
if (InvokeInst *II = dyn_cast<InvokeInst>(&MI)) {
Module *M = II->getParent()->getParent()->getParent();
Function *F = Intrinsic::getDeclaration(M, Intrinsic::donothing);
InvokeInst::Create(F, II->getNormalDest(), II->getUnwindDest(),
None, "", II->getParent());
}
return EraseInstFromFunction(MI);
}
return nullptr;
}
static Instruction *
tryToMoveFreeBeforeNullTest(CallInst &FI) {
Value *Op = FI.getArgOperand(0);
BasicBlock *FreeInstrBB = FI.getParent();
BasicBlock *PredBB = FreeInstrBB->getSinglePredecessor();
if (!PredBB)
return nullptr;
if (FreeInstrBB->size() != 2)
return nullptr;
BasicBlock *SuccBB;
if (!match(FreeInstrBB->getTerminator(), m_UnconditionalBr(SuccBB)))
return nullptr;
TerminatorInst *TI = PredBB->getTerminator();
BasicBlock *TrueBB, *FalseBB;
ICmpInst::Predicate Pred;
if (!match(TI, m_Br(m_ICmp(Pred, m_Specific(Op), m_Zero()), TrueBB, FalseBB)))
return nullptr;
if (Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE)
return nullptr;
if (SuccBB != (Pred == ICmpInst::ICMP_EQ ? TrueBB : FalseBB))
return nullptr;
assert(FreeInstrBB == (Pred == ICmpInst::ICMP_EQ ? FalseBB : TrueBB) &&
"Broken CFG: missing edge from predecessor to successor");
FI.moveBefore(TI);
return &FI;
}
Instruction *InstCombiner::visitFree(CallInst &FI) {
Value *Op = FI.getArgOperand(0);
if (isa<UndefValue>(Op)) {
Builder->CreateStore(ConstantInt::getTrue(FI.getContext()),
UndefValue::get(Type::getInt1PtrTy(FI.getContext())));
return EraseInstFromFunction(FI);
}
if (isa<ConstantPointerNull>(Op))
return EraseInstFromFunction(FI);
if (MinimizeSize)
if (Instruction *I = tryToMoveFreeBeforeNullTest(FI))
return I;
return nullptr;
}
Instruction *InstCombiner::visitReturnInst(ReturnInst &RI) {
if (RI.getNumOperands() == 0) return nullptr;
Value *ResultOp = RI.getOperand(0);
Type *VTy = ResultOp->getType();
if (!VTy->isIntegerTy())
return nullptr;
unsigned BitWidth = VTy->getPrimitiveSizeInBits();
APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0);
computeKnownBits(ResultOp, KnownZero, KnownOne, 0, &RI);
if ((KnownZero|KnownOne).isAllOnesValue())
RI.setOperand(0, Constant::getIntegerValue(VTy, KnownOne));
return nullptr;
}
Instruction *InstCombiner::visitBranchInst(BranchInst &BI) {
Value *X = nullptr;
BasicBlock *TrueDest;
BasicBlock *FalseDest;
if (match(&BI, m_Br(m_Not(m_Value(X)), TrueDest, FalseDest)) &&
!isa<Constant>(X)) {
BI.setCondition(X);
BI.swapSuccessors();
return &BI;
}
if (BI.isConditional() &&
BI.getSuccessor(0) == BI.getSuccessor(1) &&
!isa<UndefValue>(BI.getCondition())) {
BI.setCondition(UndefValue::get(BI.getCondition()->getType()));
return &BI;
}
FCmpInst::Predicate FPred; Value *Y;
if (match(&BI, m_Br(m_FCmp(FPred, m_Value(X), m_Value(Y)),
TrueDest, FalseDest)) &&
BI.getCondition()->hasOneUse())
if (FPred == FCmpInst::FCMP_ONE || FPred == FCmpInst::FCMP_OLE ||
FPred == FCmpInst::FCMP_OGE) {
FCmpInst *Cond = cast<FCmpInst>(BI.getCondition());
Cond->setPredicate(FCmpInst::getInversePredicate(FPred));
BI.swapSuccessors();
Worklist.Add(Cond);
return &BI;
}
ICmpInst::Predicate IPred;
if (match(&BI, m_Br(m_ICmp(IPred, m_Value(X), m_Value(Y)),
TrueDest, FalseDest)) &&
BI.getCondition()->hasOneUse())
if (IPred == ICmpInst::ICMP_NE || IPred == ICmpInst::ICMP_ULE ||
IPred == ICmpInst::ICMP_SLE || IPred == ICmpInst::ICMP_UGE ||
IPred == ICmpInst::ICMP_SGE) {
ICmpInst *Cond = cast<ICmpInst>(BI.getCondition());
Cond->setPredicate(ICmpInst::getInversePredicate(IPred));
BI.swapSuccessors();
Worklist.Add(Cond);
return &BI;
}
return nullptr;
}
Instruction *InstCombiner::visitSwitchInst(SwitchInst &SI) {
Value *Cond = SI.getCondition();
unsigned BitWidth = cast<IntegerType>(Cond->getType())->getBitWidth();
APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0);
computeKnownBits(Cond, KnownZero, KnownOne, 0, &SI);
unsigned LeadingKnownZeros = KnownZero.countLeadingOnes();
unsigned LeadingKnownOnes = KnownOne.countLeadingOnes();
for (auto &C : SI.cases()) {
LeadingKnownZeros = std::min(
LeadingKnownZeros, C.getCaseValue()->getValue().countLeadingZeros());
LeadingKnownOnes = std::min(
LeadingKnownOnes, C.getCaseValue()->getValue().countLeadingOnes());
}
unsigned NewWidth = BitWidth - std::max(LeadingKnownZeros, LeadingKnownOnes);
bool TruncCond = false;
if (NewWidth > 0 && BitWidth > NewWidth &&
NewWidth >= DL.getLargestLegalIntTypeSize()) {
TruncCond = true;
IntegerType *Ty = IntegerType::get(SI.getContext(), NewWidth);
Builder->SetInsertPoint(&SI);
Value *NewCond = Builder->CreateTrunc(SI.getCondition(), Ty, "trunc");
SI.setCondition(NewCond);
for (auto &C : SI.cases())
static_cast<SwitchInst::CaseIt *>(&C)->setValue(ConstantInt::get(
SI.getContext(), C.getCaseValue()->getValue().trunc(NewWidth)));
}
if (Instruction *I = dyn_cast<Instruction>(Cond)) {
if (I->getOpcode() == Instruction::Add)
if (ConstantInt *AddRHS = dyn_cast<ConstantInt>(I->getOperand(1))) {
for (SwitchInst::CaseIt i = SI.case_begin(), e = SI.case_end();
i != e; ++i) {
ConstantInt* CaseVal = i.getCaseValue();
Constant *LHS = CaseVal;
if (TruncCond)
LHS = LeadingKnownZeros
? ConstantExpr::getZExt(CaseVal, Cond->getType())
: ConstantExpr::getSExt(CaseVal, Cond->getType());
Constant* NewCaseVal = ConstantExpr::getSub(LHS, AddRHS);
assert(isa<ConstantInt>(NewCaseVal) &&
"Result of expression should be constant");
i.setValue(cast<ConstantInt>(NewCaseVal));
}
SI.setCondition(I->getOperand(0));
Worklist.Add(I);
return &SI;
}
}
return TruncCond ? &SI : nullptr;
}
Instruction *InstCombiner::visitExtractValueInst(ExtractValueInst &EV) {
Value *Agg = EV.getAggregateOperand();
if (!EV.hasIndices())
return ReplaceInstUsesWith(EV, Agg);
if (Value *V =
SimplifyExtractValueInst(Agg, EV.getIndices(), DL, TLI, DT, AC))
return ReplaceInstUsesWith(EV, V);
if (InsertValueInst *IV = dyn_cast<InsertValueInst>(Agg)) {
const unsigned *exti, *exte, *insi, *inse;
for (exti = EV.idx_begin(), insi = IV->idx_begin(),
exte = EV.idx_end(), inse = IV->idx_end();
exti != exte && insi != inse;
++exti, ++insi) {
if (*insi != *exti)
return ExtractValueInst::Create(IV->getAggregateOperand(),
EV.getIndices());
}
if (exti == exte && insi == inse)
return ReplaceInstUsesWith(EV, IV->getInsertedValueOperand());
if (exti == exte) {
Value *NewEV = Builder->CreateExtractValue(IV->getAggregateOperand(),
EV.getIndices());
return InsertValueInst::Create(NewEV, IV->getInsertedValueOperand(),
makeArrayRef(insi, inse));
}
if (insi == inse)
return ExtractValueInst::Create(IV->getInsertedValueOperand(),
makeArrayRef(exti, exte));
}
if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Agg)) {
if (II->hasOneUse()) {
switch (II->getIntrinsicID()) {
case Intrinsic::uadd_with_overflow:
case Intrinsic::sadd_with_overflow:
if (*EV.idx_begin() == 0) { Value *LHS = II->getArgOperand(0), *RHS = II->getArgOperand(1);
ReplaceInstUsesWith(*II, UndefValue::get(II->getType()));
EraseInstFromFunction(*II);
return BinaryOperator::CreateAdd(LHS, RHS);
}
if (II->getIntrinsicID() == Intrinsic::uadd_with_overflow)
if (ConstantInt *CI = dyn_cast<ConstantInt>(II->getArgOperand(1)))
return new ICmpInst(ICmpInst::ICMP_UGT, II->getArgOperand(0),
ConstantExpr::getNot(CI));
break;
case Intrinsic::usub_with_overflow:
case Intrinsic::ssub_with_overflow:
if (*EV.idx_begin() == 0) { Value *LHS = II->getArgOperand(0), *RHS = II->getArgOperand(1);
ReplaceInstUsesWith(*II, UndefValue::get(II->getType()));
EraseInstFromFunction(*II);
return BinaryOperator::CreateSub(LHS, RHS);
}
break;
case Intrinsic::umul_with_overflow:
case Intrinsic::smul_with_overflow:
if (*EV.idx_begin() == 0) { Value *LHS = II->getArgOperand(0), *RHS = II->getArgOperand(1);
ReplaceInstUsesWith(*II, UndefValue::get(II->getType()));
EraseInstFromFunction(*II);
return BinaryOperator::CreateMul(LHS, RHS);
}
break;
default:
break;
}
}
}
if (LoadInst *L = dyn_cast<LoadInst>(Agg))
if (L->isSimple() && L->hasOneUse()) {
SmallVector<Value*, 4> Indices;
Indices.push_back(Builder->getInt32(0));
for (ExtractValueInst::idx_iterator I = EV.idx_begin(), E = EV.idx_end();
I != E; ++I)
Indices.push_back(Builder->getInt32(*I));
Builder->SetInsertPoint(L);
Value *GEP = Builder->CreateInBoundsGEP(L->getType(),
L->getPointerOperand(), Indices);
return ReplaceInstUsesWith(EV, Builder->CreateLoad(GEP));
}
return nullptr;
}
static bool isCatchAll(EHPersonality Personality, Constant *TypeInfo) {
switch (Personality) {
case EHPersonality::GNU_C:
return false;
case EHPersonality::Unknown:
return false;
case EHPersonality::GNU_Ada:
return false;
case EHPersonality::GNU_CXX:
case EHPersonality::GNU_ObjC:
case EHPersonality::MSVC_X86SEH:
case EHPersonality::MSVC_Win64SEH:
case EHPersonality::MSVC_CXX:
case EHPersonality::CoreCLR:
return TypeInfo->isNullValue();
}
llvm_unreachable("invalid enum");
}
static bool shorter_filter(const Value *LHS, const Value *RHS) {
return
cast<ArrayType>(LHS->getType())->getNumElements()
<
cast<ArrayType>(RHS->getType())->getNumElements();
}
Instruction *InstCombiner::visitLandingPadInst(LandingPadInst &LI) {
EHPersonality Personality =
classifyEHPersonality(LI.getParent()->getParent()->getPersonalityFn());
bool MakeNewInstruction = false; SmallVector<Constant *, 16> NewClauses; bool CleanupFlag = LI.isCleanup();
SmallPtrSet<Value *, 16> AlreadyCaught; for (unsigned i = 0, e = LI.getNumClauses(); i != e; ++i) {
bool isLastClause = i + 1 == e;
if (LI.isCatch(i)) {
Constant *CatchClause = LI.getClause(i);
Constant *TypeInfo = CatchClause->stripPointerCasts();
if (AlreadyCaught.insert(TypeInfo).second) {
NewClauses.push_back(CatchClause);
} else {
MakeNewInstruction = true;
}
if (isCatchAll(Personality, TypeInfo)) {
if (!isLastClause)
MakeNewInstruction = true;
CleanupFlag = false;
break;
}
} else {
assert(LI.isFilter(i) && "Unsupported landingpad clause!");
Constant *FilterClause = LI.getClause(i);
ArrayType *FilterType = cast<ArrayType>(FilterClause->getType());
unsigned NumTypeInfos = FilterType->getNumElements();
if (!NumTypeInfos) {
NewClauses.push_back(FilterClause);
if (!isLastClause)
MakeNewInstruction = true;
CleanupFlag = false;
break;
}
bool MakeNewFilter = false; SmallVector<Constant *, 16> NewFilterElts; if (isa<ConstantAggregateZero>(FilterClause)) {
assert(NumTypeInfos > 0 && "Should have handled empty filter already!");
Constant *TypeInfo =
Constant::getNullValue(FilterType->getElementType());
if (isCatchAll(Personality, TypeInfo)) {
MakeNewInstruction = true;
continue;
}
NewFilterElts.push_back(TypeInfo);
if (NumTypeInfos > 1)
MakeNewFilter = true;
} else {
ConstantArray *Filter = cast<ConstantArray>(FilterClause);
SmallPtrSet<Value *, 16> SeenInFilter; NewFilterElts.reserve(NumTypeInfos);
bool SawCatchAll = false;
for (unsigned j = 0; j != NumTypeInfos; ++j) {
Constant *Elt = Filter->getOperand(j);
Constant *TypeInfo = Elt->stripPointerCasts();
if (isCatchAll(Personality, TypeInfo)) {
SawCatchAll = true;
break;
}
if (SeenInFilter.insert(TypeInfo).second)
NewFilterElts.push_back(cast<Constant>(Elt));
}
if (SawCatchAll) {
MakeNewInstruction = true;
continue;
}
if (NewFilterElts.size() < NumTypeInfos)
MakeNewFilter = true;
}
if (MakeNewFilter) {
FilterType = ArrayType::get(FilterType->getElementType(),
NewFilterElts.size());
FilterClause = ConstantArray::get(FilterType, NewFilterElts);
MakeNewInstruction = true;
}
NewClauses.push_back(FilterClause);
if (MakeNewFilter && !NewFilterElts.size()) {
assert(MakeNewInstruction && "New filter but not a new instruction!");
CleanupFlag = false;
break;
}
}
}
for (unsigned i = 0, e = NewClauses.size(); i + 1 < e; ) {
unsigned j;
for (j = i; j != e; ++j)
if (!isa<ArrayType>(NewClauses[j]->getType()))
break;
for (unsigned k = i; k + 1 < j; ++k)
if (shorter_filter(NewClauses[k+1], NewClauses[k])) {
std::stable_sort(NewClauses.begin() + i, NewClauses.begin() + j,
shorter_filter);
MakeNewInstruction = true;
break;
}
i = j + 1;
}
for (unsigned i = 0; i + 1 < NewClauses.size(); ++i) {
Value *Filter = NewClauses[i];
ArrayType *FTy = dyn_cast<ArrayType>(Filter->getType());
if (!FTy)
continue;
unsigned FElts = FTy->getNumElements();
for (unsigned j = NewClauses.size() - 1; j != i; --j) {
Value *LFilter = NewClauses[j];
ArrayType *LTy = dyn_cast<ArrayType>(LFilter->getType());
if (!LTy)
continue;
SmallVectorImpl<Constant *>::iterator J = NewClauses.begin() + j;
if (!FElts) {
NewClauses.erase(J);
MakeNewInstruction = true;
continue;
}
unsigned LElts = LTy->getNumElements();
if (FElts > LElts)
continue;
if (isa<ConstantAggregateZero>(LFilter)) { if (isa<ConstantAggregateZero>(Filter)) {
assert(FElts <= LElts && "Should have handled this case earlier!");
NewClauses.erase(J);
MakeNewInstruction = true;
}
continue;
}
ConstantArray *LArray = cast<ConstantArray>(LFilter);
if (isa<ConstantAggregateZero>(Filter)) { assert(FElts > 0 && "Should have eliminated the empty filter earlier!");
for (unsigned l = 0; l != LElts; ++l)
if (LArray->getOperand(l)->isNullValue()) {
NewClauses.erase(J);
MakeNewInstruction = true;
break;
}
continue;
}
ConstantArray *FArray = cast<ConstantArray>(Filter);
bool AllFound = true;
for (unsigned f = 0; f != FElts; ++f) {
Value *FTypeInfo = FArray->getOperand(f)->stripPointerCasts();
AllFound = false;
for (unsigned l = 0; l != LElts; ++l) {
Value *LTypeInfo = LArray->getOperand(l)->stripPointerCasts();
if (LTypeInfo == FTypeInfo) {
AllFound = true;
break;
}
}
if (!AllFound)
break;
}
if (AllFound) {
NewClauses.erase(J);
MakeNewInstruction = true;
}
}
}
if (MakeNewInstruction) {
LandingPadInst *NLI = LandingPadInst::Create(LI.getType(),
NewClauses.size());
for (unsigned i = 0, e = NewClauses.size(); i != e; ++i)
NLI->addClause(NewClauses[i]);
if (NewClauses.empty())
CleanupFlag = true;
NLI->setCleanup(CleanupFlag);
return NLI;
}
if (LI.isCleanup() != CleanupFlag) {
assert(!CleanupFlag && "Adding a cleanup, not removing one?!");
LI.setCleanup(CleanupFlag);
return &LI;
}
return nullptr;
}
static bool TryToSinkInstruction(Instruction *I, BasicBlock *DestBlock) {
assert(I->hasOneUse() && "Invariants didn't hold!");
if (isa<PHINode>(I) || I->isEHPad() || I->mayHaveSideEffects() ||
isa<TerminatorInst>(I))
return false;
if (isa<AllocaInst>(I) && I->getParent() ==
&DestBlock->getParent()->getEntryBlock())
return false;
if (auto *CI = dyn_cast<CallInst>(I)) {
if (CI->isConvergent())
return false;
}
if (I->mayReadFromMemory()) {
for (BasicBlock::iterator Scan = I->getIterator(),
E = I->getParent()->end();
Scan != E; ++Scan)
if (Scan->mayWriteToMemory())
return false;
}
BasicBlock::iterator InsertPos = DestBlock->getFirstInsertionPt();
I->moveBefore(&*InsertPos);
++NumSunkInst;
return true;
}
bool InstCombiner::run() {
while (!Worklist.isEmpty()) {
Instruction *I = Worklist.RemoveOne();
if (I == nullptr) continue;
if (isInstructionTriviallyDead(I, TLI)) {
DEBUG(dbgs() << "IC: DCE: " << *I << '\n');
EraseInstFromFunction(*I);
++NumDeadInst;
MadeIRChange = true;
continue;
}
if (!I->use_empty() &&
(I->getNumOperands() == 0 || isa<Constant>(I->getOperand(0)))) {
if (Constant *C = ConstantFoldInstruction(I, DL, TLI)) {
DEBUG(dbgs() << "IC: ConstFold to: " << *C << " from: " << *I << '\n');
ReplaceInstUsesWith(*I, C);
++NumConstProp;
EraseInstFromFunction(*I);
MadeIRChange = true;
continue;
}
}
if (!I->use_empty() && I->getType()->isIntegerTy()) {
unsigned BitWidth = I->getType()->getScalarSizeInBits();
APInt KnownZero(BitWidth, 0);
APInt KnownOne(BitWidth, 0);
computeKnownBits(I, KnownZero, KnownOne, 0, I);
if ((KnownZero | KnownOne).isAllOnesValue()) {
Constant *C = ConstantInt::get(I->getContext(), KnownOne);
DEBUG(dbgs() << "IC: ConstFold (all bits known) to: " << *C <<
" from: " << *I << '\n');
ReplaceInstUsesWith(*I, C);
++NumConstProp;
EraseInstFromFunction(*I);
MadeIRChange = true;
continue;
}
}
if (I->hasOneUse()) {
BasicBlock *BB = I->getParent();
Instruction *UserInst = cast<Instruction>(*I->user_begin());
BasicBlock *UserParent;
if (PHINode *PN = dyn_cast<PHINode>(UserInst))
UserParent = PN->getIncomingBlock(*I->use_begin());
else
UserParent = UserInst->getParent();
if (UserParent != BB) {
bool UserIsSuccessor = false;
for (succ_iterator SI = succ_begin(BB), E = succ_end(BB); SI != E; ++SI)
if (*SI == UserParent) {
UserIsSuccessor = true;
break;
}
if (UserIsSuccessor && UserParent->getSinglePredecessor()) {
if (TryToSinkInstruction(I, UserParent)) {
MadeIRChange = true;
for (Use &U : I->operands())
if (Instruction *OpI = dyn_cast<Instruction>(U.get()))
Worklist.Add(OpI);
}
}
}
}
Builder->SetInsertPoint(I);
Builder->SetCurrentDebugLocation(I->getDebugLoc());
#ifndef NDEBUG
std::string OrigI;
#endif
DEBUG(raw_string_ostream SS(OrigI); I->print(SS); OrigI = SS.str(););
DEBUG(dbgs() << "IC: Visiting: " << OrigI << '\n');
if (Instruction *Result = visit(*I)) {
++NumCombined;
if (Result != I) {
DEBUG(dbgs() << "IC: Old = " << *I << '\n'
<< " New = " << *Result << '\n');
if (I->getDebugLoc())
Result->setDebugLoc(I->getDebugLoc());
I->replaceAllUsesWith(Result);
Result->takeName(I);
Worklist.Add(Result);
Worklist.AddUsersToWorkList(*Result);
BasicBlock *InstParent = I->getParent();
BasicBlock::iterator InsertPos = I->getIterator();
if (!isa<PHINode>(Result) && isa<PHINode>(InsertPos))
InsertPos = InstParent->getFirstInsertionPt();
InstParent->getInstList().insert(InsertPos, Result);
EraseInstFromFunction(*I);
} else {
#ifndef NDEBUG
DEBUG(dbgs() << "IC: Mod = " << OrigI << '\n'
<< " New = " << *I << '\n');
#endif
if (isInstructionTriviallyDead(I, TLI)) {
EraseInstFromFunction(*I);
} else {
Worklist.Add(I);
Worklist.AddUsersToWorkList(*I);
}
}
MadeIRChange = true;
}
}
Worklist.Zap();
return MadeIRChange;
}
static bool AddReachableCodeToWorklist(BasicBlock *BB, const DataLayout &DL,
SmallPtrSetImpl<BasicBlock *> &Visited,
InstCombineWorklist &ICWorklist,
const TargetLibraryInfo *TLI) {
bool MadeIRChange = false;
SmallVector<BasicBlock*, 256> Worklist;
Worklist.push_back(BB);
SmallVector<Instruction*, 128> InstrsForInstCombineWorklist;
DenseMap<ConstantExpr*, Constant*> FoldedConstants;
do {
BB = Worklist.pop_back_val();
if (!Visited.insert(BB).second)
continue;
for (BasicBlock::iterator BBI = BB->begin(), E = BB->end(); BBI != E; ) {
Instruction *Inst = &*BBI++;
if (isInstructionTriviallyDead(Inst, TLI)) {
++NumDeadInst;
DEBUG(dbgs() << "IC: DCE: " << *Inst << '\n');
Inst->eraseFromParent();
continue;
}
if (!Inst->use_empty() &&
(Inst->getNumOperands() == 0 || isa<Constant>(Inst->getOperand(0))))
if (Constant *C = ConstantFoldInstruction(Inst, DL, TLI)) {
DEBUG(dbgs() << "IC: ConstFold to: " << *C << " from: "
<< *Inst << '\n');
Inst->replaceAllUsesWith(C);
++NumConstProp;
Inst->eraseFromParent();
continue;
}
for (User::op_iterator i = Inst->op_begin(), e = Inst->op_end(); i != e;
++i) {
ConstantExpr *CE = dyn_cast<ConstantExpr>(i);
if (CE == nullptr)
continue;
Constant *&FoldRes = FoldedConstants[CE];
if (!FoldRes)
FoldRes = ConstantFoldConstantExpression(CE, DL, TLI);
if (!FoldRes)
FoldRes = CE;
if (FoldRes != CE) {
*i = FoldRes;
MadeIRChange = true;
}
}
InstrsForInstCombineWorklist.push_back(Inst);
}
TerminatorInst *TI = BB->getTerminator();
if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
if (BI->isConditional() && isa<ConstantInt>(BI->getCondition())) {
bool CondVal = cast<ConstantInt>(BI->getCondition())->getZExtValue();
BasicBlock *ReachableBB = BI->getSuccessor(!CondVal);
Worklist.push_back(ReachableBB);
continue;
}
} else if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
if (ConstantInt *Cond = dyn_cast<ConstantInt>(SI->getCondition())) {
for (SwitchInst::CaseIt i = SI->case_begin(), e = SI->case_end();
i != e; ++i)
if (i.getCaseValue() == Cond) {
BasicBlock *ReachableBB = i.getCaseSuccessor();
Worklist.push_back(ReachableBB);
continue;
}
Worklist.push_back(SI->getDefaultDest());
continue;
}
}
for (BasicBlock *SuccBB : TI->successors())
Worklist.push_back(SuccBB);
} while (!Worklist.empty());
ICWorklist.AddInitialGroup(InstrsForInstCombineWorklist);
return MadeIRChange;
}
static bool prepareICWorklistFromFunction(Function &F, const DataLayout &DL,
TargetLibraryInfo *TLI,
InstCombineWorklist &ICWorklist) {
bool MadeIRChange = false;
SmallPtrSet<BasicBlock *, 64> Visited;
MadeIRChange |=
AddReachableCodeToWorklist(&F.front(), DL, Visited, ICWorklist, TLI);
for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) {
if (Visited.count(&*BB))
continue;
Instruction *EndInst = BB->getTerminator(); while (EndInst != BB->begin()) {
Instruction *Inst = &*--EndInst->getIterator();
if (!Inst->use_empty() && !Inst->getType()->isTokenTy())
Inst->replaceAllUsesWith(UndefValue::get(Inst->getType()));
if (Inst->isEHPad()) {
EndInst = Inst;
continue;
}
if (!isa<DbgInfoIntrinsic>(Inst)) {
++NumDeadInst;
MadeIRChange = true;
}
if (!Inst->getType()->isTokenTy())
Inst->eraseFromParent();
}
}
return MadeIRChange;
}
static bool
combineInstructionsOverFunction(Function &F, InstCombineWorklist &Worklist,
AliasAnalysis *AA, AssumptionCache &AC,
TargetLibraryInfo &TLI, DominatorTree &DT,
LoopInfo *LI = nullptr) {
auto &DL = F.getParent()->getDataLayout();
IRBuilder<true, TargetFolder, InstCombineIRInserter> Builder(
F.getContext(), TargetFolder(DL), InstCombineIRInserter(Worklist, &AC));
bool DbgDeclaresChanged = LowerDbgDeclare(F);
int Iteration = 0;
for (;;) {
++Iteration;
DEBUG(dbgs() << "\n\nINSTCOMBINE ITERATION #" << Iteration << " on "
<< F.getName() << "\n");
bool Changed = false;
if (prepareICWorklistFromFunction(F, DL, &TLI, Worklist))
Changed = true;
InstCombiner IC(Worklist, &Builder, F.optForMinSize(),
AA, &AC, &TLI, &DT, DL, LI);
if (IC.run())
Changed = true;
if (!Changed)
break;
}
return DbgDeclaresChanged || Iteration > 1;
}
PreservedAnalyses InstCombinePass::run(Function &F,
AnalysisManager<Function> *AM) {
auto &AC = AM->getResult<AssumptionAnalysis>(F);
auto &DT = AM->getResult<DominatorTreeAnalysis>(F);
auto &TLI = AM->getResult<TargetLibraryAnalysis>(F);
auto *LI = AM->getCachedResult<LoopAnalysis>(F);
if (!combineInstructionsOverFunction(F, Worklist, nullptr, AC, TLI, DT, LI))
return PreservedAnalyses::all();
PreservedAnalyses PA;
PA.preserve<DominatorTreeAnalysis>();
return PA;
}
namespace {
class InstructionCombiningPass : public FunctionPass {
InstCombineWorklist Worklist;
public:
static char ID;
InstructionCombiningPass() : FunctionPass(ID) {
initializeInstructionCombiningPassPass(*PassRegistry::getPassRegistry());
}
void getAnalysisUsage(AnalysisUsage &AU) const override;
bool runOnFunction(Function &F) override;
};
}
void InstructionCombiningPass::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesCFG();
AU.addRequired<AAResultsWrapperPass>();
AU.addRequired<AssumptionCacheTracker>();
AU.addRequired<TargetLibraryInfoWrapperPass>();
AU.addRequired<DominatorTreeWrapperPass>();
AU.addPreserved<DominatorTreeWrapperPass>();
AU.addPreserved<GlobalsAAWrapperPass>();
}
bool InstructionCombiningPass::runOnFunction(Function &F) {
if (skipOptnoneFunction(F))
return false;
auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
auto *LIWP = getAnalysisIfAvailable<LoopInfoWrapperPass>();
auto *LI = LIWP ? &LIWP->getLoopInfo() : nullptr;
return combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, DT, LI);
}
char InstructionCombiningPass::ID = 0;
INITIALIZE_PASS_BEGIN(InstructionCombiningPass, "instcombine",
"Combine redundant instructions", false, false)
INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
INITIALIZE_PASS_END(InstructionCombiningPass, "instcombine",
"Combine redundant instructions", false, false)
void llvm::initializeInstCombine(PassRegistry &Registry) {
initializeInstructionCombiningPassPass(Registry);
}
void LLVMInitializeInstCombine(LLVMPassRegistryRef R) {
initializeInstructionCombiningPassPass(*unwrap(R));
}
FunctionPass *llvm::createInstructionCombiningPass() {
return new InstructionCombiningPass();
}