InstCombineShifts.cpp [plain text]
#include "InstCombine.h"
#include "llvm/IntrinsicInst.h"
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Support/PatternMatch.h"
using namespace llvm;
using namespace PatternMatch;
Instruction *InstCombiner::commonShiftTransforms(BinaryOperator &I) {
assert(I.getOperand(1)->getType() == I.getOperand(0)->getType());
Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
if (SimplifyDemandedInstructionBits(I))
return &I;
if (isa<Constant>(Op0))
if (SelectInst *SI = dyn_cast<SelectInst>(Op1))
if (Instruction *R = FoldOpIntoSelect(I, SI))
return R;
if (ConstantInt *CUI = dyn_cast<ConstantInt>(Op1))
if (Instruction *Res = FoldShiftByConstant(Op0, CUI, I))
return Res;
Value *A; const APInt *B;
if (Op1->hasOneUse() && match(Op1, m_SRem(m_Value(A), m_Power2(B)))) {
Value *Rem = Builder->CreateAnd(A, ConstantInt::get(I.getType(), *B-1),
Op1->getName());
I.setOperand(1, Rem);
return &I;
}
return 0;
}
static bool CanEvaluateShifted(Value *V, unsigned NumBits, bool isLeftShift,
InstCombiner &IC) {
if (isa<Constant>(V))
return true;
Instruction *I = dyn_cast<Instruction>(V);
if (!I) return false;
ConstantInt *CI = 0;
if ((isLeftShift && match(I, m_LShr(m_Value(), m_ConstantInt(CI)))) ||
(!isLeftShift && match(I, m_Shl(m_Value(), m_ConstantInt(CI))))) {
if (CI->getZExtValue() == NumBits) {
#if 0
uint32_t OrigBitWidth = OrigTy->getScalarSizeInBits();
uint32_t BitWidth = Ty->getScalarSizeInBits();
if (MaskedValueIsZero(I->getOperand(0),
APInt::getHighBitsSet(OrigBitWidth, OrigBitWidth-BitWidth)) &&
CI->getLimitedValue(BitWidth) < BitWidth) {
return CanEvaluateTruncated(I->getOperand(0), Ty);
}
#endif
}
}
if (!I->hasOneUse()) return false;
switch (I->getOpcode()) {
default: return false;
case Instruction::And:
case Instruction::Or:
case Instruction::Xor:
return CanEvaluateShifted(I->getOperand(0), NumBits, isLeftShift, IC) &&
CanEvaluateShifted(I->getOperand(1), NumBits, isLeftShift, IC);
case Instruction::Shl: {
CI = dyn_cast<ConstantInt>(I->getOperand(1));
if (CI == 0) return false;
if (isLeftShift) return true;
if (CI->getValue() == NumBits) return true;
unsigned TypeWidth = I->getType()->getScalarSizeInBits();
if (CI->getZExtValue() > NumBits) {
unsigned LowBits = TypeWidth - CI->getZExtValue();
if (MaskedValueIsZero(I->getOperand(0),
APInt::getLowBitsSet(TypeWidth, NumBits) << LowBits))
return true;
}
return false;
}
case Instruction::LShr: {
CI = dyn_cast<ConstantInt>(I->getOperand(1));
if (CI == 0) return false;
if (!isLeftShift) return true;
if (CI->getValue() == NumBits) return true;
unsigned TypeWidth = I->getType()->getScalarSizeInBits();
if (CI->getZExtValue() > NumBits) {
unsigned LowBits = CI->getZExtValue() - NumBits;
if (MaskedValueIsZero(I->getOperand(0),
APInt::getLowBitsSet(TypeWidth, NumBits) << LowBits))
return true;
}
return false;
}
case Instruction::Select: {
SelectInst *SI = cast<SelectInst>(I);
return CanEvaluateShifted(SI->getTrueValue(), NumBits, isLeftShift, IC) &&
CanEvaluateShifted(SI->getFalseValue(), NumBits, isLeftShift, IC);
}
case Instruction::PHI: {
PHINode *PN = cast<PHINode>(I);
for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
if (!CanEvaluateShifted(PN->getIncomingValue(i), NumBits, isLeftShift,IC))
return false;
return true;
}
}
}
static Value *GetShiftedValue(Value *V, unsigned NumBits, bool isLeftShift,
InstCombiner &IC) {
if (Constant *C = dyn_cast<Constant>(V)) {
if (isLeftShift)
V = IC.Builder->CreateShl(C, NumBits);
else
V = IC.Builder->CreateLShr(C, NumBits);
if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
V = ConstantFoldConstantExpression(CE, IC.getTargetData());
return V;
}
Instruction *I = cast<Instruction>(V);
IC.Worklist.Add(I);
switch (I->getOpcode()) {
default: assert(0 && "Inconsistency with CanEvaluateShifted");
case Instruction::And:
case Instruction::Or:
case Instruction::Xor:
I->setOperand(0, GetShiftedValue(I->getOperand(0), NumBits,isLeftShift,IC));
I->setOperand(1, GetShiftedValue(I->getOperand(1), NumBits,isLeftShift,IC));
return I;
case Instruction::Shl: {
unsigned TypeWidth = I->getType()->getScalarSizeInBits();
ConstantInt *CI = cast<ConstantInt>(I->getOperand(1));
if (isLeftShift) {
unsigned NewShAmt = NumBits+CI->getZExtValue();
if (NewShAmt >= TypeWidth)
return Constant::getNullValue(I->getType());
I->setOperand(1, ConstantInt::get(I->getType(), NewShAmt));
return I;
}
if (CI->getValue() == NumBits) {
APInt Mask(APInt::getLowBitsSet(TypeWidth, TypeWidth - NumBits));
V = IC.Builder->CreateAnd(I->getOperand(0),
ConstantInt::get(I->getContext(), Mask));
if (Instruction *VI = dyn_cast<Instruction>(V)) {
VI->moveBefore(I);
VI->takeName(I);
}
return V;
}
assert(CI->getZExtValue() > NumBits);
I->setOperand(1, ConstantInt::get(I->getType(),
CI->getZExtValue() - NumBits));
return I;
}
case Instruction::LShr: {
unsigned TypeWidth = I->getType()->getScalarSizeInBits();
ConstantInt *CI = cast<ConstantInt>(I->getOperand(1));
if (!isLeftShift) {
unsigned NewShAmt = NumBits+CI->getZExtValue();
if (NewShAmt >= TypeWidth)
return Constant::getNullValue(I->getType());
I->setOperand(1, ConstantInt::get(I->getType(), NewShAmt));
return I;
}
if (CI->getValue() == NumBits) {
APInt Mask(APInt::getHighBitsSet(TypeWidth, TypeWidth - NumBits));
V = IC.Builder->CreateAnd(I->getOperand(0),
ConstantInt::get(I->getContext(), Mask));
if (Instruction *VI = dyn_cast<Instruction>(V)) {
VI->moveBefore(I);
VI->takeName(I);
}
return V;
}
assert(CI->getZExtValue() > NumBits);
I->setOperand(1, ConstantInt::get(I->getType(),
CI->getZExtValue() - NumBits));
return I;
}
case Instruction::Select:
I->setOperand(1, GetShiftedValue(I->getOperand(1), NumBits,isLeftShift,IC));
I->setOperand(2, GetShiftedValue(I->getOperand(2), NumBits,isLeftShift,IC));
return I;
case Instruction::PHI: {
PHINode *PN = cast<PHINode>(I);
for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
PN->setIncomingValue(i, GetShiftedValue(PN->getIncomingValue(i),
NumBits, isLeftShift, IC));
return PN;
}
}
}
Instruction *InstCombiner::FoldShiftByConstant(Value *Op0, ConstantInt *Op1,
BinaryOperator &I) {
bool isLeftShift = I.getOpcode() == Instruction::Shl;
if (I.getOpcode() != Instruction::AShr &&
CanEvaluateShifted(Op0, Op1->getZExtValue(), isLeftShift, *this)) {
DEBUG(dbgs() << "ICE: GetShiftedValue propagating shift through expression"
" to eliminate shift:\n IN: " << *Op0 << "\n SH: " << I <<"\n");
return ReplaceInstUsesWith(I,
GetShiftedValue(Op0, Op1->getZExtValue(), isLeftShift, *this));
}
uint32_t TypeBits = Op0->getType()->getScalarSizeInBits();
if (Op1->uge(TypeBits)) {
if (I.getOpcode() != Instruction::AShr)
return ReplaceInstUsesWith(I, Constant::getNullValue(Op0->getType()));
I.setOperand(1, ConstantInt::get(I.getType(), TypeBits-1));
return &I;
}
if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Op0))
if (BO->getOpcode() == Instruction::Mul && isLeftShift)
if (Constant *BOOp = dyn_cast<Constant>(BO->getOperand(1)))
return BinaryOperator::CreateMul(BO->getOperand(0),
ConstantExpr::getShl(BOOp, Op1));
if (SelectInst *SI = dyn_cast<SelectInst>(Op0))
if (Instruction *R = FoldOpIntoSelect(I, SI))
return R;
if (isa<PHINode>(Op0))
if (Instruction *NV = FoldOpIntoPhi(I))
return NV;
if (TruncInst *TI = dyn_cast<TruncInst>(Op0)) {
Instruction *TrOp = dyn_cast<Instruction>(TI->getOperand(0));
if (TrOp && I.isLogicalShift() && TrOp->isShift() &&
isa<ConstantInt>(TrOp->getOperand(1))) {
Constant *ShAmt = ConstantExpr::getZExt(Op1, TrOp->getType());
Value *NSh = Builder->CreateBinOp(I.getOpcode(), TrOp, ShAmt,I.getName());
unsigned SrcSize = TrOp->getType()->getScalarSizeInBits();
unsigned DstSize = TI->getType()->getScalarSizeInBits();
APInt MaskV(APInt::getLowBitsSet(SrcSize, DstSize));
if (I.getOpcode() == Instruction::Shl)
MaskV <<= Op1->getZExtValue();
else {
assert(I.getOpcode() == Instruction::LShr && "Unknown logical shift");
MaskV = MaskV.lshr(Op1->getZExtValue());
}
Value *And = Builder->CreateAnd(NSh,
ConstantInt::get(I.getContext(), MaskV),
TI->getName());
return new TruncInst(And, I.getType());
}
}
if (Op0->hasOneUse()) {
if (BinaryOperator *Op0BO = dyn_cast<BinaryOperator>(Op0)) {
Value *V1, *V2;
ConstantInt *CC;
switch (Op0BO->getOpcode()) {
default: break;
case Instruction::Add:
case Instruction::And:
case Instruction::Or:
case Instruction::Xor: {
if (isLeftShift && Op0BO->getOperand(1)->hasOneUse() &&
match(Op0BO->getOperand(1), m_Shr(m_Value(V1),
m_Specific(Op1)))) {
Value *YS = Builder->CreateShl(Op0BO->getOperand(0), Op1, Op0BO->getName());
Value *X = Builder->CreateBinOp(Op0BO->getOpcode(), YS, V1,
Op0BO->getOperand(1)->getName());
uint32_t Op1Val = Op1->getLimitedValue(TypeBits);
return BinaryOperator::CreateAnd(X, ConstantInt::get(I.getContext(),
APInt::getHighBitsSet(TypeBits, TypeBits-Op1Val)));
}
Value *Op0BOOp1 = Op0BO->getOperand(1);
if (isLeftShift && Op0BOOp1->hasOneUse() &&
match(Op0BOOp1,
m_And(m_Shr(m_Value(V1), m_Specific(Op1)),
m_ConstantInt(CC))) &&
cast<BinaryOperator>(Op0BOOp1)->getOperand(0)->hasOneUse()) {
Value *YS = Builder->CreateShl(Op0BO->getOperand(0), Op1,
Op0BO->getName());
Value *XM = Builder->CreateAnd(V1, ConstantExpr::getShl(CC, Op1),
V1->getName()+".mask");
return BinaryOperator::Create(Op0BO->getOpcode(), YS, XM);
}
}
case Instruction::Sub: {
if (isLeftShift && Op0BO->getOperand(0)->hasOneUse() &&
match(Op0BO->getOperand(0), m_Shr(m_Value(V1),
m_Specific(Op1)))) {
Value *YS = Builder->CreateShl(Op0BO->getOperand(1), Op1, Op0BO->getName());
Value *X = Builder->CreateBinOp(Op0BO->getOpcode(), V1, YS,
Op0BO->getOperand(0)->getName());
uint32_t Op1Val = Op1->getLimitedValue(TypeBits);
return BinaryOperator::CreateAnd(X, ConstantInt::get(I.getContext(),
APInt::getHighBitsSet(TypeBits, TypeBits-Op1Val)));
}
if (isLeftShift && Op0BO->getOperand(0)->hasOneUse() &&
match(Op0BO->getOperand(0),
m_And(m_Shr(m_Value(V1), m_Value(V2)),
m_ConstantInt(CC))) && V2 == Op1 &&
cast<BinaryOperator>(Op0BO->getOperand(0))
->getOperand(0)->hasOneUse()) {
Value *YS = Builder->CreateShl(Op0BO->getOperand(1), Op1, Op0BO->getName());
Value *XM = Builder->CreateAnd(V1, ConstantExpr::getShl(CC, Op1),
V1->getName()+".mask");
return BinaryOperator::Create(Op0BO->getOpcode(), XM, YS);
}
break;
}
}
if (ConstantInt *Op0C = dyn_cast<ConstantInt>(Op0BO->getOperand(1))) {
bool isValid = true; bool highBitSet = false;
switch (Op0BO->getOpcode()) {
default: isValid = false; break; case Instruction::Add:
isValid = isLeftShift;
break;
case Instruction::Or:
case Instruction::Xor:
highBitSet = false;
break;
case Instruction::And:
highBitSet = true;
break;
}
if (isValid && I.getOpcode() == Instruction::AShr)
isValid = Op0C->getValue()[TypeBits-1] == highBitSet;
if (isValid) {
Constant *NewRHS = ConstantExpr::get(I.getOpcode(), Op0C, Op1);
Value *NewShift =
Builder->CreateBinOp(I.getOpcode(), Op0BO->getOperand(0), Op1);
NewShift->takeName(Op0BO);
return BinaryOperator::Create(Op0BO->getOpcode(), NewShift,
NewRHS);
}
}
}
}
BinaryOperator *ShiftOp = dyn_cast<BinaryOperator>(Op0);
if (ShiftOp && !ShiftOp->isShift())
ShiftOp = 0;
if (ShiftOp && isa<ConstantInt>(ShiftOp->getOperand(1))) {
ConstantInt *ShiftAmt1C = cast<ConstantInt>(ShiftOp->getOperand(1));
uint32_t ShiftAmt1 = ShiftAmt1C->getLimitedValue(TypeBits);
uint32_t ShiftAmt2 = Op1->getLimitedValue(TypeBits);
assert(ShiftAmt2 != 0 && "Should have been simplified earlier");
if (ShiftAmt1 == 0) return 0; Value *X = ShiftOp->getOperand(0);
uint32_t AmtSum = ShiftAmt1+ShiftAmt2;
const IntegerType *Ty = cast<IntegerType>(I.getType());
if (I.getOpcode() == ShiftOp->getOpcode()) {
if (AmtSum >= TypeBits) {
if (I.getOpcode() != Instruction::AShr)
return ReplaceInstUsesWith(I, Constant::getNullValue(I.getType()));
AmtSum = TypeBits-1; }
return BinaryOperator::Create(I.getOpcode(), X,
ConstantInt::get(Ty, AmtSum));
}
if (ShiftAmt1 == ShiftAmt2) {
if (I.getOpcode() == Instruction::Shl &&
ShiftOp->getOpcode() != Instruction::Shl) {
APInt Mask(APInt::getHighBitsSet(TypeBits, TypeBits - ShiftAmt1));
return BinaryOperator::CreateAnd(X,
ConstantInt::get(I.getContext(),Mask));
}
if (I.getOpcode() == Instruction::LShr &&
ShiftOp->getOpcode() == Instruction::Shl) {
APInt Mask(APInt::getLowBitsSet(TypeBits, TypeBits - ShiftAmt1));
return BinaryOperator::CreateAnd(X,
ConstantInt::get(I.getContext(), Mask));
}
} else if (ShiftAmt1 < ShiftAmt2) {
uint32_t ShiftDiff = ShiftAmt2-ShiftAmt1;
if (I.getOpcode() == Instruction::Shl &&
ShiftOp->getOpcode() != Instruction::Shl) {
assert(ShiftOp->getOpcode() == Instruction::LShr ||
ShiftOp->getOpcode() == Instruction::AShr);
Value *Shift = Builder->CreateShl(X, ConstantInt::get(Ty, ShiftDiff));
APInt Mask(APInt::getHighBitsSet(TypeBits, TypeBits - ShiftAmt2));
return BinaryOperator::CreateAnd(Shift,
ConstantInt::get(I.getContext(),Mask));
}
if (I.getOpcode() == Instruction::LShr &&
ShiftOp->getOpcode() == Instruction::Shl) {
assert(ShiftOp->getOpcode() == Instruction::Shl);
Value *Shift = Builder->CreateLShr(X, ConstantInt::get(Ty, ShiftDiff));
APInt Mask(APInt::getLowBitsSet(TypeBits, TypeBits - ShiftAmt2));
return BinaryOperator::CreateAnd(Shift,
ConstantInt::get(I.getContext(),Mask));
}
} else {
assert(ShiftAmt2 < ShiftAmt1);
uint32_t ShiftDiff = ShiftAmt1-ShiftAmt2;
if (I.getOpcode() == Instruction::Shl &&
ShiftOp->getOpcode() != Instruction::Shl) {
Value *Shift = Builder->CreateBinOp(ShiftOp->getOpcode(), X,
ConstantInt::get(Ty, ShiftDiff));
APInt Mask(APInt::getHighBitsSet(TypeBits, TypeBits - ShiftAmt2));
return BinaryOperator::CreateAnd(Shift,
ConstantInt::get(I.getContext(),Mask));
}
if (I.getOpcode() == Instruction::LShr &&
ShiftOp->getOpcode() == Instruction::Shl) {
Value *Shift = Builder->CreateShl(X, ConstantInt::get(Ty, ShiftDiff));
APInt Mask(APInt::getLowBitsSet(TypeBits, TypeBits - ShiftAmt2));
return BinaryOperator::CreateAnd(Shift,
ConstantInt::get(I.getContext(),Mask));
}
}
}
return 0;
}
Instruction *InstCombiner::visitShl(BinaryOperator &I) {
if (Value *V = SimplifyShlInst(I.getOperand(0), I.getOperand(1),
I.hasNoSignedWrap(), I.hasNoUnsignedWrap(),
TD))
return ReplaceInstUsesWith(I, V);
if (Instruction *V = commonShiftTransforms(I))
return V;
if (ConstantInt *Op1C = dyn_cast<ConstantInt>(I.getOperand(1))) {
unsigned ShAmt = Op1C->getZExtValue();
if (!I.hasNoUnsignedWrap() &&
MaskedValueIsZero(I.getOperand(0),
APInt::getHighBitsSet(Op1C->getBitWidth(), ShAmt))) {
I.setHasNoUnsignedWrap();
return &I;
}
if (!I.hasNoSignedWrap() &&
ComputeNumSignBits(I.getOperand(0)) > ShAmt) {
I.setHasNoSignedWrap();
return &I;
}
}
Constant *C1, *C2;
Value *A;
if (match(I.getOperand(0), m_OneUse(m_Shl(m_Constant(C1), m_Value(A)))) &&
match(I.getOperand(1), m_Constant(C2)))
return BinaryOperator::CreateShl(ConstantExpr::getShl(C1, C2), A);
return 0;
}
Instruction *InstCombiner::visitLShr(BinaryOperator &I) {
if (Value *V = SimplifyLShrInst(I.getOperand(0), I.getOperand(1),
I.isExact(), TD))
return ReplaceInstUsesWith(I, V);
if (Instruction *R = commonShiftTransforms(I))
return R;
Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
if (ConstantInt *Op1C = dyn_cast<ConstantInt>(Op1)) {
unsigned ShAmt = Op1C->getZExtValue();
if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Op0)) {
unsigned BitWidth = Op0->getType()->getScalarSizeInBits();
if ((II->getIntrinsicID() == Intrinsic::ctlz ||
II->getIntrinsicID() == Intrinsic::cttz ||
II->getIntrinsicID() == Intrinsic::ctpop) &&
isPowerOf2_32(BitWidth) && Log2_32(BitWidth) == ShAmt) {
bool isCtPop = II->getIntrinsicID() == Intrinsic::ctpop;
Constant *RHS = ConstantInt::getSigned(Op0->getType(), isCtPop ? -1:0);
Value *Cmp = Builder->CreateICmpEQ(II->getArgOperand(0), RHS);
return new ZExtInst(Cmp, II->getType());
}
}
if (!I.isExact() &&
MaskedValueIsZero(Op0,APInt::getLowBitsSet(Op1C->getBitWidth(),ShAmt))){
I.setIsExact();
return &I;
}
}
return 0;
}
Instruction *InstCombiner::visitAShr(BinaryOperator &I) {
if (Value *V = SimplifyAShrInst(I.getOperand(0), I.getOperand(1),
I.isExact(), TD))
return ReplaceInstUsesWith(I, V);
if (Instruction *R = commonShiftTransforms(I))
return R;
Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
if (ConstantInt *Op1C = dyn_cast<ConstantInt>(Op1)) {
unsigned ShAmt = Op1C->getZExtValue();
Value *X;
if (match(Op0, m_Shl(m_Value(X), m_Specific(Op1)))) {
if (cast<OverflowingBinaryOperator>(Op0)->hasNoSignedWrap())
return ReplaceInstUsesWith(I, X);
if (ZExtInst *ZI = dyn_cast<ZExtInst>(X)) {
uint32_t SrcBits = ZI->getOperand(0)->getType()->getScalarSizeInBits();
uint32_t DestBits = ZI->getType()->getScalarSizeInBits();
if (Op1C->getZExtValue() == DestBits-SrcBits)
return new SExtInst(ZI->getOperand(0), ZI->getType());
}
}
if (!I.isExact() &&
MaskedValueIsZero(Op0,APInt::getLowBitsSet(Op1C->getBitWidth(),ShAmt))){
I.setIsExact();
return &I;
}
}
if (MaskedValueIsZero(Op0,
APInt::getSignBit(I.getType()->getScalarSizeInBits())))
return BinaryOperator::CreateLShr(Op0, Op1);
unsigned NumSignBits = ComputeNumSignBits(Op0);
if (NumSignBits == Op0->getType()->getScalarSizeInBits())
return ReplaceInstUsesWith(I, Op0);
return 0;
}