#include "LegalizeTypes.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
using namespace llvm;
#define DEBUG_TYPE "legalize-types"
static cl::opt<bool>
EnableExpensiveChecks("enable-legalize-types-checking", cl::Hidden);
void DAGTypeLegalizer::PerformExpensiveChecks() {
SmallVector<SDNode*, 16> NewNodes;
for (SDNode &Node : DAG.allnodes()) {
if (Node.getNodeId() == NewNode)
NewNodes.push_back(&Node);
for (unsigned i = 0, e = Node.getNumValues(); i != e; ++i) {
SDValue Res(&Node, i);
bool Failed = false;
unsigned Mapped = 0;
if (ReplacedValues.find(Res) != ReplacedValues.end()) {
Mapped |= 1;
for (SDNode::use_iterator UI = Node.use_begin(), UE = Node.use_end();
UI != UE; ++UI)
if (UI.getUse().getResNo() == i)
assert(UI->getNodeId() == NewNode &&
"Remapped value has non-trivial use!");
SDValue NewVal = ReplacedValues[Res];
DenseMap<SDValue, SDValue>::iterator I = ReplacedValues.find(NewVal);
while (I != ReplacedValues.end()) {
NewVal = I->second;
I = ReplacedValues.find(NewVal);
}
assert(NewVal.getNode()->getNodeId() != NewNode &&
"ReplacedValues maps to a new node!");
}
if (PromotedIntegers.find(Res) != PromotedIntegers.end())
Mapped |= 2;
if (SoftenedFloats.find(Res) != SoftenedFloats.end())
Mapped |= 4;
if (ScalarizedVectors.find(Res) != ScalarizedVectors.end())
Mapped |= 8;
if (ExpandedIntegers.find(Res) != ExpandedIntegers.end())
Mapped |= 16;
if (ExpandedFloats.find(Res) != ExpandedFloats.end())
Mapped |= 32;
if (SplitVectors.find(Res) != SplitVectors.end())
Mapped |= 64;
if (WidenedVectors.find(Res) != WidenedVectors.end())
Mapped |= 128;
if (Node.getNodeId() != Processed) {
if ((Node.getNodeId() == NewNode && Mapped > 1) ||
(Node.getNodeId() != NewNode && Mapped != 0)) {
dbgs() << "Unprocessed value in a map!";
Failed = true;
}
} else if (isTypeLegal(Res.getValueType()) || IgnoreNodeResults(&Node)) {
if (Mapped > 1) {
dbgs() << "Value with legal type was transformed!";
Failed = true;
}
} else {
if (Mapped == 0) {
dbgs() << "Processed value not in any map!";
Failed = true;
} else if (Mapped & (Mapped - 1)) {
dbgs() << "Value in multiple maps!";
Failed = true;
}
}
if (Failed) {
if (Mapped & 1)
dbgs() << " ReplacedValues";
if (Mapped & 2)
dbgs() << " PromotedIntegers";
if (Mapped & 4)
dbgs() << " SoftenedFloats";
if (Mapped & 8)
dbgs() << " ScalarizedVectors";
if (Mapped & 16)
dbgs() << " ExpandedIntegers";
if (Mapped & 32)
dbgs() << " ExpandedFloats";
if (Mapped & 64)
dbgs() << " SplitVectors";
if (Mapped & 128)
dbgs() << " WidenedVectors";
dbgs() << "\n";
llvm_unreachable(nullptr);
}
}
}
for (unsigned i = 0, e = NewNodes.size(); i != e; ++i) {
SDNode *N = NewNodes[i];
for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
UI != UE; ++UI)
assert(UI->getNodeId() == NewNode && "NewNode used by non-NewNode!");
}
}
bool DAGTypeLegalizer::run() {
bool Changed = false;
HandleSDNode Dummy(DAG.getRoot());
Dummy.setNodeId(Unanalyzed);
DAG.setRoot(SDValue());
for (SDNode &Node : DAG.allnodes()) {
if (Node.getNumOperands() == 0) {
Node.setNodeId(ReadyToProcess);
Worklist.push_back(&Node);
} else {
Node.setNodeId(Unanalyzed);
}
}
while (!Worklist.empty()) {
#ifndef XDEBUG
if (EnableExpensiveChecks)
#endif
PerformExpensiveChecks();
SDNode *N = Worklist.back();
Worklist.pop_back();
assert(N->getNodeId() == ReadyToProcess &&
"Node should be ready if on worklist!");
if (IgnoreNodeResults(N))
goto ScanOperands;
for (unsigned i = 0, NumResults = N->getNumValues(); i < NumResults; ++i) {
EVT ResultVT = N->getValueType(i);
switch (getTypeAction(ResultVT)) {
case TargetLowering::TypeLegal:
break;
case TargetLowering::TypePromoteInteger:
PromoteIntegerResult(N, i);
Changed = true;
goto NodeDone;
case TargetLowering::TypeExpandInteger:
ExpandIntegerResult(N, i);
Changed = true;
goto NodeDone;
case TargetLowering::TypeSoftenFloat:
SoftenFloatResult(N, i);
Changed = true;
goto NodeDone;
case TargetLowering::TypeExpandFloat:
ExpandFloatResult(N, i);
Changed = true;
goto NodeDone;
case TargetLowering::TypeScalarizeVector:
ScalarizeVectorResult(N, i);
Changed = true;
goto NodeDone;
case TargetLowering::TypeSplitVector:
SplitVectorResult(N, i);
Changed = true;
goto NodeDone;
case TargetLowering::TypeWidenVector:
WidenVectorResult(N, i);
Changed = true;
goto NodeDone;
case TargetLowering::TypePromoteFloat:
PromoteFloatResult(N, i);
Changed = true;
goto NodeDone;
}
}
ScanOperands:
{
unsigned NumOperands = N->getNumOperands();
bool NeedsReanalyzing = false;
unsigned i;
for (i = 0; i != NumOperands; ++i) {
if (IgnoreNodeResults(N->getOperand(i).getNode()))
continue;
EVT OpVT = N->getOperand(i).getValueType();
switch (getTypeAction(OpVT)) {
case TargetLowering::TypeLegal:
continue;
case TargetLowering::TypePromoteInteger:
NeedsReanalyzing = PromoteIntegerOperand(N, i);
Changed = true;
break;
case TargetLowering::TypeExpandInteger:
NeedsReanalyzing = ExpandIntegerOperand(N, i);
Changed = true;
break;
case TargetLowering::TypeSoftenFloat:
NeedsReanalyzing = SoftenFloatOperand(N, i);
Changed = true;
break;
case TargetLowering::TypeExpandFloat:
NeedsReanalyzing = ExpandFloatOperand(N, i);
Changed = true;
break;
case TargetLowering::TypeScalarizeVector:
NeedsReanalyzing = ScalarizeVectorOperand(N, i);
Changed = true;
break;
case TargetLowering::TypeSplitVector:
NeedsReanalyzing = SplitVectorOperand(N, i);
Changed = true;
break;
case TargetLowering::TypeWidenVector:
NeedsReanalyzing = WidenVectorOperand(N, i);
Changed = true;
break;
case TargetLowering::TypePromoteFloat:
NeedsReanalyzing = PromoteFloatOperand(N, i);
Changed = true;
break;
}
break;
}
if (NeedsReanalyzing) {
assert(N->getNodeId() == ReadyToProcess && "Node ID recalculated?");
N->setNodeId(NewNode);
SDNode *M = AnalyzeNewNode(N);
if (M == N)
continue;
assert(N->getNumValues() == M->getNumValues() &&
"Node morphing changed the number of results!");
for (unsigned i = 0, e = N->getNumValues(); i != e; ++i)
ReplaceValueWith(SDValue(N, i), SDValue(M, i));
assert(N->getNodeId() == NewNode && "Unexpected node state!");
continue;
}
if (i == NumOperands) {
DEBUG(dbgs() << "Legally typed node: "; N->dump(&DAG); dbgs() << "\n");
}
}
NodeDone:
assert(N->getNodeId() == ReadyToProcess && "Node ID recalculated?");
N->setNodeId(Processed);
for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end();
UI != E; ++UI) {
SDNode *User = *UI;
int NodeId = User->getNodeId();
if (NodeId > 0) {
User->setNodeId(NodeId-1);
if (NodeId-1 == ReadyToProcess)
Worklist.push_back(User);
continue;
}
if (NodeId == NewNode)
continue;
assert(NodeId == Unanalyzed && "Unknown node ID!");
User->setNodeId(User->getNumOperands() - 1);
if (User->getNumOperands() == 1)
Worklist.push_back(User);
}
}
#ifndef XDEBUG
if (EnableExpensiveChecks)
#endif
PerformExpensiveChecks();
DAG.setRoot(Dummy.getValue());
DAG.RemoveDeadNodes();
#ifndef NDEBUG
for (SDNode &Node : DAG.allnodes()) {
bool Failed = false;
if (!IgnoreNodeResults(&Node))
for (unsigned i = 0, NumVals = Node.getNumValues(); i < NumVals; ++i)
if (!isTypeLegal(Node.getValueType(i))) {
dbgs() << "Result type " << i << " illegal!\n";
Failed = true;
}
for (unsigned i = 0, NumOps = Node.getNumOperands(); i < NumOps; ++i)
if (!IgnoreNodeResults(Node.getOperand(i).getNode()) &&
!isTypeLegal(Node.getOperand(i).getValueType())) {
dbgs() << "Operand type " << i << " illegal!\n";
Failed = true;
}
if (Node.getNodeId() != Processed) {
if (Node.getNodeId() == NewNode)
dbgs() << "New node not analyzed?\n";
else if (Node.getNodeId() == Unanalyzed)
dbgs() << "Unanalyzed node not noticed?\n";
else if (Node.getNodeId() > 0)
dbgs() << "Operand not processed?\n";
else if (Node.getNodeId() == ReadyToProcess)
dbgs() << "Not added to worklist?\n";
Failed = true;
}
if (Failed) {
Node.dump(&DAG); dbgs() << "\n";
llvm_unreachable(nullptr);
}
}
#endif
return Changed;
}
SDNode *DAGTypeLegalizer::AnalyzeNewNode(SDNode *N) {
if (N->getNodeId() != NewNode && N->getNodeId() != Unanalyzed)
return N;
ExpungeNode(N);
SmallVector<SDValue, 8> NewOps;
unsigned NumProcessed = 0;
for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
SDValue OrigOp = N->getOperand(i);
SDValue Op = OrigOp;
AnalyzeNewValue(Op);
if (Op.getNode()->getNodeId() == Processed)
++NumProcessed;
if (!NewOps.empty()) {
NewOps.push_back(Op);
} else if (Op != OrigOp) {
NewOps.append(N->op_begin(), N->op_begin() + i);
NewOps.push_back(Op);
}
}
if (!NewOps.empty()) {
SDNode *M = DAG.UpdateNodeOperands(N, NewOps);
if (M != N) {
N->setNodeId(NewNode);
if (M->getNodeId() != NewNode && M->getNodeId() != Unanalyzed)
return M;
N = M;
ExpungeNode(N);
}
}
N->setNodeId(N->getNumOperands() - NumProcessed);
if (N->getNodeId() == ReadyToProcess)
Worklist.push_back(N);
return N;
}
void DAGTypeLegalizer::AnalyzeNewValue(SDValue &Val) {
Val.setNode(AnalyzeNewNode(Val.getNode()));
if (Val.getNode()->getNodeId() == Processed)
RemapValue(Val);
}
void DAGTypeLegalizer::ExpungeNode(SDNode *N) {
if (N->getNodeId() != NewNode)
return;
unsigned i, e;
for (i = 0, e = N->getNumValues(); i != e; ++i)
if (ReplacedValues.find(SDValue(N, i)) != ReplacedValues.end())
break;
if (i == e)
return;
for (DenseMap<SDValue, SDValue>::iterator I = PromotedIntegers.begin(),
E = PromotedIntegers.end(); I != E; ++I) {
assert(I->first.getNode() != N);
RemapValue(I->second);
}
for (DenseMap<SDValue, SDValue>::iterator I = SoftenedFloats.begin(),
E = SoftenedFloats.end(); I != E; ++I) {
assert(I->first.getNode() != N);
RemapValue(I->second);
}
for (DenseMap<SDValue, SDValue>::iterator I = ScalarizedVectors.begin(),
E = ScalarizedVectors.end(); I != E; ++I) {
assert(I->first.getNode() != N);
RemapValue(I->second);
}
for (DenseMap<SDValue, SDValue>::iterator I = WidenedVectors.begin(),
E = WidenedVectors.end(); I != E; ++I) {
assert(I->first.getNode() != N);
RemapValue(I->second);
}
for (DenseMap<SDValue, std::pair<SDValue, SDValue> >::iterator
I = ExpandedIntegers.begin(), E = ExpandedIntegers.end(); I != E; ++I){
assert(I->first.getNode() != N);
RemapValue(I->second.first);
RemapValue(I->second.second);
}
for (DenseMap<SDValue, std::pair<SDValue, SDValue> >::iterator
I = ExpandedFloats.begin(), E = ExpandedFloats.end(); I != E; ++I) {
assert(I->first.getNode() != N);
RemapValue(I->second.first);
RemapValue(I->second.second);
}
for (DenseMap<SDValue, std::pair<SDValue, SDValue> >::iterator
I = SplitVectors.begin(), E = SplitVectors.end(); I != E; ++I) {
assert(I->first.getNode() != N);
RemapValue(I->second.first);
RemapValue(I->second.second);
}
for (DenseMap<SDValue, SDValue>::iterator I = ReplacedValues.begin(),
E = ReplacedValues.end(); I != E; ++I)
RemapValue(I->second);
for (unsigned i = 0, e = N->getNumValues(); i != e; ++i)
ReplacedValues.erase(SDValue(N, i));
}
void DAGTypeLegalizer::RemapValue(SDValue &N) {
DenseMap<SDValue, SDValue>::iterator I = ReplacedValues.find(N);
if (I != ReplacedValues.end()) {
RemapValue(I->second);
N = I->second;
}
}
namespace {
class NodeUpdateListener : public SelectionDAG::DAGUpdateListener {
DAGTypeLegalizer &DTL;
SmallSetVector<SDNode*, 16> &NodesToAnalyze;
public:
explicit NodeUpdateListener(DAGTypeLegalizer &dtl,
SmallSetVector<SDNode*, 16> &nta)
: SelectionDAG::DAGUpdateListener(dtl.getDAG()),
DTL(dtl), NodesToAnalyze(nta) {}
void NodeDeleted(SDNode *N, SDNode *E) override {
assert(N->getNodeId() != DAGTypeLegalizer::ReadyToProcess &&
N->getNodeId() != DAGTypeLegalizer::Processed &&
"Invalid node ID for RAUW deletion!");
assert(E && "Node not replaced?");
DTL.NoteDeletion(N, E);
NodesToAnalyze.remove(N);
if (E->getNodeId() == DAGTypeLegalizer::NewNode)
NodesToAnalyze.insert(E);
}
void NodeUpdated(SDNode *N) override {
assert(N->getNodeId() != DAGTypeLegalizer::ReadyToProcess &&
N->getNodeId() != DAGTypeLegalizer::Processed &&
"Invalid node ID for RAUW deletion!");
N->setNodeId(DAGTypeLegalizer::NewNode);
NodesToAnalyze.insert(N);
}
};
}
void DAGTypeLegalizer::ReplaceValueWith(SDValue From, SDValue To) {
assert(From.getNode() != To.getNode() && "Potential legalization loop!");
ExpungeNode(From.getNode());
AnalyzeNewValue(To);
SmallSetVector<SDNode*, 16> NodesToAnalyze;
NodeUpdateListener NUL(*this, NodesToAnalyze);
do {
DAG.ReplaceAllUsesOfValueWith(From, To);
ReplacedValues[From] = To;
while (!NodesToAnalyze.empty()) {
SDNode *N = NodesToAnalyze.back();
NodesToAnalyze.pop_back();
if (N->getNodeId() != DAGTypeLegalizer::NewNode)
continue;
SDNode *M = AnalyzeNewNode(N);
if (M != N) {
assert(M->getNodeId() != NewNode && "Analysis resulted in NewNode!");
assert(N->getNumValues() == M->getNumValues() &&
"Node morphing changed the number of results!");
for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) {
SDValue OldVal(N, i);
SDValue NewVal(M, i);
if (M->getNodeId() == Processed)
RemapValue(NewVal);
DAG.ReplaceAllUsesOfValueWith(OldVal, NewVal);
ReplacedValues[OldVal] = NewVal;
}
}
}
} while (!From.use_empty());
}
void DAGTypeLegalizer::SetPromotedInteger(SDValue Op, SDValue Result) {
assert(Result.getValueType() ==
TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
"Invalid type for promoted integer");
AnalyzeNewValue(Result);
SDValue &OpEntry = PromotedIntegers[Op];
assert(!OpEntry.getNode() && "Node is already promoted!");
OpEntry = Result;
}
void DAGTypeLegalizer::SetSoftenedFloat(SDValue Op, SDValue Result) {
assert(Result.getValueType() ==
TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
"Invalid type for softened float");
AnalyzeNewValue(Result);
SDValue &OpEntry = SoftenedFloats[Op];
assert(!OpEntry.getNode() && "Node is already converted to integer!");
OpEntry = Result;
}
void DAGTypeLegalizer::SetPromotedFloat(SDValue Op, SDValue Result) {
assert(Result.getValueType() ==
TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
"Invalid type for promoted float");
AnalyzeNewValue(Result);
SDValue &OpEntry = PromotedFloats[Op];
assert(!OpEntry.getNode() && "Node is already promoted!");
OpEntry = Result;
}
void DAGTypeLegalizer::SetScalarizedVector(SDValue Op, SDValue Result) {
assert(Result.getValueType().getSizeInBits() >=
Op.getValueType().getVectorElementType().getSizeInBits() &&
"Invalid type for scalarized vector");
AnalyzeNewValue(Result);
SDValue &OpEntry = ScalarizedVectors[Op];
assert(!OpEntry.getNode() && "Node is already scalarized!");
OpEntry = Result;
}
void DAGTypeLegalizer::GetExpandedInteger(SDValue Op, SDValue &Lo,
SDValue &Hi) {
std::pair<SDValue, SDValue> &Entry = ExpandedIntegers[Op];
RemapValue(Entry.first);
RemapValue(Entry.second);
assert(Entry.first.getNode() && "Operand isn't expanded");
Lo = Entry.first;
Hi = Entry.second;
}
void DAGTypeLegalizer::SetExpandedInteger(SDValue Op, SDValue Lo,
SDValue Hi) {
assert(Lo.getValueType() ==
TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
Hi.getValueType() == Lo.getValueType() &&
"Invalid type for expanded integer");
AnalyzeNewValue(Lo);
AnalyzeNewValue(Hi);
std::pair<SDValue, SDValue> &Entry = ExpandedIntegers[Op];
assert(!Entry.first.getNode() && "Node already expanded");
Entry.first = Lo;
Entry.second = Hi;
}
void DAGTypeLegalizer::GetExpandedFloat(SDValue Op, SDValue &Lo,
SDValue &Hi) {
std::pair<SDValue, SDValue> &Entry = ExpandedFloats[Op];
RemapValue(Entry.first);
RemapValue(Entry.second);
assert(Entry.first.getNode() && "Operand isn't expanded");
Lo = Entry.first;
Hi = Entry.second;
}
void DAGTypeLegalizer::SetExpandedFloat(SDValue Op, SDValue Lo,
SDValue Hi) {
assert(Lo.getValueType() ==
TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
Hi.getValueType() == Lo.getValueType() &&
"Invalid type for expanded float");
AnalyzeNewValue(Lo);
AnalyzeNewValue(Hi);
std::pair<SDValue, SDValue> &Entry = ExpandedFloats[Op];
assert(!Entry.first.getNode() && "Node already expanded");
Entry.first = Lo;
Entry.second = Hi;
}
void DAGTypeLegalizer::GetSplitVector(SDValue Op, SDValue &Lo,
SDValue &Hi) {
std::pair<SDValue, SDValue> &Entry = SplitVectors[Op];
RemapValue(Entry.first);
RemapValue(Entry.second);
assert(Entry.first.getNode() && "Operand isn't split");
Lo = Entry.first;
Hi = Entry.second;
}
void DAGTypeLegalizer::SetSplitVector(SDValue Op, SDValue Lo,
SDValue Hi) {
assert(Lo.getValueType().getVectorElementType() ==
Op.getValueType().getVectorElementType() &&
2*Lo.getValueType().getVectorNumElements() ==
Op.getValueType().getVectorNumElements() &&
Hi.getValueType() == Lo.getValueType() &&
"Invalid type for split vector");
AnalyzeNewValue(Lo);
AnalyzeNewValue(Hi);
std::pair<SDValue, SDValue> &Entry = SplitVectors[Op];
assert(!Entry.first.getNode() && "Node already split");
Entry.first = Lo;
Entry.second = Hi;
}
void DAGTypeLegalizer::SetWidenedVector(SDValue Op, SDValue Result) {
assert(Result.getValueType() ==
TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
"Invalid type for widened vector");
AnalyzeNewValue(Result);
SDValue &OpEntry = WidenedVectors[Op];
assert(!OpEntry.getNode() && "Node already widened!");
OpEntry = Result;
}
SDValue DAGTypeLegalizer::BitConvertToInteger(SDValue Op) {
unsigned BitWidth = Op.getValueType().getSizeInBits();
return DAG.getNode(ISD::BITCAST, SDLoc(Op),
EVT::getIntegerVT(*DAG.getContext(), BitWidth), Op);
}
SDValue DAGTypeLegalizer::BitConvertVectorToIntegerVector(SDValue Op) {
assert(Op.getValueType().isVector() && "Only applies to vectors!");
unsigned EltWidth = Op.getValueType().getVectorElementType().getSizeInBits();
EVT EltNVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth);
unsigned NumElts = Op.getValueType().getVectorNumElements();
return DAG.getNode(ISD::BITCAST, SDLoc(Op),
EVT::getVectorVT(*DAG.getContext(), EltNVT, NumElts), Op);
}
SDValue DAGTypeLegalizer::CreateStackStoreLoad(SDValue Op,
EVT DestVT) {
SDLoc dl(Op);
SDValue StackPtr = DAG.CreateStackTemporary(Op.getValueType(), DestVT);
SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op, StackPtr,
MachinePointerInfo(), false, false, 0);
return DAG.getLoad(DestVT, dl, Store, StackPtr, MachinePointerInfo(),
false, false, false, 0);
}
bool DAGTypeLegalizer::CustomLowerNode(SDNode *N, EVT VT, bool LegalizeResult) {
if (TLI.getOperationAction(N->getOpcode(), VT) != TargetLowering::Custom)
return false;
SmallVector<SDValue, 8> Results;
if (LegalizeResult)
TLI.ReplaceNodeResults(N, Results, DAG);
else
TLI.LowerOperationWrapper(N, Results, DAG);
if (Results.empty())
return false;
if (Results.size() == N->getNumValues() + 1 && LegalizeResult) {
SetExpandedInteger(SDValue(N, 0), Results[0], Results[1]);
if (N->getNumValues() > 1)
ReplaceValueWith(SDValue(N, 1), Results[2]);
return true;
}
assert(Results.size() == N->getNumValues() &&
"Custom lowering returned the wrong number of results!");
for (unsigned i = 0, e = Results.size(); i != e; ++i) {
ReplaceValueWith(SDValue(N, i), Results[i]);
}
return true;
}
bool DAGTypeLegalizer::CustomWidenLowerNode(SDNode *N, EVT VT) {
if (TLI.getOperationAction(N->getOpcode(), VT) != TargetLowering::Custom)
return false;
SmallVector<SDValue, 8> Results;
TLI.ReplaceNodeResults(N, Results, DAG);
if (Results.empty())
return false;
assert(Results.size() == N->getNumValues() &&
"Custom lowering returned the wrong number of results!");
for (unsigned i = 0, e = Results.size(); i != e; ++i)
SetWidenedVector(SDValue(N, i), Results[i]);
return true;
}
SDValue DAGTypeLegalizer::DisintegrateMERGE_VALUES(SDNode *N, unsigned ResNo) {
for (unsigned i = 0, e = N->getNumValues(); i != e; ++i)
if (i != ResNo)
ReplaceValueWith(SDValue(N, i), SDValue(N->getOperand(i)));
return SDValue(N->getOperand(ResNo));
}
void DAGTypeLegalizer::GetPairElements(SDValue Pair,
SDValue &Lo, SDValue &Hi) {
SDLoc dl(Pair);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Pair.getValueType());
Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, NVT, Pair,
DAG.getIntPtrConstant(0, dl));
Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, NVT, Pair,
DAG.getIntPtrConstant(1, dl));
}
SDValue DAGTypeLegalizer::GetVectorElementPointer(SDValue VecPtr, EVT EltVT,
SDValue Index) {
SDLoc dl(Index);
Index = DAG.getZExtOrTrunc(Index, dl, TLI.getPointerTy(DAG.getDataLayout()));
unsigned EltSize = EltVT.getSizeInBits() / 8; assert(EltSize * 8 == EltVT.getSizeInBits() &&
"Converting bits to bytes lost precision");
Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
DAG.getConstant(EltSize, dl, Index.getValueType()));
return DAG.getNode(ISD::ADD, dl, Index.getValueType(), Index, VecPtr);
}
SDValue DAGTypeLegalizer::JoinIntegers(SDValue Lo, SDValue Hi) {
SDLoc dlHi(Hi);
SDLoc dlLo(Lo);
EVT LVT = Lo.getValueType();
EVT HVT = Hi.getValueType();
EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
LVT.getSizeInBits() + HVT.getSizeInBits());
Lo = DAG.getNode(ISD::ZERO_EXTEND, dlLo, NVT, Lo);
Hi = DAG.getNode(ISD::ANY_EXTEND, dlHi, NVT, Hi);
Hi = DAG.getNode(ISD::SHL, dlHi, NVT, Hi,
DAG.getConstant(LVT.getSizeInBits(), dlHi,
TLI.getPointerTy(DAG.getDataLayout())));
return DAG.getNode(ISD::OR, dlHi, NVT, Lo, Hi);
}
SDValue DAGTypeLegalizer::LibCallify(RTLIB::Libcall LC, SDNode *N,
bool isSigned) {
unsigned NumOps = N->getNumOperands();
SDLoc dl(N);
if (NumOps == 0) {
return TLI.makeLibCall(DAG, LC, N->getValueType(0), None, isSigned,
dl).first;
} else if (NumOps == 1) {
SDValue Op = N->getOperand(0);
return TLI.makeLibCall(DAG, LC, N->getValueType(0), Op, isSigned,
dl).first;
} else if (NumOps == 2) {
SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
return TLI.makeLibCall(DAG, LC, N->getValueType(0), Ops, isSigned,
dl).first;
}
SmallVector<SDValue, 8> Ops(NumOps);
for (unsigned i = 0; i < NumOps; ++i)
Ops[i] = N->getOperand(i);
return TLI.makeLibCall(DAG, LC, N->getValueType(0), Ops, isSigned, dl).first;
}
std::pair<SDValue, SDValue>
DAGTypeLegalizer::ExpandChainLibCall(RTLIB::Libcall LC,
SDNode *Node,
bool isSigned) {
SDValue InChain = Node->getOperand(0);
TargetLowering::ArgListTy Args;
TargetLowering::ArgListEntry Entry;
for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
EVT ArgVT = Node->getOperand(i).getValueType();
Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
Entry.Node = Node->getOperand(i);
Entry.Ty = ArgTy;
Entry.isSExt = isSigned;
Entry.isZExt = !isSigned;
Args.push_back(Entry);
}
SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
TLI.getPointerTy(DAG.getDataLayout()));
Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
TargetLowering::CallLoweringInfo CLI(DAG);
CLI.setDebugLoc(SDLoc(Node)).setChain(InChain)
.setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args), 0)
.setSExtResult(isSigned).setZExtResult(!isSigned);
std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
return CallInfo;
}
SDValue DAGTypeLegalizer::PromoteTargetBoolean(SDValue Bool, EVT ValVT) {
SDLoc dl(Bool);
EVT BoolVT = getSetCCResultType(ValVT);
ISD::NodeType ExtendCode =
TargetLowering::getExtendForContent(TLI.getBooleanContents(ValVT));
return DAG.getNode(ExtendCode, dl, BoolVT, Bool);
}
void DAGTypeLegalizer::SplitInteger(SDValue Op,
EVT LoVT, EVT HiVT,
SDValue &Lo, SDValue &Hi) {
SDLoc dl(Op);
assert(LoVT.getSizeInBits() + HiVT.getSizeInBits() ==
Op.getValueType().getSizeInBits() && "Invalid integer splitting!");
Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Op);
Hi = DAG.getNode(ISD::SRL, dl, Op.getValueType(), Op,
DAG.getConstant(LoVT.getSizeInBits(), dl,
TLI.getPointerTy(DAG.getDataLayout())));
Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
}
void DAGTypeLegalizer::SplitInteger(SDValue Op,
SDValue &Lo, SDValue &Hi) {
EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(),
Op.getValueType().getSizeInBits()/2);
SplitInteger(Op, HalfVT, HalfVT, Lo, Hi);
}
bool SelectionDAG::LegalizeTypes() {
return DAGTypeLegalizer(*this).run();
}