diff options
Diffstat (limited to 'lib/CodeGen/SelectionDAG/SelectionDAG.cpp')
| -rw-r--r-- | lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 494 |
1 files changed, 324 insertions, 170 deletions
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 5477e7b..6a8c92e 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -31,7 +31,6 @@ #include "llvm/CodeGen/PseudoSourceValue.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetData.h" -#include "llvm/Target/TargetFrameInfo.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetSelectionDAGInfo.h" #include "llvm/Target/TargetOptions.h" @@ -44,7 +43,7 @@ #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/System/Mutex.h" +#include "llvm/Support/Mutex.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" @@ -111,7 +110,7 @@ bool ConstantFPSDNode::isValueValidForType(EVT VT, /// BUILD_VECTOR where all of the elements are ~0 or undef. bool ISD::isBuildVectorAllOnes(const SDNode *N) { // Look through a bit convert. - if (N->getOpcode() == ISD::BIT_CONVERT) + if (N->getOpcode() == ISD::BITCAST) N = N->getOperand(0).getNode(); if (N->getOpcode() != ISD::BUILD_VECTOR) return false; @@ -152,7 +151,7 @@ bool ISD::isBuildVectorAllOnes(const SDNode *N) { /// BUILD_VECTOR where all of the elements are 0 or undef. bool ISD::isBuildVectorAllZeros(const SDNode *N) { // Look through a bit convert. - if (N->getOpcode() == ISD::BIT_CONVERT) + if (N->getOpcode() == ISD::BITCAST) N = N->getOperand(0).getNode(); if (N->getOpcode() != ISD::BUILD_VECTOR) return false; @@ -199,6 +198,8 @@ bool ISD::isScalarToVector(const SDNode *N) { if (N->getOperand(0).getOpcode() == ISD::UNDEF) return false; unsigned NumElems = N->getNumOperands(); + if (NumElems == 1) + return false; for (unsigned i = 1; i < NumElems; ++i) { SDValue V = N->getOperand(i); if (V.getOpcode() != ISD::UNDEF) @@ -489,7 +490,7 @@ encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile, /// doNotCSE - Return true if CSE should not be performed for this node. static bool doNotCSE(SDNode *N) { - if (N->getValueType(0) == MVT::Flag) + if (N->getValueType(0) == MVT::Glue) return true; // Never CSE anything that produces a flag. switch (N->getOpcode()) { @@ -501,7 +502,7 @@ static bool doNotCSE(SDNode *N) { // Check that remaining values produced are not flags. for (unsigned i = 1, e = N->getNumValues(); i != e; ++i) - if (N->getValueType(i) == MVT::Flag) + if (N->getValueType(i) == MVT::Glue) return true; // Never CSE anything that produces a flag. return false; @@ -609,9 +610,6 @@ void SelectionDAG::DeallocateNode(SDNode *N) { bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { bool Erased = false; switch (N->getOpcode()) { - case ISD::EntryToken: - llvm_unreachable("EntryToken should not be in CSEMaps!"); - return false; case ISD::HANDLENODE: return false; // noop. case ISD::CONDCODE: assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && @@ -641,6 +639,8 @@ bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { } default: // Remove it from the CSE Map. + assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); + assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); Erased = CSEMap.RemoveNode(N); break; } @@ -648,7 +648,7 @@ bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { // Verify that the node was actually in one of the CSE maps, unless it has a // flag result (which cannot be CSE'd) or is one of the special cases that are // not subject to CSE. - if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Flag && + if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue && !N->isMachineOpcode() && !doNotCSE(N)) { N->dump(this); dbgs() << "\n"; @@ -743,8 +743,9 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, return Node; } -/// VerifyNode - Sanity check the given node. Aborts if it is invalid. -void SelectionDAG::VerifyNode(SDNode *N) { +#ifndef NDEBUG +/// VerifyNodeCommon - Sanity check the given node. Aborts if it is invalid. +static void VerifyNodeCommon(SDNode *N) { switch (N->getOpcode()) { default: break; @@ -778,6 +779,44 @@ void SelectionDAG::VerifyNode(SDNode *N) { } } +/// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid. +static void VerifySDNode(SDNode *N) { + // The SDNode allocators cannot be used to allocate nodes with fields that are + // not present in an SDNode! + assert(!isa<MemSDNode>(N) && "Bad MemSDNode!"); + assert(!isa<ShuffleVectorSDNode>(N) && "Bad ShuffleVectorSDNode!"); + assert(!isa<ConstantSDNode>(N) && "Bad ConstantSDNode!"); + assert(!isa<ConstantFPSDNode>(N) && "Bad ConstantFPSDNode!"); + assert(!isa<GlobalAddressSDNode>(N) && "Bad GlobalAddressSDNode!"); + assert(!isa<FrameIndexSDNode>(N) && "Bad FrameIndexSDNode!"); + assert(!isa<JumpTableSDNode>(N) && "Bad JumpTableSDNode!"); + assert(!isa<ConstantPoolSDNode>(N) && "Bad ConstantPoolSDNode!"); + assert(!isa<BasicBlockSDNode>(N) && "Bad BasicBlockSDNode!"); + assert(!isa<SrcValueSDNode>(N) && "Bad SrcValueSDNode!"); + assert(!isa<MDNodeSDNode>(N) && "Bad MDNodeSDNode!"); + assert(!isa<RegisterSDNode>(N) && "Bad RegisterSDNode!"); + assert(!isa<BlockAddressSDNode>(N) && "Bad BlockAddressSDNode!"); + assert(!isa<EHLabelSDNode>(N) && "Bad EHLabelSDNode!"); + assert(!isa<ExternalSymbolSDNode>(N) && "Bad ExternalSymbolSDNode!"); + assert(!isa<CondCodeSDNode>(N) && "Bad CondCodeSDNode!"); + assert(!isa<CvtRndSatSDNode>(N) && "Bad CvtRndSatSDNode!"); + assert(!isa<VTSDNode>(N) && "Bad VTSDNode!"); + assert(!isa<MachineSDNode>(N) && "Bad MachineSDNode!"); + + VerifyNodeCommon(N); +} + +/// VerifyMachineNode - Sanity check the given MachineNode. Aborts if it is +/// invalid. +static void VerifyMachineNode(SDNode *N) { + // The MachineNode allocators cannot be used to allocate nodes with fields + // that are not present in a MachineNode! + // Currently there are no such nodes. + + VerifyNodeCommon(N); +} +#endif // NDEBUG + /// getEVTAlignment - Compute the default alignment value for the /// given type. /// @@ -1315,7 +1354,7 @@ SDValue SelectionDAG::getEHLabel(DebugLoc dl, SDValue Root, MCSymbol *Label) { void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - + SDNode *N = new (NodeAllocator) EHLabelSDNode(dl, Root, Label); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); @@ -1365,11 +1404,11 @@ SDValue SelectionDAG::getMDNode(const MDNode *MD) { FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), 0, 0); ID.AddPointer(MD); - + void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - + SDNode *N = new (NodeAllocator) MDNodeSDNode(MD); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); @@ -1381,7 +1420,7 @@ SDValue SelectionDAG::getMDNode(const MDNode *MD) { /// the target's desired shift amount type. SDValue SelectionDAG::getShiftAmountOperand(SDValue Op) { EVT OpTy = Op.getValueType(); - MVT ShTy = TLI.getShiftAmountTy(); + MVT ShTy = TLI.getShiftAmountTy(OpTy); if (OpTy == ShTy || OpTy.isVector()) return Op; ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; @@ -1613,7 +1652,7 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, // Also compute a conserative estimate for high known-0 bits. // More trickiness is possible, but this is sufficient for the // interesting case of alignment computation. - KnownOne.clear(); + KnownOne.clearAllBits(); unsigned TrailZ = KnownZero.countTrailingOnes() + KnownZero2.countTrailingOnes(); unsigned LeadZ = std::max(KnownZero.countLeadingOnes() + @@ -1636,8 +1675,8 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, AllOnes, KnownZero2, KnownOne2, Depth+1); unsigned LeadZ = KnownZero2.countLeadingOnes(); - KnownOne2.clear(); - KnownZero2.clear(); + KnownOne2.clearAllBits(); + KnownZero2.clearAllBits(); ComputeMaskedBits(Op.getOperand(1), AllOnes, KnownZero2, KnownOne2, Depth+1); unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros(); @@ -1765,7 +1804,7 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, // If the sign extended bits are demanded, we know that the sign // bit is demanded. - InSignBit.zext(BitWidth); + InSignBit = InSignBit.zext(BitWidth); if (NewBits.getBoolValue()) InputDemandedBits |= InSignBit; @@ -1792,7 +1831,7 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, case ISD::CTPOP: { unsigned LowBits = Log2_32(BitWidth)+1; KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits); - KnownOne.clear(); + KnownOne.clearAllBits(); return; } case ISD::LOAD: { @@ -1808,13 +1847,12 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, EVT InVT = Op.getOperand(0).getValueType(); unsigned InBits = InVT.getScalarType().getSizeInBits(); APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits) & Mask; - APInt InMask = Mask; - InMask.trunc(InBits); - KnownZero.trunc(InBits); - KnownOne.trunc(InBits); + APInt InMask = Mask.trunc(InBits); + KnownZero = KnownZero.trunc(InBits); + KnownOne = KnownOne.trunc(InBits); ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); - KnownZero.zext(BitWidth); - KnownOne.zext(BitWidth); + KnownZero = KnownZero.zext(BitWidth); + KnownOne = KnownOne.zext(BitWidth); KnownZero |= NewBits; return; } @@ -1823,16 +1861,15 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, unsigned InBits = InVT.getScalarType().getSizeInBits(); APInt InSignBit = APInt::getSignBit(InBits); APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits) & Mask; - APInt InMask = Mask; - InMask.trunc(InBits); + APInt InMask = Mask.trunc(InBits); // If any of the sign extended bits are demanded, we know that the sign // bit is demanded. Temporarily set this bit in the mask for our callee. if (NewBits.getBoolValue()) InMask |= InSignBit; - KnownZero.trunc(InBits); - KnownOne.trunc(InBits); + KnownZero = KnownZero.trunc(InBits); + KnownOne = KnownOne.trunc(InBits); ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); // Note if the sign bit is known to be zero or one. @@ -1844,13 +1881,12 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, // If the sign bit wasn't actually demanded by our caller, we don't // want it set in the KnownZero and KnownOne result values. Reset the // mask and reapply it to the result values. - InMask = Mask; - InMask.trunc(InBits); + InMask = Mask.trunc(InBits); KnownZero &= InMask; KnownOne &= InMask; - KnownZero.zext(BitWidth); - KnownOne.zext(BitWidth); + KnownZero = KnownZero.zext(BitWidth); + KnownOne = KnownOne.zext(BitWidth); // If the sign bit is known zero or one, the top bits match. if (SignBitKnownZero) @@ -1862,26 +1898,24 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, case ISD::ANY_EXTEND: { EVT InVT = Op.getOperand(0).getValueType(); unsigned InBits = InVT.getScalarType().getSizeInBits(); - APInt InMask = Mask; - InMask.trunc(InBits); - KnownZero.trunc(InBits); - KnownOne.trunc(InBits); + APInt InMask = Mask.trunc(InBits); + KnownZero = KnownZero.trunc(InBits); + KnownOne = KnownOne.trunc(InBits); ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); - KnownZero.zext(BitWidth); - KnownOne.zext(BitWidth); + KnownZero = KnownZero.zext(BitWidth); + KnownOne = KnownOne.zext(BitWidth); return; } case ISD::TRUNCATE: { EVT InVT = Op.getOperand(0).getValueType(); unsigned InBits = InVT.getScalarType().getSizeInBits(); - APInt InMask = Mask; - InMask.zext(InBits); - KnownZero.zext(InBits); - KnownOne.zext(InBits); + APInt InMask = Mask.zext(InBits); + KnownZero = KnownZero.zext(InBits); + KnownOne = KnownOne.zext(InBits); ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); - KnownZero.trunc(BitWidth); - KnownOne.trunc(BitWidth); + KnownZero = KnownZero.trunc(BitWidth); + KnownOne = KnownOne.trunc(BitWidth); break; } case ISD::AssertZext: { @@ -1921,7 +1955,8 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, } } // fall through - case ISD::ADD: { + case ISD::ADD: + case ISD::ADDE: { // Output known-0 bits are known if clear or set in both the low clear bits // common to both LHS & RHS. For example, 8+(X<<3) is known to have the // low 3 bits clear. @@ -1936,7 +1971,17 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, KnownZeroOut = std::min(KnownZeroOut, KnownZero2.countTrailingOnes()); - KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut); + if (Op.getOpcode() == ISD::ADD) { + KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut); + return; + } + + // With ADDE, a carry bit may be added in, so we can only use this + // information if we know (at least) that the low two bits are clear. We + // then return to the caller that the low bit is unknown but that other bits + // are known zero. + if (KnownZeroOut >= 2) // ADDE + KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut); return; } case ISD::SREM: @@ -1991,10 +2036,19 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, uint32_t Leaders = std::max(KnownZero.countLeadingOnes(), KnownZero2.countLeadingOnes()); - KnownOne.clear(); + KnownOne.clearAllBits(); KnownZero = APInt::getHighBitsSet(BitWidth, Leaders) & Mask; return; } + case ISD::FrameIndex: + case ISD::TargetFrameIndex: + if (unsigned Align = InferPtrAlignment(Op)) { + // The low bits are known zero if the pointer is aligned. + KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align)); + return; + } + break; + default: // Allow the target to implement this method for its nodes. if (Op.getOpcode() >= ISD::BUILTIN_OP_END) { @@ -2034,12 +2088,7 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ case ISD::Constant: { const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue(); - // If negative, return # leading ones. - if (Val.isNegative()) - return Val.countLeadingOnes(); - - // Return # leading zeros. - return Val.countLeadingZeros(); + return Val.getNumSignBits(); } case ISD::SIGN_EXTEND: @@ -2234,6 +2283,25 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros())); } +/// isBaseWithConstantOffset - Return true if the specified operand is an +/// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an +/// ISD::OR with a ConstantSDNode that is guaranteed to have the same +/// semantics as an ADD. This handles the equivalence: +/// X|Cst == X+Cst iff X&Cst = 0. +bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { + if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || + !isa<ConstantSDNode>(Op.getOperand(1))) + return false; + + if (Op.getOpcode() == ISD::OR && + !MaskedValueIsZero(Op.getOperand(0), + cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue())) + return false; + + return true; +} + + bool SelectionDAG::isKnownNeverNaN(SDValue Op) const { // If we're told that NaNs won't happen, assume they won't. if (NoNaNsFPMath) @@ -2295,7 +2363,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT) { AllNodes.push_back(N); #ifndef NDEBUG - VerifyNode(N); + VerifySDNode(N); #endif return SDValue(N, 0); } @@ -2308,23 +2376,22 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, switch (Opcode) { default: break; case ISD::SIGN_EXTEND: - return getConstant(APInt(Val).sextOrTrunc(VT.getSizeInBits()), VT); + return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT); case ISD::ANY_EXTEND: case ISD::ZERO_EXTEND: case ISD::TRUNCATE: - return getConstant(APInt(Val).zextOrTrunc(VT.getSizeInBits()), VT); + return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT); case ISD::UINT_TO_FP: case ISD::SINT_TO_FP: { - const uint64_t zero[] = {0, 0}; // No compile time operations on ppcf128. if (VT == MVT::ppcf128) break; - APFloat apf = APFloat(APInt(VT.getSizeInBits(), 2, zero)); + APFloat apf(APInt::getNullValue(VT.getSizeInBits())); (void)apf.convertFromAPInt(Val, Opcode==ISD::SINT_TO_FP, APFloat::rmNearestTiesToEven); return getConstantFP(apf, VT); } - case ISD::BIT_CONVERT: + case ISD::BITCAST: if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) return getConstantFP(Val.bitsToFloat(), VT); else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) @@ -2375,7 +2442,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, APInt api(VT.getSizeInBits(), 2, x); return getConstant(api, VT); } - case ISD::BIT_CONVERT: + case ISD::BITCAST: if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT); else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) @@ -2477,13 +2544,13 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, return Operand.getNode()->getOperand(0); } break; - case ISD::BIT_CONVERT: + case ISD::BITCAST: // Basic sanity checking. assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits() - && "Cannot BIT_CONVERT between types of different sizes!"); + && "Cannot BITCAST between types of different sizes!"); if (VT == Operand.getValueType()) return Operand; // noop conversion. - if (OpOpcode == ISD::BIT_CONVERT) // bitconv(bitconv(x)) -> bitconv(x) - return getNode(ISD::BIT_CONVERT, DL, VT, Operand.getOperand(0)); + if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) + return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0)); if (OpOpcode == ISD::UNDEF) return getUNDEF(VT); break; @@ -2519,7 +2586,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDNode *N; SDVTList VTs = getVTList(VT); - if (VT != MVT::Flag) { // Don't CSE flag producing nodes + if (VT != MVT::Glue) { // Don't CSE flag producing nodes FoldingSetNodeID ID; SDValue Ops[1] = { Operand }; AddNodeIDNode(ID, Opcode, VTs, Ops, 1); @@ -2535,7 +2602,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, AllNodes.push_back(N); #ifndef NDEBUG - VerifyNode(N); + VerifySDNode(N); #endif return SDValue(N, 0); } @@ -2676,6 +2743,13 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, "Shift operators return type must be the same as their first arg"); assert(VT.isInteger() && N2.getValueType().isInteger() && "Shifts only work on integers"); + // Verify that the shift amount VT is bit enough to hold valid shift + // amounts. This catches things like trying to shift an i1024 value by an + // i8, which is easy to fall into in generic code that uses + // TLI.getShiftAmount(). + assert(N2.getValueType().getSizeInBits() >= + Log2_32_Ceil(N1.getValueType().getSizeInBits()) && + "Invalid use of small shift amount with oversized value!"); // Always fold shifts of i1 values so the code generator doesn't need to // handle them. Since we know the size of the shift has to be less than the @@ -2820,11 +2894,30 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, return getConstant(ShiftedVal.trunc(ElementSize), VT); } break; - case ISD::EXTRACT_SUBVECTOR: - if (N1.getValueType() == VT) // Trivial extraction. - return N1; + case ISD::EXTRACT_SUBVECTOR: { + SDValue Index = N2; + if (VT.isSimple() && N1.getValueType().isSimple()) { + assert(VT.isVector() && N1.getValueType().isVector() && + "Extract subvector VTs must be a vectors!"); + assert(VT.getVectorElementType() == N1.getValueType().getVectorElementType() && + "Extract subvector VTs must have the same element type!"); + assert(VT.getSimpleVT() <= N1.getValueType().getSimpleVT() && + "Extract subvector must be from larger vector to smaller vector!"); + + if (isa<ConstantSDNode>(Index.getNode())) { + assert((VT.getVectorNumElements() + + cast<ConstantSDNode>(Index.getNode())->getZExtValue() + <= N1.getValueType().getVectorNumElements()) + && "Extract subvector overflow!"); + } + + // Trivial extraction. + if (VT.getSimpleVT() == N1.getValueType().getSimpleVT()) + return N1; + } break; } + } if (N1C) { if (N2C) { @@ -2961,7 +3054,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, // Memoize this node if possible. SDNode *N; SDVTList VTs = getVTList(VT); - if (VT != MVT::Flag) { + if (VT != MVT::Glue) { SDValue Ops[] = { N1, N2 }; FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTs, Ops, 2); @@ -2977,7 +3070,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, AllNodes.push_back(N); #ifndef NDEBUG - VerifyNode(N); + VerifySDNode(N); #endif return SDValue(N, 0); } @@ -3019,7 +3112,31 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, case ISD::VECTOR_SHUFFLE: llvm_unreachable("should use getVectorShuffle constructor!"); break; - case ISD::BIT_CONVERT: + case ISD::INSERT_SUBVECTOR: { + SDValue Index = N3; + if (VT.isSimple() && N1.getValueType().isSimple() + && N2.getValueType().isSimple()) { + assert(VT.isVector() && N1.getValueType().isVector() && + N2.getValueType().isVector() && + "Insert subvector VTs must be a vectors"); + assert(VT == N1.getValueType() && + "Dest and insert subvector source types must match!"); + assert(N2.getValueType().getSimpleVT() <= N1.getValueType().getSimpleVT() && + "Insert subvector must be from smaller vector to larger vector!"); + if (isa<ConstantSDNode>(Index.getNode())) { + assert((N2.getValueType().getVectorNumElements() + + cast<ConstantSDNode>(Index.getNode())->getZExtValue() + <= VT.getVectorNumElements()) + && "Insert subvector overflow!"); + } + + // Trivial insertion. + if (VT.getSimpleVT() == N2.getValueType().getSimpleVT()) + return N2; + } + break; + } + case ISD::BITCAST: // Fold bit_convert nodes from a type to themselves. if (N1.getValueType() == VT) return N1; @@ -3029,7 +3146,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, // Memoize node if it doesn't produce a flag. SDNode *N; SDVTList VTs = getVTList(VT); - if (VT != MVT::Flag) { + if (VT != MVT::Glue) { SDValue Ops[] = { N1, N2, N3 }; FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTs, Ops, 3); @@ -3045,7 +3162,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, AllNodes.push_back(N); #ifndef NDEBUG - VerifyNode(N); + VerifySDNode(N); #endif return SDValue(N, 0); } @@ -3087,6 +3204,17 @@ SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { &ArgChains[0], ArgChains.size()); } +/// SplatByte - Distribute ByteVal over NumBits bits. +static APInt SplatByte(unsigned NumBits, uint8_t ByteVal) { + APInt Val = APInt(NumBits, ByteVal); + unsigned Shift = 8; + for (unsigned i = NumBits; i > 8; i >>= 1) { + Val = (Val << Shift) | Val; + Shift <<= 1; + } + return Val; +} + /// getMemsetValue - Vectorized representation of the memset value /// operand. static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, @@ -3095,27 +3223,18 @@ static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, unsigned NumBits = VT.getScalarType().getSizeInBits(); if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) { - APInt Val = APInt(NumBits, C->getZExtValue() & 255); - unsigned Shift = 8; - for (unsigned i = NumBits; i > 8; i >>= 1) { - Val = (Val << Shift) | Val; - Shift <<= 1; - } + APInt Val = SplatByte(NumBits, C->getZExtValue() & 255); if (VT.isInteger()) return DAG.getConstant(Val, VT); return DAG.getConstantFP(APFloat(Val), VT); } - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value); - unsigned Shift = 8; - for (unsigned i = NumBits; i > 8; i >>= 1) { - Value = DAG.getNode(ISD::OR, dl, VT, - DAG.getNode(ISD::SHL, dl, VT, Value, - DAG.getConstant(Shift, - TLI.getShiftAmountTy())), - Value); - Shift <<= 1; + if (NumBits > 8) { + // Use a multiplication with 0x010101... to extend the input to the + // required length. + APInt Magic = SplatByte(NumBits, 0x01); + Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT)); } return Value; @@ -3136,7 +3255,7 @@ static SDValue getMemsetStringVal(EVT VT, DebugLoc dl, SelectionDAG &DAG, else if (VT.isVector()) { unsigned NumElts = VT.getVectorNumElements(); MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; - return DAG.getNode(ISD::BIT_CONVERT, dl, VT, + return DAG.getNode(ISD::BITCAST, dl, VT, DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts))); } else @@ -3233,15 +3352,6 @@ static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, if (VT.bitsGT(LVT)) VT = LVT; } - - // If we're optimizing for size, and there is a limit, bump the maximum number - // of operations inserted down to 4. This is a wild guess that approximates - // the size of a call to memcpy or memset (3 arguments + call). - if (Limit != ~0U) { - const Function *F = DAG.getMachineFunction().getFunction(); - if (F->hasFnAttr(Attribute::OptimizeForSize)) - Limit = 4; - } unsigned NumMemOps = 0; while (Size != 0) { @@ -3288,7 +3398,9 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, const TargetLowering &TLI = DAG.getTargetLoweringInfo(); std::vector<EVT> MemOps; bool DstAlignCanChange = false; - MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); + MachineFunction &MF = DAG.getMachineFunction(); + MachineFrameInfo *MFI = MF.getFrameInfo(); + bool OptSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize); FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) DstAlignCanChange = true; @@ -3298,8 +3410,8 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, std::string Str; bool CopyFromStr = isMemSrcFromString(Src, Str); bool isZeroStr = CopyFromStr && Str.empty(); - unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(); - + unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); + if (!FindOptimalMemOpLowering(MemOps, Limit, Size, (DstAlignCanChange ? 0 : Align), (isZeroStr ? 0 : SrcAlign), @@ -3345,7 +3457,7 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, // FIXME does the case above also need this? EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); assert(NVT.bitsGE(VT)); - Value = DAG.getExtLoad(ISD::EXTLOAD, NVT, dl, Chain, + Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain, getMemBasePlusOffset(Src, SrcOff, DAG), SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false, MinAlign(SrcAlign, SrcOff)); @@ -3379,14 +3491,16 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, const TargetLowering &TLI = DAG.getTargetLoweringInfo(); std::vector<EVT> MemOps; bool DstAlignCanChange = false; - MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); + MachineFunction &MF = DAG.getMachineFunction(); + MachineFrameInfo *MFI = MF.getFrameInfo(); + bool OptSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize); FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) DstAlignCanChange = true; unsigned SrcAlign = DAG.InferPtrAlignment(Src); if (Align > SrcAlign) SrcAlign = Align; - unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(); + unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); if (!FindOptimalMemOpLowering(MemOps, Limit, Size, (DstAlignCanChange ? 0 : Align), @@ -3455,13 +3569,15 @@ static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, const TargetLowering &TLI = DAG.getTargetLoweringInfo(); std::vector<EVT> MemOps; bool DstAlignCanChange = false; - MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); + MachineFunction &MF = DAG.getMachineFunction(); + MachineFrameInfo *MFI = MF.getFrameInfo(); + bool OptSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize); FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) DstAlignCanChange = true; bool NonScalarIntSafe = isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue(); - if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(), + if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize), Size, (DstAlignCanChange ? 0 : Align), 0, NonScalarIntSafe, false, DAG, TLI)) return SDValue(); @@ -3480,16 +3596,34 @@ static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, SmallVector<SDValue, 8> OutChains; uint64_t DstOff = 0; unsigned NumMemOps = MemOps.size(); + + // Find the largest store and generate the bit pattern for it. + EVT LargestVT = MemOps[0]; + for (unsigned i = 1; i < NumMemOps; i++) + if (MemOps[i].bitsGT(LargestVT)) + LargestVT = MemOps[i]; + SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl); + for (unsigned i = 0; i < NumMemOps; i++) { EVT VT = MemOps[i]; - unsigned VTSize = VT.getSizeInBits() / 8; - SDValue Value = getMemsetValue(Src, VT, DAG, dl); + + // If this store is smaller than the largest store see whether we can get + // the smaller value for free with a truncate. + SDValue Value = MemSetValue; + if (VT.bitsLT(LargestVT)) { + if (!LargestVT.isVector() && !VT.isVector() && + TLI.isTruncateFree(LargestVT, VT)) + Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); + else + Value = getMemsetValue(Src, VT, DAG, dl); + } + assert(Value.getValueType() == VT && "Value with wrong type."); SDValue Store = DAG.getStore(Chain, dl, Value, getMemBasePlusOffset(Dst, DstOff, DAG), DstPtrInfo.getWithOffset(DstOff), - isVol, false, 0); + isVol, false, Align); OutChains.push_back(Store); - DstOff += VTSize; + DstOff += VT.getSizeInBits() / 8; } return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, @@ -3641,7 +3775,7 @@ SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst, if (Result.getNode()) return Result; - // Emit a library call. + // Emit a library call. const Type *IntPtrTy = TLI.getTargetData()->getIntPtrType(*getContext()); TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; @@ -3775,7 +3909,6 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, } /// getMergeValues - Create a MERGE_VALUES node from the given operands. -/// Allowed to return something different (and simpler) if Simplify is true. SDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps, DebugLoc dl) { if (NumOps == 1) @@ -3837,7 +3970,7 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, // Memoize the node unless it returns a flag. MemIntrinsicSDNode *N; - if (VTList.VTs[VTList.NumVTs-1] != MVT::Flag) { + if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); void *IP = 0; @@ -3871,7 +4004,7 @@ static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) { !isa<ConstantSDNode>(Ptr.getOperand(1)) || !isa<FrameIndexSDNode>(Ptr.getOperand(0))) return MachinePointerInfo(); - + int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); return MachinePointerInfo::getFixedStack(FI, Offset+ cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue()); @@ -3889,7 +4022,7 @@ static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) { return InferPointerInfo(Ptr); return MachinePointerInfo(); } - + SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, @@ -3906,12 +4039,12 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, Flags |= MachineMemOperand::MOVolatile; if (isNonTemporal) Flags |= MachineMemOperand::MONonTemporal; - + // If we don't have a PtrInfo, infer the trivial frame index case to simplify // clients. if (PtrInfo.V == 0) PtrInfo = InferPointerInfo(Ptr, Offset); - + MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment, @@ -3920,7 +4053,7 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, } SDValue -SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, +SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, DebugLoc dl, SDValue Chain, SDValue Ptr, SDValue Offset, EVT MemVT, MachineMemOperand *MMO) { @@ -3975,7 +4108,7 @@ SDValue SelectionDAG::getLoad(EVT VT, DebugLoc dl, PtrInfo, VT, isVolatile, isNonTemporal, Alignment, TBAAInfo); } -SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, EVT VT, DebugLoc dl, +SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, DebugLoc dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, bool isVolatile, bool isNonTemporal, @@ -4011,7 +4144,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, Flags |= MachineMemOperand::MOVolatile; if (isNonTemporal) Flags |= MachineMemOperand::MONonTemporal; - + if (PtrInfo.V == 0) PtrInfo = InferPointerInfo(Ptr); @@ -4060,7 +4193,7 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, Flags |= MachineMemOperand::MOVolatile; if (isNonTemporal) Flags |= MachineMemOperand::MONonTemporal; - + if (PtrInfo.V == 0) PtrInfo = InferPointerInfo(Ptr); @@ -4193,7 +4326,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, SDNode *N; SDVTList VTs = getVTList(VT); - if (VT != MVT::Flag) { + if (VT != MVT::Glue) { FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTs, Ops, NumOps); void *IP = 0; @@ -4209,7 +4342,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, AllNodes.push_back(N); #ifndef NDEBUG - VerifyNode(N); + VerifySDNode(N); #endif return SDValue(N, 0); } @@ -4259,7 +4392,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, // Memoize the node unless it returns a flag. SDNode *N; - if (VTList.VTs[VTList.NumVTs-1] != MVT::Flag) { + if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); void *IP = 0; @@ -4291,7 +4424,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, } AllNodes.push_back(N); #ifndef NDEBUG - VerifyNode(N); + VerifySDNode(N); #endif return SDValue(N, 0); } @@ -4668,7 +4801,7 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, unsigned NumOps) { // If an identical node already exists, use it. void *IP = 0; - if (VTs.VTs[VTs.NumVTs-1] != MVT::Flag) { + if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { FoldingSetNodeID ID; AddNodeIDNode(ID, Opc, VTs, Ops, NumOps); if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP)) @@ -4868,9 +5001,9 @@ SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, MachineSDNode * SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc DL, SDVTList VTs, const SDValue *Ops, unsigned NumOps) { - bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Flag; + bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; MachineSDNode *N; - void *IP; + void *IP = 0; if (DoCSE) { FoldingSetNodeID ID; @@ -4899,7 +5032,7 @@ SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc DL, SDVTList VTs, AllNodes.push_back(N); #ifndef NDEBUG - VerifyNode(N); + VerifyMachineNode(N); #endif return N; } @@ -4930,7 +5063,7 @@ SelectionDAG::getTargetInsertSubreg(int SRIdx, DebugLoc DL, EVT VT, /// else return NULL. SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, const SDValue *Ops, unsigned NumOps) { - if (VTList.VTs[VTList.NumVTs-1] != MVT::Flag) { + if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); void *IP = 0; @@ -5363,6 +5496,29 @@ void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) { SD->setHasDebugValue(true); } +/// TransferDbgValues - Transfer SDDbgValues. +void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) { + if (From == To || !From.getNode()->getHasDebugValue()) + return; + SDNode *FromNode = From.getNode(); + SDNode *ToNode = To.getNode(); + SmallVector<SDDbgValue *, 2> &DVs = GetDbgValues(FromNode); + SmallVector<SDDbgValue *, 2> ClonedDVs; + for (SmallVector<SDDbgValue *, 2>::iterator I = DVs.begin(), E = DVs.end(); + I != E; ++I) { + SDDbgValue *Dbg = *I; + if (Dbg->getKind() == SDDbgValue::SDNODE) { + SDDbgValue *Clone = getDbgValue(Dbg->getMDPtr(), ToNode, To.getResNo(), + Dbg->getOffset(), Dbg->getDebugLoc(), + Dbg->getOrder()); + ClonedDVs.push_back(Clone); + } + } + for (SmallVector<SDDbgValue *, 2>::iterator I = ClonedDVs.begin(), + E = ClonedDVs.end(); I != E; ++I) + AddDbgValue(*I, ToNode, false); +} + //===----------------------------------------------------------------------===// // SDNode Class //===----------------------------------------------------------------------===// @@ -5390,7 +5546,7 @@ MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT memvt, } MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, - const SDValue *Ops, unsigned NumOps, EVT memvt, + const SDValue *Ops, unsigned NumOps, EVT memvt, MachineMemOperand *mmo) : SDNode(Opc, dl, VTs, Ops, NumOps), MemoryVT(memvt), MMO(mmo) { @@ -5409,7 +5565,7 @@ void SDNode::Profile(FoldingSetNodeID &ID) const { namespace { struct EVTArray { std::vector<EVT> VTs; - + EVTArray() { VTs.reserve(MVT::LAST_VALUETYPE); for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i) @@ -5501,8 +5657,8 @@ bool SDNode::isOperandOf(SDNode *N) const { /// reachesChainWithoutSideEffects - Return true if this operand (which must /// be a chain) reaches the specified operand without crossing any -/// side-effecting instructions on any chain path. In practice, this looks -/// through token factors and non-volatile loads. In order to remain efficient, +/// side-effecting instructions on any chain path. In practice, this looks +/// through token factors and non-volatile loads. In order to remain efficient, /// this only looks a couple of nodes in, it does not do an exhaustive search. bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth) const { @@ -5714,6 +5870,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::INSERT_VECTOR_ELT: return "insert_vector_elt"; case ISD::EXTRACT_VECTOR_ELT: return "extract_vector_elt"; case ISD::CONCAT_VECTORS: return "concat_vectors"; + case ISD::INSERT_SUBVECTOR: return "insert_subvector"; case ISD::EXTRACT_SUBVECTOR: return "extract_subvector"; case ISD::SCALAR_TO_VECTOR: return "scalar_to_vector"; case ISD::VECTOR_SHUFFLE: return "vector_shuffle"; @@ -5747,7 +5904,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::UINT_TO_FP: return "uint_to_fp"; case ISD::FP_TO_SINT: return "fp_to_sint"; case ISD::FP_TO_UINT: return "fp_to_uint"; - case ISD::BIT_CONVERT: return "bit_convert"; + case ISD::BITCAST: return "bit_convert"; case ISD::FP16_TO_FP32: return "fp16_to_fp32"; case ISD::FP32_TO_FP16: return "fp32_to_fp16"; @@ -5959,12 +6116,7 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { OS << LBB->getName() << " "; OS << (const void*)BBDN->getBasicBlock() << ">"; } else if (const RegisterSDNode *R = dyn_cast<RegisterSDNode>(this)) { - if (G && R->getReg() && - TargetRegisterInfo::isPhysicalRegister(R->getReg())) { - OS << " %" << G->getTarget().getRegisterInfo()->getName(R->getReg()); - } else { - OS << " %reg" << R->getReg(); - } + OS << ' ' << PrintReg(R->getReg(), G ? G->getTarget().getRegisterInfo() :0); } else if (const ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(this)) { OS << "'" << ES->getSymbol() << "'"; @@ -6010,7 +6162,7 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { const char *AM = getIndexedModeName(ST->getAddressingMode()); if (*AM) OS << ", " << AM; - + OS << ">"; } else if (const MemSDNode* M = dyn_cast<MemSDNode>(this)) { OS << "<" << *M->getMemOperand() << ">"; @@ -6061,7 +6213,7 @@ void SDNode::print(raw_ostream &OS, const SelectionDAG *G) const { static void printrWithDepthHelper(raw_ostream &OS, const SDNode *N, const SelectionDAG *G, unsigned depth, - unsigned indent) + unsigned indent) { if (depth == 0) return; @@ -6074,6 +6226,9 @@ static void printrWithDepthHelper(raw_ostream &OS, const SDNode *N, return; for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { + // Don't follow chain operands. + if (N->getOperand(i).getValueType() == MVT::Other) + continue; OS << '\n'; printrWithDepthHelper(OS, N->getOperand(i).getNode(), G, depth-1, indent+2); } @@ -6082,11 +6237,11 @@ static void printrWithDepthHelper(raw_ostream &OS, const SDNode *N, void SDNode::printrWithDepth(raw_ostream &OS, const SelectionDAG *G, unsigned depth) const { printrWithDepthHelper(OS, this, G, depth, 0); -} +} void SDNode::printrFull(raw_ostream &OS, const SelectionDAG *G) const { // Don't print impossibly deep things. - printrWithDepth(OS, G, 100); + printrWithDepth(OS, G, 10); } void SDNode::dumprWithDepth(const SelectionDAG *G, unsigned depth) const { @@ -6095,8 +6250,8 @@ void SDNode::dumprWithDepth(const SelectionDAG *G, unsigned depth) const { void SDNode::dumprFull(const SelectionDAG *G) const { // Don't print impossibly deep things. - dumprWithDepth(G, 100); -} + dumprWithDepth(G, 10); +} static void DumpNodes(const SDNode *N, unsigned indent, const SelectionDAG *G) { for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) @@ -6180,10 +6335,10 @@ SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { } -/// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a -/// location that is 'Dist' units away from the location that the 'Base' load +/// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a +/// location that is 'Dist' units away from the location that the 'Base' load /// is loading from. -bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, +bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, int Dist) const { if (LD->getChain() != Base->getChain()) return false; @@ -6204,11 +6359,11 @@ bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, if (FS != BFS || FS != (int)Bytes) return false; return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes); } - if (Loc.getOpcode() == ISD::ADD && Loc.getOperand(0) == BaseLoc) { - ConstantSDNode *V = dyn_cast<ConstantSDNode>(Loc.getOperand(1)); - if (V && (V->getSExtValue() == Dist*Bytes)) - return true; - } + + // Handle X+C + if (isBaseWithConstantOffset(Loc) && Loc.getOperand(0) == BaseLoc && + cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue() == Dist*Bytes) + return true; const GlobalValue *GV1 = NULL; const GlobalValue *GV2 = NULL; @@ -6249,15 +6404,14 @@ unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { int64_t FrameOffset = 0; if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) { FrameIdx = FI->getIndex(); - } else if (Ptr.getOpcode() == ISD::ADD && - isa<ConstantSDNode>(Ptr.getOperand(1)) && + } else if (isBaseWithConstantOffset(Ptr) && isa<FrameIndexSDNode>(Ptr.getOperand(0))) { + // Handle FI+Cst FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); FrameOffset = Ptr.getConstantOperandVal(1); } if (FrameIdx != (1 << 31)) { - // FIXME: Handle FI+CST. const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo(); unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx), FrameOffset); @@ -6378,7 +6532,7 @@ bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, if (OpVal.getOpcode() == ISD::UNDEF) SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize); else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) - SplatValue |= APInt(CN->getAPIntValue()).zextOrTrunc(EltBitSize). + SplatValue |= CN->getAPIntValue().zextOrTrunc(EltBitSize). zextOrTrunc(sz) << BitPos; else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos; @@ -6393,10 +6547,10 @@ bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, while (sz > 8) { unsigned HalfSize = sz / 2; - APInt HighValue = APInt(SplatValue).lshr(HalfSize).trunc(HalfSize); - APInt LowValue = APInt(SplatValue).trunc(HalfSize); - APInt HighUndef = APInt(SplatUndef).lshr(HalfSize).trunc(HalfSize); - APInt LowUndef = APInt(SplatUndef).trunc(HalfSize); + APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize); + APInt LowValue = SplatValue.trunc(HalfSize); + APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize); + APInt LowUndef = SplatUndef.trunc(HalfSize); // If the two halves do not match (ignoring undef bits), stop here. if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || @@ -6436,7 +6590,7 @@ static void checkForCyclesHelper(const SDNode *N, // If this node has already been checked, don't check it again. if (Checked.count(N)) return; - + // If a node has already been visited on this depth-first walk, reject it as // a cycle. if (!Visited.insert(N)) { @@ -6445,10 +6599,10 @@ static void checkForCyclesHelper(const SDNode *N, errs() << "Detected cycle in SelectionDAG\n"; abort(); } - + for(unsigned i = 0, e = N->getNumOperands(); i != e; ++i) checkForCyclesHelper(N->getOperand(i).getNode(), Visited, Checked); - + Checked.insert(N); Visited.erase(N); } |