Improve support for vector casts in LLVM IR and CodeGen.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@54784 91177308-0d34-0410-b5e6-96231b3b80d8

4 files changed, 221 insertions, 88 deletions

diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index bdeffba..9682a2f 100644
--- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -3146,6 +3146,10 @@ SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) {
   MVT VT = N->getValueType(0);
   MVT EVT = N->getValueType(0);
 
+  // This transformation isn't valid for vector loads.
+  if (VT.isVector())
+    return SDValue();
+
   // Special case: SIGN_EXTEND_INREG is basically truncating to EVT then
   // extended to VT.
   if (Opc == ISD::SIGN_EXTEND_INREG) {
diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index d8d45d0..55cff68 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -201,6 +201,7 @@ private:
   SDValue EmitStackConvert(SDValue SrcOp, MVT SlotVT, MVT DestVT);
   SDValue ExpandBUILD_VECTOR(SDNode *Node);
   SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
+  SDValue LegalizeINT_TO_FP(SDValue Result, bool isSigned, MVT DestTy, SDValue Op);
   SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, MVT DestVT);
   SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, MVT DestVT, bool isSigned);
   SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, MVT DestVT, bool isSigned);
@@ -3623,51 +3624,8 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
   case ISD::SINT_TO_FP:
   case ISD::UINT_TO_FP: {
     bool isSigned = Node->getOpcode() == ISD::SINT_TO_FP;
-    switch (getTypeAction(Node->getOperand(0).getValueType())) {
-    case Legal:
-      switch (TLI.getOperationAction(Node->getOpcode(),
-                                     Node->getOperand(0).getValueType())) {
-      default: assert(0 && "Unknown operation action!");
-      case TargetLowering::Custom:
-        isCustom = true;
-        // FALLTHROUGH
-      case TargetLowering::Legal:
-        Tmp1 = LegalizeOp(Node->getOperand(0));
-        Result = DAG.UpdateNodeOperands(Result, Tmp1);
-        if (isCustom) {
-          Tmp1 = TLI.LowerOperation(Result, DAG);
-          if (Tmp1.Val) Result = Tmp1;
-        }
-        break;
-      case TargetLowering::Expand:
-        Result = ExpandLegalINT_TO_FP(isSigned,
-                                      LegalizeOp(Node->getOperand(0)),
-                                      Node->getValueType(0));
-        break;
-      case TargetLowering::Promote:
-        Result = PromoteLegalINT_TO_FP(LegalizeOp(Node->getOperand(0)),
-                                       Node->getValueType(0),
-                                       isSigned);
-        break;
-      }
-      break;
-    case Expand:
-      Result = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP,
-                             Node->getValueType(0), Node->getOperand(0));
-      break;
-    case Promote:
-      Tmp1 = PromoteOp(Node->getOperand(0));
-      if (isSigned) {
-        Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, Tmp1.getValueType(),
-                 Tmp1, DAG.getValueType(Node->getOperand(0).getValueType()));
-      } else {
-        Tmp1 = DAG.getZeroExtendInReg(Tmp1,
-                                      Node->getOperand(0).getValueType());
-      }
-      Result = DAG.UpdateNodeOperands(Result, Tmp1);
-      Result = LegalizeOp(Result);  // The 'op' is not necessarily legal!
-      break;
-    }
+    Result = LegalizeINT_TO_FP(Result, isSigned,
+                               Node->getValueType(0), Node->getOperand(0));
     break;
   }
   case ISD::TRUNCATE:
@@ -5250,6 +5208,62 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
   return Result;
 }
 
+/// LegalizeINT_TO_FP - Legalize a [US]INT_TO_FP operation.
+///
+SDValue SelectionDAGLegalize::
+LegalizeINT_TO_FP(SDValue Result, bool isSigned, MVT DestTy, SDValue Op) {
+  bool isCustom = false;
+  SDValue Tmp1;
+  switch (getTypeAction(Op.getValueType())) {
+  case Legal:
+    switch (TLI.getOperationAction(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
+                                   Op.getValueType())) {
+    default: assert(0 && "Unknown operation action!");
+    case TargetLowering::Custom:
+      isCustom = true;
+      // FALLTHROUGH
+    case TargetLowering::Legal:
+      Tmp1 = LegalizeOp(Op);
+      if (Result.Val)
+        Result = DAG.UpdateNodeOperands(Result, Tmp1);
+      else
+        Result = DAG.getNode(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
+                             DestTy, Tmp1);
+      if (isCustom) {
+        Tmp1 = TLI.LowerOperation(Result, DAG);
+        if (Tmp1.Val) Result = Tmp1;
+      }
+      break;
+    case TargetLowering::Expand:
+      Result = ExpandLegalINT_TO_FP(isSigned, LegalizeOp(Op), DestTy);
+      break;
+    case TargetLowering::Promote:
+      Result = PromoteLegalINT_TO_FP(LegalizeOp(Op), DestTy, isSigned);
+      break;
+    }
+    break;
+  case Expand:
+    Result = ExpandIntToFP(isSigned, DestTy, Op);
+    break;
+  case Promote:
+    Tmp1 = PromoteOp(Op);
+    if (isSigned) {
+      Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, Tmp1.getValueType(),
+               Tmp1, DAG.getValueType(Op.getValueType()));
+    } else {
+      Tmp1 = DAG.getZeroExtendInReg(Tmp1,
+                                    Op.getValueType());
+    }
+    if (Result.Val)
+      Result = DAG.UpdateNodeOperands(Result, Tmp1);
+    else
+      Result = DAG.getNode(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
+                           DestTy, Tmp1);
+    Result = LegalizeOp(Result);  // The 'op' is not necessarily legal!
+    break;
+  }
+  return Result;
+}
 
 /// ExpandIntToFP - Expand a [US]INT_TO_FP operation.
 ///
@@ -5258,6 +5272,26 @@ ExpandIntToFP(bool isSigned, MVT DestTy, SDValue Source) {
   MVT SourceVT = Source.getValueType();
   bool ExpandSource = getTypeAction(SourceVT) == Expand;
 
+  // Expand unsupported int-to-fp vector casts by unrolling them.
+  if (DestTy.isVector()) {
+    if (!ExpandSource)
+      return LegalizeOp(UnrollVectorOp(Source));
+    MVT DestEltTy = DestTy.getVectorElementType();
+    if (DestTy.getVectorNumElements() == 1) {
+      SDValue Scalar = ScalarizeVectorOp(Source);
+      SDValue Result = LegalizeINT_TO_FP(SDValue(), isSigned,
+                                         DestEltTy, Scalar);
+      return DAG.getNode(ISD::BUILD_VECTOR, DestTy, Result);
+    }
+    SDValue Lo, Hi;
+    SplitVectorOp(Source, Lo, Hi);
+    MVT SplitDestTy = MVT::getVectorVT(DestEltTy,
+                                       DestTy.getVectorNumElements() / 2);
+    SDValue LoResult = LegalizeINT_TO_FP(SDValue(), isSigned, SplitDestTy, Lo);
+    SDValue HiResult = LegalizeINT_TO_FP(SDValue(), isSigned, SplitDestTy, Hi);
+    return LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, DestTy, LoResult, HiResult));
+  }
+
   // Special case for i32 source to take advantage of UINTTOFP_I32_F32, etc.
   if (!isSigned && SourceVT != MVT::i32) {
     // The integer value loaded will be incorrectly if the 'sign bit' of the
@@ -5863,12 +5897,13 @@ void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){
     SDValue Ch  = LD->getChain();    // Legalize the chain.
     SDValue Ptr = LD->getBasePtr();  // Legalize the pointer.
     ISD::LoadExtType ExtType = LD->getExtensionType();
+    const Value *SV = LD->getSrcValue();
     int SVOffset = LD->getSrcValueOffset();
     unsigned Alignment = LD->getAlignment();
     bool isVolatile = LD->isVolatile();
 
     if (ExtType == ISD::NON_EXTLOAD) {
-      Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), SVOffset,
+      Lo = DAG.getLoad(NVT, Ch, Ptr, SV, SVOffset,
                        isVolatile, Alignment);
       if (VT == MVT::f32 || VT == MVT::f64) {
         // f32->i32 or f64->i64 one to one expansion.
@@ -5886,7 +5921,7 @@ void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){
                         DAG.getIntPtrConstant(IncrementSize));
       SVOffset += IncrementSize;
       Alignment = MinAlign(Alignment, IncrementSize);
-      Hi = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), SVOffset,
+      Hi = DAG.getLoad(NVT, Ch, Ptr, SV, SVOffset,
                        isVolatile, Alignment);
 
       // Build a factor node to remember that this load is independent of the
@@ -5904,7 +5939,7 @@ void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){
       if ((VT == MVT::f64 && EVT == MVT::f32) ||
           (VT == MVT::ppcf128 && (EVT==MVT::f64 || EVT==MVT::f32))) {
         // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND
-        SDValue Load = DAG.getLoad(EVT, Ch, Ptr, LD->getSrcValue(),
+        SDValue Load = DAG.getLoad(EVT, Ch, Ptr, SV,
                                      SVOffset, isVolatile, Alignment);
         // Remember that we legalized the chain.
         AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Load.getValue(1)));
@@ -5913,10 +5948,10 @@ void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){
       }
     
       if (EVT == NVT)
-        Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(),
+        Lo = DAG.getLoad(NVT, Ch, Ptr, SV,
                          SVOffset, isVolatile, Alignment);
       else
-        Lo = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, LD->getSrcValue(),
+        Lo = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, SV,
                             SVOffset, EVT, isVolatile,
                             Alignment);
     
@@ -6817,6 +6852,7 @@ void SelectionDAGLegalize::SplitVectorOp(SDValue Op, SDValue &Lo,
     Hi = DAG.getNode(Node->getOpcode(), NewVT_Hi, LH, RH);
     break;
   }
+  case ISD::FP_ROUND:
   case ISD::FPOWI: {
     SDValue L, H;
     SplitVectorOp(Node->getOperand(0), L, H);
@@ -6836,7 +6872,12 @@ void SelectionDAGLegalize::SplitVectorOp(SDValue Op, SDValue &Lo,
   case ISD::FP_TO_SINT:
   case ISD::FP_TO_UINT:
   case ISD::SINT_TO_FP:
-  case ISD::UINT_TO_FP: {
+  case ISD::UINT_TO_FP:
+  case ISD::TRUNCATE:
+  case ISD::ANY_EXTEND:
+  case ISD::SIGN_EXTEND:
+  case ISD::ZERO_EXTEND:
+  case ISD::FP_EXTEND: {
     SDValue L, H;
     SplitVectorOp(Node->getOperand(0), L, H);
 
@@ -6848,18 +6889,31 @@ void SelectionDAGLegalize::SplitVectorOp(SDValue Op, SDValue &Lo,
     LoadSDNode *LD = cast<LoadSDNode>(Node);
     SDValue Ch = LD->getChain();
     SDValue Ptr = LD->getBasePtr();
+    ISD::LoadExtType ExtType = LD->getExtensionType();
     const Value *SV = LD->getSrcValue();
     int SVOffset = LD->getSrcValueOffset();
+    MVT MemoryVT = LD->getMemoryVT();
     unsigned Alignment = LD->getAlignment();
     bool isVolatile = LD->isVolatile();
 
-    Lo = DAG.getLoad(NewVT_Lo, Ch, Ptr, SV, SVOffset, isVolatile, Alignment);
-    unsigned IncrementSize = NewNumElts_Lo * NewEltVT.getSizeInBits()/8;
+    assert(LD->isUnindexed() && "Indexed vector loads are not supported yet!");
+    SDValue Offset = DAG.getNode(ISD::UNDEF, Ptr.getValueType());
+
+    MVT MemNewEltVT = MemoryVT.getVectorElementType();
+    MVT MemNewVT_Lo = MVT::getVectorVT(MemNewEltVT, NewNumElts_Lo);
+    MVT MemNewVT_Hi = MVT::getVectorVT(MemNewEltVT, NewNumElts_Hi);
+
+    Lo = DAG.getLoad(ISD::UNINDEXED, ExtType,
+                     NewVT_Lo, Ch, Ptr, Offset,
+                     SV, SVOffset, MemNewVT_Lo, isVolatile, Alignment);
+    unsigned IncrementSize = NewNumElts_Lo * MemNewEltVT.getSizeInBits()/8;
     Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
                       DAG.getIntPtrConstant(IncrementSize));
     SVOffset += IncrementSize;
     Alignment = MinAlign(Alignment, IncrementSize);
-    Hi = DAG.getLoad(NewVT_Hi, Ch, Ptr, SV, SVOffset, isVolatile, Alignment);
+    Hi = DAG.getLoad(ISD::UNINDEXED, ExtType,
+                     NewVT_Hi, Ch, Ptr, Offset,
+                     SV, SVOffset, MemNewVT_Hi, isVolatile, Alignment);
     
     // Build a factor node to remember that this load is independent of the
     // other one.
@@ -6951,11 +7005,21 @@ SDValue SelectionDAGLegalize::ScalarizeVectorOp(SDValue Op) {
   case ISD::FSQRT:
   case ISD::FSIN:
   case ISD::FCOS:
+  case ISD::FP_TO_SINT:
+  case ISD::FP_TO_UINT:
+  case ISD::SINT_TO_FP:
+  case ISD::UINT_TO_FP:
+  case ISD::SIGN_EXTEND:
+  case ISD::ZERO_EXTEND:
+  case ISD::ANY_EXTEND:
+  case ISD::TRUNCATE:
+  case ISD::FP_EXTEND:
     Result = DAG.getNode(Node->getOpcode(),
                          NewVT, 
                          ScalarizeVectorOp(Node->getOperand(0)));
     break;
   case ISD::FPOWI:
+  case ISD::FP_ROUND:
     Result = DAG.getNode(Node->getOpcode(),
                          NewVT, 
                          ScalarizeVectorOp(Node->getOperand(0)),
@@ -6965,11 +7029,20 @@ SDValue SelectionDAGLegalize::ScalarizeVectorOp(SDValue Op) {
     LoadSDNode *LD = cast<LoadSDNode>(Node);
     SDValue Ch = LegalizeOp(LD->getChain());     // Legalize the chain.
     SDValue Ptr = LegalizeOp(LD->getBasePtr());  // Legalize the pointer.
-    
+    ISD::LoadExtType ExtType = LD->getExtensionType();
     const Value *SV = LD->getSrcValue();
     int SVOffset = LD->getSrcValueOffset();
-    Result = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset,
-                         LD->isVolatile(), LD->getAlignment());
+    MVT MemoryVT = LD->getMemoryVT();
+    unsigned Alignment = LD->getAlignment();
+    bool isVolatile = LD->isVolatile();
+
+    assert(LD->isUnindexed() && "Indexed vector loads are not supported yet!");
+    SDValue Offset = DAG.getNode(ISD::UNDEF, Ptr.getValueType());
+    
+    Result = DAG.getLoad(ISD::UNINDEXED, ExtType,
+                         NewVT, Ch, Ptr, Offset, SV, SVOffset,
+                         MemoryVT.getVectorElementType(),
+                         isVolatile, Alignment);
 
     // Remember that we legalized the chain.
     AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1)));
diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index e0e40e3..9275fc2 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -115,11 +115,16 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
 }
 
 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
-  assert(ISD::isNormalLoad(N) && "Extending load of one-element vector?");
-  SDValue Result = DAG.getLoad(N->getValueType(0).getVectorElementType(),
-                                 N->getChain(), N->getBasePtr(),
-                                 N->getSrcValue(), N->getSrcValueOffset(),
-                                 N->isVolatile(), N->getAlignment());
+  assert(N->isUnindexed() && "Indexed vector load?");
+
+  SDValue Result = DAG.getLoad(ISD::UNINDEXED, N->getExtensionType(),
+                               N->getValueType(0).getVectorElementType(),
+                               N->getChain(), N->getBasePtr(),
+                               DAG.getNode(ISD::UNDEF,
+                                           N->getBasePtr().getValueType()),
+                               N->getSrcValue(), N->getSrcValueOffset(),
+                               N->getMemoryVT().getVectorElementType(),
+                               N->isVolatile(), N->getAlignment());
 
   // Legalized the chain result - switch anything that used the old chain to
   // use the new one.
@@ -232,8 +237,17 @@ SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
 /// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be
 /// scalarized, it must be <1 x ty>.  Just store the element.
 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
-  assert(ISD::isNormalStore(N) && "Truncating store of one-element vector?");
+  assert(N->isUnindexed() && "Indexed store of one-element vector?");
   assert(OpNo == 1 && "Do not know how to scalarize this operand!");
+
+  if (N->isTruncatingStore())
+    return DAG.getTruncStore(N->getChain(),
+                             GetScalarizedVector(N->getOperand(1)),
+                             N->getBasePtr(),
+                             N->getSrcValue(), N->getSrcValueOffset(),
+                             N->getMemoryVT().getVectorElementType(),
+                             N->isVolatile(), N->getAlignment());
+
   return DAG.getStore(N->getChain(), GetScalarizedVector(N->getOperand(1)),
                       N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(),
                       N->isVolatile(), N->getAlignment());
@@ -460,33 +474,34 @@ void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
   MVT LoVT, HiVT;
   GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT);
 
+  ISD::LoadExtType ExtType = LD->getExtensionType();
   SDValue Ch = LD->getChain();
   SDValue Ptr = LD->getBasePtr();
+  SDValue Offset = DAG.getNode(ISD::UNDEF, Ptr.getValueType());
   const Value *SV = LD->getSrcValue();
   int SVOffset = LD->getSrcValueOffset();
+  MVT MemoryVT = LD->getMemoryVT();
   unsigned Alignment = LD->getAlignment();
   bool isVolatile = LD->isVolatile();
 
-  Lo = DAG.getLoad(LoVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment);
+  MVT LoMemVT, HiMemVT;
+  GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT);
 
-  if (LD->getExtensionType() == ISD::NON_EXTLOAD) {
-    unsigned IncrementSize = LoVT.getSizeInBits()/8;
-    Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
-                      DAG.getIntPtrConstant(IncrementSize));
-    SVOffset += IncrementSize;
-    Alignment = MinAlign(Alignment, IncrementSize);
-    Hi = DAG.getLoad(HiVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment);
+  Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, Ch, Ptr, Offset,
+                   SV, SVOffset, LoMemVT, isVolatile, Alignment);
 
-    // Build a factor node to remember that this load is independent of the
-    // other one.
-    Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
-                     Hi.getValue(1));
-  } else {
-    assert(LD->getExtensionType() == ISD::EXTLOAD &&
-           "Unsupported vector extending load!");
-    Hi = DAG.getNode(ISD::UNDEF, HiVT);
-    Ch = Lo.getValue(1);
-  }
+  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
+  Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
+                    DAG.getIntPtrConstant(IncrementSize));
+  SVOffset += IncrementSize;
+  Alignment = MinAlign(Alignment, IncrementSize);
+  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, Ch, Ptr, Offset,
+                   SV, SVOffset, HiMemVT, isVolatile, Alignment);
+
+  // Build a factor node to remember that this load is independent of the
+  // other one.
+  Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
+                   Hi.getValue(1));
 
   // Legalized the chain result - switch anything that used the old chain to
   // use the new one.
@@ -679,27 +694,44 @@ SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
 }
 
 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
-  assert(ISD::isNormalStore(N) && "Truncating store of vector?");
+  assert(N->isUnindexed() && "Indexed store of vector?");
   assert(OpNo == 1 && "Can only split the stored value");
 
+  bool isTruncating = N->isTruncatingStore();
   SDValue Ch  = N->getChain();
   SDValue Ptr = N->getBasePtr();
   int SVOffset = N->getSrcValueOffset();
+  MVT MemoryVT = N->getMemoryVT();
   unsigned Alignment = N->getAlignment();
   bool isVol = N->isVolatile();
   SDValue Lo, Hi;
   GetSplitVector(N->getOperand(1), Lo, Hi);
 
-  unsigned IncrementSize = Lo.getValueType().getSizeInBits()/8;
+  MVT LoMemVT, HiMemVT;
+  GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT);
 
-  Lo = DAG.getStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset, isVol, Alignment);
+  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
+
+  if (isTruncating)
+    Lo = DAG.getTruncStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset,
+                           LoMemVT, isVol, Alignment);
+  else
+    Lo = DAG.getStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset,
+                      isVol, Alignment);
 
   // Increment the pointer to the other half.
   Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
                     DAG.getIntPtrConstant(IncrementSize));
 
-  Hi = DAG.getStore(Ch, Hi, Ptr, N->getSrcValue(), SVOffset+IncrementSize,
-                    isVol, MinAlign(Alignment, IncrementSize));
+  if (isTruncating)
+    Hi = DAG.getTruncStore(Ch, Hi, Ptr,
+                           N->getSrcValue(), SVOffset+IncrementSize,
+                           HiMemVT,
+                           isVol, MinAlign(Alignment, IncrementSize));
+  else
+    Hi = DAG.getStore(Ch, Hi, Ptr, N->getSrcValue(), SVOffset+IncrementSize,
+                      isVol, MinAlign(Alignment, IncrementSize));
+
   return DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
 }
 
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 9eab89f..80dda1f 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -2067,7 +2067,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT, SDValue Operand) {
   unsigned OpOpcode = Operand.Val->getOpcode();
   switch (Opcode) {
   case ISD::TokenFactor:
-    return Operand;         // Factor of one node?  No need.
+  case ISD::CONCAT_VECTORS:
+    return Operand;         // Factor or concat of one node?  No need.
   case ISD::FP_ROUND: assert(0 && "Invalid method to make FP_ROUND node");
   case ISD::FP_EXTEND:
     assert(VT.isFloatingPoint() &&
@@ -2196,6 +2197,16 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT,
     if (N1.getOpcode() == ISD::EntryToken) return N2;
     if (N2.getOpcode() == ISD::EntryToken) return N1;
     break;
+  case ISD::CONCAT_VECTORS:
+    // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
+    // one big BUILD_VECTOR.
+    if (N1.getOpcode() == ISD::BUILD_VECTOR &&
+        N2.getOpcode() == ISD::BUILD_VECTOR) {
+      SmallVector<SDValue, 16> Elts(N1.Val->op_begin(), N1.Val->op_end());
+      Elts.insert(Elts.end(), N2.Val->op_begin(), N2.Val->op_end());
+      return getNode(ISD::BUILD_VECTOR, VT, &Elts[0], Elts.size());
+    }
+    break;
   case ISD::AND:
     assert(VT.isInteger() && N1.getValueType() == N2.getValueType() &&
            N1.getValueType() == VT && "Binary operator types must match!");
@@ -2541,6 +2552,18 @@ SDValue SelectionDAG::getNode(unsigned Opcode, MVT VT,
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val);
   ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.Val);
   switch (Opcode) {
+  case ISD::CONCAT_VECTORS:
+    // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
+    // one big BUILD_VECTOR.
+    if (N1.getOpcode() == ISD::BUILD_VECTOR &&
+        N2.getOpcode() == ISD::BUILD_VECTOR &&
+        N3.getOpcode() == ISD::BUILD_VECTOR) {
+      SmallVector<SDValue, 16> Elts(N1.Val->op_begin(), N1.Val->op_end());
+      Elts.insert(Elts.end(), N2.Val->op_begin(), N2.Val->op_end());
+      Elts.insert(Elts.end(), N3.Val->op_begin(), N3.Val->op_end());
+      return getNode(ISD::BUILD_VECTOR, VT, &Elts[0], Elts.size());
+    }
+    break;
   case ISD::SETCC: {
     // Use FoldSetCC to simplify SETCC's.
     SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get());
@@ -3179,7 +3202,8 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
   } else {
     // Extending load.
     if (VT.isVector())
-      assert(EVT == VT.getVectorElementType() && "Invalid vector extload!");
+      assert(EVT.getVectorNumElements() == VT.getVectorNumElements() &&
+             "Invalid vector extload!");
     else
       assert(EVT.bitsLT(VT) &&
              "Should only be an extending load, not truncating!");