LegalizeTypes support for EXTRACT_VECTOR_ELT. The

approach taken is different to that in LegalizeDAG
when it is a question of expanding or promoting the
result type: for example, if extracting an i64 from
a <2 x i64>, when i64 needs expanding, it bitcasts
the vector to <4 x i32>, extracts the appropriate
two i32's, and uses those for the Lo and Hi parts.
Likewise, when extracting an i16 from a <4 x i16>,
and i16 needs promoting, it bitcasts the vector to
<2 x i32>, extracts the appropriate i32, twiddles
the bits if necessary, and uses that as the promoted
value.  This puts more pressure on bitcast legalization,
and I've added the appropriate cases.  They needed to
be added anyway since users can generate such bitcasts
too if they want to.  Also, when considering various
cases (Legal, Promote, Expand, Scalarize, Split) it is
a pain that expand can correspond to Expand, Scalarize
or Split, so I've changed the LegalizeTypes enum so it
lists those different cases - now Expand only means
splitting a scalar in two.
The code produced is the same as by LegalizeDAG for
all relevant testcases, except for
2007-10-31-extractelement-i64.ll, where the code seems
to have improved (see below; can an expert please tell
me if it is better or not).
Before < vs after >.

<       subl    $92, %esp
<       movaps  %xmm0, 64(%esp)
<       movaps  %xmm0, (%esp)
<       movl    4(%esp), %eax
<       movl    %eax, 28(%esp)
<       movl    (%esp), %eax
<       movl    %eax, 24(%esp)
<       movq    24(%esp), %mm0
<       movq    %mm0, 56(%esp)
---
>       subl    $44, %esp
>       movaps  %xmm0, 16(%esp)
>       pshufd  $1, %xmm0, %xmm1
>       movd    %xmm1, 4(%esp)
>       movd    %xmm0, (%esp)
>       movq    (%esp), %mm0
>       movq    %mm0, 8(%esp)

<       subl    $92, %esp
<       movaps  %xmm0, 64(%esp)
<       movaps  %xmm0, (%esp)
<       movl    12(%esp), %eax
<       movl    %eax, 28(%esp)
<       movl    8(%esp), %eax
<       movl    %eax, 24(%esp)
<       movq    24(%esp), %mm0
<       movq    %mm0, 56(%esp)
---
>       subl    $44, %esp
>       movaps  %xmm0, 16(%esp)
>       pshufd  $3, %xmm0, %xmm1
>       movd    %xmm1, 4(%esp)
>       movhlps %xmm0, %xmm0
>       movd    %xmm0, (%esp)
>       movq    (%esp), %mm0
>       movq    %mm0, 8(%esp)

<       subl    $92, %esp
<       movaps  %xmm0, 64(%esp)
---
>       subl    $44, %esp

<       movl    16(%esp), %eax
<       movl    %eax, 48(%esp)
<       movl    20(%esp), %eax
<       movl    %eax, 52(%esp)
<       movaps  %xmm0, (%esp)
<       movl    4(%esp), %eax
<       movl    %eax, 60(%esp)
<       movl    (%esp), %eax
<       movl    %eax, 56(%esp)
---
>       pshufd  $1, %xmm0, %xmm1
>       movd    %xmm1, 4(%esp)
>       movd    %xmm0, (%esp)
>       movd    %xmm1, 12(%esp)
>       movd    %xmm0, 8(%esp)

<       subl    $92, %esp
<       movaps  %xmm0, 64(%esp)
---
>       subl    $44, %esp

<       movl    24(%esp), %eax
<       movl    %eax, 48(%esp)
<       movl    28(%esp), %eax
<       movl    %eax, 52(%esp)
<       movaps  %xmm0, (%esp)
<       movl    12(%esp), %eax
<       movl    %eax, 60(%esp)
<       movl    8(%esp), %eax
<       movl    %eax, 56(%esp)
---
>       pshufd  $3, %xmm0, %xmm1
>       movd    %xmm1, 4(%esp)
>       movhlps %xmm0, %xmm0
>       movd    %xmm0, (%esp)
>       movd    %xmm1, 12(%esp)
>       movd    %xmm0, 8(%esp)


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@47672 91177308-0d34-0410-b5e6-96231b3b80d8

6 files changed, 363 insertions, 59 deletions

diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
index cfaf0a9..43acb5b 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
@@ -68,25 +68,26 @@ void DAGTypeLegalizer::run() {
     unsigned NumResults = N->getNumValues();
     do {
       MVT::ValueType ResultVT = N->getValueType(i);
-      LegalizeAction Action = getTypeAction(ResultVT);
-      if (Action == Promote) {
+      switch (getTypeAction(ResultVT)) {
+      default:
+        assert(false && "Unknown action!");
+      case Legal:
+        break;
+      case Promote:
         PromoteResult(N, i);
         goto NodeDone;
-      } else if (Action == Expand) {
-        // Expand can mean 1) split integer in half 2) scalarize single-element
-        // vector 3) split vector in half.
-        if (!MVT::isVector(ResultVT))
-          ExpandResult(N, i);
-        else if (MVT::getVectorNumElements(ResultVT) == 1)
-          ScalarizeResult(N, i);     // Scalarize the single-element vector.
-        else
-          SplitResult(N, i);         // Split the vector in half.
+      case Expand:
+        ExpandResult(N, i);
+        goto NodeDone;
+      case Scalarize:
+        ScalarizeResult(N, i);
+        goto NodeDone;
+      case Split:
+        SplitResult(N, i);
         goto NodeDone;
-      } else {
-        assert(Action == Legal && "Unknown action!");
       }
     } while (++i < NumResults);
-    
+
     // Scan the operand list for the node, handling any nodes with operands that
     // are illegal.
     {
@@ -94,25 +95,25 @@ void DAGTypeLegalizer::run() {
     bool NeedsRevisit = false;
     for (i = 0; i != NumOperands; ++i) {
       MVT::ValueType OpVT = N->getOperand(i).getValueType();
-      LegalizeAction Action = getTypeAction(OpVT);
-      if (Action == Promote) {
+      switch (getTypeAction(OpVT)) {
+      default:
+        assert(false && "Unknown action!");
+      case Legal:
+        continue;
+      case Promote:
         NeedsRevisit = PromoteOperand(N, i);
         break;
-      } else if (Action == Expand) {
-        // Expand can mean 1) split integer in half 2) scalarize single-element
-        // vector 3) split vector in half.
-        if (!MVT::isVector(OpVT)) {
-          NeedsRevisit = ExpandOperand(N, i);
-        } else if (MVT::getVectorNumElements(OpVT) == 1) {
-          // Scalarize the single-element vector.
-          NeedsRevisit = ScalarizeOperand(N, i);
-        } else {
-          NeedsRevisit = SplitOperand(N, i); // Split the vector in half.
-        }
+      case Expand:
+        NeedsRevisit = ExpandOperand(N, i);
+        break;
+      case Scalarize:
+        NeedsRevisit = ScalarizeOperand(N, i);
+        break;
+      case Split:
+        NeedsRevisit = SplitOperand(N, i);
         break;
-      } else {
-        assert(Action == Legal && "Unknown action!");
       }
+      break;
     }
 
     // If the node needs revisiting, don't add all users to the worklist etc.
@@ -432,7 +433,7 @@ SDOperand DAGTypeLegalizer::HandleMemIntrinsic(SDNode *N) {
   case Legal: break;
   case Promote: Op2 = GetPromotedOp(Op2); break;
   }
-  
+
   // The length could have any action required.
   SDOperand Length = N->getOperand(3);
   switch (getTypeAction(Length.getValueType())) {
@@ -444,21 +445,21 @@ SDOperand DAGTypeLegalizer::HandleMemIntrinsic(SDNode *N) {
     GetExpandedOp(Length, Length, Dummy);
     break;
   }
-  
+
   SDOperand Align = N->getOperand(4);
   switch (getTypeAction(Align.getValueType())) {
   default: assert(0 && "Unknown action for memop operand");
   case Legal: break;
   case Promote: Align = GetPromotedZExtOp(Align); break;
   }
-  
+
   SDOperand AlwaysInline = N->getOperand(5);
   switch (getTypeAction(AlwaysInline.getValueType())) {
   default: assert(0 && "Unknown action for memop operand");
   case Legal: break;
   case Promote: AlwaysInline = GetPromotedZExtOp(AlwaysInline); break;
   }
-  
+
   SDOperand Ops[] = { Chain, Ptr, Op2, Length, Align, AlwaysInline };
   return DAG.UpdateNodeOperands(SDOperand(N, 0), Ops, 6);
 }
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 863242d..1c20969 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -61,27 +61,46 @@ private:
   enum LegalizeAction {
     Legal,      // The target natively supports this type.
     Promote,    // This type should be executed in a larger type.
-    Expand      // This type should be split into two types of half the size.
+    Expand,     // This type should be split into two types of half the size.
+    Scalarize,  // Replace this one-element vector type with its element type.
+    Split       // This vector type should be split into smaller vectors.
   };
-  
+
   /// ValueTypeActions - This is a bitvector that contains two bits for each
   /// simple value type, where the two bits correspond to the LegalizeAction
-  /// enum.  This can be queried with "getTypeAction(VT)".
+  /// enum from TargetLowering.  This can be queried with "getTypeAction(VT)".
   TargetLowering::ValueTypeActionImpl ValueTypeActions;
   
   /// getTypeAction - Return how we should legalize values of this type, either
-  /// it is already legal or we need to expand it into multiple registers of
-  /// smaller integer type, or we need to promote it to a larger type.
+  /// it is already legal, or we need to promote it to a larger integer type, or
+  /// we need to expand it into multiple registers of a smaller integer type, or
+  /// we need to scalarize a one-element vector type into the element type, or
+  /// we need to split a vector type into smaller vector types.
   LegalizeAction getTypeAction(MVT::ValueType VT) const {
-    return (LegalizeAction)ValueTypeActions.getTypeAction(VT);
+    switch (ValueTypeActions.getTypeAction(VT)) {
+    default:
+      assert(false && "Unknown legalize action!");
+    case TargetLowering::Legal:
+      return Legal;
+    case TargetLowering::Promote:
+      return Promote;
+    case TargetLowering::Expand:
+      // Expand can mean 1) split integer in half 2) scalarize single-element
+      // vector 3) split vector in two.
+      if (!MVT::isVector(VT))
+        return Expand;
+      else if (MVT::getVectorNumElements(VT) == 1)
+        return Scalarize;
+      else
+        return Split;
+    }
   }
-  
+
   /// isTypeLegal - Return true if this type is legal on this target.
-  ///
   bool isTypeLegal(MVT::ValueType VT) const {
-    return getTypeAction(VT) == Legal;
+    return ValueTypeActions.getTypeAction(VT) == TargetLowering::Legal;
   }
-  
+
   /// PromotedNodes - For nodes that are below legal width, this map indicates
   /// what promoted value to use.
   DenseMap<SDOperand, SDOperand> PromotedNodes;
@@ -159,11 +178,13 @@ private:
     
   // Result Promotion.
   void PromoteResult(SDNode *N, unsigned ResNo);
+  SDOperand PromoteResult_BIT_CONVERT(SDNode *N);
   SDOperand PromoteResult_BUILD_PAIR(SDNode *N);
   SDOperand PromoteResult_Constant(SDNode *N);
   SDOperand PromoteResult_CTLZ(SDNode *N);
   SDOperand PromoteResult_CTPOP(SDNode *N);
   SDOperand PromoteResult_CTTZ(SDNode *N);
+  SDOperand PromoteResult_EXTRACT_VECTOR_ELT(SDNode *N);
   SDOperand PromoteResult_FP_ROUND(SDNode *N);
   SDOperand PromoteResult_FP_TO_XINT(SDNode *N);
   SDOperand PromoteResult_INT_EXTEND(SDNode *N);
@@ -219,6 +240,7 @@ private:
   void ExpandResult_CTLZ       (SDNode *N, SDOperand &Lo, SDOperand &Hi);
   void ExpandResult_CTPOP      (SDNode *N, SDOperand &Lo, SDOperand &Hi);
   void ExpandResult_CTTZ       (SDNode *N, SDOperand &Lo, SDOperand &Hi);
+  void ExpandResult_EXTRACT_VECTOR_ELT(SDNode *N, SDOperand &Lo, SDOperand &Hi);
   void ExpandResult_LOAD       (LoadSDNode *N, SDOperand &Lo, SDOperand &Hi);
   void ExpandResult_MERGE_VALUES(SDNode *N, SDOperand &Lo, SDOperand &Hi);
   void ExpandResult_SIGN_EXTEND(SDNode *N, SDOperand &Lo, SDOperand &Hi);
@@ -283,6 +305,7 @@ private:
 
   // Operand Vector Scalarization: <1 x ty> -> ty.
   bool ScalarizeOperand(SDNode *N, unsigned OpNo);
+  SDOperand ScalarizeOp_BIT_CONVERT(SDNode *N);
   SDOperand ScalarizeOp_EXTRACT_VECTOR_ELT(SDNode *N);
   SDOperand ScalarizeOp_STORE(StoreSDNode *N, unsigned OpNo);
 
@@ -313,7 +336,9 @@ private:
   // Operand Vector Scalarization: <128 x ty> -> 2 x <64 x ty>.
   bool SplitOperand(SDNode *N, unsigned OpNo);
 
+  SDOperand SplitOp_BIT_CONVERT(SDNode *N);
   SDOperand SplitOp_EXTRACT_SUBVECTOR(SDNode *N);
+  SDOperand SplitOp_EXTRACT_VECTOR_ELT(SDNode *N);
   SDOperand SplitOp_RET(SDNode *N, unsigned OpNo);
   SDOperand SplitOp_STORE(StoreSDNode *N, unsigned OpNo);
   SDOperand SplitOp_VECTOR_SHUFFLE(SDNode *N, unsigned OpNo);
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesExpand.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesExpand.cpp
index 0852d76..3d9bbc2 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypesExpand.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypesExpand.cpp
@@ -86,6 +86,10 @@ void DAGTypeLegalizer::ExpandResult(SDNode *N, unsigned ResNo) {
   case ISD::CTLZ:        ExpandResult_CTLZ(N, Lo, Hi); break;
   case ISD::CTPOP:       ExpandResult_CTPOP(N, Lo, Hi); break;
   case ISD::CTTZ:        ExpandResult_CTTZ(N, Lo, Hi); break;
+
+  case ISD::EXTRACT_VECTOR_ELT:
+    ExpandResult_EXTRACT_VECTOR_ELT(N, Lo, Hi);
+    break;
   }
 
   // If Lo/Hi is null, the sub-method took care of registering results etc.
@@ -147,7 +151,7 @@ void DAGTypeLegalizer::ExpandResult_ANY_EXTEND(SDNode *N,
     // For example, extension of an i48 to an i64.  The operand type necessarily
     // promotes to the result type, so will end up being expanded too.
     assert(getTypeAction(Op.getValueType()) == Promote &&
-           "Don't know how to expand this result!");
+           "Only know how to promote this result!");
     SDOperand Res = GetPromotedOp(Op);
     assert(Res.getValueType() == N->getValueType(0) &&
            "Operand over promoted?");
@@ -168,7 +172,7 @@ void DAGTypeLegalizer::ExpandResult_ZERO_EXTEND(SDNode *N,
     // For example, extension of an i48 to an i64.  The operand type necessarily
     // promotes to the result type, so will end up being expanded too.
     assert(getTypeAction(Op.getValueType()) == Promote &&
-           "Don't know how to expand this result!");
+           "Only know how to promote this result!");
     SDOperand Res = GetPromotedOp(Op);
     assert(Res.getValueType() == N->getValueType(0) &&
            "Operand over promoted?");
@@ -195,7 +199,7 @@ void DAGTypeLegalizer::ExpandResult_SIGN_EXTEND(SDNode *N,
     // For example, extension of an i48 to an i64.  The operand type necessarily
     // promotes to the result type, so will end up being expanded too.
     assert(getTypeAction(Op.getValueType()) == Promote &&
-           "Don't know how to expand this result!");
+           "Only know how to promote this result!");
     SDOperand Res = GetPromotedOp(Op);
     assert(Res.getValueType() == N->getValueType(0) &&
            "Operand over promoted?");
@@ -239,6 +243,8 @@ void DAGTypeLegalizer::ExpandResult_TRUNCATE(SDNode *N,
 void DAGTypeLegalizer::ExpandResult_BIT_CONVERT(SDNode *N,
                                                 SDOperand &Lo, SDOperand &Hi) {
   // Lower the bit-convert to a store/load from the stack, then expand the load.
+  // TODO: If the operand also needs expansion then this could be turned into
+  // conversion of the expanded pieces.  But there needs to be a testcase first!
   SDOperand Op = CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
   ExpandResult_LOAD(cast<LoadSDNode>(Op.Val), Lo, Hi);
 }
@@ -666,6 +672,41 @@ void DAGTypeLegalizer::ExpandResult_CTTZ(SDNode *N,
   Hi = DAG.getConstant(0, NVT);
 }
 
+void DAGTypeLegalizer::ExpandResult_EXTRACT_VECTOR_ELT(SDNode *N,
+                                                       SDOperand &Lo,
+                                                       SDOperand &Hi) {
+  SDOperand OldVec = N->getOperand(0);
+  unsigned OldElts = MVT::getVectorNumElements(OldVec.getValueType());
+
+  // Convert to a vector of the expanded element type, for example
+  // <2 x i64> -> <4 x i32>.
+  MVT::ValueType OldVT = N->getValueType(0);
+  MVT::ValueType NewVT = TLI.getTypeToTransformTo(OldVT);
+  assert(MVT::getSizeInBits(OldVT) == 2 * MVT::getSizeInBits(NewVT) &&
+         "Do not know how to handle this expansion!");
+
+  SDOperand NewVec = DAG.getNode(ISD::BIT_CONVERT,
+                                 MVT::getVectorType(NewVT, 2 * OldElts),
+                                 OldVec);
+
+  // Extract the elements at 2 * Idx and 2 * Idx + 1 from the new vector.
+  SDOperand Idx = N->getOperand(1);
+
+  // Make sure the type of Idx is big enough to hold the new values.
+  if (MVT::getSizeInBits(Idx.getValueType()) < 32)
+    Idx = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Idx);
+
+  Idx = DAG.getNode(ISD::ADD, Idx.getValueType(), Idx, Idx);
+  Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewVT, NewVec, Idx);
+
+  Idx = DAG.getNode(ISD::ADD, Idx.getValueType(), Idx,
+                    DAG.getConstant(1, Idx.getValueType()));
+  Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewVT, NewVec, Idx);
+
+  if (TLI.isBigEndian())
+    std::swap(Lo, Hi);
+}
+
 /// ExpandShiftByConstant - N is a shift by a value that needs to be expanded,
 /// and the shift amount is a constant 'Amt'.  Expand the operation.
 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, unsigned Amt, 
@@ -898,6 +939,28 @@ SDOperand DAGTypeLegalizer::ExpandOperand_TRUNCATE(SDNode *N) {
 }
 
 SDOperand DAGTypeLegalizer::ExpandOperand_BIT_CONVERT(SDNode *N) {
+  if (MVT::isVector(N->getValueType(0))) {
+    // An illegal integer type is being converted to a legal vector type.
+    // Make a two element vector out of the expanded parts and convert that
+    // instead, but only if the new vector type is legal (otherwise there
+    // is no point, and it might create expansion loops).  For example, on
+    // x86 this turns v1i64 = BIT_CONVERT i64 into v1i64 = BIT_CONVERT v2i32.
+    MVT::ValueType OVT = N->getOperand(0).getValueType();
+    MVT::ValueType NVT = MVT::getVectorType(TLI.getTypeToTransformTo(OVT), 2);
+
+    if (isTypeLegal(NVT)) {
+      SDOperand Parts[2];
+      GetExpandedOp(N->getOperand(0), Parts[0], Parts[1]);
+
+      if (TLI.isBigEndian())
+        std::swap(Parts[0], Parts[1]);
+
+      SDOperand Vec = DAG.getNode(ISD::BUILD_VECTOR, NVT, Parts, 2);
+      return DAG.getNode(ISD::BIT_CONVERT, N->getValueType(0), Vec);
+    }
+  }
+
+  // Otherwise, store to a temporary and load out again as the new type.
   return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
 }
 
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesPromote.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesPromote.cpp
index c4eb059..f3232fc 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypesPromote.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypesPromote.cpp
@@ -50,6 +50,7 @@ void DAGTypeLegalizer::PromoteResult(SDNode *N, unsigned ResNo) {
   case ISD::SETCC:    Result = PromoteResult_SETCC(N); break;
   case ISD::LOAD:     Result = PromoteResult_LOAD(cast<LoadSDNode>(N)); break;
   case ISD::BUILD_PAIR:  Result = PromoteResult_BUILD_PAIR(N); break;
+  case ISD::BIT_CONVERT: Result = PromoteResult_BIT_CONVERT(N); break;
 
   case ISD::AND:
   case ISD::OR:
@@ -74,8 +75,12 @@ void DAGTypeLegalizer::PromoteResult(SDNode *N, unsigned ResNo) {
   case ISD::CTLZ:     Result = PromoteResult_CTLZ(N); break;
   case ISD::CTPOP:    Result = PromoteResult_CTPOP(N); break;
   case ISD::CTTZ:     Result = PromoteResult_CTTZ(N); break;
+
+  case ISD::EXTRACT_VECTOR_ELT:
+    Result = PromoteResult_EXTRACT_VECTOR_ELT(N);
+    break;
   }      
-  
+
   // If Result is null, the sub-method took care of registering the result.
   if (Result.Val)
     SetPromotedOp(SDOperand(N, ResNo), Result);
@@ -214,6 +219,65 @@ SDOperand DAGTypeLegalizer::PromoteResult_BUILD_PAIR(SDNode *N) {
   return DAG.getNode(ISD::OR, NVT, Lo, Hi);
 }
 
+SDOperand DAGTypeLegalizer::PromoteResult_BIT_CONVERT(SDNode *N) {
+  SDOperand InOp = N->getOperand(0);
+  MVT::ValueType InVT = InOp.getValueType();
+  MVT::ValueType NInVT = TLI.getTypeToTransformTo(InVT);
+  MVT::ValueType OutVT = TLI.getTypeToTransformTo(N->getValueType(0));
+
+  switch (getTypeAction(InVT)) {
+  default:
+    assert(false && "Unknown type action!");
+    break;
+  case Legal:
+    break;
+  case Promote:
+    if (MVT::getSizeInBits(OutVT) == MVT::getSizeInBits(NInVT))
+      // The input promotes to the same size.  Convert the promoted value.
+      return DAG.getNode(ISD::BIT_CONVERT, OutVT, GetPromotedOp(InOp));
+    break;
+  case Expand:
+    break;
+  case Scalarize:
+    // Convert the element to an integer and promote it by hand.
+    InOp = DAG.getNode(ISD::BIT_CONVERT,
+                       MVT::getIntegerType(MVT::getSizeInBits(InVT)),
+                       GetScalarizedOp(InOp));
+    InOp = DAG.getNode(ISD::ANY_EXTEND,
+                       MVT::getIntegerType(MVT::getSizeInBits(OutVT)), InOp);
+    return DAG.getNode(ISD::BIT_CONVERT, OutVT, InOp);
+  case Split:
+    // For example, i32 = BIT_CONVERT v2i16 on alpha.  Convert the split
+    // pieces of the input into integers and reassemble in the final type.
+    SDOperand Lo, Hi;
+    GetSplitOp(N->getOperand(0), Lo, Hi);
+
+    unsigned LoBits = MVT::getSizeInBits(Lo.getValueType());
+    Lo = DAG.getNode(ISD::BIT_CONVERT, MVT::getIntegerType(LoBits), Lo);
+
+    unsigned HiBits = MVT::getSizeInBits(Hi.getValueType());
+    Hi = DAG.getNode(ISD::BIT_CONVERT, MVT::getIntegerType(HiBits), Hi);
+
+    if (TLI.isBigEndian())
+      std::swap(Lo, Hi);
+
+    MVT::ValueType TargetTy = MVT::getIntegerType(MVT::getSizeInBits(OutVT));
+    Hi = DAG.getNode(ISD::ANY_EXTEND, TargetTy, Hi);
+    Hi = DAG.getNode(ISD::SHL, TargetTy, Hi,
+                     DAG.getConstant(MVT::getSizeInBits(Lo.getValueType()),
+                                     TLI.getShiftAmountTy()));
+    Lo = DAG.getNode(ISD::ZERO_EXTEND, TargetTy, Lo);
+
+    return DAG.getNode(ISD::BIT_CONVERT, OutVT,
+                       DAG.getNode(ISD::OR, TargetTy, Lo, Hi));
+  }
+
+  // Otherwise, lower the bit-convert to a store/load from the stack, then
+  // promote the load.
+  SDOperand Op = CreateStackStoreLoad(InOp, N->getValueType(0));
+  return PromoteResult_LOAD(cast<LoadSDNode>(Op.Val));
+}
+
 SDOperand DAGTypeLegalizer::PromoteResult_SimpleIntBinOp(SDNode *N) {
   // The input may have strange things in the top bits of the registers, but
   // these operations don't care.  They may have weird bits going out, but
@@ -315,6 +379,51 @@ SDOperand DAGTypeLegalizer::PromoteResult_CTTZ(SDNode *N) {
   return DAG.getNode(ISD::CTTZ, NVT, Op);
 }
 
+SDOperand DAGTypeLegalizer::PromoteResult_EXTRACT_VECTOR_ELT(SDNode *N) {
+  MVT::ValueType OldVT = N->getValueType(0);
+  SDOperand OldVec = N->getOperand(0);
+  unsigned OldElts = MVT::getVectorNumElements(OldVec.getValueType());
+
+  if (OldElts == 1) {
+    assert(!isTypeLegal(OldVec.getValueType()) &&
+           "Legal one-element vector of a type needing promotion!");
+    // It is tempting to follow GetScalarizedOp by a call to GetPromotedOp,
+    // but this would be wrong because the scalarized value may not yet have
+    // been processed.
+    return DAG.getNode(ISD::ANY_EXTEND, TLI.getTypeToTransformTo(OldVT),
+                       GetScalarizedOp(OldVec));
+  }
+
+  // Convert to a vector half as long with an element type of twice the width,
+  // for example <4 x i16> -> <2 x i32>.
+  assert(!(OldElts & 1) && "Odd length vectors not supported!");
+  MVT::ValueType NewVT = MVT::getIntegerType(2 * MVT::getSizeInBits(OldVT));
+  assert(!MVT::isExtendedVT(OldVT) && !MVT::isExtendedVT(NewVT));
+
+  SDOperand NewVec = DAG.getNode(ISD::BIT_CONVERT,
+                                 MVT::getVectorType(NewVT, OldElts / 2),
+                                 OldVec);
+
+  // Extract the element at OldIdx / 2 from the new vector.
+  SDOperand OldIdx = N->getOperand(1);
+  SDOperand NewIdx = DAG.getNode(ISD::SRL, OldIdx.getValueType(), OldIdx,
+                                 DAG.getConstant(1, TLI.getShiftAmountTy()));
+  SDOperand Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewVT, NewVec, NewIdx);
+
+  // Select the appropriate half of the element: Lo if OldIdx was even,
+  // Hi if it was odd.
+  SDOperand Lo = Elt;
+  SDOperand Hi = DAG.getNode(ISD::SRL, NewVT, Elt,
+                             DAG.getConstant(MVT::getSizeInBits(OldVT),
+                                             TLI.getShiftAmountTy()));
+  if (TLI.isBigEndian())
+    std::swap(Lo, Hi);
+
+  SDOperand Odd = DAG.getNode(ISD::AND, OldIdx.getValueType(), OldIdx,
+                              DAG.getConstant(1, TLI.getShiftAmountTy()));
+  return DAG.getNode(ISD::SELECT, NewVT, Odd, Hi, Lo);
+}
+
 //===----------------------------------------------------------------------===//
 //  Operand Promotion
 //===----------------------------------------------------------------------===//
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesScalarize.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesScalarize.cpp
index 72c1f48..6610693 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypesScalarize.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypesScalarize.cpp
@@ -176,6 +176,9 @@ bool DAGTypeLegalizer::ScalarizeOperand(SDNode *N, unsigned OpNo) {
       assert(0 && "Do not know how to scalarize this operator's operand!");
       abort();
       
+    case ISD::BIT_CONVERT:
+      Res = ScalarizeOp_BIT_CONVERT(N); break;
+
     case ISD::EXTRACT_VECTOR_ELT:
       Res = ScalarizeOp_EXTRACT_VECTOR_ELT(N); break;
 
@@ -204,6 +207,13 @@ bool DAGTypeLegalizer::ScalarizeOperand(SDNode *N, unsigned OpNo) {
   return false;
 }
 
+/// ScalarizeOp_BIT_CONVERT - If the value to convert is a vector that needs
+/// to be scalarized, it must be <1 x ty>.  Convert the element instead.
+SDOperand DAGTypeLegalizer::ScalarizeOp_BIT_CONVERT(SDNode *N) {
+  SDOperand Elt = GetScalarizedOp(N->getOperand(0));
+  return DAG.getNode(ISD::BIT_CONVERT, N->getValueType(0), Elt);
+}
+
 /// ScalarizeOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to be
 /// scalarized, it must be <1 x ty>, so just return the element, ignoring the
 /// index.
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesSplit.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesSplit.cpp
index 80718d3..c0168f0 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypesSplit.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypesSplit.cpp
@@ -172,7 +172,6 @@ void DAGTypeLegalizer::SplitRes_INSERT_VECTOR_ELT(SDNode *N, SDOperand &Lo,
   else
     Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, Hi.getValueType(), Hi, ScalarOp,
                      DAG.getConstant(Index - LoNumElts, TLI.getPointerTy()));
-  
 }
 
 void DAGTypeLegalizer::SplitRes_VECTOR_SHUFFLE(SDNode *N, 
@@ -253,22 +252,50 @@ void DAGTypeLegalizer::SplitRes_BIT_CONVERT(SDNode *N,
                                             SDOperand &Lo, SDOperand &Hi) {
   // We know the result is a vector.  The input may be either a vector or a
   // scalar value.
+  MVT::ValueType LoVT, HiVT;
+  GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
+
   SDOperand InOp = N->getOperand(0);
-  if (MVT::isVector(InOp.getValueType()) &&
-      MVT::getVectorNumElements(InOp.getValueType()) != 1) {
-    // If this is a vector, split the vector and convert each of the pieces now.
-    GetSplitOp(InOp, Lo, Hi);
-    
-    MVT::ValueType LoVT, HiVT;
-    GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
+  MVT::ValueType InVT = InOp.getValueType();
+  MVT::ValueType NewInVT = TLI.getTypeToTransformTo(InVT);
 
+  switch (getTypeAction(InVT)) {
+  default:
+    assert(false && "Unknown type action!");
+  case Legal:
+    break;
+  case Promote:
+    break;
+  case Scalarize:
+    // While it is tempting to extract the scalarized operand, check whether it
+    // needs expansion, and if so process it in the Expand case below, there is
+    // no guarantee that the scalarized operand has been processed yet.  If it
+    // hasn't then the call to GetExpandedOp will abort.  So just give up.
+    break;
+  case Expand:
+    // A scalar to vector conversion, where the scalar needs expansion.
+    // Check that the vector is being split in two.
+    if (MVT::getSizeInBits(NewInVT) == MVT::getSizeInBits(LoVT)) {
+      // Convert each expanded piece of the scalar now.
+      GetExpandedOp(InOp, Lo, Hi);
+      if (TLI.isBigEndian())
+        std::swap(Lo, Hi);
+      Lo = DAG.getNode(ISD::BIT_CONVERT, LoVT, Lo);
+      Hi = DAG.getNode(ISD::BIT_CONVERT, HiVT, Hi);
+      return;
+    }
+    break;
+  case Split:
+    // If the input is a vector that needs to be split, convert each split
+    // piece of the input now.
+    GetSplitOp(InOp, Lo, Hi);
     Lo = DAG.getNode(ISD::BIT_CONVERT, LoVT, Lo);
     Hi = DAG.getNode(ISD::BIT_CONVERT, HiVT, Hi);
     return;
   }
-  
+
   // Lower the bit-convert to a store/load from the stack, then split the load.
-  SDOperand Op = CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
+  SDOperand Op = CreateStackStoreLoad(InOp, N->getValueType(0));
   SplitRes_LOAD(cast<LoadSDNode>(Op.Val), Lo, Hi);
 }
 
@@ -340,8 +367,11 @@ bool DAGTypeLegalizer::SplitOperand(SDNode *N, unsigned OpNo) {
     case ISD::STORE: Res = SplitOp_STORE(cast<StoreSDNode>(N), OpNo); break;
     case ISD::RET:   Res = SplitOp_RET(N, OpNo); break;
 
-    case ISD::EXTRACT_SUBVECTOR: Res = SplitOp_EXTRACT_SUBVECTOR(N); break;
-    case ISD::VECTOR_SHUFFLE:    Res = SplitOp_VECTOR_SHUFFLE(N, OpNo); break;
+    case ISD::BIT_CONVERT: Res = SplitOp_BIT_CONVERT(N); break;
+
+    case ISD::EXTRACT_VECTOR_ELT: Res = SplitOp_EXTRACT_VECTOR_ELT(N); break;
+    case ISD::EXTRACT_SUBVECTOR:  Res = SplitOp_EXTRACT_SUBVECTOR(N); break;
+    case ISD::VECTOR_SHUFFLE:     Res = SplitOp_VECTOR_SHUFFLE(N, OpNo); break;
     }
   }
   
@@ -402,6 +432,72 @@ SDOperand DAGTypeLegalizer::SplitOp_RET(SDNode *N, unsigned OpNo) {
   return DAG.getNode(ISD::RET, MVT::Other, Chain, Lo, Sign, Hi, Sign);
 }
 
+SDOperand DAGTypeLegalizer::SplitOp_BIT_CONVERT(SDNode *N) {
+  // For example, i64 = BIT_CONVERT v4i16 on alpha.  Typically the vector will
+  // end up being split all the way down to individual components.  Convert the
+  // split pieces into integers and reassemble.
+  SDOperand Lo, Hi;
+  GetSplitOp(N->getOperand(0), Lo, Hi);
+
+  unsigned LoBits = MVT::getSizeInBits(Lo.getValueType());
+  Lo = DAG.getNode(ISD::BIT_CONVERT, MVT::getIntegerType(LoBits), Lo);
+
+  unsigned HiBits = MVT::getSizeInBits(Hi.getValueType());
+  Hi = DAG.getNode(ISD::BIT_CONVERT, MVT::getIntegerType(HiBits), Hi);
+
+  if (TLI.isBigEndian())
+    std::swap(Lo, Hi);
+
+  assert(LoBits == HiBits && "Do not know how to assemble odd sized vectors!");
+
+  return DAG.getNode(ISD::BIT_CONVERT, N->getValueType(0),
+                     DAG.getNode(ISD::BUILD_PAIR,
+                                 MVT::getIntegerType(LoBits+HiBits), Lo, Hi));
+}
+
+SDOperand DAGTypeLegalizer::SplitOp_EXTRACT_VECTOR_ELT(SDNode *N) {
+  SDOperand Vec = N->getOperand(0);
+  SDOperand Idx = N->getOperand(1);
+  MVT::ValueType VecVT = Vec.getValueType();
+
+  if (isa<ConstantSDNode>(Idx)) {
+    uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getValue();
+    assert(IdxVal < MVT::getVectorNumElements(VecVT) &&
+           "Invalid vector index!");
+
+    SDOperand Lo, Hi;
+    GetSplitOp(Vec, Lo, Hi);
+
+    uint64_t LoElts = MVT::getVectorNumElements(Lo.getValueType());
+
+    if (IdxVal < LoElts)
+      return DAG.UpdateNodeOperands(SDOperand(N, 0), Lo, Idx);
+    else
+      return DAG.UpdateNodeOperands(SDOperand(N, 0), Hi,
+                                    DAG.getConstant(IdxVal - LoElts,
+                                                    Idx.getValueType()));
+  }
+
+  // Store the vector to the stack and load back the required element.
+  SDOperand StackPtr = DAG.CreateStackTemporary(VecVT);
+  SDOperand Store = DAG.getStore(DAG.getEntryNode(), Vec, StackPtr, NULL, 0);
+
+  // Add the offset to the index.
+  MVT::ValueType EltVT = MVT::getVectorElementType(VecVT);
+  unsigned EltSize = MVT::getSizeInBits(EltVT)/8; // FIXME: should be ABI size.
+  Idx = DAG.getNode(ISD::MUL, Idx.getValueType(), Idx,
+                    DAG.getConstant(EltSize, Idx.getValueType()));
+
+  if (MVT::getSizeInBits(Idx.getValueType()) >
+      MVT::getSizeInBits(TLI.getPointerTy()))
+    Idx = DAG.getNode(ISD::TRUNCATE, TLI.getPointerTy(), Idx);
+  else
+    Idx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), Idx);
+
+  StackPtr = DAG.getNode(ISD::ADD, Idx.getValueType(), Idx, StackPtr);
+  return DAG.getLoad(EltVT, Store, StackPtr, NULL, 0);
+}
+
 SDOperand DAGTypeLegalizer::SplitOp_EXTRACT_SUBVECTOR(SDNode *N) {
   // We know that the extracted result type is legal.  For now, assume the index
   // is a constant.