Generalize MVT::ValueType and associated functions to be able to represent

extended vector types. Remove the special SDNode opcodes used for pre-legalize
vector operations, and the special MVT::Vector type used with them. Adjust
lowering and legalize to work with the normal SDNode kinds instead, and to
use the normal MVT functions to work with vector types instead of using the
two special operands that the pre-legalize nodes held.

This allows pre-legalize and post-legalize DAGs, and the code that operates
on them, to be more consistent. Pre-legalize vector operators can be handled
more consistently with scalar operators. And, -view-dag-combine1-dags and
-view-legalize-dags now look prettier for vector code.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@37719 91177308-0d34-0410-b5e6-96231b3b80d8

12 files changed, 951 insertions, 1163 deletions

diff --git a/include/llvm/CodeGen/CallingConvLower.h b/include/llvm/CodeGen/CallingConvLower.h
index 4308c37..959d052 100644
--- a/include/llvm/CodeGen/CallingConvLower.h
+++ b/include/llvm/CodeGen/CallingConvLower.h
@@ -48,10 +48,10 @@ private:
   LocInfo HTP : 7;
   
   /// ValVT - The type of the value being assigned.
-  MVT::ValueType ValVT : 8;
+  MVT::ValueType ValVT;
 
   /// LocVT - The type of the location being assigned to.
-  MVT::ValueType LocVT : 8;
+  MVT::ValueType LocVT;
 public:
     
   static CCValAssign getReg(unsigned ValNo, MVT::ValueType ValVT,
diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h
index 56cb17d..9388df1 100644
--- a/include/llvm/CodeGen/SelectionDAG.h
+++ b/include/llvm/CodeGen/SelectionDAG.h
@@ -319,8 +319,6 @@ public:
                        unsigned Alignment=0);
   SDOperand getIndexedLoad(SDOperand OrigLoad, SDOperand Base,
                            SDOperand Offset, ISD::MemIndexedMode AM);
-  SDOperand getVecLoad(unsigned Count, MVT::ValueType VT, SDOperand Chain, 
-                       SDOperand Ptr, SDOperand SV);
 
   /// getStore - Helper function to build ISD::STORE nodes.
   ///
diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h
index 5a7d47f..792859a 100644
--- a/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -237,58 +237,30 @@ namespace ISD {
     // FCOPYSIGN(f32, f64) is allowed.
     FCOPYSIGN,
 
-    /// VBUILD_VECTOR(ELT1, ELT2, ELT3, ELT4,...,  COUNT,TYPE) - Return a vector
-    /// with the specified, possibly variable, elements.  The number of elements
-    /// is required to be a power of two.
-    VBUILD_VECTOR,
-
     /// BUILD_VECTOR(ELT1, ELT2, ELT3, ELT4,...) - Return a vector
     /// with the specified, possibly variable, elements.  The number of elements
     /// is required to be a power of two.
     BUILD_VECTOR,
     
-    /// VINSERT_VECTOR_ELT(VECTOR, VAL, IDX,  COUNT,TYPE) - Given a vector
-    /// VECTOR, an element ELEMENT, and a (potentially variable) index IDX,
-    /// return a vector with the specified element of VECTOR replaced with VAL.
-    /// COUNT and TYPE specify the type of vector, as is standard for V* nodes.
-    VINSERT_VECTOR_ELT,
-    
-    /// INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR (a legal packed
-    /// type) with the element at IDX replaced with VAL.
+    /// INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element
+    /// at IDX replaced with VAL.
     INSERT_VECTOR_ELT,
 
-    /// VEXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR
-    /// (an MVT::Vector value) identified by the (potentially variable) element
-    /// number IDX.
-    VEXTRACT_VECTOR_ELT,
-    
     /// EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR
-    /// (a legal vector type vector) identified by the (potentially variable)
-    /// element number IDX.
+    /// identified by the (potentially variable) element number IDX.
     EXTRACT_VECTOR_ELT,
     
-    /// VCONCAT_VECTORS(VECTOR0, VECTOR1, ..., COUNT,TYPE) - Given a number of
-    /// values of MVT::Vector type with the same length and element type, this
-    /// produces a concatenated MVT::Vector result value, with length equal to
-    /// the sum of the input vectors.  This can only be used before
-    /// legalization.
-    VCONCAT_VECTORS,
+    /// CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of
+    /// vector type with the same length and element type, this produces a
+    /// concatenated vector result value, with length equal to the sum of the
+    /// input vectors.
+    CONCAT_VECTORS,
     
-    /// VEXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an
-    /// MVT::Vector value) starting with the (potentially variable)
-    /// element number IDX, which must be a multiple of the result vector
-    /// length.  This can only be used before legalization.
-    VEXTRACT_SUBVECTOR,
+    /// EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an
+    /// vector value) starting with the (potentially variable) element number
+    /// IDX, which must be a multiple of the result vector length.
+    EXTRACT_SUBVECTOR,
     
-    /// VVECTOR_SHUFFLE(VEC1, VEC2, SHUFFLEVEC, COUNT,TYPE) - Returns a vector,
-    /// of the same type as VEC1/VEC2.  SHUFFLEVEC is a VBUILD_VECTOR of
-    /// constant int values that indicate which value each result element will
-    /// get.  The elements of VEC1/VEC2 are enumerated in order.  This is quite
-    /// similar to the Altivec 'vperm' instruction, except that the indices must
-    /// be constants and are in terms of the element size of VEC1/VEC2, not in
-    /// terms of bytes.
-    VVECTOR_SHUFFLE,
-
     /// VECTOR_SHUFFLE(VEC1, VEC2, SHUFFLEVEC) - Returns a vector, of the same
     /// type as VEC1/VEC2.  SHUFFLEVEC is a BUILD_VECTOR of constant int values
     /// (regardless of whether its datatype is legal or not) that indicate
@@ -298,34 +270,6 @@ namespace ISD {
     /// of the element size of VEC1/VEC2, not in terms of bytes.
     VECTOR_SHUFFLE,
     
-    /// X = VBIT_CONVERT(Y)  and X = VBIT_CONVERT(Y, COUNT,TYPE) - This node
-    /// represents a conversion from or to an ISD::Vector type.
-    ///
-    /// This is lowered to a BIT_CONVERT of the appropriate input/output types.
-    /// The input and output are required to have the same size and at least one
-    /// is required to be a vector (if neither is a vector, just use
-    /// BIT_CONVERT).
-    ///
-    /// If the result is a vector, this takes three operands (like any other
-    /// vector producer) which indicate the size and type of the vector result.
-    /// Otherwise it takes one input.
-    VBIT_CONVERT,
-    
-    /// BINOP(LHS, RHS,  COUNT,TYPE)
-    /// Simple abstract vector operators.  Unlike the integer and floating point
-    /// binary operators, these nodes also take two additional operands:
-    /// a constant element count, and a value type node indicating the type of
-    /// the elements.  The order is op0, op1, count, type.  All vector opcodes,
-    /// including VLOAD and VConstant must currently have count and type as
-    /// their last two operands.
-    VADD, VSUB, VMUL, VSDIV, VUDIV,
-    VAND, VOR, VXOR,
-    
-    /// VSELECT(COND,LHS,RHS,  COUNT,TYPE) - Select for MVT::Vector values.
-    /// COND is a boolean value.  This node return LHS if COND is true, RHS if
-    /// COND is false.
-    VSELECT,
-    
     /// SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a
     /// scalar value into the low element of the resultant vector type.  The top
     /// elements of the vector are undefined.
@@ -432,11 +376,6 @@ namespace ISD {
     // indexed memory ops).
     LOAD, STORE,
     
-    // Abstract vector version of LOAD.  VLOAD has a constant element count as
-    // the first operand, followed by a value type node indicating the type of
-    // the elements, a token chain, a pointer operand, and a SRCVALUE node.
-    VLOAD,
-
     // TRUNCSTORE - This operators truncates (for integer) or rounds (for FP) a
     // value and stores it to memory in one operation.  This can be used for
     // either integer or floating point operands.  The first four operands of
diff --git a/include/llvm/CodeGen/ValueTypes.h b/include/llvm/CodeGen/ValueTypes.h
index a42cee1..c998386 100644
--- a/include/llvm/CodeGen/ValueTypes.h
+++ b/include/llvm/CodeGen/ValueTypes.h
@@ -17,16 +17,17 @@
 #define LLVM_CODEGEN_VALUETYPES_H
 
 #include <cassert>
+#include <string>
 #include "llvm/Support/DataTypes.h"
 
 namespace llvm {
   class Type;
 
-/// MVT namespace - This namespace defines the ValueType enum, which contains
-/// the various low-level value types.
+/// MVT namespace - This namespace defines the SimpleValueType enum, which
+/// contains the various low-level value types, and the ValueType typedef.
 ///
 namespace MVT {  // MVT = Machine Value Types
-  enum ValueType {
+  enum SimpleValueType {
     // If you change this numbering, you must change the values in ValueTypes.td
     // well!
     Other          =   0,   // This is a non-standard value
@@ -45,10 +46,6 @@ namespace MVT {  // MVT = Machine Value Types
 
     isVoid         =  12,   // This has no value
     
-    Vector         =  13,   // This is an abstract vector type, which will
-                            // be expanded into a target vector type, or scalars
-                            // if no matching vector type is available.
-
     v8i8           =  14,   //  8 x i8
     v4i16          =  15,   //  4 x i16
     v2i32          =  16,   //  2 x i32
@@ -76,64 +73,55 @@ namespace MVT {  // MVT = Machine Value Types
     iPTR           = 255
   };
 
-  /// MVT::isInteger - Return true if this is a simple integer, or a packed
-  /// vector integer type.
-  static inline bool isInteger(ValueType VT) {
-    return (VT >= i1 && VT <= i128) || (VT >= v8i8 && VT <= v2i64);
+  /// MVT::ValueType - This type holds low-level value types. Valid values
+  /// include any of the values in the SimpleValueType enum, or any value
+  /// returned from a function in the MVT namespace that has a ValueType
+  /// return type. Any value type equal to one of the SimpleValueType enum
+  /// values is a "simple" value type. All other value types are "extended".
+  ///
+  /// Note that simple doesn't necessary mean legal for the target machine.
+  /// All legal value types must be simple, but often there are some simple
+  /// value types that are not legal.
+  typedef uint32_t ValueType;
+
+  static const int SimpleTypeBits = 8;
+
+  static const uint32_t SimpleTypeMask =
+    (~uint32_t(0) << (32 - SimpleTypeBits)) >> (32 - SimpleTypeBits);
+
+  /// MVT::isExtendedValueType - Test if the given ValueType is extended
+  /// (as opposed to being simple).
+  static inline bool isExtendedValueType(ValueType VT) {
+    return VT & ~SimpleTypeMask;
   }
 
-  /// MVT::isFloatingPoint - Return true if this is a simple FP, or a packed
-  /// vector FP type.
-  static inline bool isFloatingPoint(ValueType VT) {
-    return (VT >= f32 && VT <= f128) || (VT >= v2f32 && VT <= v2f64);
+  /// MVT::isInteger - Return true if this is an integer, or a vector integer
+  /// type.
+  static inline bool isInteger(ValueType VT) {
+    ValueType SVT = VT & SimpleTypeMask;
+    return (SVT >= i1 && SVT <= i128) || (SVT >= v8i8 && SVT <= v2i64);
   }
   
-  /// MVT::isVector - Return true if this is a packed vector type (i.e. not 
-  /// MVT::Vector).
-  static inline bool isVector(ValueType VT) {
-    return VT >= FIRST_VECTOR_VALUETYPE && VT <= LAST_VECTOR_VALUETYPE;
+  /// MVT::isFloatingPoint - Return true if this is an FP, or a vector FP type.
+  static inline bool isFloatingPoint(ValueType VT) {
+    ValueType SVT = VT & SimpleTypeMask;
+    return (SVT >= f32 && SVT <= f128) || (SVT >= v2f32 && SVT <= v2f64);
   }
   
-  /// MVT::getSizeInBits - Return the size of the specified value type in bits.
-  ///
-  static inline unsigned getSizeInBits(ValueType VT) {
-    switch (VT) {
-    default: assert(0 && "ValueType has no known size!");
-    case MVT::i1  :  return 1;
-    case MVT::i8  :  return 8;
-    case MVT::i16 :  return 16;
-    case MVT::f32 :
-    case MVT::i32 :  return 32;
-    case MVT::f64 :
-    case MVT::i64 :
-    case MVT::v8i8:
-    case MVT::v4i16:
-    case MVT::v2i32: 
-    case MVT::v1i64:
-    case MVT::v2f32: return 64;
-    case MVT::f80 :  return 80;
-    case MVT::f128:
-    case MVT::i128: 
-    case MVT::v16i8:
-    case MVT::v8i16:
-    case MVT::v4i32:
-    case MVT::v2i64:
-    case MVT::v4f32:
-    case MVT::v2f64: return 128;
-    }
+  /// MVT::isVector - Return true if this is a vector value type.
+  static inline bool isVector(ValueType VT) {
+    return (VT >= FIRST_VECTOR_VALUETYPE && VT <= LAST_VECTOR_VALUETYPE) ||
+           isExtendedValueType(VT);
   }
   
-  /// MVT::getVectorType - Returns the ValueType that represents a vector
-  /// NumElements in length, where each element is of type VT.  If there is no
-  /// ValueType that represents this vector, a ValueType of Other is returned.
-  ///
-  ValueType getVectorType(ValueType VT, unsigned NumElements);
-    
-  /// MVT::getVectorElementType - Given a packed vector type, return the type of
+  /// MVT::getVectorElementType - Given a vector type, return the type of
   /// each element.
   static inline ValueType getVectorElementType(ValueType VT) {
     switch (VT) {
-    default: assert(0 && "Invalid vector type!");
+    default:
+      if (isExtendedValueType(VT))
+        return VT & SimpleTypeMask;
+      assert(0 && "Invalid vector type!");
     case v8i8 :
     case v16i8: return i8;
     case v4i16:
@@ -148,11 +136,14 @@ namespace MVT {  // MVT = Machine Value Types
     }
   }
   
-  /// MVT::getVectorNumElements - Given a packed vector type, return the number
-  /// of elements it contains.
+  /// MVT::getVectorNumElements - Given a vector type, return the
+  /// number of elements it contains.
   static inline unsigned getVectorNumElements(ValueType VT) {
     switch (VT) {
-    default: assert(0 && "Invalid vector type!");
+    default:
+      if (isExtendedValueType(VT))
+        return ((VT & ~SimpleTypeMask) >> SimpleTypeBits) - 1;
+      assert(0 && "Invalid vector type!");
     case v16i8: return 16;
     case v8i8 :
     case v8i16: return 8;
@@ -167,11 +158,84 @@ namespace MVT {  // MVT = Machine Value Types
     }
   }
   
+  /// MVT::getSizeInBits - Return the size of the specified value type
+  /// in bits.
+  ///
+  static inline unsigned getSizeInBits(ValueType VT) {
+    switch (VT) {
+    default:
+      if (isExtendedValueType(VT))
+        return getSizeInBits(getVectorElementType(VT)) *
+               getVectorNumElements(VT);
+      assert(0 && "ValueType has no known size!");
+    case MVT::i1  :  return 1;
+    case MVT::i8  :  return 8;
+    case MVT::i16 :  return 16;
+    case MVT::f32 :
+    case MVT::i32 :  return 32;
+    case MVT::f64 :
+    case MVT::i64 :
+    case MVT::v8i8:
+    case MVT::v4i16:
+    case MVT::v2i32: 
+    case MVT::v1i64:
+    case MVT::v2f32: return 64;
+    case MVT::f80 :  return 80;
+    case MVT::f128:
+    case MVT::i128: 
+    case MVT::v16i8:
+    case MVT::v8i16:
+    case MVT::v4i32:
+    case MVT::v2i64:
+    case MVT::v4f32:
+    case MVT::v2f64: return 128;
+    }
+  }
+  
+  /// MVT::getVectorType - Returns the ValueType that represents a vector
+  /// NumElements in length, where each element is of type VT.
+  ///
+  static inline ValueType getVectorType(ValueType VT, unsigned NumElements) {
+    switch (VT) {
+    default:
+      break;
+    case MVT::i8:
+      if (NumElements == 8)  return MVT::v8i8;
+      if (NumElements == 16) return MVT::v16i8;
+      break;
+    case MVT::i16:
+      if (NumElements == 4)  return MVT::v4i16;
+      if (NumElements == 8)  return MVT::v8i16;
+      break;
+    case MVT::i32:
+      if (NumElements == 2)  return MVT::v2i32;
+      if (NumElements == 4)  return MVT::v4i32;
+      break;
+    case MVT::i64:
+      if (NumElements == 1)  return MVT::v1i64;
+      if (NumElements == 2)  return MVT::v2i64;
+      break;
+    case MVT::f32:
+      if (NumElements == 2)  return MVT::v2f32;
+      if (NumElements == 4)  return MVT::v4f32;
+      break;
+    case MVT::f64:
+      if (NumElements == 2)  return MVT::v2f64;
+      break;
+    }
+    ValueType Result = VT | ((NumElements + 1) << SimpleTypeBits);
+    assert(getVectorElementType(Result) == VT &&
+           "Bad vector element type!");
+    assert(getVectorNumElements(Result) == NumElements &&
+           "Bad vector length!");
+    return Result;
+  }
+
   /// MVT::getIntVectorWithNumElements - Return any integer vector type that has
   /// the specified number of elements.
   static inline ValueType getIntVectorWithNumElements(unsigned NumElts) {
     switch (NumElts) {
-    default: assert(0 && "Invalid vector type!");
+    default: return getVectorType(i8, NumElts);
     case  1: return v1i64;
     case  2: return v2i32;
     case  4: return v4i16;
@@ -196,7 +260,7 @@ namespace MVT {  // MVT = Machine Value Types
 
   /// MVT::getValueTypeString - This function returns value type as a string,
   /// e.g. "i32".
-  const char *getValueTypeString(ValueType VT);
+  std::string getValueTypeString(ValueType VT);
 
   /// MVT::getTypeForValueType - This method returns an LLVM type corresponding
   /// to the specified ValueType.  For integer types, this returns an unsigned
@@ -204,9 +268,8 @@ namespace MVT {  // MVT = Machine Value Types
   const Type *getTypeForValueType(ValueType VT);
   
   /// MVT::getValueType - Return the value type corresponding to the specified
-  /// type.  This returns all vectors as MVT::Vector and all pointers as
-  /// MVT::iPTR.  If HandleUnknown is true, unknown types are returned as Other,
-  /// otherwise they are invalid.
+  /// type.  This returns all pointers as MVT::iPTR.  If HandleUnknown is true,
+  /// unknown types are returned as Other, otherwise they are invalid.
   ValueType getValueType(const Type *Ty, bool HandleUnknown = false);
 }
 
diff --git a/include/llvm/Target/TargetLowering.h b/include/llvm/Target/TargetLowering.h
index 5b98667..88682f4 100644
--- a/include/llvm/Target/TargetLowering.h
+++ b/include/llvm/Target/TargetLowering.h
@@ -120,6 +120,7 @@ public:
   /// getRegClassFor - Return the register class that should be used for the
   /// specified value type.  This may only be called on legal types.
   TargetRegisterClass *getRegClassFor(MVT::ValueType VT) const {
+    assert(!MVT::isExtendedValueType(VT));
     TargetRegisterClass *RC = RegClassForVT[VT];
     assert(RC && "This value type is not natively supported!");
     return RC;
@@ -129,7 +130,9 @@ public:
   /// specified value type.  This means that it has a register that directly
   /// holds it without promotions or expansions.
   bool isTypeLegal(MVT::ValueType VT) const {
-    return RegClassForVT[VT] != 0;
+    return !MVT::isExtendedValueType(VT) ?
+           RegClassForVT[VT] != 0 :
+           false;
   }
 
   class ValueTypeActionImpl {
@@ -147,9 +150,12 @@ public:
     }
     
     LegalizeAction getTypeAction(MVT::ValueType VT) const {
-      return (LegalizeAction)((ValueTypeActions[VT>>4] >> ((2*VT) & 31)) & 3);
+      return !MVT::isExtendedValueType(VT) ?
+             (LegalizeAction)((ValueTypeActions[VT>>4] >> ((2*VT) & 31)) & 3) :
+             Expand;
     }
     void setTypeAction(MVT::ValueType VT, LegalizeAction Action) {
+      assert(!MVT::isExtendedValueType(VT));
       assert(unsigned(VT >> 4) < 
              sizeof(ValueTypeActions)/sizeof(ValueTypeActions[0]));
       ValueTypeActions[VT>>4] |= Action << ((VT*2) & 31);
@@ -175,6 +181,10 @@ public:
   /// to get to the smaller register. For illegal floating point types, this
   /// returns the integer type to transform to.
   MVT::ValueType getTypeToTransformTo(MVT::ValueType VT) const {
+    if (MVT::isExtendedValueType(VT))
+      return MVT::getVectorType(MVT::getVectorElementType(VT),
+                                MVT::getVectorNumElements(VT) / 2);
+
     return TransformToType[VT];
   }
   
@@ -183,12 +193,13 @@ public:
   /// that are larger than the largest integer register or illegal floating
   /// point types), this returns the largest legal type it will be expanded to.
   MVT::ValueType getTypeToExpandTo(MVT::ValueType VT) const {
+    assert(!MVT::isExtendedValueType(VT));
     while (true) {
       switch (getTypeAction(VT)) {
       case Legal:
         return VT;
       case Expand:
-        VT = TransformToType[VT];
+        VT = getTypeToTransformTo(VT);
         break;
       default:
         assert(false && "Type is not legal nor is it to be expanded!");
@@ -199,17 +210,17 @@ public:
   }
 
   /// getVectorTypeBreakdown - Vector types are broken down into some number of
-  /// legal first class types.  For example, <8 x float> maps to 2 MVT::v4f32
+  /// legal first class types.  For example, MVT::v8f32 maps to 2 MVT::v4f32
   /// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack.
-  /// Similarly, <2 x long> turns into 4 MVT::i32 values with both PPC and X86.
+  /// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86.
   ///
   /// This method returns the number of registers needed, and the VT for each
   /// register.  It also returns the VT of the VectorType elements before they
   /// are promoted/expanded.
   ///
-  unsigned getVectorTypeBreakdown(const VectorType *PTy, 
-                                  MVT::ValueType &PTyElementVT,
-                                  MVT::ValueType &PTyLegalElementVT) const;
+  unsigned getVectorTypeBreakdown(MVT::ValueType VT, 
+                                  MVT::ValueType &ElementVT,
+                                  MVT::ValueType &LegalElementVT) const;
   
   typedef std::vector<double>::const_iterator legal_fpimm_iterator;
   legal_fpimm_iterator legal_fpimm_begin() const {
@@ -242,7 +253,9 @@ public:
   /// expanded to some other code sequence, or the target has a custom expander
   /// for it.
   LegalizeAction getOperationAction(unsigned Op, MVT::ValueType VT) const {
-    return (LegalizeAction)((OpActions[Op] >> (2*VT)) & 3);
+    return !MVT::isExtendedValueType(VT) ?
+           (LegalizeAction)((OpActions[Op] >> (2*VT)) & 3) :
+           Expand;
   }
   
   /// isOperationLegal - Return true if the specified operation is legal on this
@@ -257,7 +270,9 @@ public:
   /// expanded to some other code sequence, or the target has a custom expander
   /// for it.
   LegalizeAction getLoadXAction(unsigned LType, MVT::ValueType VT) const {
-    return (LegalizeAction)((LoadXActions[LType] >> (2*VT)) & 3);
+    return !MVT::isExtendedValueType(VT) ?
+           (LegalizeAction)((LoadXActions[LType] >> (2*VT)) & 3) :
+           Expand;
   }
   
   /// isLoadXLegal - Return true if the specified load with extension is legal
@@ -272,7 +287,9 @@ public:
   /// expanded to some other code sequence, or the target has a custom expander
   /// for it.
   LegalizeAction getStoreXAction(MVT::ValueType VT) const {
-    return (LegalizeAction)((StoreXActions >> (2*VT)) & 3);
+    return !MVT::isExtendedValueType(VT) ?
+           (LegalizeAction)((StoreXActions >> (2*VT)) & 3) :
+           Expand;
   }
   
   /// isStoreXLegal - Return true if the specified store with truncation is
@@ -287,7 +304,9 @@ public:
   /// for it.
   LegalizeAction
   getIndexedLoadAction(unsigned IdxMode, MVT::ValueType VT) const {
-    return (LegalizeAction)((IndexedModeActions[0][IdxMode] >> (2*VT)) & 3);
+    return !MVT::isExtendedValueType(VT) ?
+           (LegalizeAction)((IndexedModeActions[0][IdxMode] >> (2*VT)) & 3) :
+           Expand;
   }
 
   /// isIndexedLoadLegal - Return true if the specified indexed load is legal
@@ -303,7 +322,9 @@ public:
   /// for it.
   LegalizeAction
   getIndexedStoreAction(unsigned IdxMode, MVT::ValueType VT) const {
-    return (LegalizeAction)((IndexedModeActions[1][IdxMode] >> (2*VT)) & 3);
+    return !MVT::isExtendedValueType(VT) ?
+           (LegalizeAction)((IndexedModeActions[1][IdxMode] >> (2*VT)) & 3) :
+           Expand;
   }  
   
   /// isIndexedStoreLegal - Return true if the specified indexed load is legal
@@ -352,7 +373,11 @@ public:
   /// registers, but may be more than one for types (like i64) that are split
   /// into pieces.
   unsigned getNumRegisters(MVT::ValueType VT) const {
-    return NumRegistersForVT[VT];
+    if (!MVT::isExtendedValueType(VT))
+      return NumRegistersForVT[VT];
+           
+    MVT::ValueType VT1, VT2;
+    return getVectorTypeBreakdown(VT, VT1, VT2);
   }
   
   /// hasTargetDAGCombine - If true, the target has custom DAG combine
@@ -648,6 +673,7 @@ protected:
   /// regclass for the specified value type.  This indicates the selector can
   /// handle values of that class natively.
   void addRegisterClass(MVT::ValueType VT, TargetRegisterClass *RC) {
+    assert(!MVT::isExtendedValueType(VT));
     AvailableRegClasses.push_back(std::make_pair(VT, RC));
     RegClassForVT[VT] = RC;
   }
diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 32ab5d3..cb7276e 100644
--- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -227,7 +227,7 @@ namespace {
     SDOperand visitAND(SDNode *N);
     SDOperand visitOR(SDNode *N);
     SDOperand visitXOR(SDNode *N);
-    SDOperand visitVBinOp(SDNode *N, ISD::NodeType IntOp, ISD::NodeType FPOp);
+    SDOperand SimplifyVBinOp(SDNode *N);
     SDOperand visitSHL(SDNode *N);
     SDOperand visitSRA(SDNode *N);
     SDOperand visitSRL(SDNode *N);
@@ -243,7 +243,6 @@ namespace {
     SDOperand visitSIGN_EXTEND_INREG(SDNode *N);
     SDOperand visitTRUNCATE(SDNode *N);
     SDOperand visitBIT_CONVERT(SDNode *N);
-    SDOperand visitVBIT_CONVERT(SDNode *N);
     SDOperand visitFADD(SDNode *N);
     SDOperand visitFSUB(SDNode *N);
     SDOperand visitFMUL(SDNode *N);
@@ -264,10 +263,9 @@ namespace {
     SDOperand visitLOAD(SDNode *N);
     SDOperand visitSTORE(SDNode *N);
     SDOperand visitINSERT_VECTOR_ELT(SDNode *N);
-    SDOperand visitVINSERT_VECTOR_ELT(SDNode *N);
-    SDOperand visitVBUILD_VECTOR(SDNode *N);
+    SDOperand visitBUILD_VECTOR(SDNode *N);
+    SDOperand visitCONCAT_VECTORS(SDNode *N);
     SDOperand visitVECTOR_SHUFFLE(SDNode *N);
-    SDOperand visitVVECTOR_SHUFFLE(SDNode *N);
 
     SDOperand XformToShuffleWithZero(SDNode *N);
     SDOperand ReassociateOps(unsigned Opc, SDOperand LHS, SDOperand RHS);
@@ -280,7 +278,7 @@ namespace {
                                bool NotExtCompare = false);
     SDOperand SimplifySetCC(MVT::ValueType VT, SDOperand N0, SDOperand N1,
                             ISD::CondCode Cond, bool foldBooleans = true);
-    SDOperand ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(SDNode *, MVT::ValueType);
+    SDOperand ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *, MVT::ValueType);
     SDOperand BuildSDIV(SDNode *N);
     SDOperand BuildUDIV(SDNode *N);
     SDNode *MatchRotate(SDOperand LHS, SDOperand RHS);
@@ -657,7 +655,6 @@ SDOperand DAGCombiner::visit(SDNode *N) {
   case ISD::SIGN_EXTEND_INREG:  return visitSIGN_EXTEND_INREG(N);
   case ISD::TRUNCATE:           return visitTRUNCATE(N);
   case ISD::BIT_CONVERT:        return visitBIT_CONVERT(N);
-  case ISD::VBIT_CONVERT:       return visitVBIT_CONVERT(N);
   case ISD::FADD:               return visitFADD(N);
   case ISD::FSUB:               return visitFSUB(N);
   case ISD::FMUL:               return visitFMUL(N);
@@ -678,18 +675,9 @@ SDOperand DAGCombiner::visit(SDNode *N) {
   case ISD::LOAD:               return visitLOAD(N);
   case ISD::STORE:              return visitSTORE(N);
   case ISD::INSERT_VECTOR_ELT:  return visitINSERT_VECTOR_ELT(N);
-  case ISD::VINSERT_VECTOR_ELT: return visitVINSERT_VECTOR_ELT(N);
-  case ISD::VBUILD_VECTOR:      return visitVBUILD_VECTOR(N);
+  case ISD::BUILD_VECTOR:       return visitBUILD_VECTOR(N);
+  case ISD::CONCAT_VECTORS:     return visitCONCAT_VECTORS(N);
   case ISD::VECTOR_SHUFFLE:     return visitVECTOR_SHUFFLE(N);
-  case ISD::VVECTOR_SHUFFLE:    return visitVVECTOR_SHUFFLE(N);
-  case ISD::VADD:               return visitVBinOp(N, ISD::ADD , ISD::FADD);
-  case ISD::VSUB:               return visitVBinOp(N, ISD::SUB , ISD::FSUB);
-  case ISD::VMUL:               return visitVBinOp(N, ISD::MUL , ISD::FMUL);
-  case ISD::VSDIV:              return visitVBinOp(N, ISD::SDIV, ISD::FDIV);
-  case ISD::VUDIV:              return visitVBinOp(N, ISD::UDIV, ISD::UDIV);
-  case ISD::VAND:               return visitVBinOp(N, ISD::AND , ISD::AND);
-  case ISD::VOR:                return visitVBinOp(N, ISD::OR  , ISD::OR);
-  case ISD::VXOR:               return visitVBinOp(N, ISD::XOR , ISD::XOR);
   }
   return SDOperand();
 }
@@ -856,6 +844,10 @@ SDOperand DAGCombiner::visitADD(SDNode *N) {
   ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
   MVT::ValueType VT = N0.getValueType();
+
+  // fold vector ops
+  SDOperand FoldedVOp = SimplifyVBinOp(N);
+  if (FoldedVOp.Val) return FoldedVOp;
   
   // fold (add c1, c2) -> c1+c2
   if (N0C && N1C)
@@ -928,6 +920,10 @@ SDOperand DAGCombiner::visitADD(SDNode *N) {
     if (Result.Val) return Result;
   }
 
+  // If either operand is undef, the result is undef
+  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+    return DAG.getNode(ISD::UNDEF, VT);
+
   return SDOperand();
 }
 
@@ -1004,6 +1000,10 @@ SDOperand DAGCombiner::visitSUB(SDNode *N) {
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val);
   MVT::ValueType VT = N0.getValueType();
   
+  // fold vector ops
+  SDOperand FoldedVOp = SimplifyVBinOp(N);
+  if (FoldedVOp.Val) return FoldedVOp;
+  
   // fold (sub x, x) -> 0
   if (N0 == N1)
     return DAG.getConstant(0, N->getValueType(0));
@@ -1024,6 +1024,10 @@ SDOperand DAGCombiner::visitSUB(SDNode *N) {
     SDOperand Result = combineSelectAndUse(N, N1, N0, DAG);
     if (Result.Val) return Result;
   }
+  // If either operand is undef, the result is undef
+  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+    return DAG.getNode(ISD::UNDEF, VT);
+
   return SDOperand();
 }
 
@@ -1034,6 +1038,10 @@ SDOperand DAGCombiner::visitMUL(SDNode *N) {
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
   MVT::ValueType VT = N0.getValueType();
   
+  // fold vector ops
+  SDOperand FoldedVOp = SimplifyVBinOp(N);
+  if (FoldedVOp.Val) return FoldedVOp;
+  
   // fold (mul c1, c2) -> c1*c2
   if (N0C && N1C)
     return DAG.getNode(ISD::MUL, VT, N0, N1);
@@ -1098,6 +1106,11 @@ SDOperand DAGCombiner::visitMUL(SDNode *N) {
   SDOperand RMUL = ReassociateOps(ISD::MUL, N0, N1);
   if (RMUL.Val != 0)
     return RMUL;
+
+  // If either operand is undef, the result is undef
+  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+    return DAG.getNode(ISD::UNDEF, VT);
+
   return SDOperand();
 }
 
@@ -1108,6 +1121,10 @@ SDOperand DAGCombiner::visitSDIV(SDNode *N) {
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val);
   MVT::ValueType VT = N->getValueType(0);
 
+  // fold vector ops
+  SDOperand FoldedVOp = SimplifyVBinOp(N);
+  if (FoldedVOp.Val) return FoldedVOp;
+  
   // fold (sdiv c1, c2) -> c1/c2
   if (N0C && N1C && !N1C->isNullValue())
     return DAG.getNode(ISD::SDIV, VT, N0, N1);
@@ -1162,6 +1179,11 @@ SDOperand DAGCombiner::visitSDIV(SDNode *N) {
     SDOperand Op = BuildSDIV(N);
     if (Op.Val) return Op;
   }
+
+  // If either operand is undef, the result is undef
+  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+    return DAG.getNode(ISD::UNDEF, VT);
+
   return SDOperand();
 }
 
@@ -1172,6 +1194,10 @@ SDOperand DAGCombiner::visitUDIV(SDNode *N) {
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val);
   MVT::ValueType VT = N->getValueType(0);
   
+  // fold vector ops
+  SDOperand FoldedVOp = SimplifyVBinOp(N);
+  if (FoldedVOp.Val) return FoldedVOp;
+  
   // fold (udiv c1, c2) -> c1/c2
   if (N0C && N1C && !N1C->isNullValue())
     return DAG.getNode(ISD::UDIV, VT, N0, N1);
@@ -1198,6 +1224,11 @@ SDOperand DAGCombiner::visitUDIV(SDNode *N) {
     SDOperand Op = BuildUDIV(N);
     if (Op.Val) return Op;
   }
+
+  // If either operand is undef, the result is undef
+  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+    return DAG.getNode(ISD::UNDEF, VT);
+
   return SDOperand();
 }
 
@@ -1229,6 +1260,10 @@ SDOperand DAGCombiner::visitSREM(SDNode *N) {
     return Sub;
   }
   
+  // If either operand is undef, the result is undef
+  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+    return DAG.getNode(ISD::UNDEF, VT);
+
   return SDOperand();
 }
 
@@ -1267,6 +1302,10 @@ SDOperand DAGCombiner::visitUREM(SDNode *N) {
     return Sub;
   }
   
+  // If either operand is undef, the result is undef
+  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+    return DAG.getNode(ISD::UNDEF, VT);
+
   return SDOperand();
 }
 
@@ -1274,6 +1313,7 @@ SDOperand DAGCombiner::visitMULHS(SDNode *N) {
   SDOperand N0 = N->getOperand(0);
   SDOperand N1 = N->getOperand(1);
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+  MVT::ValueType VT = N->getValueType(0);
   
   // fold (mulhs x, 0) -> 0
   if (N1C && N1C->isNullValue())
@@ -1283,6 +1323,10 @@ SDOperand DAGCombiner::visitMULHS(SDNode *N) {
     return DAG.getNode(ISD::SRA, N0.getValueType(), N0, 
                        DAG.getConstant(MVT::getSizeInBits(N0.getValueType())-1,
                                        TLI.getShiftAmountTy()));
+  // If either operand is undef, the result is undef
+  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+    return DAG.getNode(ISD::UNDEF, VT);
+
   return SDOperand();
 }
 
@@ -1290,6 +1334,7 @@ SDOperand DAGCombiner::visitMULHU(SDNode *N) {
   SDOperand N0 = N->getOperand(0);
   SDOperand N1 = N->getOperand(1);
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
+  MVT::ValueType VT = N->getValueType(0);
   
   // fold (mulhu x, 0) -> 0
   if (N1C && N1C->isNullValue())
@@ -1297,6 +1342,10 @@ SDOperand DAGCombiner::visitMULHU(SDNode *N) {
   // fold (mulhu x, 1) -> 0
   if (N1C && N1C->getValue() == 1)
     return DAG.getConstant(0, N0.getValueType());
+  // If either operand is undef, the result is undef
+  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+    return DAG.getNode(ISD::UNDEF, VT);
+
   return SDOperand();
 }
 
@@ -1336,6 +1385,10 @@ SDOperand DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) {
     return DAG.getNode(N0.getOpcode(), VT, ORNode, N0.getOperand(1));
   }
   
+  // If either operand is undef, the result is undef
+  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+    return DAG.getNode(ISD::UNDEF, VT);
+
   return SDOperand();
 }
 
@@ -1347,6 +1400,10 @@ SDOperand DAGCombiner::visitAND(SDNode *N) {
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
   MVT::ValueType VT = N1.getValueType();
   
+  // fold vector ops
+  SDOperand FoldedVOp = SimplifyVBinOp(N);
+  if (FoldedVOp.Val) return FoldedVOp;
+  
   // fold (and c1, c2) -> c1&c2
   if (N0C && N1C)
     return DAG.getNode(ISD::AND, VT, N0, N1);
@@ -1528,6 +1585,10 @@ SDOperand DAGCombiner::visitOR(SDNode *N) {
   MVT::ValueType VT = N1.getValueType();
   unsigned OpSizeInBits = MVT::getSizeInBits(VT);
   
+  // fold vector ops
+  SDOperand FoldedVOp = SimplifyVBinOp(N);
+  if (FoldedVOp.Val) return FoldedVOp;
+  
   // fold (or c1, c2) -> c1|c2
   if (N0C && N1C)
     return DAG.getNode(ISD::OR, VT, N0, N1);
@@ -1807,6 +1868,10 @@ SDOperand DAGCombiner::visitXOR(SDNode *N) {
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
   MVT::ValueType VT = N0.getValueType();
   
+  // fold vector ops
+  SDOperand FoldedVOp = SimplifyVBinOp(N);
+  if (FoldedVOp.Val) return FoldedVOp;
+  
   // fold (xor c1, c2) -> c1^c2
   if (N0C && N1C)
     return DAG.getNode(ISD::XOR, VT, N0, N1);
@@ -2742,6 +2807,30 @@ SDOperand DAGCombiner::visitBIT_CONVERT(SDNode *N) {
   SDOperand N0 = N->getOperand(0);
   MVT::ValueType VT = N->getValueType(0);
 
+  // If the input is a BUILD_VECTOR with all constant elements, fold this now.
+  // Only do this before legalize, since afterward the target may be depending
+  // on the bitconvert.
+  // First check to see if this is all constant.
+  if (!AfterLegalize &&
+      N0.getOpcode() == ISD::BUILD_VECTOR && N0.Val->hasOneUse() &&
+      MVT::isVector(VT)) {
+    bool isSimple = true;
+    for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i)
+      if (N0.getOperand(i).getOpcode() != ISD::UNDEF &&
+          N0.getOperand(i).getOpcode() != ISD::Constant &&
+          N0.getOperand(i).getOpcode() != ISD::ConstantFP) {
+        isSimple = false; 
+        break;
+      }
+        
+    MVT::ValueType DestEltVT = MVT::getVectorElementType(N->getValueType(0));
+    assert(!MVT::isVector(DestEltVT) &&
+           "Element type of vector ValueType must not be vector!");
+    if (isSimple) {
+      return ConstantFoldBIT_CONVERTofBUILD_VECTOR(N0.Val, DestEltVT);
+    }
+  }
+  
   // If the input is a constant, let getNode() fold it.
   if (isa<ConstantSDNode>(N0) || isa<ConstantFPSDNode>(N0)) {
     SDOperand Res = DAG.getNode(ISD::BIT_CONVERT, VT, N0);
@@ -2774,37 +2863,11 @@ SDOperand DAGCombiner::visitBIT_CONVERT(SDNode *N) {
   return SDOperand();
 }
 
-SDOperand DAGCombiner::visitVBIT_CONVERT(SDNode *N) {
-  SDOperand N0 = N->getOperand(0);
-  MVT::ValueType VT = N->getValueType(0);
-
-  // If the input is a VBUILD_VECTOR with all constant elements, fold this now.
-  // First check to see if this is all constant.
-  if (N0.getOpcode() == ISD::VBUILD_VECTOR && N0.Val->hasOneUse() &&
-      VT == MVT::Vector) {
-    bool isSimple = true;
-    for (unsigned i = 0, e = N0.getNumOperands()-2; i != e; ++i)
-      if (N0.getOperand(i).getOpcode() != ISD::UNDEF &&
-          N0.getOperand(i).getOpcode() != ISD::Constant &&
-          N0.getOperand(i).getOpcode() != ISD::ConstantFP) {
-        isSimple = false; 
-        break;
-      }
-        
-    MVT::ValueType DestEltVT = cast<VTSDNode>(N->getOperand(2))->getVT();
-    if (isSimple && !MVT::isVector(DestEltVT)) {
-      return ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(N0.Val, DestEltVT);
-    }
-  }
-  
-  return SDOperand();
-}
-
-/// ConstantFoldVBIT_CONVERTofVBUILD_VECTOR - We know that BV is a vbuild_vector
+/// ConstantFoldBIT_CONVERTofBUILD_VECTOR - We know that BV is a build_vector
 /// node with Constant, ConstantFP or Undef operands.  DstEltVT indicates the 
 /// destination element value type.
 SDOperand DAGCombiner::
-ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) {
+ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) {
   MVT::ValueType SrcEltVT = BV->getOperand(0).getValueType();
   
   // If this is already the right type, we're done.
@@ -2817,13 +2880,14 @@ ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) {
   // type, convert each element.  This handles FP<->INT cases.
   if (SrcBitSize == DstBitSize) {
     SmallVector<SDOperand, 8> Ops;
-    for (unsigned i = 0, e = BV->getNumOperands()-2; i != e; ++i) {
+    for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
       Ops.push_back(DAG.getNode(ISD::BIT_CONVERT, DstEltVT, BV->getOperand(i)));
       AddToWorkList(Ops.back().Val);
     }
-    Ops.push_back(*(BV->op_end()-2)); // Add num elements.
-    Ops.push_back(DAG.getValueType(DstEltVT));
-    return DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &Ops[0], Ops.size());
+    MVT::ValueType VT =
+      MVT::getVectorType(DstEltVT,
+                         MVT::getVectorNumElements(BV->getValueType(0)));
+    return DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size());
   }
   
   // Otherwise, we're growing or shrinking the elements.  To avoid having to
@@ -2834,7 +2898,7 @@ ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) {
     // same sizes.
     assert((SrcEltVT == MVT::f32 || SrcEltVT == MVT::f64) && "Unknown FP VT!");
     MVT::ValueType IntVT = SrcEltVT == MVT::f32 ? MVT::i32 : MVT::i64;
-    BV = ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(BV, IntVT).Val;
+    BV = ConstantFoldBIT_CONVERTofBUILD_VECTOR(BV, IntVT).Val;
     SrcEltVT = IntVT;
   }
   
@@ -2843,10 +2907,10 @@ ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) {
   if (MVT::isFloatingPoint(DstEltVT)) {
     assert((DstEltVT == MVT::f32 || DstEltVT == MVT::f64) && "Unknown FP VT!");
     MVT::ValueType TmpVT = DstEltVT == MVT::f32 ? MVT::i32 : MVT::i64;
-    SDNode *Tmp = ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(BV, TmpVT).Val;
+    SDNode *Tmp = ConstantFoldBIT_CONVERTofBUILD_VECTOR(BV, TmpVT).Val;
     
     // Next, convert to FP elements of the same size.
-    return ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(Tmp, DstEltVT);
+    return ConstantFoldBIT_CONVERTofBUILD_VECTOR(Tmp, DstEltVT);
   }
   
   // Okay, we know the src/dst types are both integers of differing types.
@@ -2856,7 +2920,7 @@ ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) {
     unsigned NumInputsPerOutput = DstBitSize/SrcBitSize;
     
     SmallVector<SDOperand, 8> Ops;
-    for (unsigned i = 0, e = BV->getNumOperands()-2; i != e;
+    for (unsigned i = 0, e = BV->getNumOperands(); i != e;
          i += NumInputsPerOutput) {
       bool isLE = TLI.isLittleEndian();
       uint64_t NewBits = 0;
@@ -2877,16 +2941,16 @@ ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) {
         Ops.push_back(DAG.getConstant(NewBits, DstEltVT));
     }
 
-    Ops.push_back(DAG.getConstant(Ops.size(), MVT::i32)); // Add num elements.
-    Ops.push_back(DAG.getValueType(DstEltVT));            // Add element size.
-    return DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &Ops[0], Ops.size());
+    MVT::ValueType VT = MVT::getVectorType(DstEltVT,
+                                           Ops.size());
+    return DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size());
   }
   
   // Finally, this must be the case where we are shrinking elements: each input
   // turns into multiple outputs.
   unsigned NumOutputsPerInput = SrcBitSize/DstBitSize;
   SmallVector<SDOperand, 8> Ops;
-  for (unsigned i = 0, e = BV->getNumOperands()-2; i != e; ++i) {
+  for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
     if (BV->getOperand(i).getOpcode() == ISD::UNDEF) {
       for (unsigned j = 0; j != NumOutputsPerInput; ++j)
         Ops.push_back(DAG.getNode(ISD::UNDEF, DstEltVT));
@@ -2904,9 +2968,8 @@ ConstantFoldVBIT_CONVERTofVBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) {
     if (!TLI.isLittleEndian())
       std::reverse(Ops.end()-NumOutputsPerInput, Ops.end());
   }
-  Ops.push_back(DAG.getConstant(Ops.size(), MVT::i32)); // Add num elements.
-  Ops.push_back(DAG.getValueType(DstEltVT));            // Add element size.
-  return DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &Ops[0], Ops.size());
+  MVT::ValueType VT = MVT::getVectorType(DstEltVT, Ops.size());
+  return DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size());
 }
 
 
@@ -2918,6 +2981,10 @@ SDOperand DAGCombiner::visitFADD(SDNode *N) {
   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
   MVT::ValueType VT = N->getValueType(0);
   
+  // fold vector ops
+  SDOperand FoldedVOp = SimplifyVBinOp(N);
+  if (FoldedVOp.Val) return FoldedVOp;
+  
   // fold (fadd c1, c2) -> c1+c2
   if (N0CFP && N1CFP)
     return DAG.getNode(ISD::FADD, VT, N0, N1);
@@ -2937,6 +3004,10 @@ SDOperand DAGCombiner::visitFADD(SDNode *N) {
     return DAG.getNode(ISD::FADD, VT, N0.getOperand(0),
                        DAG.getNode(ISD::FADD, VT, N0.getOperand(1), N1));
   
+  // If either operand is undef, the result is undef
+  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+    return DAG.getNode(ISD::UNDEF, VT);
+
   return SDOperand();
 }
 
@@ -2947,6 +3018,10 @@ SDOperand DAGCombiner::visitFSUB(SDNode *N) {
   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
   MVT::ValueType VT = N->getValueType(0);
   
+  // fold vector ops
+  SDOperand FoldedVOp = SimplifyVBinOp(N);
+  if (FoldedVOp.Val) return FoldedVOp;
+  
   // fold (fsub c1, c2) -> c1-c2
   if (N0CFP && N1CFP)
     return DAG.getNode(ISD::FSUB, VT, N0, N1);
@@ -2954,6 +3029,10 @@ SDOperand DAGCombiner::visitFSUB(SDNode *N) {
   if (isNegatibleForFree(N1))
     return DAG.getNode(ISD::FADD, VT, N0, GetNegatedExpression(N1, DAG));
   
+  // If either operand is undef, the result is undef
+  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+    return DAG.getNode(ISD::UNDEF, VT);
+
   return SDOperand();
 }
 
@@ -2964,6 +3043,10 @@ SDOperand DAGCombiner::visitFMUL(SDNode *N) {
   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
   MVT::ValueType VT = N->getValueType(0);
 
+  // fold vector ops
+  SDOperand FoldedVOp = SimplifyVBinOp(N);
+  if (FoldedVOp.Val) return FoldedVOp;
+  
   // fold (fmul c1, c2) -> c1*c2
   if (N0CFP && N1CFP)
     return DAG.getNode(ISD::FMUL, VT, N0, N1);
@@ -2994,6 +3077,10 @@ SDOperand DAGCombiner::visitFMUL(SDNode *N) {
     return DAG.getNode(ISD::FMUL, VT, N0.getOperand(0),
                        DAG.getNode(ISD::FMUL, VT, N0.getOperand(1), N1));
   
+  // If either operand is undef, the result is undef
+  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+    return DAG.getNode(ISD::UNDEF, VT);
+
   return SDOperand();
 }
 
@@ -3004,6 +3091,10 @@ SDOperand DAGCombiner::visitFDIV(SDNode *N) {
   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
   MVT::ValueType VT = N->getValueType(0);
 
+  // fold vector ops
+  SDOperand FoldedVOp = SimplifyVBinOp(N);
+  if (FoldedVOp.Val) return FoldedVOp;
+  
   // fold (fdiv c1, c2) -> c1/c2
   if (N0CFP && N1CFP)
     return DAG.getNode(ISD::FDIV, VT, N0, N1);
@@ -3020,6 +3111,10 @@ SDOperand DAGCombiner::visitFDIV(SDNode *N) {
     }
   }
   
+  // If either operand is undef, the result is undef
+  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+    return DAG.getNode(ISD::UNDEF, VT);
+
   return SDOperand();
 }
 
@@ -3033,6 +3128,11 @@ SDOperand DAGCombiner::visitFREM(SDNode *N) {
   // fold (frem c1, c2) -> fmod(c1,c2)
   if (N0CFP && N1CFP)
     return DAG.getNode(ISD::FREM, VT, N0, N1);
+
+  // If either operand is undef, the result is undef
+  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
+    return DAG.getNode(ISD::UNDEF, VT);
+
   return SDOperand();
 }
 
@@ -3735,53 +3835,32 @@ SDOperand DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
   return SDOperand();
 }
 
-SDOperand DAGCombiner::visitVINSERT_VECTOR_ELT(SDNode *N) {
-  SDOperand InVec = N->getOperand(0);
-  SDOperand InVal = N->getOperand(1);
-  SDOperand EltNo = N->getOperand(2);
-  SDOperand NumElts = N->getOperand(3);
-  SDOperand EltType = N->getOperand(4);
-  
-  // If the invec is a VBUILD_VECTOR and if EltNo is a constant, build a new
-  // vector with the inserted element.
-  if (InVec.getOpcode() == ISD::VBUILD_VECTOR && isa<ConstantSDNode>(EltNo)) {
-    unsigned Elt = cast<ConstantSDNode>(EltNo)->getValue();
-    SmallVector<SDOperand, 8> Ops(InVec.Val->op_begin(), InVec.Val->op_end());
-    if (Elt < Ops.size()-2)
-      Ops[Elt] = InVal;
-    return DAG.getNode(ISD::VBUILD_VECTOR, InVec.getValueType(),
-                       &Ops[0], Ops.size());
-  }
-  
-  return SDOperand();
-}
-
-SDOperand DAGCombiner::visitVBUILD_VECTOR(SDNode *N) {
-  unsigned NumInScalars = N->getNumOperands()-2;
-  SDOperand NumElts = N->getOperand(NumInScalars);
-  SDOperand EltType = N->getOperand(NumInScalars+1);
+SDOperand DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
+  unsigned NumInScalars = N->getNumOperands();
+  MVT::ValueType VT = N->getValueType(0);
+  unsigned NumElts = MVT::getVectorNumElements(VT);
+  MVT::ValueType EltType = MVT::getVectorElementType(VT);
 
-  // Check to see if this is a VBUILD_VECTOR of a bunch of VEXTRACT_VECTOR_ELT
-  // operations.  If so, and if the EXTRACT_ELT vector inputs come from at most
-  // two distinct vectors, turn this into a shuffle node.
+  // Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT
+  // operations.  If so, and if the EXTRACT_VECTOR_ELT vector inputs come from
+  // at most two distinct vectors, turn this into a shuffle node.
   SDOperand VecIn1, VecIn2;
   for (unsigned i = 0; i != NumInScalars; ++i) {
     // Ignore undef inputs.
     if (N->getOperand(i).getOpcode() == ISD::UNDEF) continue;
     
-    // If this input is something other than a VEXTRACT_VECTOR_ELT with a
+    // If this input is something other than a EXTRACT_VECTOR_ELT with a
     // constant index, bail out.
-    if (N->getOperand(i).getOpcode() != ISD::VEXTRACT_VECTOR_ELT ||
+    if (N->getOperand(i).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
         !isa<ConstantSDNode>(N->getOperand(i).getOperand(1))) {
       VecIn1 = VecIn2 = SDOperand(0, 0);
       break;
     }
     
-    // If the input vector type disagrees with the result of the vbuild_vector,
+    // If the input vector type disagrees with the result of the build_vector,
     // we can't make a shuffle.
     SDOperand ExtractedFromVec = N->getOperand(i).getOperand(0);
-    if (*(ExtractedFromVec.Val->op_end()-2) != NumElts ||
-        *(ExtractedFromVec.Val->op_end()-1) != EltType) {
+    if (ExtractedFromVec.getValueType() != VT) {
       VecIn1 = VecIn2 = SDOperand(0, 0);
       break;
     }
@@ -3825,158 +3904,49 @@ SDOperand DAGCombiner::visitVBUILD_VECTOR(SDNode *N) {
     }
     
     // Add count and size info.
-    BuildVecIndices.push_back(NumElts);
-    BuildVecIndices.push_back(DAG.getValueType(TLI.getPointerTy()));
+    MVT::ValueType BuildVecVT =
+      MVT::getVectorType(TLI.getPointerTy(), NumElts);
     
-    // Return the new VVECTOR_SHUFFLE node.
+    // Return the new VECTOR_SHUFFLE node.
     SDOperand Ops[5];
     Ops[0] = VecIn1;
     if (VecIn2.Val) {
       Ops[1] = VecIn2;
     } else {
-      // Use an undef vbuild_vector as input for the second operand.
+      // Use an undef build_vector as input for the second operand.
       std::vector<SDOperand> UnOps(NumInScalars,
                                    DAG.getNode(ISD::UNDEF, 
-                                           cast<VTSDNode>(EltType)->getVT()));
-      UnOps.push_back(NumElts);
-      UnOps.push_back(EltType);
-      Ops[1] = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector,
+                                               EltType));
+      Ops[1] = DAG.getNode(ISD::BUILD_VECTOR, VT,
                            &UnOps[0], UnOps.size());
       AddToWorkList(Ops[1].Val);
     }
-    Ops[2] = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector,
+    Ops[2] = DAG.getNode(ISD::BUILD_VECTOR, BuildVecVT,
                          &BuildVecIndices[0], BuildVecIndices.size());
-    Ops[3] = NumElts;
-    Ops[4] = EltType;
-    return DAG.getNode(ISD::VVECTOR_SHUFFLE, MVT::Vector, Ops, 5);
+    return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, Ops, 3);
   }
   
   return SDOperand();
 }
 
-SDOperand DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
-  SDOperand ShufMask = N->getOperand(2);
-  unsigned NumElts = ShufMask.getNumOperands();
+SDOperand DAGCombiner::visitCONCAT_VECTORS(SDNode *N) {
+  // TODO: Check to see if this is a CONCAT_VECTORS of a bunch of
+  // EXTRACT_SUBVECTOR operations.  If so, and if the EXTRACT_SUBVECTOR vector
+  // inputs come from at most two distinct vectors, turn this into a shuffle
+  // node.
 
-  // If the shuffle mask is an identity operation on the LHS, return the LHS.
-  bool isIdentity = true;
-  for (unsigned i = 0; i != NumElts; ++i) {
-    if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF &&
-        cast<ConstantSDNode>(ShufMask.getOperand(i))->getValue() != i) {
-      isIdentity = false;
-      break;
-    }
-  }
-  if (isIdentity) return N->getOperand(0);
-
-  // If the shuffle mask is an identity operation on the RHS, return the RHS.
-  isIdentity = true;
-  for (unsigned i = 0; i != NumElts; ++i) {
-    if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF &&
-        cast<ConstantSDNode>(ShufMask.getOperand(i))->getValue() != i+NumElts) {
-      isIdentity = false;
-      break;
-    }
-  }
-  if (isIdentity) return N->getOperand(1);
-
-  // Check if the shuffle is a unary shuffle, i.e. one of the vectors is not
-  // needed at all.
-  bool isUnary = true;
-  bool isSplat = true;
-  int VecNum = -1;
-  unsigned BaseIdx = 0;
-  for (unsigned i = 0; i != NumElts; ++i)
-    if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF) {
-      unsigned Idx = cast<ConstantSDNode>(ShufMask.getOperand(i))->getValue();
-      int V = (Idx < NumElts) ? 0 : 1;
-      if (VecNum == -1) {
-        VecNum = V;
-        BaseIdx = Idx;
-      } else {
-        if (BaseIdx != Idx)
-          isSplat = false;
-        if (VecNum != V) {
-          isUnary = false;
-          break;
-        }
-      }
-    }
-
-  SDOperand N0 = N->getOperand(0);
-  SDOperand N1 = N->getOperand(1);
-  // Normalize unary shuffle so the RHS is undef.
-  if (isUnary && VecNum == 1)
-    std::swap(N0, N1);
-
-  // If it is a splat, check if the argument vector is a build_vector with
-  // all scalar elements the same.
-  if (isSplat) {
-    SDNode *V = N0.Val;
-    if (V->getOpcode() == ISD::BIT_CONVERT)
-      V = V->getOperand(0).Val;
-    if (V->getOpcode() == ISD::BUILD_VECTOR) {
-      unsigned NumElems = V->getNumOperands()-2;
-      if (NumElems > BaseIdx) {
-        SDOperand Base;
-        bool AllSame = true;
-        for (unsigned i = 0; i != NumElems; ++i) {
-          if (V->getOperand(i).getOpcode() != ISD::UNDEF) {
-            Base = V->getOperand(i);
-            break;
-          }
-        }
-        // Splat of <u, u, u, u>, return <u, u, u, u>
-        if (!Base.Val)
-          return N0;
-        for (unsigned i = 0; i != NumElems; ++i) {
-          if (V->getOperand(i).getOpcode() != ISD::UNDEF &&
-              V->getOperand(i) != Base) {
-            AllSame = false;
-            break;
-          }
-        }
-        // Splat of <x, x, x, x>, return <x, x, x, x>
-        if (AllSame)
-          return N0;
-      }
-    }
+  // If we only have one input vector, we don't need to do any concatenation.
+  if (N->getNumOperands() == 1) {
+    return N->getOperand(0);
   }
 
-  // If it is a unary or the LHS and the RHS are the same node, turn the RHS
-  // into an undef.
-  if (isUnary || N0 == N1) {
-    if (N0.getOpcode() == ISD::UNDEF)
-      return DAG.getNode(ISD::UNDEF, N->getValueType(0));
-    // Check the SHUFFLE mask, mapping any inputs from the 2nd operand into the
-    // first operand.
-    SmallVector<SDOperand, 8> MappedOps;
-    for (unsigned i = 0, e = ShufMask.getNumOperands(); i != e; ++i) {
-      if (ShufMask.getOperand(i).getOpcode() == ISD::UNDEF ||
-          cast<ConstantSDNode>(ShufMask.getOperand(i))->getValue() < NumElts) {
-        MappedOps.push_back(ShufMask.getOperand(i));
-      } else {
-        unsigned NewIdx = 
-           cast<ConstantSDNode>(ShufMask.getOperand(i))->getValue() - NumElts;
-        MappedOps.push_back(DAG.getConstant(NewIdx, MVT::i32));
-      }
-    }
-    ShufMask = DAG.getNode(ISD::BUILD_VECTOR, ShufMask.getValueType(),
-                           &MappedOps[0], MappedOps.size());
-    AddToWorkList(ShufMask.Val);
-    return DAG.getNode(ISD::VECTOR_SHUFFLE, N->getValueType(0),
-                       N0, 
-                       DAG.getNode(ISD::UNDEF, N->getValueType(0)),
-                       ShufMask);
-  }
- 
   return SDOperand();
 }
 
-SDOperand DAGCombiner::visitVVECTOR_SHUFFLE(SDNode *N) {
+SDOperand DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
   SDOperand ShufMask = N->getOperand(2);
-  unsigned NumElts = ShufMask.getNumOperands()-2;
-  
+  unsigned NumElts = ShufMask.getNumOperands();
+
   // If the shuffle mask is an identity operation on the LHS, return the LHS.
   bool isIdentity = true;
   for (unsigned i = 0; i != NumElts; ++i) {
@@ -3987,7 +3957,7 @@ SDOperand DAGCombiner::visitVVECTOR_SHUFFLE(SDNode *N) {
     }
   }
   if (isIdentity) return N->getOperand(0);
-  
+
   // If the shuffle mask is an identity operation on the RHS, return the RHS.
   isIdentity = true;
   for (unsigned i = 0; i != NumElts; ++i) {
@@ -4033,19 +4003,17 @@ SDOperand DAGCombiner::visitVVECTOR_SHUFFLE(SDNode *N) {
   if (isSplat) {
     SDNode *V = N0.Val;
 
-    // If this is a vbit convert that changes the element type of the vector but
+    // If this is a bit convert that changes the element type of the vector but
     // not the number of vector elements, look through it.  Be careful not to
     // look though conversions that change things like v4f32 to v2f64.
-    if (V->getOpcode() == ISD::VBIT_CONVERT) {
+    if (V->getOpcode() == ISD::BIT_CONVERT) {
       SDOperand ConvInput = V->getOperand(0);
-      if (ConvInput.getValueType() == MVT::Vector &&
-          NumElts ==
-          ConvInput.getConstantOperandVal(ConvInput.getNumOperands()-2))
+      if (MVT::getVectorNumElements(ConvInput.getValueType()) == NumElts)
         V = ConvInput.Val;
     }
 
-    if (V->getOpcode() == ISD::VBUILD_VECTOR) {
-      unsigned NumElems = V->getNumOperands()-2;
+    if (V->getOpcode() == ISD::BUILD_VECTOR) {
+      unsigned NumElems = V->getNumOperands();
       if (NumElems > BaseIdx) {
         SDOperand Base;
         bool AllSame = true;
@@ -4088,48 +4056,33 @@ SDOperand DAGCombiner::visitVVECTOR_SHUFFLE(SDNode *N) {
         MappedOps.push_back(DAG.getConstant(NewIdx, MVT::i32));
       }
     }
-    // Add the type/#elts values.
-    MappedOps.push_back(ShufMask.getOperand(NumElts));
-    MappedOps.push_back(ShufMask.getOperand(NumElts+1));
-
-    ShufMask = DAG.getNode(ISD::VBUILD_VECTOR, ShufMask.getValueType(),
+    ShufMask = DAG.getNode(ISD::BUILD_VECTOR, ShufMask.getValueType(),
                            &MappedOps[0], MappedOps.size());
     AddToWorkList(ShufMask.Val);
-    
-    // Build the undef vector.
-    SDOperand UDVal = DAG.getNode(ISD::UNDEF, MappedOps[0].getValueType());
-    for (unsigned i = 0; i != NumElts; ++i)
-      MappedOps[i] = UDVal;
-    MappedOps[NumElts  ] = *(N0.Val->op_end()-2);
-    MappedOps[NumElts+1] = *(N0.Val->op_end()-1);
-    UDVal = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector,
-                        &MappedOps[0], MappedOps.size());
-    
-    return DAG.getNode(ISD::VVECTOR_SHUFFLE, MVT::Vector, 
-                       N0, UDVal, ShufMask,
-                       MappedOps[NumElts], MappedOps[NumElts+1]);
+    return DAG.getNode(ISD::VECTOR_SHUFFLE, N->getValueType(0),
+                       N0,
+                       DAG.getNode(ISD::UNDEF, N->getValueType(0)),
+                       ShufMask);
   }
-  
+ 
   return SDOperand();
 }
 
 /// XformToShuffleWithZero - Returns a vector_shuffle if it able to transform
-/// a VAND to a vector_shuffle with the destination vector and a zero vector.
-/// e.g. VAND V, <0xffffffff, 0, 0xffffffff, 0>. ==>
+/// an AND to a vector_shuffle with the destination vector and a zero vector.
+/// e.g. AND V, <0xffffffff, 0, 0xffffffff, 0>. ==>
 ///      vector_shuffle V, Zero, <0, 4, 2, 4>
 SDOperand DAGCombiner::XformToShuffleWithZero(SDNode *N) {
   SDOperand LHS = N->getOperand(0);
   SDOperand RHS = N->getOperand(1);
-  if (N->getOpcode() == ISD::VAND) {
-    SDOperand DstVecSize = *(LHS.Val->op_end()-2);
-    SDOperand DstVecEVT  = *(LHS.Val->op_end()-1);
-    if (RHS.getOpcode() == ISD::VBIT_CONVERT)
+  if (N->getOpcode() == ISD::AND) {
+    if (RHS.getOpcode() == ISD::BIT_CONVERT)
       RHS = RHS.getOperand(0);
-    if (RHS.getOpcode() == ISD::VBUILD_VECTOR) {
+    if (RHS.getOpcode() == ISD::BUILD_VECTOR) {
       std::vector<SDOperand> IdxOps;
       unsigned NumOps = RHS.getNumOperands();
-      unsigned NumElts = NumOps-2;
-      MVT::ValueType EVT = cast<VTSDNode>(RHS.getOperand(NumOps-1))->getVT();
+      unsigned NumElts = NumOps;
+      MVT::ValueType EVT = MVT::getVectorElementType(RHS.getValueType());
       for (unsigned i = 0; i != NumElts; ++i) {
         SDOperand Elt = RHS.getOperand(i);
         if (!isa<ConstantSDNode>(Elt))
@@ -4146,30 +4099,21 @@ SDOperand DAGCombiner::XformToShuffleWithZero(SDNode *N) {
       if (!TLI.isVectorClearMaskLegal(IdxOps, EVT, DAG))
         return SDOperand();
 
-      // Return the new VVECTOR_SHUFFLE node.
-      SDOperand NumEltsNode = DAG.getConstant(NumElts, MVT::i32);
-      SDOperand EVTNode = DAG.getValueType(EVT);
+      // Return the new VECTOR_SHUFFLE node.
+      MVT::ValueType VT = MVT::getVectorType(EVT, NumElts);
       std::vector<SDOperand> Ops;
-      LHS = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, LHS, NumEltsNode,
-                        EVTNode);
+      LHS = DAG.getNode(ISD::BIT_CONVERT, VT, LHS);
       Ops.push_back(LHS);
       AddToWorkList(LHS.Val);
       std::vector<SDOperand> ZeroOps(NumElts, DAG.getConstant(0, EVT));
-      ZeroOps.push_back(NumEltsNode);
-      ZeroOps.push_back(EVTNode);
-      Ops.push_back(DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector,
+      Ops.push_back(DAG.getNode(ISD::BUILD_VECTOR, VT,
                                 &ZeroOps[0], ZeroOps.size()));
-      IdxOps.push_back(NumEltsNode);
-      IdxOps.push_back(EVTNode);
-      Ops.push_back(DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector,
+      Ops.push_back(DAG.getNode(ISD::BUILD_VECTOR, VT,
                                 &IdxOps[0], IdxOps.size()));
-      Ops.push_back(NumEltsNode);
-      Ops.push_back(EVTNode);
-      SDOperand Result = DAG.getNode(ISD::VVECTOR_SHUFFLE, MVT::Vector,
+      SDOperand Result = DAG.getNode(ISD::VECTOR_SHUFFLE, VT,
                                      &Ops[0], Ops.size());
-      if (NumEltsNode != DstVecSize || EVTNode != DstVecEVT) {
-        Result = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Result,
-                             DstVecSize, DstVecEVT);
+      if (VT != LHS.getValueType()) {
+        Result = DAG.getNode(ISD::BIT_CONVERT, LHS.getValueType(), Result);
       }
       return Result;
     }
@@ -4177,24 +4121,28 @@ SDOperand DAGCombiner::XformToShuffleWithZero(SDNode *N) {
   return SDOperand();
 }
 
-/// visitVBinOp - Visit a binary vector operation, like VADD.  IntOp indicates
-/// the scalar operation of the vop if it is operating on an integer vector
-/// (e.g. ADD) and FPOp indicates the FP version (e.g. FADD).
-SDOperand DAGCombiner::visitVBinOp(SDNode *N, ISD::NodeType IntOp, 
-                                   ISD::NodeType FPOp) {
-  MVT::ValueType EltType = cast<VTSDNode>(*(N->op_end()-1))->getVT();
-  ISD::NodeType ScalarOp = MVT::isInteger(EltType) ? IntOp : FPOp;
+/// SimplifyVBinOp - Visit a binary vector operation, like ADD.
+SDOperand DAGCombiner::SimplifyVBinOp(SDNode *N) {
+  // After legalize, the target may be depending on adds and other
+  // binary ops to provide legal ways to construct constants or other
+  // things. Simplifying them may result in a loss of legality.
+  if (AfterLegalize) return SDOperand();
+
+  MVT::ValueType VT = N->getValueType(0);
+  if (!MVT::isVector(VT)) return SDOperand();
+
+  MVT::ValueType EltType = MVT::getVectorElementType(VT);
   SDOperand LHS = N->getOperand(0);
   SDOperand RHS = N->getOperand(1);
   SDOperand Shuffle = XformToShuffleWithZero(N);
   if (Shuffle.Val) return Shuffle;
 
-  // If the LHS and RHS are VBUILD_VECTOR nodes, see if we can constant fold
+  // If the LHS and RHS are BUILD_VECTOR nodes, see if we can constant fold
   // this operation.
-  if (LHS.getOpcode() == ISD::VBUILD_VECTOR && 
-      RHS.getOpcode() == ISD::VBUILD_VECTOR) {
+  if (LHS.getOpcode() == ISD::BUILD_VECTOR && 
+      RHS.getOpcode() == ISD::BUILD_VECTOR) {
     SmallVector<SDOperand, 8> Ops;
-    for (unsigned i = 0, e = LHS.getNumOperands()-2; i != e; ++i) {
+    for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
       SDOperand LHSOp = LHS.getOperand(i);
       SDOperand RHSOp = RHS.getOperand(i);
       // If these two elements can't be folded, bail out.
@@ -4206,14 +4154,15 @@ SDOperand DAGCombiner::visitVBinOp(SDNode *N, ISD::NodeType IntOp,
            RHSOp.getOpcode() != ISD::ConstantFP))
         break;
       // Can't fold divide by zero.
-      if (N->getOpcode() == ISD::VSDIV || N->getOpcode() == ISD::VUDIV) {
+      if (N->getOpcode() == ISD::SDIV || N->getOpcode() == ISD::UDIV ||
+          N->getOpcode() == ISD::FDIV) {
         if ((RHSOp.getOpcode() == ISD::Constant &&
              cast<ConstantSDNode>(RHSOp.Val)->isNullValue()) ||
             (RHSOp.getOpcode() == ISD::ConstantFP &&
              !cast<ConstantFPSDNode>(RHSOp.Val)->getValue()))
           break;
       }
-      Ops.push_back(DAG.getNode(ScalarOp, EltType, LHSOp, RHSOp));
+      Ops.push_back(DAG.getNode(N->getOpcode(), EltType, LHSOp, RHSOp));
       AddToWorkList(Ops.back().Val);
       assert((Ops.back().getOpcode() == ISD::UNDEF ||
               Ops.back().getOpcode() == ISD::Constant ||
@@ -4221,10 +4170,9 @@ SDOperand DAGCombiner::visitVBinOp(SDNode *N, ISD::NodeType IntOp,
              "Scalar binop didn't fold!");
     }
     
-    if (Ops.size() == LHS.getNumOperands()-2) {
-      Ops.push_back(*(LHS.Val->op_end()-2));
-      Ops.push_back(*(LHS.Val->op_end()-1));
-      return DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &Ops[0], Ops.size());
+    if (Ops.size() == LHS.getNumOperands()) {
+      MVT::ValueType VT = LHS.getValueType();
+      return DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size());
     }
   }
   
diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 5735744..5f41452 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -111,10 +111,10 @@ class VISIBILITY_HIDDEN SelectionDAGLegalize {
   /// to avoid splitting the same node more than once.
   std::map<SDOperand, std::pair<SDOperand, SDOperand> > SplitNodes;
   
-  /// PackedNodes - For nodes that need to be packed from MVT::Vector types to
-  /// concrete vector types, this contains the mapping of ones we have already
+  /// ScalarizedNodes - For nodes that need to be converted from vector types to
+  /// scalar types, this contains the mapping of ones we have already
   /// processed to the result.
-  std::map<SDOperand, SDOperand> PackedNodes;
+  std::map<SDOperand, SDOperand> ScalarizedNodes;
   
   void AddLegalizedOperand(SDOperand From, SDOperand To) {
     LegalizedNodes.insert(std::make_pair(From, To));
@@ -149,7 +149,7 @@ public:
   void LegalizeDAG();
 
 private:
-  /// HandleOp - Legalize, Promote, Expand or Pack the specified operand as
+  /// HandleOp - Legalize, Promote, or Expand the specified operand as
   /// appropriate for its type.
   void HandleOp(SDOperand Op);
     
@@ -173,15 +173,13 @@ private:
   /// types.
   void ExpandOp(SDOperand O, SDOperand &Lo, SDOperand &Hi);
 
-  /// SplitVectorOp - Given an operand of MVT::Vector type, break it down into
-  /// two smaller values of MVT::Vector type.
+  /// SplitVectorOp - Given an operand of vector type, break it down into
+  /// two smaller values.
   void SplitVectorOp(SDOperand O, SDOperand &Lo, SDOperand &Hi);
   
-  /// PackVectorOp - Given an operand of MVT::Vector type, convert it into the
-  /// equivalent operation that returns a packed value (e.g. MVT::V4F32).  When
-  /// this is called, we know that PackedVT is the right type for the result and
-  /// we know that this type is legal for the target.
-  SDOperand PackVectorOp(SDOperand O, MVT::ValueType PackedVT);
+  /// ScalarizeVectorOp - Given an operand of vector type, convert it into the
+  /// equivalent operation that returns a scalar value.
+  SDOperand ScalarizeVectorOp(SDOperand O);
   
   /// isShuffleLegal - Return true if a vector shuffle is legal with the
   /// specified mask and type.  Targets can specify exactly which masks they
@@ -224,8 +222,7 @@ private:
   void ExpandShiftParts(unsigned NodeOp, SDOperand Op, SDOperand Amt,
                         SDOperand &Lo, SDOperand &Hi);
 
-  SDOperand LowerVEXTRACT_VECTOR_ELT(SDOperand Op);
-  SDOperand LowerVEXTRACT_SUBVECTOR(SDOperand Op);
+  SDOperand ExpandEXTRACT_SUBVECTOR(SDOperand Op);
   SDOperand ExpandEXTRACT_VECTOR_ELT(SDOperand Op);
   
   SDOperand getIntPtrConstant(uint64_t Val) {
@@ -279,22 +276,6 @@ SDNode *SelectionDAGLegalize::isShuffleLegal(MVT::ValueType VT,
   return TLI.isShuffleMaskLegal(Mask, VT) ? Mask.Val : 0;
 }
 
-/// getScalarizedOpcode - Return the scalar opcode that corresponds to the
-/// specified vector opcode.
-static unsigned getScalarizedOpcode(unsigned VecOp, MVT::ValueType VT) {
-  switch (VecOp) {
-  default: assert(0 && "Don't know how to scalarize this opcode!");
-  case ISD::VADD:  return MVT::isInteger(VT) ? ISD::ADD : ISD::FADD;
-  case ISD::VSUB:  return MVT::isInteger(VT) ? ISD::SUB : ISD::FSUB;
-  case ISD::VMUL:  return MVT::isInteger(VT) ? ISD::MUL : ISD::FMUL;
-  case ISD::VSDIV: return MVT::isInteger(VT) ? ISD::SDIV: ISD::FDIV;
-  case ISD::VUDIV: return MVT::isInteger(VT) ? ISD::UDIV: ISD::FDIV;
-  case ISD::VAND:  return MVT::isInteger(VT) ? ISD::AND : 0;
-  case ISD::VOR:   return MVT::isInteger(VT) ? ISD::OR  : 0;
-  case ISD::VXOR:  return MVT::isInteger(VT) ? ISD::XOR : 0;
-  }
-}
-
 SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag)
   : TLI(dag.getTargetLoweringInfo()), DAG(dag),
     ValueTypeActions(TLI.getValueTypeActions()) {
@@ -369,7 +350,7 @@ void SelectionDAGLegalize::LegalizeDAG() {
   LegalizedNodes.clear();
   PromotedNodes.clear();
   SplitNodes.clear();
-  PackedNodes.clear();
+  ScalarizedNodes.clear();
 
   // Remove dead nodes now.
   DAG.RemoveDeadNodes();
@@ -473,38 +454,29 @@ bool SelectionDAGLegalize::LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest,
   return false;
 }
 
-/// HandleOp - Legalize, Promote, Expand or Pack the specified operand as
+/// HandleOp - Legalize, Promote, or Expand the specified operand as
 /// appropriate for its type.
 void SelectionDAGLegalize::HandleOp(SDOperand Op) {
-  switch (getTypeAction(Op.getValueType())) {
+  MVT::ValueType VT = Op.getValueType();
+  switch (getTypeAction(VT)) {
   default: assert(0 && "Bad type action!");
-  case Legal:   LegalizeOp(Op); break;
-  case Promote: PromoteOp(Op);  break;
+  case Legal:   (void)LegalizeOp(Op); break;
+  case Promote: (void)PromoteOp(Op); break;
   case Expand:
-    if (Op.getValueType() != MVT::Vector) {
+    if (!MVT::isVector(VT)) {
+      // If this is an illegal scalar, expand it into its two component
+      // pieces.
       SDOperand X, Y;
       ExpandOp(Op, X, Y);
+    } else if (MVT::getVectorNumElements(VT) == 1) {
+      // If this is an illegal single element vector, convert it to a
+      // scalar operation.
+      (void)ScalarizeVectorOp(Op);
     } else {
-      SDNode *N = Op.Val;
-      unsigned NumOps = N->getNumOperands();
-      unsigned NumElements =
-        cast<ConstantSDNode>(N->getOperand(NumOps-2))->getValue();
-      MVT::ValueType EVT = cast<VTSDNode>(N->getOperand(NumOps-1))->getVT();
-      MVT::ValueType PackedVT = MVT::getVectorType(EVT, NumElements);
-      if (PackedVT != MVT::Other && TLI.isTypeLegal(PackedVT)) {
-        // In the common case, this is a legal vector type, convert it to the
-        // packed operation and type now.
-        PackVectorOp(Op, PackedVT);
-      } else if (NumElements == 1) {
-        // Otherwise, if this is a single element vector, convert it to a
-        // scalar operation.
-        PackVectorOp(Op, EVT);
-      } else {
-        // Otherwise, this is a multiple element vector that isn't supported.
-        // Split it in half and legalize both parts.
-        SDOperand X, Y;
-        SplitVectorOp(Op, X, Y);
-      }
+      // Otherwise, this is an illegal multiple element vector.
+      // Split it in half and legalize both parts.
+      SDOperand X, Y;
+      SplitVectorOp(Op, X, Y);
     }
     break;
   }
@@ -556,6 +528,8 @@ SDOperand ExpandFCOPYSIGNToBitwiseOps(SDNode *Node, MVT::ValueType NVT,
                                       SelectionDAG &DAG, TargetLowering &TLI) {
   MVT::ValueType VT = Node->getValueType(0);
   MVT::ValueType SrcVT = Node->getOperand(1).getValueType();
+  assert((SrcVT == MVT::f32 || SrcVT == MVT::f64) &&
+         "fcopysign expansion only supported for f32 and f64");
   MVT::ValueType SrcNVT = (SrcVT == MVT::f64) ? MVT::i64 : MVT::i32;
 
   // First get the sign bit of second operand.
@@ -1158,34 +1132,17 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
     break;
   
   case ISD::EXTRACT_VECTOR_ELT:
-    Tmp1 = LegalizeOp(Node->getOperand(0));
+    Tmp1 = Node->getOperand(0);
     Tmp2 = LegalizeOp(Node->getOperand(1));
     Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
-    
-    switch (TLI.getOperationAction(ISD::EXTRACT_VECTOR_ELT,
-                                   Tmp1.getValueType())) {
-    default: assert(0 && "This action is not supported yet!");
-    case TargetLowering::Legal:
-      break;
-    case TargetLowering::Custom:
-      Tmp3 = TLI.LowerOperation(Result, DAG);
-      if (Tmp3.Val) {
-        Result = Tmp3;
-        break;
-      }
-      // FALLTHROUGH
-    case TargetLowering::Expand:
-      Result = ExpandEXTRACT_VECTOR_ELT(Result);
-      break;
-    }
+    Result = ExpandEXTRACT_VECTOR_ELT(Result);
     break;
 
-  case ISD::VEXTRACT_VECTOR_ELT: 
-    Result = LegalizeOp(LowerVEXTRACT_VECTOR_ELT(Op));
-    break;
-    
-  case ISD::VEXTRACT_SUBVECTOR: 
-    Result = LegalizeOp(LowerVEXTRACT_SUBVECTOR(Op));
+  case ISD::EXTRACT_SUBVECTOR: 
+    Tmp1 = Node->getOperand(0);
+    Tmp2 = LegalizeOp(Node->getOperand(1));
+    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
+    Result = ExpandEXTRACT_SUBVECTOR(Result);
     break;
     
   case ISD::CALLSEQ_START: {
@@ -1688,7 +1645,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
         Result = DAG.UpdateNodeOperands(Result, Tmp1, LegalizeOp(Tmp2), Tmp3);
         break;
       case Expand:
-        if (Tmp2.getValueType() != MVT::Vector) {
+        if (!MVT::isVector(Tmp2.getValueType())) {
           SDOperand Lo, Hi;
           ExpandOp(Tmp2, Lo, Hi);
 
@@ -1703,20 +1660,20 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
           Result = LegalizeOp(Result);
         } else {
           SDNode *InVal = Tmp2.Val;
-          unsigned NumElems =
-            cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue();
-          MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT();
+          unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(0));
+          MVT::ValueType EVT = MVT::getVectorElementType(InVal->getValueType(0));
           
-          // Figure out if there is a Packed type corresponding to this Vector
+          // Figure out if there is a simple type corresponding to this Vector
           // type.  If so, convert to the vector type.
           MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
-          if (TVT != MVT::Other && TLI.isTypeLegal(TVT)) {
+          if (TLI.isTypeLegal(TVT)) {
             // Turn this into a return of the vector type.
-            Tmp2 = PackVectorOp(Tmp2, TVT);
+            Tmp2 = LegalizeOp(Tmp2);
             Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
           } else if (NumElems == 1) {
             // Turn this into a return of the scalar type.
-            Tmp2 = PackVectorOp(Tmp2, EVT);
+            Tmp2 = ScalarizeVectorOp(Tmp2);
+            Tmp2 = LegalizeOp(Tmp2);
             Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
             
             // FIXME: Returns of gcc generic vectors smaller than a legal type
@@ -1756,7 +1713,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
           break;
         case Expand: {
           SDOperand Lo, Hi;
-          assert(Node->getOperand(i).getValueType() != MVT::Vector &&
+          assert(!MVT::isExtendedValueType(Node->getOperand(i).getValueType())&&
                  "FIXME: TODO: implement returning non-legal vector types!");
           ExpandOp(Node->getOperand(i), Lo, Hi);
           NewValues.push_back(Lo);
@@ -1853,27 +1810,30 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
         // If this is a vector type, then we have to calculate the increment as
         // the product of the element size in bytes, and the number of elements
         // in the high half of the vector.
-        if (ST->getValue().getValueType() == MVT::Vector) {
+        if (MVT::isVector(ST->getValue().getValueType())) {
           SDNode *InVal = ST->getValue().Val;
-          unsigned NumElems =
-            cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue();
-          MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT();
+          unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(0));
+          MVT::ValueType EVT = MVT::getVectorElementType(InVal->getValueType(0));
 
-          // Figure out if there is a Packed type corresponding to this Vector
+          // Figure out if there is a simple type corresponding to this Vector
           // type.  If so, convert to the vector type.
           MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
-          if (TVT != MVT::Other && TLI.isTypeLegal(TVT)) {
+          if (TLI.isTypeLegal(TVT)) {
             // Turn this into a normal store of the vector type.
-            Tmp3 = PackVectorOp(Node->getOperand(1), TVT);
+            Tmp3 = LegalizeOp(Node->getOperand(1));
             Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
-                                  ST->getSrcValueOffset());
+                                  ST->getSrcValueOffset(),
+                                  ST->isVolatile(),
+                                  ST->getAlignment());
             Result = LegalizeOp(Result);
             break;
           } else if (NumElems == 1) {
             // Turn this into a normal store of the scalar type.
-            Tmp3 = PackVectorOp(Node->getOperand(1), EVT);
+            Tmp3 = ScalarizeVectorOp(Node->getOperand(1));
             Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
-                                  ST->getSrcValueOffset());
+                                  ST->getSrcValueOffset(),
+                                  ST->isVolatile(),
+                                  ST->getAlignment());
             // The scalarized value type may not be legal, e.g. it might require
             // promotion or expansion.  Relegalize the scalar store.
             Result = LegalizeOp(Result);
@@ -2864,6 +2824,30 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
   case ISD::BIT_CONVERT:
     if (!isTypeLegal(Node->getOperand(0).getValueType())) {
       Result = ExpandBIT_CONVERT(Node->getValueType(0), Node->getOperand(0));
+    } else if (MVT::isVector(Op.getOperand(0).getValueType())) {
+      // The input has to be a vector type, we have to either scalarize it, pack
+      // it, or convert it based on whether the input vector type is legal.
+      SDNode *InVal = Node->getOperand(0).Val;
+      unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(0));
+      MVT::ValueType EVT = MVT::getVectorElementType(InVal->getValueType(0));
+    
+      // Figure out if there is a simple type corresponding to this Vector
+      // type.  If so, convert to the vector type.
+      MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
+      if (TLI.isTypeLegal(TVT)) {
+        // Turn this into a bit convert of the packed input.
+        Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0), 
+                             LegalizeOp(Node->getOperand(0)));
+        break;
+      } else if (NumElems == 1) {
+        // Turn this into a bit convert of the scalar input.
+        Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0), 
+                             ScalarizeVectorOp(Node->getOperand(0)));
+        break;
+      } else {
+        // FIXME: UNIMP!  Store then reload
+        assert(0 && "Cast from unsupported vector type not implemented yet!");
+      }
     } else {
       switch (TLI.getOperationAction(ISD::BIT_CONVERT,
                                      Node->getOperand(0).getValueType())) {
@@ -2878,35 +2862,6 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
       }
     }
     break;
-  case ISD::VBIT_CONVERT: {
-    assert(Op.getOperand(0).getValueType() == MVT::Vector &&
-           "Can only have VBIT_CONVERT where input or output is MVT::Vector!");
-    
-    // The input has to be a vector type, we have to either scalarize it, pack
-    // it, or convert it based on whether the input vector type is legal.
-    SDNode *InVal = Node->getOperand(0).Val;
-    unsigned NumElems =
-      cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue();
-    MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT();
-    
-    // Figure out if there is a Packed type corresponding to this Vector
-    // type.  If so, convert to the vector type.
-    MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
-    if (TVT != MVT::Other && TLI.isTypeLegal(TVT)) {
-      // Turn this into a bit convert of the packed input.
-      Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0), 
-                           PackVectorOp(Node->getOperand(0), TVT));
-      break;
-    } else if (NumElems == 1) {
-      // Turn this into a bit convert of the scalar input.
-      Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0), 
-                           PackVectorOp(Node->getOperand(0), EVT));
-      break;
-    } else {
-      // FIXME: UNIMP!  Store then reload
-      assert(0 && "Cast from unsupported vector type not implemented yet!");
-    }
-  }
       
     // Conversion operators.  The source and destination have different types.
   case ISD::SINT_TO_FP:
@@ -3568,11 +3523,8 @@ SDOperand SelectionDAGLegalize::PromoteOp(SDOperand Op) {
       break;
     }
     break;
-  case ISD::VEXTRACT_VECTOR_ELT:
-    Result = PromoteOp(LowerVEXTRACT_VECTOR_ELT(Op));
-    break;
-  case ISD::VEXTRACT_SUBVECTOR:
-    Result = PromoteOp(LowerVEXTRACT_SUBVECTOR(Op));
+  case ISD::EXTRACT_SUBVECTOR:
+    Result = PromoteOp(ExpandEXTRACT_SUBVECTOR(Op));
     break;
   case ISD::EXTRACT_VECTOR_ELT:
     Result = PromoteOp(ExpandEXTRACT_VECTOR_ELT(Op));
@@ -3589,66 +3541,90 @@ SDOperand SelectionDAGLegalize::PromoteOp(SDOperand Op) {
   return Result;
 }
 
-/// LowerVEXTRACT_VECTOR_ELT - Lower a VEXTRACT_VECTOR_ELT operation into a
-/// EXTRACT_VECTOR_ELT operation, to memory operations, or to scalar code based
-/// on the vector type.  The return type of this matches the element type of the
-/// vector, which may not be legal for the target.
-SDOperand SelectionDAGLegalize::LowerVEXTRACT_VECTOR_ELT(SDOperand Op) {
+/// ExpandEXTRACT_VECTOR_ELT - Expand an EXTRACT_VECTOR_ELT operation into
+/// a legal EXTRACT_VECTOR_ELT operation, scalar code, or memory traffic,
+/// based on the vector type. The return type of this matches the element type
+/// of the vector, which may not be legal for the target.
+SDOperand SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDOperand Op) {
   // We know that operand #0 is the Vec vector.  If the index is a constant
   // or if the invec is a supported hardware type, we can use it.  Otherwise,
   // lower to a store then an indexed load.
   SDOperand Vec = Op.getOperand(0);
-  SDOperand Idx = LegalizeOp(Op.getOperand(1));
+  SDOperand Idx = Op.getOperand(1);
   
   SDNode *InVal = Vec.Val;
-  unsigned NumElems = cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue();
-  MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT();
+  MVT::ValueType TVT = InVal->getValueType(0);
+  unsigned NumElems = MVT::getVectorNumElements(TVT);
   
-  // Figure out if there is a Packed type corresponding to this Vector
-  // type.  If so, convert to the vector type.
-  MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
-  if (TVT != MVT::Other && TLI.isTypeLegal(TVT)) {
-    // Turn this into a packed extract_vector_elt operation.
-    Vec = PackVectorOp(Vec, TVT);
-    return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, Op.getValueType(), Vec, Idx);
-  } else if (NumElems == 1) {
+  switch (TLI.getOperationAction(ISD::EXTRACT_VECTOR_ELT, TVT)) {
+  default: assert(0 && "This action is not supported yet!");
+  case TargetLowering::Custom: {
+    Vec = LegalizeOp(Vec);
+    Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
+    SDOperand Tmp3 = TLI.LowerOperation(Op, DAG);
+    if (Tmp3.Val)
+      return Tmp3;
+    break;
+  }
+  case TargetLowering::Legal:
+    if (isTypeLegal(TVT)) {
+      Vec = LegalizeOp(Vec);
+      Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
+      Op = LegalizeOp(Op);
+    }
+    break;
+  case TargetLowering::Expand:
+    break;
+  }
+
+  if (NumElems == 1) {
     // This must be an access of the only element.  Return it.
-    return PackVectorOp(Vec, EVT);
-  } else if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
+    Op = ScalarizeVectorOp(Vec);
+  } else if (!TLI.isTypeLegal(TVT) && isa<ConstantSDNode>(Idx)) {
+    ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx);
     SDOperand Lo, Hi;
     SplitVectorOp(Vec, Lo, Hi);
     if (CIdx->getValue() < NumElems/2) {
       Vec = Lo;
     } else {
       Vec = Hi;
-      Idx = DAG.getConstant(CIdx->getValue() - NumElems/2, Idx.getValueType());
+      Idx = DAG.getConstant(CIdx->getValue() - NumElems/2,
+                            Idx.getValueType());
     }
-    
+  
     // It's now an extract from the appropriate high or low part.  Recurse.
     Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
-    return LowerVEXTRACT_VECTOR_ELT(Op);
+    Op = ExpandEXTRACT_VECTOR_ELT(Op);
   } else {
-    // Variable index case for extract element.
-    // FIXME: IMPLEMENT STORE/LOAD lowering.  Need alignment of stack slot!!
-    assert(0 && "unimp!");
-    return SDOperand();
+    // Store the value to a temporary stack slot, then LOAD the scalar
+    // element back out.
+    SDOperand StackPtr = CreateStackTemporary(Vec.getValueType());
+    SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Vec, StackPtr, NULL, 0);
+
+    // Add the offset to the index.
+    unsigned EltSize = MVT::getSizeInBits(Op.getValueType())/8;
+    Idx = DAG.getNode(ISD::MUL, Idx.getValueType(), Idx,
+                      DAG.getConstant(EltSize, Idx.getValueType()));
+    StackPtr = DAG.getNode(ISD::ADD, Idx.getValueType(), Idx, StackPtr);
+
+    Op = DAG.getLoad(Op.getValueType(), Ch, StackPtr, NULL, 0);
   }
+  return Op;
 }
 
-/// LowerVEXTRACT_SUBVECTOR - Lower a VEXTRACT_SUBVECTOR operation.  For now
+/// ExpandEXTRACT_SUBVECTOR - Expand a EXTRACT_SUBVECTOR operation.  For now
 /// we assume the operation can be split if it is not already legal.
-SDOperand SelectionDAGLegalize::LowerVEXTRACT_SUBVECTOR(SDOperand Op) {
+SDOperand SelectionDAGLegalize::ExpandEXTRACT_SUBVECTOR(SDOperand Op) {
   // We know that operand #0 is the Vec vector.  For now we assume the index
   // is a constant and that the extracted result is a supported hardware type.
   SDOperand Vec = Op.getOperand(0);
   SDOperand Idx = LegalizeOp(Op.getOperand(1));
   
-  SDNode *InVal = Vec.Val;
-  unsigned NumElems = cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue();
+  unsigned NumElems = MVT::getVectorNumElements(Vec.getValueType());
   
   if (NumElems == MVT::getVectorNumElements(Op.getValueType())) {
     // This must be an access of the desired vector length.  Return it.
-    return PackVectorOp(Vec, Op.getValueType());
+    return Vec;
   }
 
   ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx);
@@ -3663,30 +3639,9 @@ SDOperand SelectionDAGLegalize::LowerVEXTRACT_SUBVECTOR(SDOperand Op) {
   
   // It's now an extract from the appropriate high or low part.  Recurse.
   Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
-  return LowerVEXTRACT_SUBVECTOR(Op);
-}
-
-/// ExpandEXTRACT_VECTOR_ELT - Expand an EXTRACT_VECTOR_ELT operation into
-/// memory traffic.
-SDOperand SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDOperand Op) {
-  SDOperand Vector = Op.getOperand(0);
-  SDOperand Idx    = Op.getOperand(1);
-  
-  // If the target doesn't support this, store the value to a temporary
-  // stack slot, then LOAD the scalar element back out.
-  SDOperand StackPtr = CreateStackTemporary(Vector.getValueType());
-  SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Vector, StackPtr, NULL, 0);
-  
-  // Add the offset to the index.
-  unsigned EltSize = MVT::getSizeInBits(Op.getValueType())/8;
-  Idx = DAG.getNode(ISD::MUL, Idx.getValueType(), Idx,
-                    DAG.getConstant(EltSize, Idx.getValueType()));
-  StackPtr = DAG.getNode(ISD::ADD, Idx.getValueType(), Idx, StackPtr);
-  
-  return DAG.getLoad(Op.getValueType(), Ch, StackPtr, NULL, 0);
+  return ExpandEXTRACT_SUBVECTOR(Op);
 }
 
-
 /// LegalizeSetCCOperands - Attempts to create a legal LHS and RHS for a SETCC
 /// with condition CC on the current target.  This usually involves legalizing
 /// or promoting the arguments.  In the case where LHS and RHS must be expanded,
@@ -4748,7 +4703,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){
   SDNode *Node = Op.Val;
   assert(getTypeAction(VT) == Expand && "Not an expanded type!");
   assert(((MVT::isInteger(NVT) && NVT < VT) || MVT::isFloatingPoint(VT) ||
-         VT == MVT::Vector) &&
+         MVT::isVector(VT)) &&
          "Cannot expand to FP value or to larger int value!");
 
   // See if we already expanded it.
@@ -4870,9 +4825,10 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){
     SDOperand Ch  = LD->getChain();    // Legalize the chain.
     SDOperand Ptr = LD->getBasePtr();  // Legalize the pointer.
     ISD::LoadExtType ExtType = LD->getExtensionType();
+    unsigned SVOffset = LD->getSrcValueOffset();
 
     if (ExtType == ISD::NON_EXTLOAD) {
-      Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(),LD->getSrcValueOffset());
+      Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), SVOffset);
       if (VT == MVT::f32 || VT == MVT::f64) {
         // f32->i32 or f64->i64 one to one expansion.
         // Remember that we legalized the chain.
@@ -4887,8 +4843,8 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){
       unsigned IncrementSize = MVT::getSizeInBits(Lo.getValueType())/8;
       Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
                         getIntPtrConstant(IncrementSize));
-      // FIXME: This creates a bogus srcvalue!
-      Hi = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(),LD->getSrcValueOffset());
+      SVOffset += IncrementSize;
+      Hi = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), SVOffset);
 
       // Build a factor node to remember that this load is independent of the
       // other one.
@@ -4905,7 +4861,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){
       if (VT == MVT::f64 && EVT == MVT::f32) {
         // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND
         SDOperand Load = DAG.getLoad(EVT, Ch, Ptr, LD->getSrcValue(),
-                                     LD->getSrcValueOffset());
+                                     SVOffset);
         // Remember that we legalized the chain.
         AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Load.getValue(1)));
         ExpandOp(DAG.getNode(ISD::FP_EXTEND, VT, Load), Lo, Hi);
@@ -4914,10 +4870,10 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){
     
       if (EVT == NVT)
         Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(),
-                         LD->getSrcValueOffset());
+                         SVOffset);
       else
         Lo = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, LD->getSrcValue(),
-                            LD->getSrcValueOffset(), EVT);
+                            SVOffset, EVT);
     
       // Remember that we legalized the chain.
       AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Lo.getValue(1)));
@@ -5504,17 +5460,17 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){
   assert(isNew && "Value already expanded?!?");
 }
 
-/// SplitVectorOp - Given an operand of MVT::Vector type, break it down into
-/// two smaller values of MVT::Vector type.
+/// SplitVectorOp - Given an operand of vector type, break it down into
+/// two smaller values, still of vector type.
 void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo,
                                          SDOperand &Hi) {
-  assert(Op.getValueType() == MVT::Vector && "Cannot split non-vector type!");
+  assert(MVT::isVector(Op.getValueType()) && "Cannot split non-vector type!");
   SDNode *Node = Op.Val;
-  unsigned NumElements = cast<ConstantSDNode>(*(Node->op_end()-2))->getValue();
+  unsigned NumElements = MVT::getVectorNumElements(Node->getValueType(0));
   assert(NumElements > 1 && "Cannot split a single element vector!");
   unsigned NewNumElts = NumElements/2;
-  SDOperand NewNumEltsNode = DAG.getConstant(NewNumElts, MVT::i32);
-  SDOperand TypeNode = *(Node->op_end()-1);
+  MVT::ValueType NewEltVT = MVT::getVectorElementType(Node->getValueType(0));
+  MVT::ValueType NewVT = MVT::getVectorType(NewEltVT, NewNumElts);
   
   // See if we already split it.
   std::map<SDOperand, std::pair<SDOperand, SDOperand> >::iterator I
@@ -5531,64 +5487,73 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo,
     Node->dump(&DAG);
 #endif
     assert(0 && "Unhandled operation in SplitVectorOp!");
-  case ISD::VBUILD_VECTOR: {
+  case ISD::BUILD_PAIR:
+    Lo = Node->getOperand(0);
+    Hi = Node->getOperand(1);
+    break;
+  case ISD::BUILD_VECTOR: {
     SmallVector<SDOperand, 8> LoOps(Node->op_begin(), 
                                     Node->op_begin()+NewNumElts);
-    LoOps.push_back(NewNumEltsNode);
-    LoOps.push_back(TypeNode);
-    Lo = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &LoOps[0], LoOps.size());
+    Lo = DAG.getNode(ISD::BUILD_VECTOR, NewVT, &LoOps[0], LoOps.size());
 
     SmallVector<SDOperand, 8> HiOps(Node->op_begin()+NewNumElts, 
-                                    Node->op_end()-2);
-    HiOps.push_back(NewNumEltsNode);
-    HiOps.push_back(TypeNode);
-    Hi = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &HiOps[0], HiOps.size());
+                                    Node->op_end());
+    Hi = DAG.getNode(ISD::BUILD_VECTOR, NewVT, &HiOps[0], HiOps.size());
     break;
   }
-  case ISD::VCONCAT_VECTORS: {
-    unsigned NewNumSubvectors = (Node->getNumOperands() - 2) / 2;
-    SmallVector<SDOperand, 8> LoOps(Node->op_begin(), 
-                                    Node->op_begin()+NewNumSubvectors);
-    LoOps.push_back(NewNumEltsNode);
-    LoOps.push_back(TypeNode);
-    Lo = DAG.getNode(ISD::VCONCAT_VECTORS, MVT::Vector, &LoOps[0], LoOps.size());
+  case ISD::CONCAT_VECTORS: {
+    unsigned NewNumSubvectors = Node->getNumOperands() / 2;
+    if (NewNumSubvectors == 1) {
+      Lo = Node->getOperand(0);
+      Hi = Node->getOperand(1);
+    } else {
+      SmallVector<SDOperand, 8> LoOps(Node->op_begin(), 
+                                      Node->op_begin()+NewNumSubvectors);
+      Lo = DAG.getNode(ISD::CONCAT_VECTORS, NewVT, &LoOps[0], LoOps.size());
 
-    SmallVector<SDOperand, 8> HiOps(Node->op_begin()+NewNumSubvectors, 
-                                    Node->op_end()-2);
-    HiOps.push_back(NewNumEltsNode);
-    HiOps.push_back(TypeNode);
-    Hi = DAG.getNode(ISD::VCONCAT_VECTORS, MVT::Vector, &HiOps[0], HiOps.size());
+      SmallVector<SDOperand, 8> HiOps(Node->op_begin()+NewNumSubvectors, 
+                                      Node->op_end());
+      Hi = DAG.getNode(ISD::CONCAT_VECTORS, NewVT, &HiOps[0], HiOps.size());
+    }
     break;
   }
-  case ISD::VADD:
-  case ISD::VSUB:
-  case ISD::VMUL:
-  case ISD::VSDIV:
-  case ISD::VUDIV:
-  case ISD::VAND:
-  case ISD::VOR:
-  case ISD::VXOR: {
+  case ISD::ADD:
+  case ISD::SUB:
+  case ISD::MUL:
+  case ISD::FADD:
+  case ISD::FSUB:
+  case ISD::FMUL:
+  case ISD::SDIV:
+  case ISD::UDIV:
+  case ISD::FDIV:
+  case ISD::AND:
+  case ISD::OR:
+  case ISD::XOR: {
     SDOperand LL, LH, RL, RH;
     SplitVectorOp(Node->getOperand(0), LL, LH);
     SplitVectorOp(Node->getOperand(1), RL, RH);
     
-    Lo = DAG.getNode(Node->getOpcode(), MVT::Vector, LL, RL,
-                     NewNumEltsNode, TypeNode);
-    Hi = DAG.getNode(Node->getOpcode(), MVT::Vector, LH, RH,
-                     NewNumEltsNode, TypeNode);
+    Lo = DAG.getNode(Node->getOpcode(), NewVT, LL, RL);
+    Hi = DAG.getNode(Node->getOpcode(), NewVT, LH, RH);
     break;
   }
-  case ISD::VLOAD: {
-    SDOperand Ch = Node->getOperand(0);   // Legalize the chain.
-    SDOperand Ptr = Node->getOperand(1);  // Legalize the pointer.
-    MVT::ValueType EVT = cast<VTSDNode>(TypeNode)->getVT();
-    
-    Lo = DAG.getVecLoad(NewNumElts, EVT, Ch, Ptr, Node->getOperand(2));
-    unsigned IncrementSize = NewNumElts * MVT::getSizeInBits(EVT)/8;
+  case ISD::LOAD: {
+    LoadSDNode *LD = cast<LoadSDNode>(Node);
+    SDOperand Ch = LD->getChain();
+    SDOperand Ptr = LD->getBasePtr();
+    const Value *SV = LD->getSrcValue();
+    int SVOffset = LD->getSrcValueOffset();
+    unsigned Alignment = LD->getAlignment();
+    bool isVolatile = LD->isVolatile();
+
+    Lo = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment);
+    unsigned IncrementSize = NewNumElts * MVT::getSizeInBits(NewEltVT)/8;
     Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
                       getIntPtrConstant(IncrementSize));
-    // FIXME: This creates a bogus srcvalue!
-    Hi = DAG.getVecLoad(NewNumElts, EVT, Ch, Ptr, Node->getOperand(2));
+    SVOffset += IncrementSize;
+    if (Alignment > IncrementSize)
+      Alignment = IncrementSize;
+    Hi = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment);
     
     // Build a factor node to remember that this load is independent of the
     // other one.
@@ -5599,42 +5564,31 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo,
     AddLegalizedOperand(Op.getValue(1), LegalizeOp(TF));
     break;
   }
-  case ISD::VBIT_CONVERT: {
+  case ISD::BIT_CONVERT: {
     // We know the result is a vector.  The input may be either a vector or a
     // scalar value.
-    if (Op.getOperand(0).getValueType() != MVT::Vector) {
+    if (!MVT::isVector(Op.getOperand(0).getValueType())) {
       // Lower to a store/load.  FIXME: this could be improved probably.
       SDOperand Ptr = CreateStackTemporary(Op.getOperand(0).getValueType());
 
       SDOperand St = DAG.getStore(DAG.getEntryNode(),
                                   Op.getOperand(0), Ptr, NULL, 0);
-      MVT::ValueType EVT = cast<VTSDNode>(TypeNode)->getVT();
-      St = DAG.getVecLoad(NumElements, EVT, St, Ptr, DAG.getSrcValue(0));
+      St = DAG.getLoad(NewVT, St, Ptr, NULL, 0);
       SplitVectorOp(St, Lo, Hi);
     } else {
       // If the input is a vector type, we have to either scalarize it, pack it
       // or convert it based on whether the input vector type is legal.
       SDNode *InVal = Node->getOperand(0).Val;
-      unsigned NumElems =
-        cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue();
-      MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT();
-
-      // If the input is from a single element vector, scalarize the vector,
-      // then treat like a scalar.
-      if (NumElems == 1) {
-        SDOperand Scalar = PackVectorOp(Op.getOperand(0), EVT);
-        Scalar = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Scalar,
-                             Op.getOperand(1), Op.getOperand(2));
-        SplitVectorOp(Scalar, Lo, Hi);
-      } else {
+      unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(0));
+
+      assert(NumElems > 1);
+      {
         // Split the input vector.
         SplitVectorOp(Op.getOperand(0), Lo, Hi);
 
         // Convert each of the pieces now.
-        Lo = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Lo,
-                         NewNumEltsNode, TypeNode);
-        Hi = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Hi,
-                         NewNumEltsNode, TypeNode);
+        Lo = DAG.getNode(ISD::BIT_CONVERT, NewVT, Lo);
+        Hi = DAG.getNode(ISD::BIT_CONVERT, NewVT, Hi);
       }
       break;
     }
@@ -5648,18 +5602,18 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo,
 }
 
 
-/// PackVectorOp - Given an operand of MVT::Vector type, convert it into the
-/// equivalent operation that returns a scalar (e.g. MVT::f32) or packed value
-/// (e.g. MVT::v4f32).  When this is called, we know that PackedVT is the right
-/// type for the result.
-SDOperand SelectionDAGLegalize::PackVectorOp(SDOperand Op, 
-                                             MVT::ValueType NewVT) {
-  assert(Op.getValueType() == MVT::Vector && "Bad PackVectorOp invocation!");
+/// ScalarizeVectorOp - Given an operand of vector type, convert it into the
+/// equivalent operation that returns a scalar (e.g. F32) value.
+SDOperand SelectionDAGLegalize::ScalarizeVectorOp(SDOperand Op) {
+  assert(MVT::isVector(Op.getValueType()) &&
+         "Bad ScalarizeVectorOp invocation!");
   SDNode *Node = Op.Val;
+  MVT::ValueType NewVT = MVT::getVectorElementType(Op.getValueType());
+  assert(MVT::getVectorNumElements(Op.getValueType()) == 1);
   
-  // See if we already packed it.
-  std::map<SDOperand, SDOperand>::iterator I = PackedNodes.find(Op);
-  if (I != PackedNodes.end()) return I->second;
+  // See if we already scalarized it.
+  std::map<SDOperand, SDOperand>::iterator I = ScalarizedNodes.find(Op);
+  if (I != ScalarizedNodes.end()) return I->second;
   
   SDOperand Result;
   switch (Node->getOpcode()) {
@@ -5667,146 +5621,89 @@ SDOperand SelectionDAGLegalize::PackVectorOp(SDOperand Op,
 #ifndef NDEBUG
     Node->dump(&DAG); cerr << "\n";
 #endif
-    assert(0 && "Unknown vector operation in PackVectorOp!");
-  case ISD::VADD:
-  case ISD::VSUB:
-  case ISD::VMUL:
-  case ISD::VSDIV:
-  case ISD::VUDIV:
-  case ISD::VAND:
-  case ISD::VOR:
-  case ISD::VXOR:
-    Result = DAG.getNode(getScalarizedOpcode(Node->getOpcode(), NewVT),
+    assert(0 && "Unknown vector operation in ScalarizeVectorOp!");
+  case ISD::ADD:
+  case ISD::FADD:
+  case ISD::SUB:
+  case ISD::FSUB:
+  case ISD::MUL:
+  case ISD::FMUL:
+  case ISD::SDIV:
+  case ISD::UDIV:
+  case ISD::FDIV:
+  case ISD::SREM:
+  case ISD::UREM:
+  case ISD::FREM:
+  case ISD::AND:
+  case ISD::OR:
+  case ISD::XOR:
+    Result = DAG.getNode(Node->getOpcode(),
                          NewVT, 
-                         PackVectorOp(Node->getOperand(0), NewVT),
-                         PackVectorOp(Node->getOperand(1), NewVT));
+                         ScalarizeVectorOp(Node->getOperand(0)),
+                         ScalarizeVectorOp(Node->getOperand(1)));
     break;
-  case ISD::VLOAD: {
-    SDOperand Ch = LegalizeOp(Node->getOperand(0));   // Legalize the chain.
-    SDOperand Ptr = LegalizeOp(Node->getOperand(1));  // Legalize the pointer.
-    
-    SrcValueSDNode *SV = cast<SrcValueSDNode>(Node->getOperand(2));
-    Result = DAG.getLoad(NewVT, Ch, Ptr, SV->getValue(), SV->getOffset());
+  case ISD::FNEG:
+  case ISD::FABS:
+  case ISD::FSQRT:
+  case ISD::FSIN:
+  case ISD::FCOS:
+    Result = DAG.getNode(Node->getOpcode(),
+                         NewVT, 
+                         ScalarizeVectorOp(Node->getOperand(0)));
+    break;
+  case ISD::LOAD: {
+    LoadSDNode *LD = cast<LoadSDNode>(Node);
+    SDOperand Ch = LegalizeOp(LD->getChain());     // Legalize the chain.
+    SDOperand Ptr = LegalizeOp(LD->getBasePtr());  // Legalize the pointer.
     
+    const Value *SV = LD->getSrcValue();
+    int SVOffset = LD->getSrcValueOffset();
+    Result = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset,
+                         LD->isVolatile(), LD->getAlignment());
+
     // Remember that we legalized the chain.
     AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1)));
     break;
   }
-  case ISD::VBUILD_VECTOR:
-    if (Node->getOperand(0).getValueType() == NewVT) {
-      // Returning a scalar?
-      Result = Node->getOperand(0);
-    } else {
-      // Returning a BUILD_VECTOR?
-      
-      // If all elements of the build_vector are undefs, return an undef.
-      bool AllUndef = true;
-      for (unsigned i = 0, e = Node->getNumOperands()-2; i != e; ++i)
-        if (Node->getOperand(i).getOpcode() != ISD::UNDEF) {
-          AllUndef = false;
-          break;
-        }
-      if (AllUndef) {
-        Result = DAG.getNode(ISD::UNDEF, NewVT);
-      } else {
-        Result = DAG.getNode(ISD::BUILD_VECTOR, NewVT, Node->op_begin(),
-                             Node->getNumOperands()-2);
-      }
-    }
+  case ISD::BUILD_VECTOR:
+    Result = Node->getOperand(0);
+    break;
+  case ISD::INSERT_VECTOR_ELT:
+    // Returning the inserted scalar element.
+    Result = Node->getOperand(1);
     break;
-  case ISD::VCONCAT_VECTORS:
+  case ISD::CONCAT_VECTORS:
     assert(Node->getOperand(0).getValueType() == NewVT &&
            "Concat of non-legal vectors not yet supported!");
     Result = Node->getOperand(0);
     break;
-  case ISD::VINSERT_VECTOR_ELT:
-    if (!MVT::isVector(NewVT)) {
-      // Returning a scalar?  Must be the inserted element.
-      Result = Node->getOperand(1);
-    } else {
-      Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, NewVT,
-                           PackVectorOp(Node->getOperand(0), NewVT),
-                           Node->getOperand(1), Node->getOperand(2));
-    }
+  case ISD::VECTOR_SHUFFLE: {
+    // Figure out if the scalar is the LHS or RHS and return it.
+    SDOperand EltNum = Node->getOperand(2).getOperand(0);
+    if (cast<ConstantSDNode>(EltNum)->getValue())
+      Result = ScalarizeVectorOp(Node->getOperand(1));
+    else
+      Result = ScalarizeVectorOp(Node->getOperand(0));
     break;
-  case ISD::VEXTRACT_SUBVECTOR:
-    Result = PackVectorOp(Node->getOperand(0), NewVT);
+  }
+  case ISD::EXTRACT_SUBVECTOR:
+    Result = Node->getOperand(0);
     assert(Result.getValueType() == NewVT);
     break;
-  case ISD::VVECTOR_SHUFFLE:
-    if (!MVT::isVector(NewVT)) {
-      // Returning a scalar?  Figure out if it is the LHS or RHS and return it.
-      SDOperand EltNum = Node->getOperand(2).getOperand(0);
-      if (cast<ConstantSDNode>(EltNum)->getValue())
-        Result = PackVectorOp(Node->getOperand(1), NewVT);
-      else
-        Result = PackVectorOp(Node->getOperand(0), NewVT);
-    } else {
-      // Otherwise, return a VECTOR_SHUFFLE node.  First convert the index
-      // vector from a VBUILD_VECTOR to a BUILD_VECTOR.
-      std::vector<SDOperand> BuildVecIdx(Node->getOperand(2).Val->op_begin(),
-                                         Node->getOperand(2).Val->op_end()-2);
-      MVT::ValueType BVT = MVT::getIntVectorWithNumElements(BuildVecIdx.size());
-      SDOperand BV = DAG.getNode(ISD::BUILD_VECTOR, BVT,
-                                 Node->getOperand(2).Val->op_begin(),
-                                 Node->getOperand(2).Val->getNumOperands()-2);
-      
-      Result = DAG.getNode(ISD::VECTOR_SHUFFLE, NewVT,
-                           PackVectorOp(Node->getOperand(0), NewVT),
-                           PackVectorOp(Node->getOperand(1), NewVT), BV);
-    }
-    break;
-  case ISD::VBIT_CONVERT:
-    if (Op.getOperand(0).getValueType() != MVT::Vector)
-      Result = DAG.getNode(ISD::BIT_CONVERT, NewVT, Op.getOperand(0));
-    else {
-      // If the input is a vector type, we have to either scalarize it, pack it
-      // or convert it based on whether the input vector type is legal.
-      SDNode *InVal = Node->getOperand(0).Val;
-      unsigned NumElems =
-        cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue();
-      MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT();
-        
-      // Figure out if there is a Packed type corresponding to this Vector
-      // type.  If so, convert to the vector type.
-      MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
-      if (TVT != MVT::Other && TLI.isTypeLegal(TVT)) {
-        // Turn this into a bit convert of the packed input.
-        Result = DAG.getNode(ISD::BIT_CONVERT, NewVT, 
-                             PackVectorOp(Node->getOperand(0), TVT));
-        break;
-      } else if (NumElems == 1) {
-        // Turn this into a bit convert of the scalar input.
-        Result = DAG.getNode(ISD::BIT_CONVERT, NewVT, 
-                             PackVectorOp(Node->getOperand(0), EVT));
-        break;
-      } else {
-        // If the input vector type isn't legal, then go through memory.
-        SDOperand Ptr = CreateStackTemporary(NewVT);
-        // Get the alignment for the store.
-        const TargetData &TD = *TLI.getTargetData();
-        unsigned Align = 
-          TD.getABITypeAlignment(MVT::getTypeForValueType(NewVT));
-        
-        SDOperand St = DAG.getStore(DAG.getEntryNode(),
-                                    Node->getOperand(0), Ptr, NULL, 0, false,
-                                    Align);
-        Result = DAG.getLoad(NewVT, St, Ptr, 0, 0);
-        break;
-      }
-    }
+  case ISD::BIT_CONVERT:
+    Result = DAG.getNode(ISD::BIT_CONVERT, NewVT, Op.getOperand(0));
     break;
-  case ISD::VSELECT:
+  case ISD::SELECT:
     Result = DAG.getNode(ISD::SELECT, NewVT, Op.getOperand(0),
-                         PackVectorOp(Op.getOperand(1), NewVT),
-                         PackVectorOp(Op.getOperand(2), NewVT));
+                         ScalarizeVectorOp(Op.getOperand(1)),
+                         ScalarizeVectorOp(Op.getOperand(2)));
     break;
   }
 
   if (TLI.isTypeLegal(NewVT))
     Result = LegalizeOp(Result);
-  bool isNew = PackedNodes.insert(std::make_pair(Op, Result)).second;
-  assert(isNew && "Value already packed?");
+  bool isNew = ScalarizedNodes.insert(std::make_pair(Op, Result)).second;
+  assert(isNew && "Value already scalarized?");
   return Result;
 }
 
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index d708233..5780eff 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -673,7 +673,9 @@ SDOperand SelectionDAG::getConstant(uint64_t Val, MVT::ValueType VT, bool isT) {
 SDOperand SelectionDAG::getConstantFP(double Val, MVT::ValueType VT,
                                       bool isTarget) {
   assert(MVT::isFloatingPoint(VT) && "Cannot create integer FP constant!");
-  if (VT == MVT::f32)
+  MVT::ValueType EltVT =
+    MVT::isVector(VT) ? MVT::getVectorElementType(VT) : VT;
+  if (EltVT == MVT::f32)
     Val = (float)Val;  // Mask out extra precision.
 
   // Do the map lookup using the actual bit pattern for the floating point
@@ -681,15 +683,21 @@ SDOperand SelectionDAG::getConstantFP(double Val, MVT::ValueType VT,
   // we don't have issues with SNANs.
   unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
   FoldingSetNodeID ID;
-  AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
+  AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0);
   ID.AddDouble(Val);
   void *IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDOperand(E, 0);
-  SDNode *N = new ConstantFPSDNode(isTarget, Val, VT);
+  SDNode *N = new ConstantFPSDNode(isTarget, Val, EltVT);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  return SDOperand(N, 0);
+  SDOperand Result(N, 0);
+  if (MVT::isVector(VT)) {
+    SmallVector<SDOperand, 8> Ops;
+    Ops.assign(MVT::getVectorNumElements(VT), Result);
+    Result = getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size());
+  }
+  return Result;
 }
 
 SDOperand SelectionDAG::getGlobalAddress(const GlobalValue *GV,
@@ -1952,6 +1960,23 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
     if (EVT == VT) return N1;  // Not actually extending
     break;
   }
+  case ISD::EXTRACT_VECTOR_ELT:
+    assert(N2C && "Bad EXTRACT_VECTOR_ELT!");
+
+    // EXTRACT_VECTOR_ELT of BUILD_PAIR is often formed while lowering is
+    // expanding copies of large vectors from registers.
+    if (N1.getOpcode() == ISD::BUILD_PAIR) {
+      unsigned NewNumElts = MVT::getVectorNumElements(N1.getValueType()) / 2;
+      bool Low = N2C->getValue() < NewNumElts;
+      return getNode(ISD::EXTRACT_VECTOR_ELT, VT, N1.getOperand(!Low),
+                     Low ? N2 : getConstant(N2C->getValue() - NewNumElts,
+                                            N2.getValueType()));
+    }
+    // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
+    // expanding large vector constants.
+    if (N1.getOpcode() == ISD::BUILD_VECTOR)
+      return N1.getOperand(N2C->getValue());
+    break;
   case ISD::EXTRACT_ELEMENT:
     assert(N2C && (unsigned)N2C->getValue() < 2 && "Bad EXTRACT_ELEMENT!");
     
@@ -2045,14 +2070,10 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
            MVT::getVectorNumElements(VT) == N3.getNumOperands() &&
            "Illegal VECTOR_SHUFFLE node!");
     break;
-  case ISD::VBIT_CONVERT:
-    // Fold vbit_convert nodes from a type to themselves.
-    if (N1.getValueType() == MVT::Vector) {
-      assert(isa<ConstantSDNode>(*(N1.Val->op_end()-2)) &&
-             isa<VTSDNode>(*(N1.Val->op_end()-1)) && "Malformed vector input!");
-      if (*(N1.Val->op_end()-2) == N2 && *(N1.Val->op_end()-1) == N3)
-        return N1;
-    }
+  case ISD::BIT_CONVERT:
+    // Fold bit_convert nodes from a type to themselves.
+    if (N1.getValueType() == VT)
+      return N1;
     break;
   }
 
@@ -2095,7 +2116,7 @@ SDOperand SelectionDAG::getLoad(MVT::ValueType VT,
                                 bool isVolatile, unsigned Alignment) {
   if (Alignment == 0) { // Ensure that codegen never sees alignment 0
     const Type *Ty = 0;
-    if (VT != MVT::Vector && VT != MVT::iPTR) {
+    if (VT != MVT::iPTR) {
       Ty = MVT::getTypeForValueType(VT);
     } else if (SV) {
       const PointerType *PT = dyn_cast<PointerType>(SV->getType());
@@ -2149,7 +2170,7 @@ SDOperand SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, MVT::ValueType VT,
 
   if (Alignment == 0) { // Ensure that codegen never sees alignment 0
     const Type *Ty = 0;
-    if (VT != MVT::Vector && VT != MVT::iPTR) {
+    if (VT != MVT::iPTR) {
       Ty = MVT::getTypeForValueType(VT);
     } else if (SV) {
       const PointerType *PT = dyn_cast<PointerType>(SV->getType());
@@ -2211,14 +2232,6 @@ SelectionDAG::getIndexedLoad(SDOperand OrigLoad, SDOperand Base,
   return SDOperand(N, 0);
 }
 
-SDOperand SelectionDAG::getVecLoad(unsigned Count, MVT::ValueType EVT,
-                                   SDOperand Chain, SDOperand Ptr,
-                                   SDOperand SV) {
-  SDOperand Ops[] = { Chain, Ptr, SV, getConstant(Count, MVT::i32), 
-                      getValueType(EVT) };
-  return getNode(ISD::VLOAD, getVTList(MVT::Vector, MVT::Other), Ops, 5);
-}
-
 SDOperand SelectionDAG::getStore(SDOperand Chain, SDOperand Val,
                                  SDOperand Ptr, const Value *SV, int SVOffset,
                                  bool isVolatile, unsigned Alignment) {
@@ -2226,7 +2239,7 @@ SDOperand SelectionDAG::getStore(SDOperand Chain, SDOperand Val,
 
   if (Alignment == 0) { // Ensure that codegen never sees alignment 0
     const Type *Ty = 0;
-    if (VT != MVT::Vector && VT != MVT::iPTR) {
+    if (VT != MVT::iPTR) {
       Ty = MVT::getTypeForValueType(VT);
     } else if (SV) {
       const PointerType *PT = dyn_cast<PointerType>(SV->getType());
@@ -2271,7 +2284,7 @@ SDOperand SelectionDAG::getTruncStore(SDOperand Chain, SDOperand Val,
 
   if (Alignment == 0) { // Ensure that codegen never sees alignment 0
     const Type *Ty = 0;
-    if (VT != MVT::Vector && VT != MVT::iPTR) {
+    if (VT != MVT::iPTR) {
       Ty = MVT::getTypeForValueType(VT);
     } else if (SV) {
       const PointerType *PT = dyn_cast<PointerType>(SV->getType());
@@ -2462,7 +2475,18 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, SDVTList VTList,
 }
 
 SDVTList SelectionDAG::getVTList(MVT::ValueType VT) {
-  return makeVTList(SDNode::getValueTypeList(VT), 1);
+  if (!MVT::isExtendedValueType(VT))
+    return makeVTList(SDNode::getValueTypeList(VT), 1);
+
+  for (std::list<std::vector<MVT::ValueType> >::iterator I = VTList.begin(),
+       E = VTList.end(); I != E; ++I) {
+    if (I->size() == 1 && (*I)[0] == VT)
+      return makeVTList(&(*I)[0], 1);
+  }
+  std::vector<MVT::ValueType> V;
+  V.push_back(VT);
+  VTList.push_front(V);
+  return makeVTList(&(*VTList.begin())[0], 1);
 }
 
 SDVTList SelectionDAG::getVTList(MVT::ValueType VT1, MVT::ValueType VT2) {
@@ -2496,7 +2520,7 @@ SDVTList SelectionDAG::getVTList(MVT::ValueType VT1, MVT::ValueType VT2,
 SDVTList SelectionDAG::getVTList(const MVT::ValueType *VTs, unsigned NumVTs) {
   switch (NumVTs) {
     case 0: assert(0 && "Cannot have nodes without results!");
-    case 1: return makeVTList(SDNode::getValueTypeList(VTs[0]), 1);
+    case 1: return getVTList(VTs[0]);
     case 2: return getVTList(VTs[0], VTs[1]);
     case 3: return getVTList(VTs[0], VTs[1], VTs[2]);
     default: break;
@@ -3394,30 +3418,16 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::FDIV:   return "fdiv";
   case ISD::FREM:   return "frem";
   case ISD::FCOPYSIGN: return "fcopysign";
-  case ISD::VADD:   return "vadd";
-  case ISD::VSUB:   return "vsub";
-  case ISD::VMUL:   return "vmul";
-  case ISD::VSDIV:  return "vsdiv";
-  case ISD::VUDIV:  return "vudiv";
-  case ISD::VAND:   return "vand";
-  case ISD::VOR:    return "vor";
-  case ISD::VXOR:   return "vxor";
 
   case ISD::SETCC:       return "setcc";
   case ISD::SELECT:      return "select";
   case ISD::SELECT_CC:   return "select_cc";
-  case ISD::VSELECT:     return "vselect";
   case ISD::INSERT_VECTOR_ELT:   return "insert_vector_elt";
-  case ISD::VINSERT_VECTOR_ELT:  return "vinsert_vector_elt";
   case ISD::EXTRACT_VECTOR_ELT:  return "extract_vector_elt";
-  case ISD::VEXTRACT_VECTOR_ELT: return "vextract_vector_elt";
-  case ISD::VCONCAT_VECTORS:     return "vconcat_vectors";
-  case ISD::VEXTRACT_SUBVECTOR:  return "vextract_subvector";
+  case ISD::CONCAT_VECTORS:      return "concat_vectors";
+  case ISD::EXTRACT_SUBVECTOR:   return "extract_subvector";
   case ISD::SCALAR_TO_VECTOR:    return "scalar_to_vector";
-  case ISD::VBUILD_VECTOR:       return "vbuild_vector";
   case ISD::VECTOR_SHUFFLE:      return "vector_shuffle";
-  case ISD::VVECTOR_SHUFFLE:     return "vvector_shuffle";
-  case ISD::VBIT_CONVERT:        return "vbit_convert";
   case ISD::CARRY_FALSE:         return "carry_false";
   case ISD::ADDC:        return "addc";
   case ISD::ADDE:        return "adde";
@@ -3456,7 +3466,6 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
     // Other operators
   case ISD::LOAD:               return "load";
   case ISD::STORE:              return "store";
-  case ISD::VLOAD:              return "vload";
   case ISD::VAARG:              return "vaarg";
   case ISD::VACOPY:             return "vacopy";
   case ISD::VAEND:              return "vaend";
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index cce0a32..472ca49 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -290,15 +290,7 @@ FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli,
       if (PN->use_empty()) continue;
       
       MVT::ValueType VT = TLI.getValueType(PN->getType());
-      unsigned NumRegisters;
-      if (VT != MVT::Vector)
-        NumRegisters = TLI.getNumRegisters(VT);
-      else {
-        MVT::ValueType VT1,VT2;
-        NumRegisters = 
-          TLI.getVectorTypeBreakdown(cast<VectorType>(PN->getType()),
-                                     VT1, VT2);
-      }
+      unsigned NumRegisters = TLI.getNumRegisters(VT);
       unsigned PHIReg = ValueMap[PN];
       assert(PHIReg && "PHI node does not have an assigned virtual register!");
       const TargetInstrInfo *TII = TLI.getTargetMachine().getInstrInfo();
@@ -320,7 +312,7 @@ unsigned FunctionLoweringInfo::CreateRegForValue(const Value *V) {
   
   // If this is a vector type, figure out what type it will decompose into
   // and how many of the elements it will use.
-  if (VT == MVT::Vector) {
+  if (MVT::isVector(VT)) {
     const VectorType *PTy = cast<VectorType>(V->getType());
     unsigned NumElts = PTy->getNumElements();
     MVT::ValueType EltTy = TLI.getValueType(PTy->getElementType());
@@ -582,36 +574,30 @@ public:
   void visitInvoke(InvokeInst &I);
   void visitUnwind(UnwindInst &I);
 
-  void visitScalarBinary(User &I, unsigned OpCode);
-  void visitVectorBinary(User &I, unsigned OpCode);
-  void visitEitherBinary(User &I, unsigned ScalarOp, unsigned VectorOp);
+  void visitBinary(User &I, unsigned OpCode);
   void visitShift(User &I, unsigned Opcode);
   void visitAdd(User &I) { 
-    if (isa<VectorType>(I.getType()))
-      visitVectorBinary(I, ISD::VADD);
-    else if (I.getType()->isFloatingPoint())
-      visitScalarBinary(I, ISD::FADD);
+    if (I.getType()->isFPOrFPVector())
+      visitBinary(I, ISD::FADD);
     else
-      visitScalarBinary(I, ISD::ADD);
+      visitBinary(I, ISD::ADD);
   }
   void visitSub(User &I);
   void visitMul(User &I) {
-    if (isa<VectorType>(I.getType()))
-      visitVectorBinary(I, ISD::VMUL);
-    else if (I.getType()->isFloatingPoint())
-      visitScalarBinary(I, ISD::FMUL);
+    if (I.getType()->isFPOrFPVector())
+      visitBinary(I, ISD::FMUL);
     else
-      visitScalarBinary(I, ISD::MUL);
+      visitBinary(I, ISD::MUL);
   }
-  void visitURem(User &I) { visitScalarBinary(I, ISD::UREM); }
-  void visitSRem(User &I) { visitScalarBinary(I, ISD::SREM); }
-  void visitFRem(User &I) { visitScalarBinary(I, ISD::FREM); }
-  void visitUDiv(User &I) { visitEitherBinary(I, ISD::UDIV, ISD::VUDIV); }
-  void visitSDiv(User &I) { visitEitherBinary(I, ISD::SDIV, ISD::VSDIV); }
-  void visitFDiv(User &I) { visitEitherBinary(I, ISD::FDIV, ISD::VSDIV); }
-  void visitAnd (User &I) { visitEitherBinary(I, ISD::AND,  ISD::VAND ); }
-  void visitOr  (User &I) { visitEitherBinary(I, ISD::OR,   ISD::VOR  ); }
-  void visitXor (User &I) { visitEitherBinary(I, ISD::XOR,  ISD::VXOR ); }
+  void visitURem(User &I) { visitBinary(I, ISD::UREM); }
+  void visitSRem(User &I) { visitBinary(I, ISD::SREM); }
+  void visitFRem(User &I) { visitBinary(I, ISD::FREM); }
+  void visitUDiv(User &I) { visitBinary(I, ISD::UDIV); }
+  void visitSDiv(User &I) { visitBinary(I, ISD::SDIV); }
+  void visitFDiv(User &I) { visitBinary(I, ISD::FDIV); }
+  void visitAnd (User &I) { visitBinary(I, ISD::AND); }
+  void visitOr  (User &I) { visitBinary(I, ISD::OR); }
+  void visitXor (User &I) { visitBinary(I, ISD::XOR); }
   void visitShl (User &I) { visitShift(I, ISD::SHL); }
   void visitLShr(User &I) { visitShift(I, ISD::SRL); }
   void visitAShr(User &I) { visitShift(I, ISD::SRA); }
@@ -687,7 +673,7 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) {
       if (!isa<VectorType>(VTy))
         return N = DAG.getNode(ISD::UNDEF, VT);
 
-      // Create a VBUILD_VECTOR of undef nodes.
+      // Create a BUILD_VECTOR of undef nodes.
       const VectorType *PTy = cast<VectorType>(VTy);
       unsigned NumElements = PTy->getNumElements();
       MVT::ValueType PVT = TLI.getValueType(PTy->getElementType());
@@ -696,9 +682,8 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) {
       Ops.assign(NumElements, DAG.getNode(ISD::UNDEF, PVT));
       
       // Create a VConstant node with generic Vector type.
-      Ops.push_back(DAG.getConstant(NumElements, MVT::i32));
-      Ops.push_back(DAG.getValueType(PVT));
-      return N = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector,
+      MVT::ValueType VT = MVT::getVectorType(PVT, NumElements);
+      return N = DAG.getNode(ISD::BUILD_VECTOR, VT,
                              &Ops[0], Ops.size());
     } else if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) {
       return N = DAG.getConstantFP(CFP->getValue(), VT);
@@ -723,10 +708,9 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) {
         Ops.assign(NumElements, Op);
       }
       
-      // Create a VBUILD_VECTOR node with generic Vector type.
-      Ops.push_back(DAG.getConstant(NumElements, MVT::i32));
-      Ops.push_back(DAG.getValueType(PVT));
-      return NodeMap[V] = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &Ops[0],
+      // Create a BUILD_VECTOR node.
+      MVT::ValueType VT = MVT::getVectorType(PVT, NumElements);
+      return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0],
                                       Ops.size());
     } else {
       // Canonicalize all constant ints to be unsigned.
@@ -745,7 +729,7 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) {
   assert(InReg && "Value not in map!");
   
   // If this type is not legal, make it so now.
-  if (VT != MVT::Vector) {
+  if (!MVT::isVector(VT)) {
     if (TLI.getTypeAction(VT) == TargetLowering::Expand) {
       // Source must be expanded.  This input value is actually coming from the
       // register pair InReg and InReg+1.
@@ -757,7 +741,7 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) {
       else {
         assert(NumVals == 2 && "1 to 4 (and more) expansion not implemented!");
         N = DAG.getNode(ISD::BUILD_PAIR, VT, N,
-                       DAG.getCopyFromReg(DAG.getEntryNode(), InReg+1, DestVT));
+                        DAG.getCopyFromReg(DAG.getEntryNode(), InReg+1, DestVT));
       }
     } else {
       MVT::ValueType DestVT = TLI.getTypeToTransformTo(VT);
@@ -768,29 +752,28 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) {
           : DAG.getNode(ISD::TRUNCATE, VT, N);
     }
   } else {
-    // Otherwise, if this is a vector, make it available as a generic vector
+    // Otherwise, if this is a vector, make it available as a vector
     // here.
-    MVT::ValueType PTyElementVT, PTyLegalElementVT;
-    const VectorType *PTy = cast<VectorType>(VTy);
-    unsigned NE = TLI.getVectorTypeBreakdown(PTy, PTyElementVT,
-                                             PTyLegalElementVT);
+    MVT::ValueType ElementVT, LegalElementVT;
+    unsigned NE = TLI.getVectorTypeBreakdown(VT, ElementVT,
+                                             LegalElementVT);
 
-    // Build a VBUILD_VECTOR or VCONCAT_VECTORS with the input registers.
+    // Build a BUILD_VECTOR or CONCAT_VECTORS with the input registers.
     SmallVector<SDOperand, 8> Ops;
-    if (PTyElementVT == PTyLegalElementVT) {
-      // If the value types are legal, just VBUILD the CopyFromReg nodes.
+    if (ElementVT == LegalElementVT) {
+      // If the value types are legal, just BUILD the CopyFromReg nodes.
       for (unsigned i = 0; i != NE; ++i)
         Ops.push_back(DAG.getCopyFromReg(DAG.getEntryNode(), InReg++, 
-                                         PTyElementVT));
-    } else if (PTyElementVT < PTyLegalElementVT) {
+                                         ElementVT));
+    } else if (ElementVT < LegalElementVT) {
       // If the register was promoted, use TRUNCATE or FP_ROUND as appropriate.
       for (unsigned i = 0; i != NE; ++i) {
         SDOperand Op = DAG.getCopyFromReg(DAG.getEntryNode(), InReg++, 
-                                          PTyLegalElementVT);
-        if (MVT::isFloatingPoint(PTyElementVT))
-          Op = DAG.getNode(ISD::FP_ROUND, PTyElementVT, Op);
+                                          LegalElementVT);
+        if (MVT::isFloatingPoint(ElementVT))
+          Op = DAG.getNode(ISD::FP_ROUND, ElementVT, Op);
         else
-          Op = DAG.getNode(ISD::TRUNCATE, PTyElementVT, Op);
+          Op = DAG.getNode(ISD::TRUNCATE, ElementVT, Op);
         Ops.push_back(Op);
       }
     } else {
@@ -798,21 +781,22 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) {
       assert((NE & 1) == 0 && "Must expand into a multiple of 2 elements!");
       for (unsigned i = 0; i != NE; ++i) {
         SDOperand Op0 = DAG.getCopyFromReg(DAG.getEntryNode(), InReg++, 
-                                           PTyLegalElementVT);
+                                           LegalElementVT);
         SDOperand Op1 = DAG.getCopyFromReg(DAG.getEntryNode(), InReg++, 
-                                           PTyLegalElementVT);
-        Ops.push_back(DAG.getNode(ISD::BUILD_PAIR, PTyElementVT, Op0, Op1));
+                                           LegalElementVT);
+        Ops.push_back(DAG.getNode(ISD::BUILD_PAIR, ElementVT, Op0, Op1));
       }
     }
     
-    if (MVT::isVector(PTyElementVT)) {
-      Ops.push_back(DAG.getConstant(NE * MVT::getVectorNumElements(PTyElementVT), MVT::i32));
-      Ops.push_back(DAG.getValueType(MVT::getVectorElementType(PTyElementVT)));
-      N = DAG.getNode(ISD::VCONCAT_VECTORS, MVT::Vector, &Ops[0], Ops.size());
+    if (MVT::isVector(ElementVT)) {
+      N = DAG.getNode(ISD::CONCAT_VECTORS,
+                      MVT::getVectorType(MVT::getVectorElementType(ElementVT),
+                                         NE * MVT::getVectorNumElements(ElementVT)),
+                      &Ops[0], Ops.size());
     } else {
-      Ops.push_back(DAG.getConstant(NE, MVT::i32));
-      Ops.push_back(DAG.getValueType(PTyElementVT));
-      N = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &Ops[0], Ops.size());
+      N = DAG.getNode(ISD::BUILD_VECTOR,
+                      MVT::getVectorType(ElementVT, NE),
+                      &Ops[0], Ops.size());
     }
   }
   
@@ -1220,7 +1204,7 @@ void SelectionDAGLowering::visitJumpTableHeader(SelectionDAGISel::JumpTable &JT,
   // register so it can be used as an index into the jump table in a 
   // subsequent basic block.  This value may be smaller or larger than the
   // target's pointer type, and therefore require extension or truncating.
-  if (VT > TLI.getPointerTy())
+  if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getPointerTy()))
     SwitchOp = DAG.getNode(ISD::TRUNCATE, TLI.getPointerTy(), SUB);
   else
     SwitchOp = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), SUB);
@@ -1270,7 +1254,7 @@ void SelectionDAGLowering::visitBitTestHeader(SelectionDAGISel::BitTestBlock &B)
                                     ISD::SETUGT);
 
   SDOperand ShiftOp;
-  if (VT > TLI.getShiftAmountTy())
+  if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getShiftAmountTy()))
     ShiftOp = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), SUB);
   else
     ShiftOp = DAG.getNode(ISD::ZERO_EXTEND, TLI.getShiftAmountTy(), SUB);
@@ -1910,52 +1894,44 @@ void SelectionDAGLowering::visitSub(User &I) {
   // -0.0 - X --> fneg
   const Type *Ty = I.getType();
   if (isa<VectorType>(Ty)) {
-    visitVectorBinary(I, ISD::VSUB);
-  } else if (Ty->isFloatingPoint()) {
+    if (ConstantVector *CV = dyn_cast<ConstantVector>(I.getOperand(0))) {
+      const VectorType *DestTy = cast<VectorType>(I.getType());
+      const Type *ElTy = DestTy->getElementType();
+      unsigned VL = DestTy->getNumElements();
+      std::vector<Constant*> NZ(VL, ConstantFP::get(ElTy, -0.0));
+      Constant *CNZ = ConstantVector::get(&NZ[0], NZ.size());
+      if (CV == CNZ) {
+        SDOperand Op2 = getValue(I.getOperand(1));
+        setValue(&I, DAG.getNode(ISD::FNEG, Op2.getValueType(), Op2));
+        return;
+      }
+    }
+  }
+  if (Ty->isFloatingPoint()) {
     if (ConstantFP *CFP = dyn_cast<ConstantFP>(I.getOperand(0)))
       if (CFP->isExactlyValue(-0.0)) {
         SDOperand Op2 = getValue(I.getOperand(1));
         setValue(&I, DAG.getNode(ISD::FNEG, Op2.getValueType(), Op2));
         return;
       }
-    visitScalarBinary(I, ISD::FSUB);
-  } else 
-    visitScalarBinary(I, ISD::SUB);
+  }
+
+  visitBinary(I, Ty->isFPOrFPVector() ? ISD::FSUB : ISD::SUB);
 }
 
-void SelectionDAGLowering::visitScalarBinary(User &I, unsigned OpCode) {
+void SelectionDAGLowering::visitBinary(User &I, unsigned OpCode) {
   SDOperand Op1 = getValue(I.getOperand(0));
   SDOperand Op2 = getValue(I.getOperand(1));
   
   setValue(&I, DAG.getNode(OpCode, Op1.getValueType(), Op1, Op2));
 }
 
-void
-SelectionDAGLowering::visitVectorBinary(User &I, unsigned OpCode) {
-  assert(isa<VectorType>(I.getType()));
-  const VectorType *Ty = cast<VectorType>(I.getType());
-  SDOperand Typ = DAG.getValueType(TLI.getValueType(Ty->getElementType()));
-
-  setValue(&I, DAG.getNode(OpCode, MVT::Vector,
-                           getValue(I.getOperand(0)),
-                           getValue(I.getOperand(1)),
-                           DAG.getConstant(Ty->getNumElements(), MVT::i32),
-                           Typ));
-}
-
-void SelectionDAGLowering::visitEitherBinary(User &I, unsigned ScalarOp,
-                                             unsigned VectorOp) {
-  if (isa<VectorType>(I.getType()))
-    visitVectorBinary(I, VectorOp);
-  else
-    visitScalarBinary(I, ScalarOp);
-}
-
 void SelectionDAGLowering::visitShift(User &I, unsigned Opcode) {
   SDOperand Op1 = getValue(I.getOperand(0));
   SDOperand Op2 = getValue(I.getOperand(1));
   
-  if (TLI.getShiftAmountTy() < Op2.getValueType())
+  if (MVT::getSizeInBits(TLI.getShiftAmountTy()) <
+      MVT::getSizeInBits(Op2.getValueType()))
     Op2 = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), Op2);
   else if (TLI.getShiftAmountTy() > Op2.getValueType())
     Op2 = DAG.getNode(ISD::ANY_EXTEND, TLI.getShiftAmountTy(), Op2);
@@ -2033,14 +2009,8 @@ void SelectionDAGLowering::visitSelect(User &I) {
   SDOperand Cond     = getValue(I.getOperand(0));
   SDOperand TrueVal  = getValue(I.getOperand(1));
   SDOperand FalseVal = getValue(I.getOperand(2));
-  if (!isa<VectorType>(I.getType())) {
-    setValue(&I, DAG.getNode(ISD::SELECT, TrueVal.getValueType(), Cond,
-                             TrueVal, FalseVal));
-  } else {
-    setValue(&I, DAG.getNode(ISD::VSELECT, MVT::Vector, Cond, TrueVal, FalseVal,
-                             *(TrueVal.Val->op_end()-2),
-                             *(TrueVal.Val->op_end()-1)));
-  }
+  setValue(&I, DAG.getNode(ISD::SELECT, TrueVal.getValueType(), Cond,
+                           TrueVal, FalseVal));
 }
 
 
@@ -2140,23 +2110,6 @@ void SelectionDAGLowering::visitIntToPtr(User &I) {
 void SelectionDAGLowering::visitBitCast(User &I) { 
   SDOperand N = getValue(I.getOperand(0));
   MVT::ValueType DestVT = TLI.getValueType(I.getType());
-  if (DestVT == MVT::Vector) {
-    // This is a cast to a vector from something else.  
-    // Get information about the output vector.
-    const VectorType *DestTy = cast<VectorType>(I.getType());
-    MVT::ValueType EltVT = TLI.getValueType(DestTy->getElementType());
-    setValue(&I, DAG.getNode(ISD::VBIT_CONVERT, DestVT, N, 
-                             DAG.getConstant(DestTy->getNumElements(),MVT::i32),
-                             DAG.getValueType(EltVT)));
-    return;
-  } 
-  MVT::ValueType SrcVT = N.getValueType();
-  if (SrcVT == MVT::Vector) {
-    // This is a cast from a vctor to something else. 
-    // Get information about the input vector.
-    setValue(&I, DAG.getNode(ISD::VBIT_CONVERT, DestVT, N));
-    return;
-  }
 
   // BitCast assures us that source and destination are the same size so this 
   // is either a BIT_CONVERT or a no-op.
@@ -2172,18 +2125,16 @@ void SelectionDAGLowering::visitInsertElement(User &I) {
   SDOperand InIdx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(),
                                 getValue(I.getOperand(2)));
 
-  SDOperand Num = *(InVec.Val->op_end()-2);
-  SDOperand Typ = *(InVec.Val->op_end()-1);
-  setValue(&I, DAG.getNode(ISD::VINSERT_VECTOR_ELT, MVT::Vector,
-                           InVec, InVal, InIdx, Num, Typ));
+  setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT,
+                           TLI.getValueType(I.getType()),
+                           InVec, InVal, InIdx));
 }
 
 void SelectionDAGLowering::visitExtractElement(User &I) {
   SDOperand InVec = getValue(I.getOperand(0));
   SDOperand InIdx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(),
                                 getValue(I.getOperand(1)));
-  SDOperand Typ = *(InVec.Val->op_end()-1);
-  setValue(&I, DAG.getNode(ISD::VEXTRACT_VECTOR_ELT,
+  setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT,
                            TLI.getValueType(I.getType()), InVec, InIdx));
 }
 
@@ -2192,10 +2143,9 @@ void SelectionDAGLowering::visitShuffleVector(User &I) {
   SDOperand V2   = getValue(I.getOperand(1));
   SDOperand Mask = getValue(I.getOperand(2));
 
-  SDOperand Num = *(V1.Val->op_end()-2);
-  SDOperand Typ = *(V2.Val->op_end()-1);
-  setValue(&I, DAG.getNode(ISD::VVECTOR_SHUFFLE, MVT::Vector,
-                           V1, V2, Mask, Num, Typ));
+  setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE,
+                           TLI.getValueType(I.getType()),
+                           V1, V2, Mask));
 }
 
 
@@ -2325,15 +2275,9 @@ SDOperand SelectionDAGLowering::getLoadFrom(const Type *Ty, SDOperand Ptr,
                                             const Value *SV, SDOperand Root,
                                             bool isVolatile, 
                                             unsigned Alignment) {
-  SDOperand L;
-  if (const VectorType *PTy = dyn_cast<VectorType>(Ty)) {
-    MVT::ValueType PVT = TLI.getValueType(PTy->getElementType());
-    L = DAG.getVecLoad(PTy->getNumElements(), PVT, Root, Ptr,
-                       DAG.getSrcValue(SV));
-  } else {
-    L = DAG.getLoad(TLI.getValueType(Ty), Root, Ptr, SV, 0, 
-                    isVolatile, Alignment);
-  }
+  SDOperand L =
+    DAG.getLoad(TLI.getValueType(Ty), Root, Ptr, SV, 0, 
+                isVolatile, Alignment);
 
   if (isVolatile)
     DAG.setRoot(L.getValue(1));
@@ -2394,17 +2338,6 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I,
   // Add all operands of the call to the operand list.
   for (unsigned i = 1, e = I.getNumOperands(); i != e; ++i) {
     SDOperand Op = getValue(I.getOperand(i));
-    
-    // If this is a vector type, force it to the right vector type.
-    if (Op.getValueType() == MVT::Vector) {
-      const VectorType *OpTy = cast<VectorType>(I.getOperand(i)->getType());
-      MVT::ValueType EltVT = TLI.getValueType(OpTy->getElementType());
-      
-      MVT::ValueType VVT = MVT::getVectorType(EltVT, OpTy->getNumElements());
-      assert(VVT != MVT::Other && "Intrinsic uses a non-legal type?");
-      Op = DAG.getNode(ISD::VBIT_CONVERT, VVT, Op);
-    }
-    
     assert(TLI.isTypeLegal(Op.getValueType()) &&
            "Intrinsic uses a non-legal type?");
     Ops.push_back(Op);
@@ -2413,7 +2346,7 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I,
   std::vector<MVT::ValueType> VTs;
   if (I.getType() != Type::VoidTy) {
     MVT::ValueType VT = TLI.getValueType(I.getType());
-    if (VT == MVT::Vector) {
+    if (MVT::isVector(VT)) {
       const VectorType *DestTy = cast<VectorType>(I.getType());
       MVT::ValueType EltVT = TLI.getValueType(DestTy->getElementType());
       
@@ -2450,10 +2383,8 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I,
   }
   if (I.getType() != Type::VoidTy) {
     if (const VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
-      MVT::ValueType EVT = TLI.getValueType(PTy->getElementType());
-      Result = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Result,
-                           DAG.getConstant(PTy->getNumElements(), MVT::i32),
-                           DAG.getValueType(EVT));
+      MVT::ValueType VT = TLI.getValueType(PTy);
+      Result = DAG.getNode(ISD::BIT_CONVERT, VT, Result);
     } 
     setValue(&I, Result);
   }
@@ -2931,11 +2862,8 @@ SDOperand RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
     return Val;
   
   if (MVT::isVector(RegVT)) {
-    assert(ValueVT == MVT::Vector && "Unknown vector conversion!");
-    return DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Val, 
-                       DAG.getConstant(MVT::getVectorNumElements(RegVT),
-                                       MVT::i32),
-                       DAG.getValueType(MVT::getVectorElementType(RegVT)));
+    assert(MVT::isVector(ValueVT) && "Unknown vector conversion!");
+    return DAG.getNode(ISD::BIT_CONVERT, RegVT, Val);
   }
   
   if (MVT::isInteger(RegVT)) {
@@ -2960,8 +2888,9 @@ void RegsForValue::getCopyToRegs(SDOperand Val, SelectionDAG &DAG,
     // a promotion.
     if (RegVT != ValueVT) {
       if (MVT::isVector(RegVT)) {
-        assert(Val.getValueType() == MVT::Vector &&"Not a vector-vector cast?");
-        Val = DAG.getNode(ISD::VBIT_CONVERT, RegVT, Val);
+        assert(MVT::isVector(Val.getValueType()) &&
+               "Not a vector-vector cast?");
+        Val = DAG.getNode(ISD::BIT_CONVERT, RegVT, Val);
       } else if (MVT::isInteger(RegVT) && MVT::isInteger(Val.getValueType())) {
         if (RegVT < ValueVT)
           Val = DAG.getNode(ISD::TRUNCATE, RegVT, Val);
@@ -3631,15 +3560,12 @@ void SelectionDAGLowering::visitInlineAsm(CallInst &I) {
     
     // If the result of the inline asm is a vector, it may have the wrong
     // width/num elts.  Make sure to convert it to the right type with
-    // vbit_convert.
-    if (Val.getValueType() == MVT::Vector) {
+    // bit_convert.
+    if (MVT::isVector(Val.getValueType())) {
       const VectorType *VTy = cast<VectorType>(I.getType());
-      unsigned DesiredNumElts = VTy->getNumElements();
-      MVT::ValueType DesiredEltVT = TLI.getValueType(VTy->getElementType());
+      MVT::ValueType DesiredVT = TLI.getValueType(VTy);
       
-      Val = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Val, 
-                        DAG.getConstant(DesiredNumElts, MVT::i32),
-                        DAG.getValueType(DesiredEltVT));
+      Val = DAG.getNode(ISD::BIT_CONVERT, DesiredVT, Val);
     }
     
     setValue(&I, Val);
@@ -3826,7 +3752,7 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) {
       Ops.push_back(DAG.getConstant(Flags, MVT::i32));
       break;
     case Expand:
-      if (VT != MVT::Vector) {
+      if (!MVT::isVector(VT)) {
         // If this is a large integer, it needs to be broken up into small
         // integers.  Figure out what the destination type is and how many small
         // integers it turns into.
@@ -3848,7 +3774,8 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) {
 
         // Figure out if there is a Packed type corresponding to this Vector
         // type.  If so, convert to the vector type.
-        MVT::ValueType TVT = MVT::getVectorType(getValueType(EltTy), NumElems);
+        MVT::ValueType TVT =
+          MVT::getVectorType(getValueType(EltTy), NumElems);
         if (TVT != MVT::Other && isTypeLegal(TVT)) {
           RetVals.push_back(TVT);
           Ops.push_back(DAG.getConstant(Flags, MVT::i32));
@@ -3900,7 +3827,7 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) {
       break;
     }
     case Expand:
-      if (VT != MVT::Vector) {
+      if (!MVT::isVector(VT)) {
         // If this is a large integer or a floating point node that needs to be
         // expanded, it needs to be reassembled from small integers.  Figure out
         // what the source elt type is and how many small integers it is.
@@ -3914,13 +3841,12 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) {
 
         // Figure out if there is a Packed type corresponding to this Vector
         // type.  If so, convert to the vector type.
-        MVT::ValueType TVT = MVT::getVectorType(getValueType(EltTy), NumElems);
+        MVT::ValueType TVT =
+          MVT::getVectorType(getValueType(EltTy), NumElems);
         if (TVT != MVT::Other && isTypeLegal(TVT)) {
           SDOperand N = SDOperand(Result, i++);
-          // Handle copies from generic vectors to registers.
-          N = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, N,
-                          DAG.getConstant(NumElems, MVT::i32), 
-                          DAG.getValueType(getValueType(EltTy)));
+          // Handle copies from vectors to registers.
+          N = DAG.getNode(ISD::BIT_CONVERT, TVT, N);
           Ops.push_back(N);
         } else {
           assert(0 && "Don't support illegal by-val vector arguments yet!");
@@ -4040,7 +3966,7 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy,
       Ops.push_back(DAG.getConstant(Flags, MVT::i32));
       break;
     case Expand:
-      if (VT != MVT::Vector) {
+      if (!MVT::isVector(VT)) {
         // If this is a large integer, it needs to be broken down into small
         // integers.  Figure out what the source elt type is and how many small
         // integers it is.
@@ -4054,10 +3980,11 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy,
         
         // Figure out if there is a Packed type corresponding to this Vector
         // type.  If so, convert to the vector type.
-        MVT::ValueType TVT = MVT::getVectorType(getValueType(EltTy), NumElems);
+        MVT::ValueType TVT =
+          MVT::getVectorType(getValueType(EltTy), NumElems);
         if (TVT != MVT::Other && isTypeLegal(TVT)) {
-          // Insert a VBIT_CONVERT of the MVT::Vector type to the vector type.
-          Op = DAG.getNode(ISD::VBIT_CONVERT, TVT, Op);
+          // Insert a BIT_CONVERT of the original type to the vector type.
+          Op = DAG.getNode(ISD::BIT_CONVERT, TVT, Op);
           Ops.push_back(Op);
           Ops.push_back(DAG.getConstant(Flags, MVT::i32));
         } else {
@@ -4083,7 +4010,7 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy,
       RetTys.push_back(getTypeToTransformTo(VT));
       break;
     case Expand:
-      if (VT != MVT::Vector) {
+      if (!MVT::isVector(VT)) {
         // If this is a large integer, it needs to be reassembled from small
         // integers.  Figure out what the source elt type is and how many small
         // integers it is.
@@ -4100,7 +4027,8 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy,
         
         // Figure out if there is a Packed type corresponding to this Vector
         // type.  If so, convert to the vector type.
-        MVT::ValueType TVT = MVT::getVectorType(getValueType(EltTy), NumElems);
+        MVT::ValueType TVT =
+          MVT::getVectorType(getValueType(EltTy), NumElems);
         if (TVT != MVT::Other && isTypeLegal(TVT)) {
           RetTys.push_back(TVT);
         } else {
@@ -4129,21 +4057,20 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy,
       
       // If this value was promoted, truncate it down.
       if (ResVal.getValueType() != VT) {
-        if (VT == MVT::Vector) {
-          // Insert a VBIT_CONVERT to convert from the packed result type to the
-          // MVT::Vector type.
+        if (MVT::isVector(VT)) {
+          // Insert a BIT_CONVERT to convert from the packed result type to the
+          // new vector type.
           unsigned NumElems = cast<VectorType>(RetTy)->getNumElements();
           const Type *EltTy = cast<VectorType>(RetTy)->getElementType();
           
           // Figure out if there is a Packed type corresponding to this Vector
           // type.  If so, convert to the vector type.
-          MVT::ValueType TVT = MVT::getVectorType(getValueType(EltTy),NumElems);
+          MVT::ValueType TVT =
+            MVT::getVectorType(getValueType(EltTy),NumElems);
           if (TVT != MVT::Other && isTypeLegal(TVT)) {
-            // Insert a VBIT_CONVERT of the FORMAL_ARGUMENTS to a
-            // "N x PTyElementVT" MVT::Vector type.
-            ResVal = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, ResVal,
-                                 DAG.getConstant(NumElems, MVT::i32), 
-                                 DAG.getValueType(getValueType(EltTy)));
+            // Insert a BIT_CONVERT of the FORMAL_ARGUMENTS to a
+            // "N x PTyElementVT" vector type.
+            ResVal = DAG.getNode(ISD::BIT_CONVERT, TVT, ResVal);
           } else {
             abort();
           }
@@ -4457,40 +4384,40 @@ SDOperand SelectionDAGLowering::CopyValueToVirtualRegister(Value *V,
   MVT::ValueType DestVT = TLI.getTypeToTransformTo(SrcVT);
   if (SrcVT == DestVT) {
     return DAG.getCopyToReg(getRoot(), Reg, Op);
-  } else if (SrcVT == MVT::Vector) {
+  } else if (MVT::isVector(SrcVT)) {
     // Handle copies from generic vectors to registers.
-    MVT::ValueType PTyElementVT, PTyLegalElementVT;
-    unsigned NE = TLI.getVectorTypeBreakdown(cast<VectorType>(V->getType()),
-                                             PTyElementVT, PTyLegalElementVT);
-    uint64_t SrcVL = cast<ConstantSDNode>(*(Op.Val->op_end()-2))->getValue();
+    MVT::ValueType ElementVT, LegalElementVT;
+    unsigned NE = TLI.getVectorTypeBreakdown(SrcVT,
+                                             ElementVT, LegalElementVT);
+    uint64_t SrcVL = MVT::getVectorNumElements(SrcVT);
     
     // Loop over all of the elements of the resultant vector,
-    // VEXTRACT_VECTOR_ELT'ing or VEXTRACT_SUBVECTOR'ing them, converting them
-    // to PTyLegalElementVT, then copying them into output registers.
+    // EXTRACT_VECTOR_ELT'ing or EXTRACT_SUBVECTOR'ing them, converting them
+    // to LegalElementVT, then copying them into output registers.
     SmallVector<SDOperand, 8> OutChains;
     SDOperand Root = getRoot();
     for (unsigned i = 0; i != NE; ++i) {
-      SDOperand Elt = MVT::isVector(PTyElementVT) ?
-        DAG.getNode(ISD::VEXTRACT_SUBVECTOR, PTyElementVT,
+      SDOperand Elt = MVT::isVector(ElementVT) ?
+        DAG.getNode(ISD::EXTRACT_SUBVECTOR, ElementVT,
                     Op, DAG.getConstant(i * (SrcVL / NE), TLI.getPointerTy())) :
-        DAG.getNode(ISD::VEXTRACT_VECTOR_ELT, PTyElementVT,
+        DAG.getNode(ISD::EXTRACT_VECTOR_ELT, ElementVT,
                     Op, DAG.getConstant(i, TLI.getPointerTy()));
-      if (PTyElementVT == PTyLegalElementVT) {
+      if (ElementVT == LegalElementVT) {
         // Elements are legal.
         OutChains.push_back(DAG.getCopyToReg(Root, Reg++, Elt));
-      } else if (PTyLegalElementVT > PTyElementVT) {
+      } else if (LegalElementVT > ElementVT) {
         // Elements are promoted.
-        if (MVT::isFloatingPoint(PTyLegalElementVT))
-          Elt = DAG.getNode(ISD::FP_EXTEND, PTyLegalElementVT, Elt);
+        if (MVT::isFloatingPoint(LegalElementVT))
+          Elt = DAG.getNode(ISD::FP_EXTEND, LegalElementVT, Elt);
         else
-          Elt = DAG.getNode(ISD::ANY_EXTEND, PTyLegalElementVT, Elt);
+          Elt = DAG.getNode(ISD::ANY_EXTEND, LegalElementVT, Elt);
         OutChains.push_back(DAG.getCopyToReg(Root, Reg++, Elt));
       } else {
         // Elements are expanded.
         // The src value is expanded into multiple registers.
-        SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, PTyLegalElementVT,
+        SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, LegalElementVT,
                                    Elt, DAG.getConstant(0, TLI.getPointerTy()));
-        SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, PTyLegalElementVT,
+        SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, LegalElementVT,
                                    Elt, DAG.getConstant(1, TLI.getPointerTy()));
         OutChains.push_back(DAG.getCopyToReg(Root, Reg++, Lo));
         OutChains.push_back(DAG.getCopyToReg(Root, Reg++, Hi));
@@ -4695,15 +4622,7 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB,
       // Remember that this register needs to added to the machine PHI node as
       // the input for this MBB.
       MVT::ValueType VT = TLI.getValueType(PN->getType());
-      unsigned NumRegisters;
-      if (VT != MVT::Vector)
-        NumRegisters = TLI.getNumRegisters(VT);
-      else {
-        MVT::ValueType VT1,VT2;
-        NumRegisters = 
-          TLI.getVectorTypeBreakdown(cast<VectorType>(PN->getType()),
-                                     VT1, VT2);
-      }
+      unsigned NumRegisters = TLI.getNumRegisters(VT);
       for (unsigned i = 0, e = NumRegisters; i != e; ++i)
         PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg+i));
     }
diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index b2c9016..7ab5528 100644
--- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -199,7 +199,7 @@ static void SetValueTypeAction(MVT::ValueType VT,
     else if (VT == MVT::f64)
       TransformToType[VT] = MVT::i64;
     else {
-      assert((VT == MVT::Vector || MVT::isInteger(VT)) && VT > MVT::i8 &&
+      assert((MVT::isVector(VT) || MVT::isInteger(VT)) && VT > MVT::i8 &&
              "Cannot expand this type: target must support SOME integer reg!");
       // Expand to the next smaller integer type!
       TransformToType[VT] = (MVT::ValueType)(VT-1);
@@ -265,16 +265,18 @@ void TargetLowering::computeRegisterProperties() {
                        ValueTypeActions);
   }
   
-  // Set MVT::Vector to always be Expanded
-  SetValueTypeAction(MVT::Vector, Expand, *this, TransformToType, 
-                     ValueTypeActions);
-  
   // Loop over all of the legal vector value types, specifying an identity type
   // transformation.
   for (unsigned i = MVT::FIRST_VECTOR_VALUETYPE;
        i <= MVT::LAST_VECTOR_VALUETYPE; ++i) {
     if (isTypeLegal((MVT::ValueType)i))
       TransformToType[i] = (MVT::ValueType)i;
+    else {
+      MVT::ValueType VT1, VT2;
+      NumRegistersForVT[i] = getVectorTypeBreakdown(i, VT1, VT2);
+      SetValueTypeAction(i, Expand, *this, TransformToType,
+                         ValueTypeActions);
+    }
   }
 }
 
@@ -282,38 +284,42 @@ const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
   return NULL;
 }
 
-/// getVectorTypeBreakdown - Packed types are broken down into some number of
-/// legal first class types. For example, <8 x float> maps to 2 MVT::v4f32
+/// getVectorTypeBreakdown - Vector types are broken down into some number of
+/// legal first class types.  For example, MVT::v8f32 maps to 2 MVT::v4f32
 /// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack.
+/// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86.
 ///
-/// This method returns the number and type of the resultant breakdown.
+/// This method returns the number of registers needed, and the VT for each
+/// register.  It also returns the VT of the VectorType elements before they
+/// are promoted/expanded.
 ///
-unsigned TargetLowering::getVectorTypeBreakdown(const VectorType *PTy, 
-                                                MVT::ValueType &PTyElementVT,
-                                      MVT::ValueType &PTyLegalElementVT) const {
+unsigned TargetLowering::getVectorTypeBreakdown(MVT::ValueType VT, 
+                                                MVT::ValueType &ElementVT,
+                                      MVT::ValueType &LegalElementVT) const {
   // Figure out the right, legal destination reg to copy into.
-  unsigned NumElts = PTy->getNumElements();
-  MVT::ValueType EltTy = getValueType(PTy->getElementType());
+  unsigned NumElts = MVT::getVectorNumElements(VT);
+  MVT::ValueType EltTy = MVT::getVectorElementType(VT);
   
   unsigned NumVectorRegs = 1;
   
   // Divide the input until we get to a supported size.  This will always
   // end with a scalar if the target doesn't support vectors.
-  while (NumElts > 1 && !isTypeLegal(MVT::getVectorType(EltTy, NumElts))) {
+  while (NumElts > 1 &&
+         !isTypeLegal(MVT::getVectorType(EltTy, NumElts))) {
     NumElts >>= 1;
     NumVectorRegs <<= 1;
   }
   
-  MVT::ValueType VT = MVT::getVectorType(EltTy, NumElts);
-  if (!isTypeLegal(VT))
-    VT = EltTy;
-  PTyElementVT = VT;
-
-  MVT::ValueType DestVT = getTypeToTransformTo(VT);
-  PTyLegalElementVT = DestVT;
-  if (DestVT < VT) {
+  MVT::ValueType NewVT = MVT::getVectorType(EltTy, NumElts);
+  if (!isTypeLegal(NewVT))
+    NewVT = EltTy;
+  ElementVT = NewVT;
+
+  MVT::ValueType DestVT = getTypeToTransformTo(NewVT);
+  LegalElementVT = DestVT;
+  if (DestVT < NewVT) {
     // Value is expanded, e.g. i64 -> i16.
-    return NumVectorRegs*(MVT::getSizeInBits(VT)/MVT::getSizeInBits(DestVT));
+    return NumVectorRegs*(MVT::getSizeInBits(NewVT)/MVT::getSizeInBits(DestVT));
   } else {
     // Otherwise, promotion or legal types use the same number of registers as
     // the vector decimated to the appropriate level.
diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp
index d048cae..4c1d6ea 100644
--- a/lib/Target/X86/X86ISelLowering.cpp
+++ b/lib/Target/X86/X86ISelLowering.cpp
@@ -2493,9 +2493,14 @@ X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) {
     }
   }
 
-  if (NumNonZero == 0)
-    // Must be a mix of zero and undef. Return a zero vector.
-    return getZeroVector(VT, DAG);
+  if (NumNonZero == 0) {
+    if (NumZero == 0)
+      // All undef vector. Return an UNDEF.
+      return DAG.getNode(ISD::UNDEF, VT);
+    else
+      // A mix of zero and undef. Return a zero vector.
+      return getZeroVector(VT, DAG);
+  }
 
   // Splat is obviously ok. Let legalizer expand it to a shuffle.
   if (Values.size() == 1)
@@ -4919,7 +4924,6 @@ X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
       case MVT::i64:
         return std::make_pair(0U, X86::FR64RegisterClass);
       // Vector types.
-      case MVT::Vector:
       case MVT::v16i8:
       case MVT::v8i16:
       case MVT::v4i32:
diff --git a/lib/VMCore/ValueTypes.cpp b/lib/VMCore/ValueTypes.cpp
index e788161..751bd58 100644
--- a/lib/VMCore/ValueTypes.cpp
+++ b/lib/VMCore/ValueTypes.cpp
@@ -14,13 +14,21 @@
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/Type.h"
 #include "llvm/DerivedTypes.h"
+#include <sstream>
 using namespace llvm;
 
 /// MVT::getValueTypeString - This function returns value type as a string,
 /// e.g. "i32".
-const char *MVT::getValueTypeString(MVT::ValueType VT) {
+std::string MVT::getValueTypeString(MVT::ValueType VT) {
   switch (VT) {
-  default: assert(0 && "Invalid ValueType!");
+  default:
+    if (isExtendedValueType(VT)) {
+      std::ostringstream OS;
+      OS << "v" << getVectorNumElements(VT)
+         << getValueTypeString(getVectorElementType(VT));
+      return OS.str();
+    }
+    assert(0 && "Invalid ValueType!");
   case MVT::i1:    return "i1";
   case MVT::i8:    return "i8";
   case MVT::i16:   return "i16";
@@ -34,7 +42,6 @@ const char *MVT::getValueTypeString(MVT::ValueType VT) {
   case MVT::isVoid:return "isVoid";
   case MVT::Other: return "ch";
   case MVT::Flag:  return "flag";
-  case MVT::Vector:return "vec";
   case MVT::v8i8:  return "v8i8";
   case MVT::v4i16: return "v4i16";
   case MVT::v2i32: return "v2i32";
@@ -49,47 +56,16 @@ const char *MVT::getValueTypeString(MVT::ValueType VT) {
   }
 }
 
-/// MVT::getVectorType - Returns the ValueType that represents a vector
-/// NumElements in length, where each element is of type VT.  If there is no
-/// ValueType that represents this vector, a ValueType of Other is returned.
-///
-MVT::ValueType MVT::getVectorType(ValueType VT, unsigned NumElements) {
-  switch (VT) {
-  default: 
-    break;
-  case MVT::i8:
-    if (NumElements == 8)  return MVT::v8i8;
-    if (NumElements == 16) return MVT::v16i8;
-    break;
-  case MVT::i16:
-    if (NumElements == 4)  return MVT::v4i16;
-    if (NumElements == 8)  return MVT::v8i16;
-    break;
-  case MVT::i32:
-    if (NumElements == 2)  return MVT::v2i32;
-    if (NumElements == 4)  return MVT::v4i32;
-    break;
-  case MVT::i64:
-    if (NumElements == 1)  return MVT::v1i64;
-    if (NumElements == 2)  return MVT::v2i64;
-    break;
-  case MVT::f32:
-    if (NumElements == 2)  return MVT::v2f32;
-    if (NumElements == 4)  return MVT::v4f32;
-    break;
-  case MVT::f64:
-    if (NumElements == 2)  return MVT::v2f64;
-    break;
-  }
-  return MVT::Other;
-}
-
 /// MVT::getTypeForValueType - This method returns an LLVM type corresponding
 /// to the specified ValueType.  Note that this will abort for types that cannot
 /// be represented.
 const Type *MVT::getTypeForValueType(MVT::ValueType VT) {
   switch (VT) {
-  default: assert(0 && "ValueType does not correspond to LLVM type!");
+  default:
+    if (isExtendedValueType(VT))
+      return VectorType::get(getTypeForValueType(getVectorElementType(VT)),
+                             getVectorNumElements(VT));
+    assert(0 && "ValueType does not correspond to LLVM type!");
   case MVT::isVoid:return Type::VoidTy;
   case MVT::i1:    return Type::Int1Ty;
   case MVT::i8:    return Type::Int8Ty;
@@ -114,9 +90,8 @@ const Type *MVT::getTypeForValueType(MVT::ValueType VT) {
 }
 
 /// MVT::getValueType - Return the value type corresponding to the specified
-/// type.  This returns all vectors as MVT::Vector and all pointers as
-/// MVT::iPTR.  If HandleUnknown is true, unknown types are returned as Other,
-/// otherwise they are invalid.
+/// type.  This returns all pointers as MVT::iPTR.  If HandleUnknown is true,
+/// unknown types are returned as Other, otherwise they are invalid.
 MVT::ValueType MVT::getValueType(const Type *Ty, bool HandleUnknown) {
   switch (Ty->getTypeID()) {
   default:
@@ -141,6 +116,10 @@ MVT::ValueType MVT::getValueType(const Type *Ty, bool HandleUnknown) {
   case Type::FloatTyID:   return MVT::f32;
   case Type::DoubleTyID:  return MVT::f64;
   case Type::PointerTyID: return MVT::iPTR;
-  case Type::VectorTyID:  return MVT::Vector;
+  case Type::VectorTyID: {
+    const VectorType *VTy = cast<VectorType>(Ty);
+    return getVectorType(getValueType(VTy->getElementType(), false),
+                         VTy->getNumElements());
+  }
   }
 }