Fix a crasher trying to fold each element in a comparison between two vectors

if one of the vectors didn't have elements (such as undef). Fixes PR 6096.

Fix an issue in the constant folder where fcmp (<2 x %ty>, <2 x %ty>) would
have <2 x i1> type if constant folding was successful and i1 type if it wasn't.
This exposed a related issue in the bitcode reader.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@94069 91177308-0d34-0410-b5e6-96231b3b80d8

3 files changed, 17 insertions, 10 deletions

diff --git a/lib/Bitcode/Reader/BitcodeReader.cpp b/lib/Bitcode/Reader/BitcodeReader.cpp
index aabbc90..2549a51 100644
--- a/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -1167,7 +1167,7 @@ bool BitcodeReader::ParseConstants() {
       Constant *Op0 = ValueList.getConstantFwdRef(Record[1], OpTy);
       Constant *Op1 = ValueList.getConstantFwdRef(Record[2], OpTy);
 
-      if (OpTy->isFloatingPoint())
+      if (OpTy->isFPOrFPVector())
         V = ConstantExpr::getFCmp(Record[3], Op0, Op1);
       else
         V = ConstantExpr::getICmp(Record[3], Op0, Op1);
diff --git a/lib/VMCore/ConstantFold.cpp b/lib/VMCore/ConstantFold.cpp
index 3a24389..ddd5587 100644
--- a/lib/VMCore/ConstantFold.cpp
+++ b/lib/VMCore/ConstantFold.cpp
@@ -1673,14 +1673,15 @@ Constant *llvm::ConstantFoldCompareInstruction(LLVMContext &Context,
     SmallVector<Constant*, 16> C1Elts, C2Elts;
     C1->getVectorElements(Context, C1Elts);
     C2->getVectorElements(Context, C2Elts);
+    if (C1Elts.empty() || C2Elts.empty())
+      return 0;
 
     // If we can constant fold the comparison of each element, constant fold
     // the whole vector comparison.
     SmallVector<Constant*, 4> ResElts;
     for (unsigned i = 0, e = C1Elts.size(); i != e; ++i) {
       // Compare the elements, producing an i1 result or constant expr.
-      ResElts.push_back(
-                    ConstantExpr::getCompare(pred, C1Elts[i], C2Elts[i]));
+      ResElts.push_back(ConstantExpr::getCompare(pred, C1Elts[i], C2Elts[i]));
     }
     return ConstantVector::get(&ResElts[0], ResElts.size());
   }
diff --git a/lib/VMCore/Constants.cpp b/lib/VMCore/Constants.cpp
index cc8961f..916aac6 100644
--- a/lib/VMCore/Constants.cpp
+++ b/lib/VMCore/Constants.cpp
@@ -1630,7 +1630,7 @@ Constant *ConstantExpr::getInBoundsGetElementPtr(Constant *C,
 }
 
 Constant *
-ConstantExpr::getICmp(unsigned short pred, Constant* LHS, Constant* RHS) {
+ConstantExpr::getICmp(unsigned short pred, Constant *LHS, Constant *RHS) {
   assert(LHS->getType() == RHS->getType());
   assert(pred >= ICmpInst::FIRST_ICMP_PREDICATE && 
          pred <= ICmpInst::LAST_ICMP_PREDICATE && "Invalid ICmp Predicate");
@@ -1646,13 +1646,16 @@ ConstantExpr::getICmp(unsigned short pred, Constant* LHS, Constant* RHS) {
   // Get the key type with both the opcode and predicate
   const ExprMapKeyType Key(Instruction::ICmp, ArgVec, pred);
 
+  const Type *ResultTy = Type::getInt1Ty(LHS->getContext());
+  if (const VectorType *VT = dyn_cast<VectorType>(LHS->getType()))
+    ResultTy = VectorType::get(ResultTy, VT->getNumElements());
+
   LLVMContextImpl *pImpl = LHS->getType()->getContext().pImpl;
-  return
-      pImpl->ExprConstants.getOrCreate(Type::getInt1Ty(LHS->getContext()), Key);
+  return pImpl->ExprConstants.getOrCreate(ResultTy, Key);
 }
 
 Constant *
-ConstantExpr::getFCmp(unsigned short pred, Constant* LHS, Constant* RHS) {
+ConstantExpr::getFCmp(unsigned short pred, Constant *LHS, Constant *RHS) {
   assert(LHS->getType() == RHS->getType());
   assert(pred <= FCmpInst::LAST_FCMP_PREDICATE && "Invalid FCmp Predicate");
 
@@ -1666,10 +1669,13 @@ ConstantExpr::getFCmp(unsigned short pred, Constant* LHS, Constant* RHS) {
   ArgVec.push_back(RHS);
   // Get the key type with both the opcode and predicate
   const ExprMapKeyType Key(Instruction::FCmp, ArgVec, pred);
-  
+
+  const Type *ResultTy = Type::getInt1Ty(LHS->getContext());
+  if (const VectorType *VT = dyn_cast<VectorType>(LHS->getType()))
+    ResultTy = VectorType::get(ResultTy, VT->getNumElements());
+
   LLVMContextImpl *pImpl = LHS->getType()->getContext().pImpl;
-  return
-      pImpl->ExprConstants.getOrCreate(Type::getInt1Ty(LHS->getContext()), Key);
+  return pImpl->ExprConstants.getOrCreate(ResultTy, Key);
 }
 
 Constant *ConstantExpr::getExtractElementTy(const Type *ReqTy, Constant *Val,