Instcombine constant folding can normalize gep with negative index to index with large offset. When instcombine objsize checking transformation sees these geps where the offset seemingly point out of bound, it should just return "i don't know" rather than asserting.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@96825 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 9 insertions, 4 deletions

diff --git a/lib/Transforms/InstCombine/InstCombineCalls.cpp b/lib/Transforms/InstCombine/InstCombineCalls.cpp
index b944504..835d149 100644
--- a/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -319,7 +319,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Op1)) {
       if (GV->hasDefinitiveInitializer()) {
         Constant *C = GV->getInitializer();
-        size_t globalSize = TD->getTypeAllocSize(C->getType());
+        uint64_t globalSize = TD->getTypeAllocSize(C->getType());
         return ReplaceInstUsesWith(CI, ConstantInt::get(ReturnTy, globalSize));
       } else {
         Constant *RetVal = ConstantInt::get(ReturnTy, Min ? 0 : -1ULL);
@@ -341,16 +341,21 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
       // Get what we're pointing to and its size. 
       const PointerType *BaseType = 
         cast<PointerType>(Operand->getType());
-      size_t Size = TD->getTypeAllocSize(BaseType->getElementType());
+      uint64_t Size = TD->getTypeAllocSize(BaseType->getElementType());
       
       // Get the current byte offset into the thing. Use the original
       // operand in case we're looking through a bitcast.
       SmallVector<Value*, 8> Ops(CE->op_begin()+1, CE->op_end());
       const PointerType *OffsetType =
         cast<PointerType>(GEP->getPointerOperand()->getType());
-      size_t Offset = TD->getIndexedOffset(OffsetType, &Ops[0], Ops.size());
+      uint64_t Offset = TD->getIndexedOffset(OffsetType, &Ops[0], Ops.size());
 
-      assert(Size >= Offset);
+      if (Size < Offset) {
+        // Out of bound reference? Negative index normalized to large
+        // index? Just return "I don't know".
+        Constant *RetVal = ConstantInt::get(ReturnTy, Min ? 0 : -1ULL);
+        return ReplaceInstUsesWith(CI, RetVal);
+      }
       
       Constant *RetVal = ConstantInt::get(ReturnTy, Size-Offset);
       return ReplaceInstUsesWith(CI, RetVal);