1 files changed, 278 insertions, 3 deletions

diff --git a/src/google/protobuf/io/coded_stream_unittest.cc b/src/google/protobuf/io/coded_stream_unittest.cc
index 7d29833..0cfeb88 100644
--- a/src/google/protobuf/io/coded_stream_unittest.cc
+++ b/src/google/protobuf/io/coded_stream_unittest.cc
@@ -44,7 +44,6 @@
 #include <google/protobuf/testing/googletest.h>
 #include <gtest/gtest.h>
 #include <google/protobuf/io/zero_copy_stream_impl.h>
-#include <google/protobuf/stubs/strutil.h>
 
 
 // This declares an unsigned long long integer literal in a portable way.
@@ -125,6 +124,13 @@ namespace {
 
 class CodedStreamTest : public testing::Test {
  protected:
+  // Helper method used by tests for bytes warning. See implementation comment
+  // for further information.
+  static void SetupTotalBytesLimitWarningTest(
+      int total_bytes_limit, int warning_threshold,
+      vector<string>* out_errors, vector<string>* out_warnings);
+
+  // Buffer used during most of the tests. This assumes tests run sequentially.
   static const int kBufferSize = 1024 * 64;
   static uint8 buffer_[kBufferSize];
 };
@@ -208,6 +214,33 @@ TEST_2D(CodedStreamTest, ReadTag, kVarintCases, kBlockSizes) {
   EXPECT_EQ(kVarintCases_case.size, input.ByteCount());
 }
 
+// This is the regression test that verifies that there is no issues
+// with the empty input buffers handling.
+TEST_F(CodedStreamTest, EmptyInputBeforeEos) {
+  class In : public ZeroCopyInputStream {
+   public:
+    In() : count_(0) {}
+   private:
+    virtual bool Next(const void** data, int* size) {
+      *data = NULL;
+      *size = 0;
+      return count_++ < 2;
+    }
+    virtual void BackUp(int count)  {
+      GOOGLE_LOG(FATAL) << "Tests never call this.";
+    }
+    virtual bool Skip(int count) {
+      GOOGLE_LOG(FATAL) << "Tests never call this.";
+      return false;
+    }
+    virtual int64 ByteCount() const { return 0; }
+    int count_;
+  } in;
+  CodedInputStream input(&in);
+  input.ReadTag();
+  EXPECT_TRUE(input.ConsumedEntireMessage());
+}
+
 TEST_1D(CodedStreamTest, ExpectTag, kVarintCases) {
   // Leave one byte at the beginning of the buffer so we can read it
   // to force the first buffer to be loaded.
@@ -649,14 +682,197 @@ TEST_F(CodedStreamTest, ReadStringImpossiblyLargeFromStringOnHeap) {
   EXPECT_FALSE(coded_input.ReadString(&str, 1 << 30));
 }
 
+TEST_1D(CodedStreamTest, ReadStringReservesMemoryOnTotalLimit, kBlockSizes) {
+  memcpy(buffer_, kRawBytes, sizeof(kRawBytes));
+  ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case);
+
+  {
+    CodedInputStream coded_input(&input);
+    coded_input.SetTotalBytesLimit(sizeof(kRawBytes), sizeof(kRawBytes));
+    EXPECT_EQ(sizeof(kRawBytes), coded_input.BytesUntilTotalBytesLimit());
+
+    string str;
+    EXPECT_TRUE(coded_input.ReadString(&str, strlen(kRawBytes)));
+    EXPECT_EQ(sizeof(kRawBytes) - strlen(kRawBytes),
+              coded_input.BytesUntilTotalBytesLimit());
+    EXPECT_EQ(kRawBytes, str);
+    // TODO(liujisi): Replace with a more meaningful test (see cl/60966023).
+    EXPECT_GE(str.capacity(), strlen(kRawBytes));
+  }
+
+  EXPECT_EQ(strlen(kRawBytes), input.ByteCount());
+}
+
+TEST_1D(CodedStreamTest, ReadStringReservesMemoryOnPushedLimit, kBlockSizes) {
+  memcpy(buffer_, kRawBytes, sizeof(kRawBytes));
+  ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case);
+
+  {
+    CodedInputStream coded_input(&input);
+    coded_input.PushLimit(sizeof(buffer_));
+
+    string str;
+    EXPECT_TRUE(coded_input.ReadString(&str, strlen(kRawBytes)));
+    EXPECT_EQ(kRawBytes, str);
+    // TODO(liujisi): Replace with a more meaningful test (see cl/60966023).
+    EXPECT_GE(str.capacity(), strlen(kRawBytes));
+  }
+
+  EXPECT_EQ(strlen(kRawBytes), input.ByteCount());
+}
+
+TEST_F(CodedStreamTest, ReadStringNoReservationIfLimitsNotSet) {
+  memcpy(buffer_, kRawBytes, sizeof(kRawBytes));
+  // Buffer size in the input must be smaller than sizeof(kRawBytes),
+  // otherwise check against capacity will fail as ReadStringInline()
+  // will handle the reading and will reserve the memory as needed.
+  ArrayInputStream input(buffer_, sizeof(buffer_), 32);
+
+  {
+    CodedInputStream coded_input(&input);
+
+    string str;
+    EXPECT_TRUE(coded_input.ReadString(&str, strlen(kRawBytes)));
+    EXPECT_EQ(kRawBytes, str);
+    // Note: this check depends on string class implementation. It
+    // expects that string will allocate more than strlen(kRawBytes)
+    // if the content of kRawBytes is appended to string in small
+    // chunks.
+    // TODO(liujisi): Replace with a more meaningful test (see cl/60966023).
+    EXPECT_GE(str.capacity(), strlen(kRawBytes));
+  }
+
+  EXPECT_EQ(strlen(kRawBytes), input.ByteCount());
+}
+
+TEST_F(CodedStreamTest, ReadStringNoReservationSizeIsNegative) {
+  memcpy(buffer_, kRawBytes, sizeof(kRawBytes));
+  // Buffer size in the input must be smaller than sizeof(kRawBytes),
+  // otherwise check against capacity will fail as ReadStringInline()
+  // will handle the reading and will reserve the memory as needed.
+  ArrayInputStream input(buffer_, sizeof(buffer_), 32);
+
+  {
+    CodedInputStream coded_input(&input);
+    coded_input.PushLimit(sizeof(buffer_));
+
+    string str;
+    EXPECT_FALSE(coded_input.ReadString(&str, -1));
+    // Note: this check depends on string class implementation. It
+    // expects that string will always allocate the same amount of
+    // memory for an empty string.
+    EXPECT_EQ(string().capacity(), str.capacity());
+  }
+}
+
+TEST_F(CodedStreamTest, ReadStringNoReservationSizeIsLarge) {
+  memcpy(buffer_, kRawBytes, sizeof(kRawBytes));
+  // Buffer size in the input must be smaller than sizeof(kRawBytes),
+  // otherwise check against capacity will fail as ReadStringInline()
+  // will handle the reading and will reserve the memory as needed.
+  ArrayInputStream input(buffer_, sizeof(buffer_), 32);
+
+  {
+    CodedInputStream coded_input(&input);
+    coded_input.PushLimit(sizeof(buffer_));
+
+    string str;
+    EXPECT_FALSE(coded_input.ReadString(&str, 1 << 30));
+    EXPECT_GT(1 << 30, str.capacity());
+  }
+}
+
+TEST_F(CodedStreamTest, ReadStringNoReservationSizeIsOverTheLimit) {
+  memcpy(buffer_, kRawBytes, sizeof(kRawBytes));
+  // Buffer size in the input must be smaller than sizeof(kRawBytes),
+  // otherwise check against capacity will fail as ReadStringInline()
+  // will handle the reading and will reserve the memory as needed.
+  ArrayInputStream input(buffer_, sizeof(buffer_), 32);
+
+  {
+    CodedInputStream coded_input(&input);
+    coded_input.PushLimit(16);
+
+    string str;
+    EXPECT_FALSE(coded_input.ReadString(&str, strlen(kRawBytes)));
+    // Note: this check depends on string class implementation. It
+    // expects that string will allocate less than strlen(kRawBytes)
+    // for an empty string.
+    EXPECT_GT(strlen(kRawBytes), str.capacity());
+  }
+}
+
+TEST_F(CodedStreamTest, ReadStringNoReservationSizeIsOverTheTotalBytesLimit) {
+  memcpy(buffer_, kRawBytes, sizeof(kRawBytes));
+  // Buffer size in the input must be smaller than sizeof(kRawBytes),
+  // otherwise check against capacity will fail as ReadStringInline()
+  // will handle the reading and will reserve the memory as needed.
+  ArrayInputStream input(buffer_, sizeof(buffer_), 32);
+
+  {
+    CodedInputStream coded_input(&input);
+    coded_input.SetTotalBytesLimit(16, 16);
+
+    string str;
+    EXPECT_FALSE(coded_input.ReadString(&str, strlen(kRawBytes)));
+    // Note: this check depends on string class implementation. It
+    // expects that string will allocate less than strlen(kRawBytes)
+    // for an empty string.
+    EXPECT_GT(strlen(kRawBytes), str.capacity());
+  }
+}
+
+TEST_F(CodedStreamTest,
+       ReadStringNoReservationSizeIsOverTheClosestLimit_GlobalLimitIsCloser) {
+  memcpy(buffer_, kRawBytes, sizeof(kRawBytes));
+  // Buffer size in the input must be smaller than sizeof(kRawBytes),
+  // otherwise check against capacity will fail as ReadStringInline()
+  // will handle the reading and will reserve the memory as needed.
+  ArrayInputStream input(buffer_, sizeof(buffer_), 32);
+
+  {
+    CodedInputStream coded_input(&input);
+    coded_input.PushLimit(sizeof(buffer_));
+    coded_input.SetTotalBytesLimit(16, 16);
+
+    string str;
+    EXPECT_FALSE(coded_input.ReadString(&str, strlen(kRawBytes)));
+    // Note: this check depends on string class implementation. It
+    // expects that string will allocate less than strlen(kRawBytes)
+    // for an empty string.
+    EXPECT_GT(strlen(kRawBytes), str.capacity());
+  }
+}
+
+TEST_F(CodedStreamTest,
+       ReadStringNoReservationSizeIsOverTheClosestLimit_LocalLimitIsCloser) {
+  memcpy(buffer_, kRawBytes, sizeof(kRawBytes));
+  // Buffer size in the input must be smaller than sizeof(kRawBytes),
+  // otherwise check against capacity will fail as ReadStringInline()
+  // will handle the reading and will reserve the memory as needed.
+  ArrayInputStream input(buffer_, sizeof(buffer_), 32);
+
+  {
+    CodedInputStream coded_input(&input);
+    coded_input.PushLimit(16);
+    coded_input.SetTotalBytesLimit(sizeof(buffer_), sizeof(buffer_));
+    EXPECT_EQ(sizeof(buffer_), coded_input.BytesUntilTotalBytesLimit());
+
+    string str;
+    EXPECT_FALSE(coded_input.ReadString(&str, strlen(kRawBytes)));
+    // Note: this check depends on string class implementation. It
+    // expects that string will allocate less than strlen(kRawBytes)
+    // for an empty string.
+    EXPECT_GT(strlen(kRawBytes), str.capacity());
+  }
+}
+
 
 // -------------------------------------------------------------------
 // Skip
 
 const char kSkipTestBytes[] =
   "<Before skipping><To be skipped><After skipping>";
-const char kSkipOutputTestBytes[] =
-  "-----------------<To be skipped>----------------";
 
 TEST_1D(CodedStreamTest, SkipInput, kBlockSizes) {
   memcpy(buffer_, kSkipTestBytes, sizeof(kSkipTestBytes));
@@ -947,9 +1163,11 @@ TEST_F(CodedStreamTest, TotalBytesLimit) {
   ArrayInputStream input(buffer_, sizeof(buffer_));
   CodedInputStream coded_input(&input);
   coded_input.SetTotalBytesLimit(16, -1);
+  EXPECT_EQ(16, coded_input.BytesUntilTotalBytesLimit());
 
   string str;
   EXPECT_TRUE(coded_input.ReadString(&str, 16));
+  EXPECT_EQ(0, coded_input.BytesUntilTotalBytesLimit());
 
   vector<string> errors;
 
@@ -964,7 +1182,9 @@ TEST_F(CodedStreamTest, TotalBytesLimit) {
     "A protocol message was rejected because it was too big", errors[0]);
 
   coded_input.SetTotalBytesLimit(32, -1);
+  EXPECT_EQ(16, coded_input.BytesUntilTotalBytesLimit());
   EXPECT_TRUE(coded_input.ReadString(&str, 16));
+  EXPECT_EQ(0, coded_input.BytesUntilTotalBytesLimit());
 }
 
 TEST_F(CodedStreamTest, TotalBytesLimitNotValidMessageEnd) {
@@ -995,6 +1215,60 @@ TEST_F(CodedStreamTest, TotalBytesLimitNotValidMessageEnd) {
   EXPECT_FALSE(coded_input.ConsumedEntireMessage());
 }
 
+// This method is used by the tests below.
+// It constructs a CodedInputStream with the given limits and tries to read 2KiB
+// of data from it. Then it returns the logged errors and warnings in the given
+// vectors.
+void CodedStreamTest::SetupTotalBytesLimitWarningTest(
+    int total_bytes_limit, int warning_threshold,
+    vector<string>* out_errors, vector<string>* out_warnings) {
+  ArrayInputStream raw_input(buffer_, sizeof(buffer_), 128);
+
+  ScopedMemoryLog scoped_log;
+  {
+    CodedInputStream input(&raw_input);
+    input.SetTotalBytesLimit(total_bytes_limit, warning_threshold);
+    string str;
+    EXPECT_TRUE(input.ReadString(&str, 2048));
+  }
+
+  *out_errors = scoped_log.GetMessages(ERROR);
+  *out_warnings = scoped_log.GetMessages(WARNING);
+}
+
+TEST_F(CodedStreamTest, TotalBytesLimitWarning) {
+  vector<string> errors;
+  vector<string> warnings;
+  SetupTotalBytesLimitWarningTest(10240, 1024, &errors, &warnings);
+
+  EXPECT_EQ(0, errors.size());
+
+  ASSERT_EQ(2, warnings.size());
+  EXPECT_PRED_FORMAT2(testing::IsSubstring,
+    "Reading dangerously large protocol message.  If the message turns out to "
+    "be larger than 10240 bytes, parsing will be halted for security reasons.",
+    warnings[0]);
+  EXPECT_PRED_FORMAT2(testing::IsSubstring,
+    "The total number of bytes read was 2048",
+    warnings[1]);
+}
+
+TEST_F(CodedStreamTest, TotalBytesLimitWarningDisabled) {
+  vector<string> errors;
+  vector<string> warnings;
+
+  // Test with -1
+  SetupTotalBytesLimitWarningTest(10240, -1, &errors, &warnings);
+  EXPECT_EQ(0, errors.size());
+  EXPECT_EQ(0, warnings.size());
+
+  // Test again with -2, expecting the same result
+  SetupTotalBytesLimitWarningTest(10240, -2, &errors, &warnings);
+  EXPECT_EQ(0, errors.size());
+  EXPECT_EQ(0, warnings.size());
+}
+
+
 TEST_F(CodedStreamTest, RecursionLimit) {
   ArrayInputStream input(buffer_, sizeof(buffer_));
   CodedInputStream coded_input(&input);
@@ -1032,6 +1306,7 @@ TEST_F(CodedStreamTest, RecursionLimit) {
   EXPECT_FALSE(coded_input.IncrementRecursionDepth());     // 7
 }
 
+
 class ReallyBigInputStream : public ZeroCopyInputStream {
  public:
   ReallyBigInputStream() : backup_amount_(0), buffer_count_(0) {}