diff options
Diffstat (limited to 'python/google/protobuf/internal')
23 files changed, 6565 insertions, 0 deletions
diff --git a/python/google/protobuf/internal/__init__.py b/python/google/protobuf/internal/__init__.py new file mode 100755 index 0000000..e69de29 --- /dev/null +++ b/python/google/protobuf/internal/__init__.py diff --git a/python/google/protobuf/internal/containers.py b/python/google/protobuf/internal/containers.py new file mode 100755 index 0000000..d8a825d --- /dev/null +++ b/python/google/protobuf/internal/containers.py @@ -0,0 +1,227 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Contains container classes to represent different protocol buffer types. + +This file defines container classes which represent categories of protocol +buffer field types which need extra maintenance. Currently these categories +are: + - Repeated scalar fields - These are all repeated fields which aren't + composite (e.g. they are of simple types like int32, string, etc). + - Repeated composite fields - Repeated fields which are composite. This + includes groups and nested messages. +""" + +__author__ = 'petar@google.com (Petar Petrov)' + + +class BaseContainer(object): + + """Base container class.""" + + # Minimizes memory usage and disallows assignment to other attributes. + __slots__ = ['_message_listener', '_values'] + + def __init__(self, message_listener): + """ + Args: + message_listener: A MessageListener implementation. + The RepeatedScalarFieldContainer will call this object's + TransitionToNonempty() method when it transitions from being empty to + being nonempty. + """ + self._message_listener = message_listener + self._values = [] + + def __getitem__(self, key): + """Retrieves item by the specified key.""" + return self._values[key] + + def __len__(self): + """Returns the number of elements in the container.""" + return len(self._values) + + def __ne__(self, other): + """Checks if another instance isn't equal to this one.""" + # The concrete classes should define __eq__. + return not self == other + + +class RepeatedScalarFieldContainer(BaseContainer): + + """Simple, type-checked, list-like container for holding repeated scalars.""" + + # Disallows assignment to other attributes. + __slots__ = ['_type_checker'] + + def __init__(self, message_listener, type_checker): + """ + Args: + message_listener: A MessageListener implementation. + The RepeatedScalarFieldContainer will call this object's + TransitionToNonempty() method when it transitions from being empty to + being nonempty. + type_checker: A type_checkers.ValueChecker instance to run on elements + inserted into this container. + """ + super(RepeatedScalarFieldContainer, self).__init__(message_listener) + self._type_checker = type_checker + + def append(self, value): + """Appends an item to the list. Similar to list.append().""" + self.insert(len(self._values), value) + + def insert(self, key, value): + """Inserts the item at the specified position. Similar to list.insert().""" + self._type_checker.CheckValue(value) + self._values.insert(key, value) + self._message_listener.ByteSizeDirty() + if len(self._values) == 1: + self._message_listener.TransitionToNonempty() + + def extend(self, elem_seq): + """Extends by appending the given sequence. Similar to list.extend().""" + if not elem_seq: + return + + orig_empty = len(self._values) == 0 + new_values = [] + for elem in elem_seq: + self._type_checker.CheckValue(elem) + new_values.append(elem) + self._values.extend(new_values) + self._message_listener.ByteSizeDirty() + if orig_empty: + self._message_listener.TransitionToNonempty() + + def remove(self, elem): + """Removes an item from the list. Similar to list.remove().""" + self._values.remove(elem) + self._message_listener.ByteSizeDirty() + + def __setitem__(self, key, value): + """Sets the item on the specified position.""" + # No need to call TransitionToNonempty(), since if we're able to + # set the element at this index, we were already nonempty before + # this method was called. + self._message_listener.ByteSizeDirty() + self._type_checker.CheckValue(value) + self._values[key] = value + + def __getslice__(self, start, stop): + """Retrieves the subset of items from between the specified indices.""" + return self._values[start:stop] + + def __setslice__(self, start, stop, values): + """Sets the subset of items from between the specified indices.""" + new_values = [] + for value in values: + self._type_checker.CheckValue(value) + new_values.append(value) + self._values[start:stop] = new_values + self._message_listener.ByteSizeDirty() + + def __delitem__(self, key): + """Deletes the item at the specified position.""" + del self._values[key] + self._message_listener.ByteSizeDirty() + + def __delslice__(self, start, stop): + """Deletes the subset of items from between the specified indices.""" + del self._values[start:stop] + self._message_listener.ByteSizeDirty() + + def __eq__(self, other): + """Compares the current instance with another one.""" + if self is other: + return True + # Special case for the same type which should be common and fast. + if isinstance(other, self.__class__): + return other._values == self._values + # We are presumably comparing against some other sequence type. + return other == self._values + + +class RepeatedCompositeFieldContainer(BaseContainer): + + """Simple, list-like container for holding repeated composite fields.""" + + # Disallows assignment to other attributes. + __slots__ = ['_message_descriptor'] + + def __init__(self, message_listener, message_descriptor): + """ + Note that we pass in a descriptor instead of the generated directly, + since at the time we construct a _RepeatedCompositeFieldContainer we + haven't yet necessarily initialized the type that will be contained in the + container. + + Args: + message_listener: A MessageListener implementation. + The RepeatedCompositeFieldContainer will call this object's + TransitionToNonempty() method when it transitions from being empty to + being nonempty. + message_descriptor: A Descriptor instance describing the protocol type + that should be present in this container. We'll use the + _concrete_class field of this descriptor when the client calls add(). + """ + super(RepeatedCompositeFieldContainer, self).__init__(message_listener) + self._message_descriptor = message_descriptor + + def add(self): + new_element = self._message_descriptor._concrete_class() + new_element._SetListener(self._message_listener) + self._values.append(new_element) + self._message_listener.ByteSizeDirty() + self._message_listener.TransitionToNonempty() + return new_element + + def __getslice__(self, start, stop): + """Retrieves the subset of items from between the specified indices.""" + return self._values[start:stop] + + def __delitem__(self, key): + """Deletes the item at the specified position.""" + del self._values[key] + self._message_listener.ByteSizeDirty() + + def __delslice__(self, start, stop): + """Deletes the subset of items from between the specified indices.""" + del self._values[start:stop] + self._message_listener.ByteSizeDirty() + + def __eq__(self, other): + """Compares the current instance with another one.""" + if self is other: + return True + if not isinstance(other, self.__class__): + raise TypeError('Can only compare repeated composite fields against ' + 'other repeated composite fields.') + return self._values == other._values diff --git a/python/google/protobuf/internal/decoder.py b/python/google/protobuf/internal/decoder.py new file mode 100755 index 0000000..83d6fe0 --- /dev/null +++ b/python/google/protobuf/internal/decoder.py @@ -0,0 +1,209 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Class for decoding protocol buffer primitives. + +Contains the logic for decoding every logical protocol field type +from one of the 5 physical wire types. +""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import struct +from google.protobuf import message +from google.protobuf.internal import input_stream +from google.protobuf.internal import wire_format + + + +# Note that much of this code is ported from //net/proto/ProtocolBuffer, and +# that the interface is strongly inspired by WireFormat from the C++ proto2 +# implementation. + + +class Decoder(object): + + """Decodes logical protocol buffer fields from the wire.""" + + def __init__(self, s): + """Initializes the decoder to read from s. + + Args: + s: An immutable sequence of bytes, which must be accessible + via the Python buffer() primitive (i.e., buffer(s)). + """ + self._stream = input_stream.InputStream(s) + + def EndOfStream(self): + """Returns true iff we've reached the end of the bytes we're reading.""" + return self._stream.EndOfStream() + + def Position(self): + """Returns the 0-indexed position in |s|.""" + return self._stream.Position() + + def ReadFieldNumberAndWireType(self): + """Reads a tag from the wire. Returns a (field_number, wire_type) pair.""" + tag_and_type = self.ReadUInt32() + return wire_format.UnpackTag(tag_and_type) + + def SkipBytes(self, bytes): + """Skips the specified number of bytes on the wire.""" + self._stream.SkipBytes(bytes) + + # Note that the Read*() methods below are not exactly symmetrical with the + # corresponding Encoder.Append*() methods. Those Encoder methods first + # encode a tag, but the Read*() methods below assume that the tag has already + # been read, and that the client wishes to read a field of the specified type + # starting at the current position. + + def ReadInt32(self): + """Reads and returns a signed, varint-encoded, 32-bit integer.""" + return self._stream.ReadVarint32() + + def ReadInt64(self): + """Reads and returns a signed, varint-encoded, 64-bit integer.""" + return self._stream.ReadVarint64() + + def ReadUInt32(self): + """Reads and returns an signed, varint-encoded, 32-bit integer.""" + return self._stream.ReadVarUInt32() + + def ReadUInt64(self): + """Reads and returns an signed, varint-encoded,64-bit integer.""" + return self._stream.ReadVarUInt64() + + def ReadSInt32(self): + """Reads and returns a signed, zigzag-encoded, varint-encoded, + 32-bit integer.""" + return wire_format.ZigZagDecode(self._stream.ReadVarUInt32()) + + def ReadSInt64(self): + """Reads and returns a signed, zigzag-encoded, varint-encoded, + 64-bit integer.""" + return wire_format.ZigZagDecode(self._stream.ReadVarUInt64()) + + def ReadFixed32(self): + """Reads and returns an unsigned, fixed-width, 32-bit integer.""" + return self._stream.ReadLittleEndian32() + + def ReadFixed64(self): + """Reads and returns an unsigned, fixed-width, 64-bit integer.""" + return self._stream.ReadLittleEndian64() + + def ReadSFixed32(self): + """Reads and returns a signed, fixed-width, 32-bit integer.""" + value = self._stream.ReadLittleEndian32() + if value >= (1 << 31): + value -= (1 << 32) + return value + + def ReadSFixed64(self): + """Reads and returns a signed, fixed-width, 64-bit integer.""" + value = self._stream.ReadLittleEndian64() + if value >= (1 << 63): + value -= (1 << 64) + return value + + def ReadFloat(self): + """Reads and returns a 4-byte floating-point number.""" + serialized = self._stream.ReadBytes(4) + return struct.unpack(wire_format.FORMAT_FLOAT_LITTLE_ENDIAN, serialized)[0] + + def ReadDouble(self): + """Reads and returns an 8-byte floating-point number.""" + serialized = self._stream.ReadBytes(8) + return struct.unpack(wire_format.FORMAT_DOUBLE_LITTLE_ENDIAN, serialized)[0] + + def ReadBool(self): + """Reads and returns a bool.""" + i = self._stream.ReadVarUInt32() + return bool(i) + + def ReadEnum(self): + """Reads and returns an enum value.""" + return self._stream.ReadVarUInt32() + + def ReadString(self): + """Reads and returns a length-delimited string.""" + bytes = self.ReadBytes() + return unicode(bytes, 'utf-8') + + def ReadBytes(self): + """Reads and returns a length-delimited byte sequence.""" + length = self._stream.ReadVarUInt32() + return self._stream.ReadBytes(length) + + def ReadMessageInto(self, msg): + """Calls msg.MergeFromString() to merge + length-delimited serialized message data into |msg|. + + REQUIRES: The decoder must be positioned at the serialized "length" + prefix to a length-delmiited serialized message. + + POSTCONDITION: The decoder is positioned just after the + serialized message, and we have merged those serialized + contents into |msg|. + """ + length = self._stream.ReadVarUInt32() + sub_buffer = self._stream.GetSubBuffer(length) + num_bytes_used = msg.MergeFromString(sub_buffer) + if num_bytes_used != length: + raise message.DecodeError( + 'Submessage told to deserialize from %d-byte encoding, ' + 'but used only %d bytes' % (length, num_bytes_used)) + self._stream.SkipBytes(num_bytes_used) + + def ReadGroupInto(self, expected_field_number, group): + """Calls group.MergeFromString() to merge + END_GROUP-delimited serialized message data into |group|. + We'll raise an exception if we don't find an END_GROUP + tag immediately after the serialized message contents. + + REQUIRES: The decoder is positioned just after the START_GROUP + tag for this group. + + POSTCONDITION: The decoder is positioned just after the + END_GROUP tag for this group, and we have merged + the contents of the group into |group|. + """ + sub_buffer = self._stream.GetSubBuffer() # No a priori length limit. + num_bytes_used = group.MergeFromString(sub_buffer) + if num_bytes_used < 0: + raise message.DecodeError('Group message reported negative bytes read.') + self._stream.SkipBytes(num_bytes_used) + field_number, field_type = self.ReadFieldNumberAndWireType() + if field_type != wire_format.WIRETYPE_END_GROUP: + raise message.DecodeError('Group message did not end with an END_GROUP.') + if field_number != expected_field_number: + raise message.DecodeError('END_GROUP tag had field ' + 'number %d, was expecting field number %d' % ( + field_number, expected_field_number)) + # We're now positioned just after the END_GROUP tag. Perfect. diff --git a/python/google/protobuf/internal/decoder_test.py b/python/google/protobuf/internal/decoder_test.py new file mode 100755 index 0000000..98e4647 --- /dev/null +++ b/python/google/protobuf/internal/decoder_test.py @@ -0,0 +1,256 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Test for google.protobuf.internal.decoder.""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import struct +import unittest +from google.protobuf.internal import decoder +from google.protobuf.internal import encoder +from google.protobuf.internal import input_stream +from google.protobuf.internal import wire_format +from google.protobuf import message +import logging +import mox + + +class DecoderTest(unittest.TestCase): + + def setUp(self): + self.mox = mox.Mox() + self.mock_stream = self.mox.CreateMock(input_stream.InputStream) + self.mock_message = self.mox.CreateMock(message.Message) + + def testReadFieldNumberAndWireType(self): + # Test field numbers that will require various varint sizes. + for expected_field_number in (1, 15, 16, 2047, 2048): + for expected_wire_type in range(6): # Highest-numbered wiretype is 5. + e = encoder.Encoder() + e.AppendTag(expected_field_number, expected_wire_type) + s = e.ToString() + d = decoder.Decoder(s) + field_number, wire_type = d.ReadFieldNumberAndWireType() + self.assertEqual(expected_field_number, field_number) + self.assertEqual(expected_wire_type, wire_type) + + def ReadScalarTestHelper(self, test_name, decoder_method, expected_result, + expected_stream_method_name, + stream_method_return, *args): + """Helper for testReadScalars below. + + Calls one of the Decoder.Read*() methods and ensures that the results are + as expected. + + Args: + test_name: Name of this test, used for logging only. + decoder_method: Unbound decoder.Decoder method to call. + expected_result: Value we expect returned from decoder_method(). + expected_stream_method_name: (string) Name of the InputStream + method we expect Decoder to call to actually read the value + on the wire. + stream_method_return: Value our mocked-out stream method should + return to the decoder. + args: Additional arguments that we expect to be passed to the + stream method. + """ + logging.info('Testing %s scalar input.\n' + 'Calling %r(), and expecting that to call the ' + 'stream method %s(%r), which will return %r. Finally, ' + 'expecting the Decoder method to return %r'% ( + test_name, decoder_method, + expected_stream_method_name, args, stream_method_return, + expected_result)) + + d = decoder.Decoder('') + d._stream = self.mock_stream + if decoder_method in (decoder.Decoder.ReadString, + decoder.Decoder.ReadBytes): + self.mock_stream.ReadVarUInt32().AndReturn(len(stream_method_return)) + # We have to use names instead of methods to work around some + # mox weirdness. (ResetAll() is overzealous). + expected_stream_method = getattr(self.mock_stream, + expected_stream_method_name) + expected_stream_method(*args).AndReturn(stream_method_return) + + self.mox.ReplayAll() + result = decoder_method(d) + self.assertEqual(expected_result, result) + self.assert_(isinstance(result, type(expected_result))) + self.mox.VerifyAll() + self.mox.ResetAll() + + VAL = 1.125 # Perfectly representable as a float (no rounding error). + LITTLE_FLOAT_VAL = '\x00\x00\x90?' + LITTLE_DOUBLE_VAL = '\x00\x00\x00\x00\x00\x00\xf2?' + + def testReadScalars(self): + test_string = 'I can feel myself getting sutpider.' + scalar_tests = [ + ['int32', decoder.Decoder.ReadInt32, 0, 'ReadVarint32', 0], + ['int64', decoder.Decoder.ReadInt64, 0, 'ReadVarint64', 0], + ['uint32', decoder.Decoder.ReadUInt32, 0, 'ReadVarUInt32', 0], + ['uint64', decoder.Decoder.ReadUInt64, 0, 'ReadVarUInt64', 0], + ['fixed32', decoder.Decoder.ReadFixed32, 0xffffffff, + 'ReadLittleEndian32', 0xffffffff], + ['fixed64', decoder.Decoder.ReadFixed64, 0xffffffffffffffff, + 'ReadLittleEndian64', 0xffffffffffffffff], + ['sfixed32', decoder.Decoder.ReadSFixed32, long(-1), + 'ReadLittleEndian32', long(0xffffffff)], + ['sfixed64', decoder.Decoder.ReadSFixed64, long(-1), + 'ReadLittleEndian64', 0xffffffffffffffff], + ['float', decoder.Decoder.ReadFloat, self.VAL, + 'ReadBytes', self.LITTLE_FLOAT_VAL, 4], + ['double', decoder.Decoder.ReadDouble, self.VAL, + 'ReadBytes', self.LITTLE_DOUBLE_VAL, 8], + ['bool', decoder.Decoder.ReadBool, True, 'ReadVarUInt32', 1], + ['enum', decoder.Decoder.ReadEnum, 23, 'ReadVarUInt32', 23], + ['string', decoder.Decoder.ReadString, + unicode(test_string, 'utf-8'), 'ReadBytes', test_string, + len(test_string)], + ['utf8-string', decoder.Decoder.ReadString, + unicode(test_string, 'utf-8'), 'ReadBytes', test_string, + len(test_string)], + ['bytes', decoder.Decoder.ReadBytes, + test_string, 'ReadBytes', test_string, len(test_string)], + # We test zigzag decoding routines more extensively below. + ['sint32', decoder.Decoder.ReadSInt32, -1, 'ReadVarUInt32', 1], + ['sint64', decoder.Decoder.ReadSInt64, -1, 'ReadVarUInt64', 1], + ] + # Ensure that we're testing different Decoder methods and using + # different test names in all test cases above. + self.assertEqual(len(scalar_tests), len(set(t[0] for t in scalar_tests))) + self.assert_(len(scalar_tests) >= len(set(t[1] for t in scalar_tests))) + for args in scalar_tests: + self.ReadScalarTestHelper(*args) + + def testReadMessageInto(self): + length = 23 + def Test(simulate_error): + d = decoder.Decoder('') + d._stream = self.mock_stream + self.mock_stream.ReadVarUInt32().AndReturn(length) + sub_buffer = object() + self.mock_stream.GetSubBuffer(length).AndReturn(sub_buffer) + + if simulate_error: + self.mock_message.MergeFromString(sub_buffer).AndReturn(length - 1) + self.mox.ReplayAll() + self.assertRaises( + message.DecodeError, d.ReadMessageInto, self.mock_message) + else: + self.mock_message.MergeFromString(sub_buffer).AndReturn(length) + self.mock_stream.SkipBytes(length) + self.mox.ReplayAll() + d.ReadMessageInto(self.mock_message) + + self.mox.VerifyAll() + self.mox.ResetAll() + + Test(simulate_error=False) + Test(simulate_error=True) + + def testReadGroupInto_Success(self): + # Test both the empty and nonempty cases. + for num_bytes in (5, 0): + field_number = expected_field_number = 10 + d = decoder.Decoder('') + d._stream = self.mock_stream + sub_buffer = object() + self.mock_stream.GetSubBuffer().AndReturn(sub_buffer) + self.mock_message.MergeFromString(sub_buffer).AndReturn(num_bytes) + self.mock_stream.SkipBytes(num_bytes) + self.mock_stream.ReadVarUInt32().AndReturn(wire_format.PackTag( + field_number, wire_format.WIRETYPE_END_GROUP)) + self.mox.ReplayAll() + d.ReadGroupInto(expected_field_number, self.mock_message) + self.mox.VerifyAll() + self.mox.ResetAll() + + def ReadGroupInto_FailureTestHelper(self, bytes_read): + d = decoder.Decoder('') + d._stream = self.mock_stream + sub_buffer = object() + self.mock_stream.GetSubBuffer().AndReturn(sub_buffer) + self.mock_message.MergeFromString(sub_buffer).AndReturn(bytes_read) + return d + + def testReadGroupInto_NegativeBytesReported(self): + expected_field_number = 10 + d = self.ReadGroupInto_FailureTestHelper(bytes_read=-1) + self.mox.ReplayAll() + self.assertRaises(message.DecodeError, + d.ReadGroupInto, expected_field_number, + self.mock_message) + self.mox.VerifyAll() + + def testReadGroupInto_NoEndGroupTag(self): + field_number = expected_field_number = 10 + num_bytes = 5 + d = self.ReadGroupInto_FailureTestHelper(bytes_read=num_bytes) + self.mock_stream.SkipBytes(num_bytes) + # Right field number, wrong wire type. + self.mock_stream.ReadVarUInt32().AndReturn(wire_format.PackTag( + field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)) + self.mox.ReplayAll() + self.assertRaises(message.DecodeError, + d.ReadGroupInto, expected_field_number, + self.mock_message) + self.mox.VerifyAll() + + def testReadGroupInto_WrongFieldNumberInEndGroupTag(self): + expected_field_number = 10 + field_number = expected_field_number + 1 + num_bytes = 5 + d = self.ReadGroupInto_FailureTestHelper(bytes_read=num_bytes) + self.mock_stream.SkipBytes(num_bytes) + # Wrong field number, right wire type. + self.mock_stream.ReadVarUInt32().AndReturn(wire_format.PackTag( + field_number, wire_format.WIRETYPE_END_GROUP)) + self.mox.ReplayAll() + self.assertRaises(message.DecodeError, + d.ReadGroupInto, expected_field_number, + self.mock_message) + self.mox.VerifyAll() + + def testSkipBytes(self): + d = decoder.Decoder('') + num_bytes = 1024 + self.mock_stream.SkipBytes(num_bytes) + d._stream = self.mock_stream + self.mox.ReplayAll() + d.SkipBytes(num_bytes) + self.mox.VerifyAll() + +if __name__ == '__main__': + unittest.main() diff --git a/python/google/protobuf/internal/descriptor_test.py b/python/google/protobuf/internal/descriptor_test.py new file mode 100755 index 0000000..eb9f2be --- /dev/null +++ b/python/google/protobuf/internal/descriptor_test.py @@ -0,0 +1,113 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unittest for google.protobuf.internal.descriptor.""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import unittest +from google.protobuf import descriptor_pb2 +from google.protobuf import descriptor + +class DescriptorTest(unittest.TestCase): + + def setUp(self): + self.my_enum = descriptor.EnumDescriptor( + name='ForeignEnum', + full_name='protobuf_unittest.ForeignEnum', + filename='ForeignEnum', + values=[ + descriptor.EnumValueDescriptor(name='FOREIGN_FOO', index=0, number=4), + descriptor.EnumValueDescriptor(name='FOREIGN_BAR', index=1, number=5), + descriptor.EnumValueDescriptor(name='FOREIGN_BAZ', index=2, number=6), + ]) + self.my_message = descriptor.Descriptor( + name='NestedMessage', + full_name='protobuf_unittest.TestAllTypes.NestedMessage', + filename='some/filename/some.proto', + containing_type=None, + fields=[ + descriptor.FieldDescriptor( + name='bb', + full_name='protobuf_unittest.TestAllTypes.NestedMessage.bb', + index=0, number=1, + type=5, cpp_type=1, label=1, + default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None), + ], + nested_types=[], + enum_types=[ + self.my_enum, + ], + extensions=[]) + self.my_method = descriptor.MethodDescriptor( + name='Bar', + full_name='protobuf_unittest.TestService.Bar', + index=0, + containing_service=None, + input_type=None, + output_type=None) + self.my_service = descriptor.ServiceDescriptor( + name='TestServiceWithOptions', + full_name='protobuf_unittest.TestServiceWithOptions', + index=0, + methods=[ + self.my_method + ]) + + def testEnumFixups(self): + self.assertEqual(self.my_enum, self.my_enum.values[0].type) + + def testContainingTypeFixups(self): + self.assertEqual(self.my_message, self.my_message.fields[0].containing_type) + self.assertEqual(self.my_message, self.my_enum.containing_type) + + def testContainingServiceFixups(self): + self.assertEqual(self.my_service, self.my_method.containing_service) + + def testGetOptions(self): + self.assertEqual(self.my_enum.GetOptions(), + descriptor_pb2.EnumOptions()) + self.assertEqual(self.my_enum.values[0].GetOptions(), + descriptor_pb2.EnumValueOptions()) + self.assertEqual(self.my_message.GetOptions(), + descriptor_pb2.MessageOptions()) + self.assertEqual(self.my_message.fields[0].GetOptions(), + descriptor_pb2.FieldOptions()) + self.assertEqual(self.my_method.GetOptions(), + descriptor_pb2.MethodOptions()) + self.assertEqual(self.my_service.GetOptions(), + descriptor_pb2.ServiceOptions()) + +if __name__ == '__main__': + unittest.main() diff --git a/python/google/protobuf/internal/encoder.py b/python/google/protobuf/internal/encoder.py new file mode 100755 index 0000000..3ec3b2b --- /dev/null +++ b/python/google/protobuf/internal/encoder.py @@ -0,0 +1,280 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Class for encoding protocol message primitives. + +Contains the logic for encoding every logical protocol field type +into one of the 5 physical wire types. +""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import struct +from google.protobuf import message +from google.protobuf.internal import wire_format +from google.protobuf.internal import output_stream + + +# Note that much of this code is ported from //net/proto/ProtocolBuffer, and +# that the interface is strongly inspired by WireFormat from the C++ proto2 +# implementation. + + +class Encoder(object): + + """Encodes logical protocol buffer fields to the wire format.""" + + def __init__(self): + self._stream = output_stream.OutputStream() + + def ToString(self): + """Returns all values encoded in this object as a string.""" + return self._stream.ToString() + + # Append*NoTag methods. These are necessary for serializing packed + # repeated fields. The Append*() methods call these methods to do + # the actual serialization. + def AppendInt32NoTag(self, value): + """Appends a 32-bit integer to our buffer, varint-encoded.""" + self._stream.AppendVarint32(value) + + def AppendInt64NoTag(self, value): + """Appends a 64-bit integer to our buffer, varint-encoded.""" + self._stream.AppendVarint64(value) + + def AppendUInt32NoTag(self, unsigned_value): + """Appends an unsigned 32-bit integer to our buffer, varint-encoded.""" + self._stream.AppendVarUInt32(unsigned_value) + + def AppendUInt64NoTag(self, unsigned_value): + """Appends an unsigned 64-bit integer to our buffer, varint-encoded.""" + self._stream.AppendVarUInt64(unsigned_value) + + def AppendSInt32NoTag(self, value): + """Appends a 32-bit integer to our buffer, zigzag-encoded and then + varint-encoded. + """ + zigzag_value = wire_format.ZigZagEncode(value) + self._stream.AppendVarUInt32(zigzag_value) + + def AppendSInt64NoTag(self, value): + """Appends a 64-bit integer to our buffer, zigzag-encoded and then + varint-encoded. + """ + zigzag_value = wire_format.ZigZagEncode(value) + self._stream.AppendVarUInt64(zigzag_value) + + def AppendFixed32NoTag(self, unsigned_value): + """Appends an unsigned 32-bit integer to our buffer, in little-endian + byte-order. + """ + self._stream.AppendLittleEndian32(unsigned_value) + + def AppendFixed64NoTag(self, unsigned_value): + """Appends an unsigned 64-bit integer to our buffer, in little-endian + byte-order. + """ + self._stream.AppendLittleEndian64(unsigned_value) + + def AppendSFixed32NoTag(self, value): + """Appends a signed 32-bit integer to our buffer, in little-endian + byte-order. + """ + sign = (value & 0x80000000) and -1 or 0 + if value >> 32 != sign: + raise message.EncodeError('SFixed32 out of range: %d' % value) + self._stream.AppendLittleEndian32(value & 0xffffffff) + + def AppendSFixed64NoTag(self, value): + """Appends a signed 64-bit integer to our buffer, in little-endian + byte-order. + """ + sign = (value & 0x8000000000000000) and -1 or 0 + if value >> 64 != sign: + raise message.EncodeError('SFixed64 out of range: %d' % value) + self._stream.AppendLittleEndian64(value & 0xffffffffffffffff) + + def AppendFloatNoTag(self, value): + """Appends a floating-point number to our buffer.""" + self._stream.AppendRawBytes( + struct.pack(wire_format.FORMAT_FLOAT_LITTLE_ENDIAN, value)) + + def AppendDoubleNoTag(self, value): + """Appends a double-precision floating-point number to our buffer.""" + self._stream.AppendRawBytes( + struct.pack(wire_format.FORMAT_DOUBLE_LITTLE_ENDIAN, value)) + + def AppendBoolNoTag(self, value): + """Appends a boolean to our buffer.""" + self.AppendInt32NoTag(value) + + def AppendEnumNoTag(self, value): + """Appends an enum value to our buffer.""" + self.AppendInt32NoTag(value) + + + # All the Append*() methods below first append a tag+type pair to the buffer + # before appending the specified value. + + def AppendInt32(self, field_number, value): + """Appends a 32-bit integer to our buffer, varint-encoded.""" + self.AppendTag(field_number, wire_format.WIRETYPE_VARINT) + self.AppendInt32NoTag(value) + + def AppendInt64(self, field_number, value): + """Appends a 64-bit integer to our buffer, varint-encoded.""" + self.AppendTag(field_number, wire_format.WIRETYPE_VARINT) + self.AppendInt64NoTag(value) + + def AppendUInt32(self, field_number, unsigned_value): + """Appends an unsigned 32-bit integer to our buffer, varint-encoded.""" + self.AppendTag(field_number, wire_format.WIRETYPE_VARINT) + self.AppendUInt32NoTag(unsigned_value) + + def AppendUInt64(self, field_number, unsigned_value): + """Appends an unsigned 64-bit integer to our buffer, varint-encoded.""" + self.AppendTag(field_number, wire_format.WIRETYPE_VARINT) + self.AppendUInt64NoTag(unsigned_value) + + def AppendSInt32(self, field_number, value): + """Appends a 32-bit integer to our buffer, zigzag-encoded and then + varint-encoded. + """ + self.AppendTag(field_number, wire_format.WIRETYPE_VARINT) + self.AppendSInt32NoTag(value) + + def AppendSInt64(self, field_number, value): + """Appends a 64-bit integer to our buffer, zigzag-encoded and then + varint-encoded. + """ + self.AppendTag(field_number, wire_format.WIRETYPE_VARINT) + self.AppendSInt64NoTag(value) + + def AppendFixed32(self, field_number, unsigned_value): + """Appends an unsigned 32-bit integer to our buffer, in little-endian + byte-order. + """ + self.AppendTag(field_number, wire_format.WIRETYPE_FIXED32) + self.AppendFixed32NoTag(unsigned_value) + + def AppendFixed64(self, field_number, unsigned_value): + """Appends an unsigned 64-bit integer to our buffer, in little-endian + byte-order. + """ + self.AppendTag(field_number, wire_format.WIRETYPE_FIXED64) + self.AppendFixed64NoTag(unsigned_value) + + def AppendSFixed32(self, field_number, value): + """Appends a signed 32-bit integer to our buffer, in little-endian + byte-order. + """ + self.AppendTag(field_number, wire_format.WIRETYPE_FIXED32) + self.AppendSFixed32NoTag(value) + + def AppendSFixed64(self, field_number, value): + """Appends a signed 64-bit integer to our buffer, in little-endian + byte-order. + """ + self.AppendTag(field_number, wire_format.WIRETYPE_FIXED64) + self.AppendSFixed64NoTag(value) + + def AppendFloat(self, field_number, value): + """Appends a floating-point number to our buffer.""" + self.AppendTag(field_number, wire_format.WIRETYPE_FIXED32) + self.AppendFloatNoTag(value) + + def AppendDouble(self, field_number, value): + """Appends a double-precision floating-point number to our buffer.""" + self.AppendTag(field_number, wire_format.WIRETYPE_FIXED64) + self.AppendDoubleNoTag(value) + + def AppendBool(self, field_number, value): + """Appends a boolean to our buffer.""" + self.AppendInt32(field_number, value) + + def AppendEnum(self, field_number, value): + """Appends an enum value to our buffer.""" + self.AppendInt32(field_number, value) + + def AppendString(self, field_number, value): + """Appends a length-prefixed unicode string, encoded as UTF-8 to our buffer, + with the length varint-encoded. + """ + self.AppendBytes(field_number, value.encode('utf-8')) + + def AppendBytes(self, field_number, value): + """Appends a length-prefixed sequence of bytes to our buffer, with the + length varint-encoded. + """ + self.AppendTag(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) + self._stream.AppendVarUInt32(len(value)) + self._stream.AppendRawBytes(value) + + # TODO(robinson): For AppendGroup() and AppendMessage(), we'd really like to + # avoid the extra string copy here. We can do so if we widen the Message + # interface to be able to serialize to a stream in addition to a string. The + # challenge when thinking ahead to the Python/C API implementation of Message + # is finding a stream-like Python thing to which we can write raw bytes + # from C. I'm not sure such a thing exists(?). (array.array is pretty much + # what we want, but it's not directly exposed in the Python/C API). + + def AppendGroup(self, field_number, group): + """Appends a group to our buffer. + """ + self.AppendTag(field_number, wire_format.WIRETYPE_START_GROUP) + self._stream.AppendRawBytes(group.SerializeToString()) + self.AppendTag(field_number, wire_format.WIRETYPE_END_GROUP) + + def AppendMessage(self, field_number, msg): + """Appends a nested message to our buffer. + """ + self.AppendTag(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) + self._stream.AppendVarUInt32(msg.ByteSize()) + self._stream.AppendRawBytes(msg.SerializeToString()) + + def AppendMessageSetItem(self, field_number, msg): + """Appends an item using the message set wire format. + + The message set message looks like this: + message MessageSet { + repeated group Item = 1 { + required int32 type_id = 2; + required string message = 3; + } + } + """ + self.AppendTag(1, wire_format.WIRETYPE_START_GROUP) + self.AppendInt32(2, field_number) + self.AppendMessage(3, msg) + self.AppendTag(1, wire_format.WIRETYPE_END_GROUP) + + def AppendTag(self, field_number, wire_type): + """Appends a tag containing field number and wire type information.""" + self._stream.AppendVarUInt32(wire_format.PackTag(field_number, wire_type)) diff --git a/python/google/protobuf/internal/encoder_test.py b/python/google/protobuf/internal/encoder_test.py new file mode 100755 index 0000000..bf75ea8 --- /dev/null +++ b/python/google/protobuf/internal/encoder_test.py @@ -0,0 +1,286 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Test for google.protobuf.internal.encoder.""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import struct +import logging +import unittest +from google.protobuf.internal import wire_format +from google.protobuf.internal import encoder +from google.protobuf.internal import output_stream +from google.protobuf import message +import mox + + +class EncoderTest(unittest.TestCase): + + def setUp(self): + self.mox = mox.Mox() + self.encoder = encoder.Encoder() + self.mock_stream = self.mox.CreateMock(output_stream.OutputStream) + self.mock_message = self.mox.CreateMock(message.Message) + self.encoder._stream = self.mock_stream + + def PackTag(self, field_number, wire_type): + return wire_format.PackTag(field_number, wire_type) + + def AppendScalarTestHelper(self, test_name, encoder_method, + expected_stream_method_name, + wire_type, field_value, + expected_value=None, expected_length=None, + is_tag_test=True): + """Helper for testAppendScalars. + + Calls one of the Encoder methods, and ensures that the Encoder + in turn makes the expected calls into its OutputStream. + + Args: + test_name: Name of this test, used only for logging. + encoder_method: Callable on self.encoder. This is the Encoder + method we're testing. If is_tag_test=True, the encoder method + accepts a field_number and field_value. if is_tag_test=False, + the encoder method accepts a field_value. + expected_stream_method_name: (string) Name of the OutputStream + method we expect Encoder to call to actually put the value + on the wire. + wire_type: The WIRETYPE_* constant we expect encoder to + use in the specified encoder_method. + field_value: The value we're trying to encode. Passed + into encoder_method. + expected_value: The value we expect Encoder to pass into + the OutputStream method. If None, we expect field_value + to pass through unmodified. + expected_length: The length we expect Encoder to pass to the + AppendVarUInt32 method. If None we expect the length of the + field_value. + is_tag_test: A Boolean. If True (the default), we append the + the packed field number and wire_type to the stream before + the field value. + """ + if expected_value is None: + expected_value = field_value + + logging.info('Testing %s scalar output.\n' + 'Calling %r(%r), and expecting that to call the ' + 'stream method %s(%r).' % ( + test_name, encoder_method, field_value, + expected_stream_method_name, expected_value)) + + if is_tag_test: + field_number = 10 + # Should first append the field number and type information. + self.mock_stream.AppendVarUInt32(self.PackTag(field_number, wire_type)) + # If we're length-delimited, we should then append the length. + if wire_type == wire_format.WIRETYPE_LENGTH_DELIMITED: + if expected_length is None: + expected_length = len(field_value) + self.mock_stream.AppendVarUInt32(expected_length) + + # Should then append the value itself. + # We have to use names instead of methods to work around some + # mox weirdness. (ResetAll() is overzealous). + expected_stream_method = getattr(self.mock_stream, + expected_stream_method_name) + expected_stream_method(expected_value) + + self.mox.ReplayAll() + if is_tag_test: + encoder_method(field_number, field_value) + else: + encoder_method(field_value) + self.mox.VerifyAll() + self.mox.ResetAll() + + VAL = 1.125 # Perfectly representable as a float (no rounding error). + LITTLE_FLOAT_VAL = '\x00\x00\x90?' + LITTLE_DOUBLE_VAL = '\x00\x00\x00\x00\x00\x00\xf2?' + + def testAppendScalars(self): + utf8_bytes = '\xd0\xa2\xd0\xb5\xd1\x81\xd1\x82' + utf8_string = unicode(utf8_bytes, 'utf-8') + scalar_tests = [ + ['int32', self.encoder.AppendInt32, 'AppendVarint32', + wire_format.WIRETYPE_VARINT, 0], + ['int64', self.encoder.AppendInt64, 'AppendVarint64', + wire_format.WIRETYPE_VARINT, 0], + ['uint32', self.encoder.AppendUInt32, 'AppendVarUInt32', + wire_format.WIRETYPE_VARINT, 0], + ['uint64', self.encoder.AppendUInt64, 'AppendVarUInt64', + wire_format.WIRETYPE_VARINT, 0], + ['fixed32', self.encoder.AppendFixed32, 'AppendLittleEndian32', + wire_format.WIRETYPE_FIXED32, 0], + ['fixed64', self.encoder.AppendFixed64, 'AppendLittleEndian64', + wire_format.WIRETYPE_FIXED64, 0], + ['sfixed32', self.encoder.AppendSFixed32, 'AppendLittleEndian32', + wire_format.WIRETYPE_FIXED32, -1, 0xffffffff], + ['sfixed64', self.encoder.AppendSFixed64, 'AppendLittleEndian64', + wire_format.WIRETYPE_FIXED64, -1, 0xffffffffffffffff], + ['float', self.encoder.AppendFloat, 'AppendRawBytes', + wire_format.WIRETYPE_FIXED32, self.VAL, self.LITTLE_FLOAT_VAL], + ['double', self.encoder.AppendDouble, 'AppendRawBytes', + wire_format.WIRETYPE_FIXED64, self.VAL, self.LITTLE_DOUBLE_VAL], + ['bool', self.encoder.AppendBool, 'AppendVarint32', + wire_format.WIRETYPE_VARINT, False], + ['enum', self.encoder.AppendEnum, 'AppendVarint32', + wire_format.WIRETYPE_VARINT, 0], + ['string', self.encoder.AppendString, 'AppendRawBytes', + wire_format.WIRETYPE_LENGTH_DELIMITED, + "You're in a maze of twisty little passages, all alike."], + ['utf8-string', self.encoder.AppendString, 'AppendRawBytes', + wire_format.WIRETYPE_LENGTH_DELIMITED, utf8_string, + utf8_bytes, len(utf8_bytes)], + # We test zigzag encoding routines more extensively below. + ['sint32', self.encoder.AppendSInt32, 'AppendVarUInt32', + wire_format.WIRETYPE_VARINT, -1, 1], + ['sint64', self.encoder.AppendSInt64, 'AppendVarUInt64', + wire_format.WIRETYPE_VARINT, -1, 1], + ] + # Ensure that we're testing different Encoder methods and using + # different test names in all test cases above. + self.assertEqual(len(scalar_tests), len(set(t[0] for t in scalar_tests))) + self.assert_(len(scalar_tests) >= len(set(t[1] for t in scalar_tests))) + for args in scalar_tests: + self.AppendScalarTestHelper(*args) + + def testAppendScalarsWithoutTags(self): + scalar_no_tag_tests = [ + ['int32', self.encoder.AppendInt32NoTag, 'AppendVarint32', None, 0], + ['int64', self.encoder.AppendInt64NoTag, 'AppendVarint64', None, 0], + ['uint32', self.encoder.AppendUInt32NoTag, 'AppendVarUInt32', None, 0], + ['uint64', self.encoder.AppendUInt64NoTag, 'AppendVarUInt64', None, 0], + ['fixed32', self.encoder.AppendFixed32NoTag, + 'AppendLittleEndian32', None, 0], + ['fixed64', self.encoder.AppendFixed64NoTag, + 'AppendLittleEndian64', None, 0], + ['sfixed32', self.encoder.AppendSFixed32NoTag, + 'AppendLittleEndian32', None, 0], + ['sfixed64', self.encoder.AppendSFixed64NoTag, + 'AppendLittleEndian64', None, 0], + ['float', self.encoder.AppendFloatNoTag, + 'AppendRawBytes', None, self.VAL, self.LITTLE_FLOAT_VAL], + ['double', self.encoder.AppendDoubleNoTag, + 'AppendRawBytes', None, self.VAL, self.LITTLE_DOUBLE_VAL], + ['bool', self.encoder.AppendBoolNoTag, 'AppendVarint32', None, 0], + ['enum', self.encoder.AppendEnumNoTag, 'AppendVarint32', None, 0], + ['sint32', self.encoder.AppendSInt32NoTag, + 'AppendVarUInt32', None, -1, 1], + ['sint64', self.encoder.AppendSInt64NoTag, + 'AppendVarUInt64', None, -1, 1], + ] + + self.assertEqual(len(scalar_no_tag_tests), + len(set(t[0] for t in scalar_no_tag_tests))) + self.assert_(len(scalar_no_tag_tests) >= + len(set(t[1] for t in scalar_no_tag_tests))) + for args in scalar_no_tag_tests: + # For no tag tests, the wire_type is not used, so we put in None. + self.AppendScalarTestHelper(is_tag_test=False, *args) + + def testAppendGroup(self): + field_number = 23 + # Should first append the start-group marker. + self.mock_stream.AppendVarUInt32( + self.PackTag(field_number, wire_format.WIRETYPE_START_GROUP)) + # Should then serialize itself. + self.mock_message.SerializeToString().AndReturn('foo') + self.mock_stream.AppendRawBytes('foo') + # Should finally append the end-group marker. + self.mock_stream.AppendVarUInt32( + self.PackTag(field_number, wire_format.WIRETYPE_END_GROUP)) + + self.mox.ReplayAll() + self.encoder.AppendGroup(field_number, self.mock_message) + self.mox.VerifyAll() + + def testAppendMessage(self): + field_number = 23 + byte_size = 42 + # Should first append the field number and type information. + self.mock_stream.AppendVarUInt32( + self.PackTag(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)) + # Should then append its length. + self.mock_message.ByteSize().AndReturn(byte_size) + self.mock_stream.AppendVarUInt32(byte_size) + # Should then serialize itself to the encoder. + self.mock_message.SerializeToString().AndReturn('foo') + self.mock_stream.AppendRawBytes('foo') + + self.mox.ReplayAll() + self.encoder.AppendMessage(field_number, self.mock_message) + self.mox.VerifyAll() + + def testAppendMessageSetItem(self): + field_number = 23 + byte_size = 42 + # Should first append the field number and type information. + self.mock_stream.AppendVarUInt32( + self.PackTag(1, wire_format.WIRETYPE_START_GROUP)) + self.mock_stream.AppendVarUInt32( + self.PackTag(2, wire_format.WIRETYPE_VARINT)) + self.mock_stream.AppendVarint32(field_number) + self.mock_stream.AppendVarUInt32( + self.PackTag(3, wire_format.WIRETYPE_LENGTH_DELIMITED)) + # Should then append its length. + self.mock_message.ByteSize().AndReturn(byte_size) + self.mock_stream.AppendVarUInt32(byte_size) + # Should then serialize itself to the encoder. + self.mock_message.SerializeToString().AndReturn('foo') + self.mock_stream.AppendRawBytes('foo') + self.mock_stream.AppendVarUInt32( + self.PackTag(1, wire_format.WIRETYPE_END_GROUP)) + + self.mox.ReplayAll() + self.encoder.AppendMessageSetItem(field_number, self.mock_message) + self.mox.VerifyAll() + + def testAppendSFixed(self): + # Most of our bounds-checking is done in output_stream.py, + # but encoder.py is responsible for transforming signed + # fixed-width integers into unsigned ones, so we test here + # to ensure that we're not losing any entropy when we do + # that conversion. + field_number = 10 + self.assertRaises(message.EncodeError, self.encoder.AppendSFixed32, + 10, wire_format.UINT32_MAX + 1) + self.assertRaises(message.EncodeError, self.encoder.AppendSFixed32, + 10, -(1 << 32)) + self.assertRaises(message.EncodeError, self.encoder.AppendSFixed64, + 10, wire_format.UINT64_MAX + 1) + self.assertRaises(message.EncodeError, self.encoder.AppendSFixed64, + 10, -(1 << 64)) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/google/protobuf/internal/generator_test.py b/python/google/protobuf/internal/generator_test.py new file mode 100755 index 0000000..11fcfa0 --- /dev/null +++ b/python/google/protobuf/internal/generator_test.py @@ -0,0 +1,100 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# TODO(robinson): Flesh this out considerably. We focused on reflection_test.py +# first, since it's testing the subtler code, and since it provides decent +# indirect testing of the protocol compiler output. + +"""Unittest that directly tests the output of the pure-Python protocol +compiler. See //net/proto2/internal/reflection_test.py for a test which +further ensures that we can use Python protocol message objects as we expect. +""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import unittest +from google.protobuf import unittest_mset_pb2 +from google.protobuf import unittest_pb2 + + +class GeneratorTest(unittest.TestCase): + + def testNestedMessageDescriptor(self): + field_name = 'optional_nested_message' + proto_type = unittest_pb2.TestAllTypes + self.assertEqual( + proto_type.NestedMessage.DESCRIPTOR, + proto_type.DESCRIPTOR.fields_by_name[field_name].message_type) + + def testEnums(self): + # We test only module-level enums here. + # TODO(robinson): Examine descriptors directly to check + # enum descriptor output. + self.assertEqual(4, unittest_pb2.FOREIGN_FOO) + self.assertEqual(5, unittest_pb2.FOREIGN_BAR) + self.assertEqual(6, unittest_pb2.FOREIGN_BAZ) + + proto = unittest_pb2.TestAllTypes() + self.assertEqual(1, proto.FOO) + self.assertEqual(1, unittest_pb2.TestAllTypes.FOO) + self.assertEqual(2, proto.BAR) + self.assertEqual(2, unittest_pb2.TestAllTypes.BAR) + self.assertEqual(3, proto.BAZ) + self.assertEqual(3, unittest_pb2.TestAllTypes.BAZ) + + def testContainingTypeBehaviorForExtensions(self): + self.assertEqual(unittest_pb2.optional_int32_extension.containing_type, + unittest_pb2.TestAllExtensions.DESCRIPTOR) + self.assertEqual(unittest_pb2.TestRequired.single.containing_type, + unittest_pb2.TestAllExtensions.DESCRIPTOR) + + def testExtensionScope(self): + self.assertEqual(unittest_pb2.optional_int32_extension.extension_scope, + None) + self.assertEqual(unittest_pb2.TestRequired.single.extension_scope, + unittest_pb2.TestRequired.DESCRIPTOR) + + def testIsExtension(self): + self.assertTrue(unittest_pb2.optional_int32_extension.is_extension) + self.assertTrue(unittest_pb2.TestRequired.single.is_extension) + + message_descriptor = unittest_pb2.TestRequired.DESCRIPTOR + non_extension_descriptor = message_descriptor.fields_by_name['a'] + self.assertTrue(not non_extension_descriptor.is_extension) + + def testOptions(self): + proto = unittest_mset_pb2.TestMessageSet() + self.assertTrue(proto.DESCRIPTOR.GetOptions().message_set_wire_format) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/google/protobuf/internal/input_stream.py b/python/google/protobuf/internal/input_stream.py new file mode 100755 index 0000000..7bda17e --- /dev/null +++ b/python/google/protobuf/internal/input_stream.py @@ -0,0 +1,338 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""InputStream is the primitive interface for reading bits from the wire. + +All protocol buffer deserialization can be expressed in terms of +the InputStream primitives provided here. +""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import array +import struct +from google.protobuf import message +from google.protobuf.internal import wire_format + + +# Note that much of this code is ported from //net/proto/ProtocolBuffer, and +# that the interface is strongly inspired by CodedInputStream from the C++ +# proto2 implementation. + + +class InputStreamBuffer(object): + + """Contains all logic for reading bits, and dealing with stream position. + + If an InputStream method ever raises an exception, the stream is left + in an indeterminate state and is not safe for further use. + """ + + def __init__(self, s): + # What we really want is something like array('B', s), where elements we + # read from the array are already given to us as one-byte integers. BUT + # using array() instead of buffer() would force full string copies to result + # from each GetSubBuffer() call. + # + # So, if the N serialized bytes of a single protocol buffer object are + # split evenly between 2 child messages, and so on recursively, using + # array('B', s) instead of buffer() would incur an additional N*logN bytes + # copied during deserialization. + # + # The higher constant overhead of having to ord() for every byte we read + # from the buffer in _ReadVarintHelper() could definitely lead to worse + # performance in many real-world scenarios, even if the asymptotic + # complexity is better. However, our real answer is that the mythical + # Python/C extension module output mode for the protocol compiler will + # be blazing-fast and will eliminate most use of this class anyway. + self._buffer = buffer(s) + self._pos = 0 + + def EndOfStream(self): + """Returns true iff we're at the end of the stream. + If this returns true, then a call to any other InputStream method + will raise an exception. + """ + return self._pos >= len(self._buffer) + + def Position(self): + """Returns the current position in the stream, or equivalently, the + number of bytes read so far. + """ + return self._pos + + def GetSubBuffer(self, size=None): + """Returns a sequence-like object that represents a portion of our + underlying sequence. + + Position 0 in the returned object corresponds to self.Position() + in this stream. + + If size is specified, then the returned object ends after the + next "size" bytes in this stream. If size is not specified, + then the returned object ends at the end of this stream. + + We guarantee that the returned object R supports the Python buffer + interface (and thus that the call buffer(R) will work). + + Note that the returned buffer is read-only. + + The intended use for this method is for nested-message and nested-group + deserialization, where we want to make a recursive MergeFromString() + call on the portion of the original sequence that contains the serialized + nested message. (And we'd like to do so without making unnecessary string + copies). + + REQUIRES: size is nonnegative. + """ + # Note that buffer() doesn't perform any actual string copy. + if size is None: + return buffer(self._buffer, self._pos) + else: + if size < 0: + raise message.DecodeError('Negative size %d' % size) + return buffer(self._buffer, self._pos, size) + + def SkipBytes(self, num_bytes): + """Skip num_bytes bytes ahead, or go to the end of the stream, whichever + comes first. + + REQUIRES: num_bytes is nonnegative. + """ + if num_bytes < 0: + raise message.DecodeError('Negative num_bytes %d' % num_bytes) + self._pos += num_bytes + self._pos = min(self._pos, len(self._buffer)) + + def ReadBytes(self, size): + """Reads up to 'size' bytes from the stream, stopping early + only if we reach the end of the stream. Returns the bytes read + as a string. + """ + if size < 0: + raise message.DecodeError('Negative size %d' % size) + s = (self._buffer[self._pos : self._pos + size]) + self._pos += len(s) # Only advance by the number of bytes actually read. + return s + + def ReadLittleEndian32(self): + """Interprets the next 4 bytes of the stream as a little-endian + encoded, unsiged 32-bit integer, and returns that integer. + """ + try: + i = struct.unpack(wire_format.FORMAT_UINT32_LITTLE_ENDIAN, + self._buffer[self._pos : self._pos + 4]) + self._pos += 4 + return i[0] # unpack() result is a 1-element tuple. + except struct.error, e: + raise message.DecodeError(e) + + def ReadLittleEndian64(self): + """Interprets the next 8 bytes of the stream as a little-endian + encoded, unsiged 64-bit integer, and returns that integer. + """ + try: + i = struct.unpack(wire_format.FORMAT_UINT64_LITTLE_ENDIAN, + self._buffer[self._pos : self._pos + 8]) + self._pos += 8 + return i[0] # unpack() result is a 1-element tuple. + except struct.error, e: + raise message.DecodeError(e) + + def ReadVarint32(self): + """Reads a varint from the stream, interprets this varint + as a signed, 32-bit integer, and returns the integer. + """ + i = self.ReadVarint64() + if not wire_format.INT32_MIN <= i <= wire_format.INT32_MAX: + raise message.DecodeError('Value out of range for int32: %d' % i) + return int(i) + + def ReadVarUInt32(self): + """Reads a varint from the stream, interprets this varint + as an unsigned, 32-bit integer, and returns the integer. + """ + i = self.ReadVarUInt64() + if i > wire_format.UINT32_MAX: + raise message.DecodeError('Value out of range for uint32: %d' % i) + return i + + def ReadVarint64(self): + """Reads a varint from the stream, interprets this varint + as a signed, 64-bit integer, and returns the integer. + """ + i = self.ReadVarUInt64() + if i > wire_format.INT64_MAX: + i -= (1 << 64) + return i + + def ReadVarUInt64(self): + """Reads a varint from the stream, interprets this varint + as an unsigned, 64-bit integer, and returns the integer. + """ + i = self._ReadVarintHelper() + if not 0 <= i <= wire_format.UINT64_MAX: + raise message.DecodeError('Value out of range for uint64: %d' % i) + return i + + def _ReadVarintHelper(self): + """Helper for the various varint-reading methods above. + Reads an unsigned, varint-encoded integer from the stream and + returns this integer. + + Does no bounds checking except to ensure that we read at most as many bytes + as could possibly be present in a varint-encoded 64-bit number. + """ + result = 0 + shift = 0 + while 1: + if shift >= 64: + raise message.DecodeError('Too many bytes when decoding varint.') + try: + b = ord(self._buffer[self._pos]) + except IndexError: + raise message.DecodeError('Truncated varint.') + self._pos += 1 + result |= ((b & 0x7f) << shift) + shift += 7 + if not (b & 0x80): + return result + + +class InputStreamArray(object): + + """Contains all logic for reading bits, and dealing with stream position. + + If an InputStream method ever raises an exception, the stream is left + in an indeterminate state and is not safe for further use. + + This alternative to InputStreamBuffer is used in environments where buffer() + is unavailble, such as Google App Engine. + """ + + def __init__(self, s): + self._buffer = array.array('B', s) + self._pos = 0 + + def EndOfStream(self): + return self._pos >= len(self._buffer) + + def Position(self): + return self._pos + + def GetSubBuffer(self, size=None): + if size is None: + return self._buffer[self._pos : ].tostring() + else: + if size < 0: + raise message.DecodeError('Negative size %d' % size) + return self._buffer[self._pos : self._pos + size].tostring() + + def SkipBytes(self, num_bytes): + if num_bytes < 0: + raise message.DecodeError('Negative num_bytes %d' % num_bytes) + self._pos += num_bytes + self._pos = min(self._pos, len(self._buffer)) + + def ReadBytes(self, size): + if size < 0: + raise message.DecodeError('Negative size %d' % size) + s = self._buffer[self._pos : self._pos + size].tostring() + self._pos += len(s) # Only advance by the number of bytes actually read. + return s + + def ReadLittleEndian32(self): + try: + i = struct.unpack(wire_format.FORMAT_UINT32_LITTLE_ENDIAN, + self._buffer[self._pos : self._pos + 4]) + self._pos += 4 + return i[0] # unpack() result is a 1-element tuple. + except struct.error, e: + raise message.DecodeError(e) + + def ReadLittleEndian64(self): + try: + i = struct.unpack(wire_format.FORMAT_UINT64_LITTLE_ENDIAN, + self._buffer[self._pos : self._pos + 8]) + self._pos += 8 + return i[0] # unpack() result is a 1-element tuple. + except struct.error, e: + raise message.DecodeError(e) + + def ReadVarint32(self): + i = self.ReadVarint64() + if not wire_format.INT32_MIN <= i <= wire_format.INT32_MAX: + raise message.DecodeError('Value out of range for int32: %d' % i) + return int(i) + + def ReadVarUInt32(self): + i = self.ReadVarUInt64() + if i > wire_format.UINT32_MAX: + raise message.DecodeError('Value out of range for uint32: %d' % i) + return i + + def ReadVarint64(self): + i = self.ReadVarUInt64() + if i > wire_format.INT64_MAX: + i -= (1 << 64) + return i + + def ReadVarUInt64(self): + i = self._ReadVarintHelper() + if not 0 <= i <= wire_format.UINT64_MAX: + raise message.DecodeError('Value out of range for uint64: %d' % i) + return i + + def _ReadVarintHelper(self): + result = 0 + shift = 0 + while 1: + if shift >= 64: + raise message.DecodeError('Too many bytes when decoding varint.') + try: + b = self._buffer[self._pos] + except IndexError: + raise message.DecodeError('Truncated varint.') + self._pos += 1 + result |= ((b & 0x7f) << shift) + shift += 7 + if not (b & 0x80): + return result + + +try: + buffer('') + InputStream = InputStreamBuffer +except NotImplementedError: + # Google App Engine: dev_appserver.py + InputStream = InputStreamArray +except RuntimeError: + # Google App Engine: production + InputStream = InputStreamArray diff --git a/python/google/protobuf/internal/input_stream_test.py b/python/google/protobuf/internal/input_stream_test.py new file mode 100755 index 0000000..ecec7f7 --- /dev/null +++ b/python/google/protobuf/internal/input_stream_test.py @@ -0,0 +1,314 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Test for google.protobuf.internal.input_stream.""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import unittest +from google.protobuf import message +from google.protobuf.internal import wire_format +from google.protobuf.internal import input_stream + + +class InputStreamBufferTest(unittest.TestCase): + + def setUp(self): + self.__original_input_stream = input_stream.InputStream + input_stream.InputStream = input_stream.InputStreamBuffer + + def tearDown(self): + input_stream.InputStream = self.__original_input_stream + + def testEndOfStream(self): + stream = input_stream.InputStream('abcd') + self.assertFalse(stream.EndOfStream()) + self.assertEqual('abcd', stream.ReadBytes(10)) + self.assertTrue(stream.EndOfStream()) + + def testPosition(self): + stream = input_stream.InputStream('abcd') + self.assertEqual(0, stream.Position()) + self.assertEqual(0, stream.Position()) # No side-effects. + stream.ReadBytes(1) + self.assertEqual(1, stream.Position()) + stream.ReadBytes(1) + self.assertEqual(2, stream.Position()) + stream.ReadBytes(10) + self.assertEqual(4, stream.Position()) # Can't go past end of stream. + + def testGetSubBuffer(self): + stream = input_stream.InputStream('abcd') + # Try leaving out the size. + self.assertEqual('abcd', str(stream.GetSubBuffer())) + stream.SkipBytes(1) + # GetSubBuffer() always starts at current size. + self.assertEqual('bcd', str(stream.GetSubBuffer())) + # Try 0-size. + self.assertEqual('', str(stream.GetSubBuffer(0))) + # Negative sizes should raise an error. + self.assertRaises(message.DecodeError, stream.GetSubBuffer, -1) + # Positive sizes should work as expected. + self.assertEqual('b', str(stream.GetSubBuffer(1))) + self.assertEqual('bc', str(stream.GetSubBuffer(2))) + # Sizes longer than remaining bytes in the buffer should + # return the whole remaining buffer. + self.assertEqual('bcd', str(stream.GetSubBuffer(1000))) + + def testSkipBytes(self): + stream = input_stream.InputStream('') + # Skipping bytes when at the end of stream + # should have no effect. + stream.SkipBytes(0) + stream.SkipBytes(1) + stream.SkipBytes(2) + self.assertTrue(stream.EndOfStream()) + self.assertEqual(0, stream.Position()) + + # Try skipping within a stream. + stream = input_stream.InputStream('abcd') + self.assertEqual(0, stream.Position()) + stream.SkipBytes(1) + self.assertEqual(1, stream.Position()) + stream.SkipBytes(10) # Can't skip past the end. + self.assertEqual(4, stream.Position()) + + # Ensure that a negative skip raises an exception. + stream = input_stream.InputStream('abcd') + stream.SkipBytes(1) + self.assertRaises(message.DecodeError, stream.SkipBytes, -1) + + def testReadBytes(self): + s = 'abcd' + # Also test going past the total stream length. + for i in range(len(s) + 10): + stream = input_stream.InputStream(s) + self.assertEqual(s[:i], stream.ReadBytes(i)) + self.assertEqual(min(i, len(s)), stream.Position()) + stream = input_stream.InputStream(s) + self.assertRaises(message.DecodeError, stream.ReadBytes, -1) + + def EnsureFailureOnEmptyStream(self, input_stream_method): + """Helper for integer-parsing tests below. + Ensures that the given InputStream method raises a DecodeError + if called on a stream with no bytes remaining. + """ + stream = input_stream.InputStream('') + self.assertRaises(message.DecodeError, input_stream_method, stream) + + def testReadLittleEndian32(self): + self.EnsureFailureOnEmptyStream(input_stream.InputStream.ReadLittleEndian32) + s = '' + # Read 0. + s += '\x00\x00\x00\x00' + # Read 1. + s += '\x01\x00\x00\x00' + # Read a bunch of different bytes. + s += '\x01\x02\x03\x04' + # Read max unsigned 32-bit int. + s += '\xff\xff\xff\xff' + # Try a read with fewer than 4 bytes left in the stream. + s += '\x00\x00\x00' + stream = input_stream.InputStream(s) + self.assertEqual(0, stream.ReadLittleEndian32()) + self.assertEqual(4, stream.Position()) + self.assertEqual(1, stream.ReadLittleEndian32()) + self.assertEqual(8, stream.Position()) + self.assertEqual(0x04030201, stream.ReadLittleEndian32()) + self.assertEqual(12, stream.Position()) + self.assertEqual(wire_format.UINT32_MAX, stream.ReadLittleEndian32()) + self.assertEqual(16, stream.Position()) + self.assertRaises(message.DecodeError, stream.ReadLittleEndian32) + + def testReadLittleEndian64(self): + self.EnsureFailureOnEmptyStream(input_stream.InputStream.ReadLittleEndian64) + s = '' + # Read 0. + s += '\x00\x00\x00\x00\x00\x00\x00\x00' + # Read 1. + s += '\x01\x00\x00\x00\x00\x00\x00\x00' + # Read a bunch of different bytes. + s += '\x01\x02\x03\x04\x05\x06\x07\x08' + # Read max unsigned 64-bit int. + s += '\xff\xff\xff\xff\xff\xff\xff\xff' + # Try a read with fewer than 8 bytes left in the stream. + s += '\x00\x00\x00' + stream = input_stream.InputStream(s) + self.assertEqual(0, stream.ReadLittleEndian64()) + self.assertEqual(8, stream.Position()) + self.assertEqual(1, stream.ReadLittleEndian64()) + self.assertEqual(16, stream.Position()) + self.assertEqual(0x0807060504030201, stream.ReadLittleEndian64()) + self.assertEqual(24, stream.Position()) + self.assertEqual(wire_format.UINT64_MAX, stream.ReadLittleEndian64()) + self.assertEqual(32, stream.Position()) + self.assertRaises(message.DecodeError, stream.ReadLittleEndian64) + + def ReadVarintSuccessTestHelper(self, varints_and_ints, read_method): + """Helper for tests below that test successful reads of various varints. + + Args: + varints_and_ints: Iterable of (str, integer) pairs, where the string + gives the wire encoding and the integer gives the value we expect + to be returned by the read_method upon encountering this string. + read_method: Unbound InputStream method that is capable of reading + the encoded strings provided in the first elements of varints_and_ints. + """ + s = ''.join(s for s, i in varints_and_ints) + stream = input_stream.InputStream(s) + expected_pos = 0 + self.assertEqual(expected_pos, stream.Position()) + for s, expected_int in varints_and_ints: + self.assertEqual(expected_int, read_method(stream)) + expected_pos += len(s) + self.assertEqual(expected_pos, stream.Position()) + + def testReadVarint32Success(self): + varints_and_ints = [ + ('\x00', 0), + ('\x01', 1), + ('\x7f', 127), + ('\x80\x01', 128), + ('\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01', -1), + ('\xff\xff\xff\xff\x07', wire_format.INT32_MAX), + ('\x80\x80\x80\x80\xf8\xff\xff\xff\xff\x01', wire_format.INT32_MIN), + ] + self.ReadVarintSuccessTestHelper(varints_and_ints, + input_stream.InputStream.ReadVarint32) + + def testReadVarint32Failure(self): + self.EnsureFailureOnEmptyStream(input_stream.InputStream.ReadVarint32) + + # Try and fail to read INT32_MAX + 1. + s = '\x80\x80\x80\x80\x08' + stream = input_stream.InputStream(s) + self.assertRaises(message.DecodeError, stream.ReadVarint32) + + # Try and fail to read INT32_MIN - 1. + s = '\xfe\xff\xff\xff\xf7\xff\xff\xff\xff\x01' + stream = input_stream.InputStream(s) + self.assertRaises(message.DecodeError, stream.ReadVarint32) + + # Try and fail to read something that looks like + # a varint with more than 10 bytes. + s = '\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00' + stream = input_stream.InputStream(s) + self.assertRaises(message.DecodeError, stream.ReadVarint32) + + def testReadVarUInt32Success(self): + varints_and_ints = [ + ('\x00', 0), + ('\x01', 1), + ('\x7f', 127), + ('\x80\x01', 128), + ('\xff\xff\xff\xff\x0f', wire_format.UINT32_MAX), + ] + self.ReadVarintSuccessTestHelper(varints_and_ints, + input_stream.InputStream.ReadVarUInt32) + + def testReadVarUInt32Failure(self): + self.EnsureFailureOnEmptyStream(input_stream.InputStream.ReadVarUInt32) + # Try and fail to read UINT32_MAX + 1 + s = '\x80\x80\x80\x80\x10' + stream = input_stream.InputStream(s) + self.assertRaises(message.DecodeError, stream.ReadVarUInt32) + + # Try and fail to read something that looks like + # a varint with more than 10 bytes. + s = '\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00' + stream = input_stream.InputStream(s) + self.assertRaises(message.DecodeError, stream.ReadVarUInt32) + + def testReadVarint64Success(self): + varints_and_ints = [ + ('\x00', 0), + ('\x01', 1), + ('\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01', -1), + ('\x7f', 127), + ('\x80\x01', 128), + ('\xff\xff\xff\xff\xff\xff\xff\xff\x7f', wire_format.INT64_MAX), + ('\x80\x80\x80\x80\x80\x80\x80\x80\x80\x01', wire_format.INT64_MIN), + ] + self.ReadVarintSuccessTestHelper(varints_and_ints, + input_stream.InputStream.ReadVarint64) + + def testReadVarint64Failure(self): + self.EnsureFailureOnEmptyStream(input_stream.InputStream.ReadVarint64) + # Try and fail to read something with the mythical 64th bit set. + s = '\x80\x80\x80\x80\x80\x80\x80\x80\x80\x02' + stream = input_stream.InputStream(s) + self.assertRaises(message.DecodeError, stream.ReadVarint64) + + # Try and fail to read something that looks like + # a varint with more than 10 bytes. + s = '\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00' + stream = input_stream.InputStream(s) + self.assertRaises(message.DecodeError, stream.ReadVarint64) + + def testReadVarUInt64Success(self): + varints_and_ints = [ + ('\x00', 0), + ('\x01', 1), + ('\x7f', 127), + ('\x80\x01', 128), + ('\x80\x80\x80\x80\x80\x80\x80\x80\x80\x01', 1 << 63), + ] + self.ReadVarintSuccessTestHelper(varints_and_ints, + input_stream.InputStream.ReadVarUInt64) + + def testReadVarUInt64Failure(self): + self.EnsureFailureOnEmptyStream(input_stream.InputStream.ReadVarUInt64) + # Try and fail to read something with the mythical 64th bit set. + s = '\x80\x80\x80\x80\x80\x80\x80\x80\x80\x02' + stream = input_stream.InputStream(s) + self.assertRaises(message.DecodeError, stream.ReadVarUInt64) + + # Try and fail to read something that looks like + # a varint with more than 10 bytes. + s = '\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00' + stream = input_stream.InputStream(s) + self.assertRaises(message.DecodeError, stream.ReadVarUInt64) + + +class InputStreamArrayTest(InputStreamBufferTest): + + def setUp(self): + # Test InputStreamArray against the same tests in InputStreamBuffer + self.__original_input_stream = input_stream.InputStream + input_stream.InputStream = input_stream.InputStreamArray + + def tearDown(self): + input_stream.InputStream = self.__original_input_stream + + +if __name__ == '__main__': + unittest.main() diff --git a/python/google/protobuf/internal/message_listener.py b/python/google/protobuf/internal/message_listener.py new file mode 100755 index 0000000..4397895 --- /dev/null +++ b/python/google/protobuf/internal/message_listener.py @@ -0,0 +1,69 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Defines a listener interface for observing certain +state transitions on Message objects. + +Also defines a null implementation of this interface. +""" + +__author__ = 'robinson@google.com (Will Robinson)' + + +class MessageListener(object): + + """Listens for transitions to nonempty and for invalidations of cached + byte sizes. Meant to be registered via Message._SetListener(). + """ + + def TransitionToNonempty(self): + """Called the *first* time that this message becomes nonempty. + Implementations are free (but not required) to call this method multiple + times after the message has become nonempty. + """ + raise NotImplementedError + + def ByteSizeDirty(self): + """Called *every* time the cached byte size value + for this object is invalidated (transitions from being + "clean" to "dirty"). + """ + raise NotImplementedError + + +class NullMessageListener(object): + + """No-op MessageListener implementation.""" + + def TransitionToNonempty(self): + pass + + def ByteSizeDirty(self): + pass diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py new file mode 100755 index 0000000..df344cf --- /dev/null +++ b/python/google/protobuf/internal/message_test.py @@ -0,0 +1,53 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Tests python protocol buffers against the golden message.""" + +__author__ = 'gps@google.com (Gregory P. Smith)' + +import unittest +from google.protobuf import unittest_import_pb2 +from google.protobuf import unittest_pb2 +from google.protobuf.internal import test_util + + +class MessageTest(test_util.GoldenMessageTestCase): + + def testGoldenMessage(self): + golden_data = test_util.GoldenFile('golden_message').read() + golden_message = unittest_pb2.TestAllTypes() + golden_message.ParseFromString(golden_data) + self.ExpectAllFieldsSet(golden_message) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/google/protobuf/internal/more_extensions.proto b/python/google/protobuf/internal/more_extensions.proto new file mode 100644 index 0000000..e2d9701 --- /dev/null +++ b/python/google/protobuf/internal/more_extensions.proto @@ -0,0 +1,58 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: robinson@google.com (Will Robinson) + + +package google.protobuf.internal; + + +message TopLevelMessage { + optional ExtendedMessage submessage = 1; +} + + +message ExtendedMessage { + extensions 1 to max; +} + + +message ForeignMessage { + optional int32 foreign_message_int = 1; +} + + +extend ExtendedMessage { + optional int32 optional_int_extension = 1; + optional ForeignMessage optional_message_extension = 2; + + repeated int32 repeated_int_extension = 3; + repeated ForeignMessage repeated_message_extension = 4; +} diff --git a/python/google/protobuf/internal/more_messages.proto b/python/google/protobuf/internal/more_messages.proto new file mode 100644 index 0000000..c701b44 --- /dev/null +++ b/python/google/protobuf/internal/more_messages.proto @@ -0,0 +1,51 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: robinson@google.com (Will Robinson) + + +package google.protobuf.internal; + +// A message where tag numbers are listed out of order, to allow us to test our +// canonicalization of serialized output, which should always be in tag order. +// We also mix in some extensions for extra fun. +message OutOfOrderFields { + optional sint32 optional_sint32 = 5; + extensions 4 to 4; + optional uint32 optional_uint32 = 3; + extensions 2 to 2; + optional int32 optional_int32 = 1; +}; + + +extend OutOfOrderFields { + optional uint64 optional_uint64 = 4; + optional int64 optional_int64 = 2; +} diff --git a/python/google/protobuf/internal/output_stream.py b/python/google/protobuf/internal/output_stream.py new file mode 100755 index 0000000..6c2d6f6 --- /dev/null +++ b/python/google/protobuf/internal/output_stream.py @@ -0,0 +1,125 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""OutputStream is the primitive interface for sticking bits on the wire. + +All protocol buffer serialization can be expressed in terms of +the OutputStream primitives provided here. +""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import array +import struct +from google.protobuf import message +from google.protobuf.internal import wire_format + + + +# Note that much of this code is ported from //net/proto/ProtocolBuffer, and +# that the interface is strongly inspired by CodedOutputStream from the C++ +# proto2 implementation. + + +class OutputStream(object): + + """Contains all logic for writing bits, and ToString() to get the result.""" + + def __init__(self): + self._buffer = array.array('B') + + def AppendRawBytes(self, raw_bytes): + """Appends raw_bytes to our internal buffer.""" + self._buffer.fromstring(raw_bytes) + + def AppendLittleEndian32(self, unsigned_value): + """Appends an unsigned 32-bit integer to the internal buffer, + in little-endian byte order. + """ + if not 0 <= unsigned_value <= wire_format.UINT32_MAX: + raise message.EncodeError( + 'Unsigned 32-bit out of range: %d' % unsigned_value) + self._buffer.fromstring(struct.pack( + wire_format.FORMAT_UINT32_LITTLE_ENDIAN, unsigned_value)) + + def AppendLittleEndian64(self, unsigned_value): + """Appends an unsigned 64-bit integer to the internal buffer, + in little-endian byte order. + """ + if not 0 <= unsigned_value <= wire_format.UINT64_MAX: + raise message.EncodeError( + 'Unsigned 64-bit out of range: %d' % unsigned_value) + self._buffer.fromstring(struct.pack( + wire_format.FORMAT_UINT64_LITTLE_ENDIAN, unsigned_value)) + + def AppendVarint32(self, value): + """Appends a signed 32-bit integer to the internal buffer, + encoded as a varint. (Note that a negative varint32 will + always require 10 bytes of space.) + """ + if not wire_format.INT32_MIN <= value <= wire_format.INT32_MAX: + raise message.EncodeError('Value out of range: %d' % value) + self.AppendVarint64(value) + + def AppendVarUInt32(self, value): + """Appends an unsigned 32-bit integer to the internal buffer, + encoded as a varint. + """ + if not 0 <= value <= wire_format.UINT32_MAX: + raise message.EncodeError('Value out of range: %d' % value) + self.AppendVarUInt64(value) + + def AppendVarint64(self, value): + """Appends a signed 64-bit integer to the internal buffer, + encoded as a varint. + """ + if not wire_format.INT64_MIN <= value <= wire_format.INT64_MAX: + raise message.EncodeError('Value out of range: %d' % value) + if value < 0: + value += (1 << 64) + self.AppendVarUInt64(value) + + def AppendVarUInt64(self, unsigned_value): + """Appends an unsigned 64-bit integer to the internal buffer, + encoded as a varint. + """ + if not 0 <= unsigned_value <= wire_format.UINT64_MAX: + raise message.EncodeError('Value out of range: %d' % unsigned_value) + while True: + bits = unsigned_value & 0x7f + unsigned_value >>= 7 + if not unsigned_value: + self._buffer.append(bits) + break + self._buffer.append(0x80|bits) + + def ToString(self): + """Returns a string containing the bytes in our internal buffer.""" + return self._buffer.tostring() diff --git a/python/google/protobuf/internal/output_stream_test.py b/python/google/protobuf/internal/output_stream_test.py new file mode 100755 index 0000000..df92eec --- /dev/null +++ b/python/google/protobuf/internal/output_stream_test.py @@ -0,0 +1,178 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Test for google.protobuf.internal.output_stream.""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import unittest +from google.protobuf import message +from google.protobuf.internal import output_stream +from google.protobuf.internal import wire_format + + +class OutputStreamTest(unittest.TestCase): + + def setUp(self): + self.stream = output_stream.OutputStream() + + def testAppendRawBytes(self): + # Empty string. + self.stream.AppendRawBytes('') + self.assertEqual('', self.stream.ToString()) + + # Nonempty string. + self.stream.AppendRawBytes('abc') + self.assertEqual('abc', self.stream.ToString()) + + # Ensure that we're actually appending. + self.stream.AppendRawBytes('def') + self.assertEqual('abcdef', self.stream.ToString()) + + def AppendNumericTestHelper(self, append_fn, values_and_strings): + """For each (value, expected_string) pair in values_and_strings, + calls an OutputStream.Append*(value) method on an OutputStream and ensures + that the string written to that stream matches expected_string. + + Args: + append_fn: Unbound OutputStream method that takes an integer or + long value as input. + values_and_strings: Iterable of (value, expected_string) pairs. + """ + for conversion in (int, long): + for value, string in values_and_strings: + stream = output_stream.OutputStream() + expected_string = '' + append_fn(stream, conversion(value)) + expected_string += string + self.assertEqual(expected_string, stream.ToString()) + + def AppendOverflowTestHelper(self, append_fn, value): + """Calls an OutputStream.Append*(value) method and asserts + that the method raises message.EncodeError. + + Args: + append_fn: Unbound OutputStream method that takes an integer or + long value as input. + value: Value to pass to append_fn which should cause an + message.EncodeError. + """ + stream = output_stream.OutputStream() + self.assertRaises(message.EncodeError, append_fn, stream, value) + + def testAppendLittleEndian32(self): + append_fn = output_stream.OutputStream.AppendLittleEndian32 + values_and_expected_strings = [ + (0, '\x00\x00\x00\x00'), + (1, '\x01\x00\x00\x00'), + ((1 << 32) - 1, '\xff\xff\xff\xff'), + ] + self.AppendNumericTestHelper(append_fn, values_and_expected_strings) + + self.AppendOverflowTestHelper(append_fn, 1 << 32) + self.AppendOverflowTestHelper(append_fn, -1) + + def testAppendLittleEndian64(self): + append_fn = output_stream.OutputStream.AppendLittleEndian64 + values_and_expected_strings = [ + (0, '\x00\x00\x00\x00\x00\x00\x00\x00'), + (1, '\x01\x00\x00\x00\x00\x00\x00\x00'), + ((1 << 64) - 1, '\xff\xff\xff\xff\xff\xff\xff\xff'), + ] + self.AppendNumericTestHelper(append_fn, values_and_expected_strings) + + self.AppendOverflowTestHelper(append_fn, 1 << 64) + self.AppendOverflowTestHelper(append_fn, -1) + + def testAppendVarint32(self): + append_fn = output_stream.OutputStream.AppendVarint32 + values_and_expected_strings = [ + (0, '\x00'), + (1, '\x01'), + (127, '\x7f'), + (128, '\x80\x01'), + (-1, '\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01'), + (wire_format.INT32_MAX, '\xff\xff\xff\xff\x07'), + (wire_format.INT32_MIN, '\x80\x80\x80\x80\xf8\xff\xff\xff\xff\x01'), + ] + self.AppendNumericTestHelper(append_fn, values_and_expected_strings) + + self.AppendOverflowTestHelper(append_fn, wire_format.INT32_MAX + 1) + self.AppendOverflowTestHelper(append_fn, wire_format.INT32_MIN - 1) + + def testAppendVarUInt32(self): + append_fn = output_stream.OutputStream.AppendVarUInt32 + values_and_expected_strings = [ + (0, '\x00'), + (1, '\x01'), + (127, '\x7f'), + (128, '\x80\x01'), + (wire_format.UINT32_MAX, '\xff\xff\xff\xff\x0f'), + ] + self.AppendNumericTestHelper(append_fn, values_and_expected_strings) + + self.AppendOverflowTestHelper(append_fn, -1) + self.AppendOverflowTestHelper(append_fn, wire_format.UINT32_MAX + 1) + + def testAppendVarint64(self): + append_fn = output_stream.OutputStream.AppendVarint64 + values_and_expected_strings = [ + (0, '\x00'), + (1, '\x01'), + (127, '\x7f'), + (128, '\x80\x01'), + (-1, '\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01'), + (wire_format.INT64_MAX, '\xff\xff\xff\xff\xff\xff\xff\xff\x7f'), + (wire_format.INT64_MIN, '\x80\x80\x80\x80\x80\x80\x80\x80\x80\x01'), + ] + self.AppendNumericTestHelper(append_fn, values_and_expected_strings) + + self.AppendOverflowTestHelper(append_fn, wire_format.INT64_MAX + 1) + self.AppendOverflowTestHelper(append_fn, wire_format.INT64_MIN - 1) + + def testAppendVarUInt64(self): + append_fn = output_stream.OutputStream.AppendVarUInt64 + values_and_expected_strings = [ + (0, '\x00'), + (1, '\x01'), + (127, '\x7f'), + (128, '\x80\x01'), + (wire_format.UINT64_MAX, '\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01'), + ] + self.AppendNumericTestHelper(append_fn, values_and_expected_strings) + + self.AppendOverflowTestHelper(append_fn, -1) + self.AppendOverflowTestHelper(append_fn, wire_format.UINT64_MAX + 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/google/protobuf/internal/reflection_test.py b/python/google/protobuf/internal/reflection_test.py new file mode 100755 index 0000000..8610177 --- /dev/null +++ b/python/google/protobuf/internal/reflection_test.py @@ -0,0 +1,1976 @@ +#! /usr/bin/python +# -*- coding: utf-8 -*- +# +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unittest for reflection.py, which also indirectly tests the output of the +pure-Python protocol compiler. +""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import operator + +import unittest +# TODO(robinson): When we split this test in two, only some of these imports +# will be necessary in each test. +from google.protobuf import unittest_import_pb2 +from google.protobuf import unittest_mset_pb2 +from google.protobuf import unittest_pb2 +from google.protobuf import descriptor_pb2 +from google.protobuf import descriptor +from google.protobuf import message +from google.protobuf import reflection +from google.protobuf.internal import more_extensions_pb2 +from google.protobuf.internal import more_messages_pb2 +from google.protobuf.internal import wire_format +from google.protobuf.internal import test_util +from google.protobuf.internal import decoder + + +class ReflectionTest(unittest.TestCase): + + def assertIs(self, values, others): + self.assertEqual(len(values), len(others)) + for i in range(len(values)): + self.assertTrue(values[i] is others[i]) + + def testSimpleHasBits(self): + # Test a scalar. + proto = unittest_pb2.TestAllTypes() + self.assertTrue(not proto.HasField('optional_int32')) + self.assertEqual(0, proto.optional_int32) + # HasField() shouldn't be true if all we've done is + # read the default value. + self.assertTrue(not proto.HasField('optional_int32')) + proto.optional_int32 = 1 + # Setting a value however *should* set the "has" bit. + self.assertTrue(proto.HasField('optional_int32')) + proto.ClearField('optional_int32') + # And clearing that value should unset the "has" bit. + self.assertTrue(not proto.HasField('optional_int32')) + + def testHasBitsWithSinglyNestedScalar(self): + # Helper used to test foreign messages and groups. + # + # composite_field_name should be the name of a non-repeated + # composite (i.e., foreign or group) field in TestAllTypes, + # and scalar_field_name should be the name of an integer-valued + # scalar field within that composite. + # + # I never thought I'd miss C++ macros and templates so much. :( + # This helper is semantically just: + # + # assert proto.composite_field.scalar_field == 0 + # assert not proto.composite_field.HasField('scalar_field') + # assert not proto.HasField('composite_field') + # + # proto.composite_field.scalar_field = 10 + # old_composite_field = proto.composite_field + # + # assert proto.composite_field.scalar_field == 10 + # assert proto.composite_field.HasField('scalar_field') + # assert proto.HasField('composite_field') + # + # proto.ClearField('composite_field') + # + # assert not proto.composite_field.HasField('scalar_field') + # assert not proto.HasField('composite_field') + # assert proto.composite_field.scalar_field == 0 + # + # # Now ensure that ClearField('composite_field') disconnected + # # the old field object from the object tree... + # assert old_composite_field is not proto.composite_field + # old_composite_field.scalar_field = 20 + # assert not proto.composite_field.HasField('scalar_field') + # assert not proto.HasField('composite_field') + def TestCompositeHasBits(composite_field_name, scalar_field_name): + proto = unittest_pb2.TestAllTypes() + # First, check that we can get the scalar value, and see that it's the + # default (0), but that proto.HasField('omposite') and + # proto.composite.HasField('scalar') will still return False. + composite_field = getattr(proto, composite_field_name) + original_scalar_value = getattr(composite_field, scalar_field_name) + self.assertEqual(0, original_scalar_value) + # Assert that the composite object does not "have" the scalar. + self.assertTrue(not composite_field.HasField(scalar_field_name)) + # Assert that proto does not "have" the composite field. + self.assertTrue(not proto.HasField(composite_field_name)) + + # Now set the scalar within the composite field. Ensure that the setting + # is reflected, and that proto.HasField('composite') and + # proto.composite.HasField('scalar') now both return True. + new_val = 20 + setattr(composite_field, scalar_field_name, new_val) + self.assertEqual(new_val, getattr(composite_field, scalar_field_name)) + # Hold on to a reference to the current composite_field object. + old_composite_field = composite_field + # Assert that the has methods now return true. + self.assertTrue(composite_field.HasField(scalar_field_name)) + self.assertTrue(proto.HasField(composite_field_name)) + + # Now call the clear method... + proto.ClearField(composite_field_name) + + # ...and ensure that the "has" bits are all back to False... + composite_field = getattr(proto, composite_field_name) + self.assertTrue(not composite_field.HasField(scalar_field_name)) + self.assertTrue(not proto.HasField(composite_field_name)) + # ...and ensure that the scalar field has returned to its default. + self.assertEqual(0, getattr(composite_field, scalar_field_name)) + + # Finally, ensure that modifications to the old composite field object + # don't have any effect on the parent. + # + # (NOTE that when we clear the composite field in the parent, we actually + # don't recursively clear down the tree. Instead, we just disconnect the + # cleared composite from the tree.) + self.assertTrue(old_composite_field is not composite_field) + setattr(old_composite_field, scalar_field_name, new_val) + self.assertTrue(not composite_field.HasField(scalar_field_name)) + self.assertTrue(not proto.HasField(composite_field_name)) + self.assertEqual(0, getattr(composite_field, scalar_field_name)) + + # Test simple, single-level nesting when we set a scalar. + TestCompositeHasBits('optionalgroup', 'a') + TestCompositeHasBits('optional_nested_message', 'bb') + TestCompositeHasBits('optional_foreign_message', 'c') + TestCompositeHasBits('optional_import_message', 'd') + + def testReferencesToNestedMessage(self): + proto = unittest_pb2.TestAllTypes() + nested = proto.optional_nested_message + del proto + # A previous version had a bug where this would raise an exception when + # hitting a now-dead weak reference. + nested.bb = 23 + + def testDisconnectingNestedMessageBeforeSettingField(self): + proto = unittest_pb2.TestAllTypes() + nested = proto.optional_nested_message + proto.ClearField('optional_nested_message') # Should disconnect from parent + self.assertTrue(nested is not proto.optional_nested_message) + nested.bb = 23 + self.assertTrue(not proto.HasField('optional_nested_message')) + self.assertEqual(0, proto.optional_nested_message.bb) + + def testHasBitsWhenModifyingRepeatedFields(self): + # Test nesting when we add an element to a repeated field in a submessage. + proto = unittest_pb2.TestNestedMessageHasBits() + proto.optional_nested_message.nestedmessage_repeated_int32.append(5) + self.assertEqual( + [5], proto.optional_nested_message.nestedmessage_repeated_int32) + self.assertTrue(proto.HasField('optional_nested_message')) + + # Do the same test, but with a repeated composite field within the + # submessage. + proto.ClearField('optional_nested_message') + self.assertTrue(not proto.HasField('optional_nested_message')) + proto.optional_nested_message.nestedmessage_repeated_foreignmessage.add() + self.assertTrue(proto.HasField('optional_nested_message')) + + def testHasBitsForManyLevelsOfNesting(self): + # Test nesting many levels deep. + recursive_proto = unittest_pb2.TestMutualRecursionA() + self.assertTrue(not recursive_proto.HasField('bb')) + self.assertEqual(0, recursive_proto.bb.a.bb.a.bb.optional_int32) + self.assertTrue(not recursive_proto.HasField('bb')) + recursive_proto.bb.a.bb.a.bb.optional_int32 = 5 + self.assertEqual(5, recursive_proto.bb.a.bb.a.bb.optional_int32) + self.assertTrue(recursive_proto.HasField('bb')) + self.assertTrue(recursive_proto.bb.HasField('a')) + self.assertTrue(recursive_proto.bb.a.HasField('bb')) + self.assertTrue(recursive_proto.bb.a.bb.HasField('a')) + self.assertTrue(recursive_proto.bb.a.bb.a.HasField('bb')) + self.assertTrue(not recursive_proto.bb.a.bb.a.bb.HasField('a')) + self.assertTrue(recursive_proto.bb.a.bb.a.bb.HasField('optional_int32')) + + def testSingularListFields(self): + proto = unittest_pb2.TestAllTypes() + proto.optional_fixed32 = 1 + proto.optional_int32 = 5 + proto.optional_string = 'foo' + self.assertEqual( + [ (proto.DESCRIPTOR.fields_by_name['optional_int32' ], 5), + (proto.DESCRIPTOR.fields_by_name['optional_fixed32'], 1), + (proto.DESCRIPTOR.fields_by_name['optional_string' ], 'foo') ], + proto.ListFields()) + + def testRepeatedListFields(self): + proto = unittest_pb2.TestAllTypes() + proto.repeated_fixed32.append(1) + proto.repeated_int32.append(5) + proto.repeated_int32.append(11) + proto.repeated_string.extend(['foo', 'bar']) + proto.repeated_string.extend([]) + proto.repeated_string.append('baz') + proto.repeated_string.extend(str(x) for x in xrange(2)) + proto.optional_int32 = 21 + self.assertEqual( + [ (proto.DESCRIPTOR.fields_by_name['optional_int32' ], 21), + (proto.DESCRIPTOR.fields_by_name['repeated_int32' ], [5, 11]), + (proto.DESCRIPTOR.fields_by_name['repeated_fixed32'], [1]), + (proto.DESCRIPTOR.fields_by_name['repeated_string' ], + ['foo', 'bar', 'baz', '0', '1']) ], + proto.ListFields()) + + def testSingularListExtensions(self): + proto = unittest_pb2.TestAllExtensions() + proto.Extensions[unittest_pb2.optional_fixed32_extension] = 1 + proto.Extensions[unittest_pb2.optional_int32_extension ] = 5 + proto.Extensions[unittest_pb2.optional_string_extension ] = 'foo' + self.assertEqual( + [ (unittest_pb2.optional_int32_extension , 5), + (unittest_pb2.optional_fixed32_extension, 1), + (unittest_pb2.optional_string_extension , 'foo') ], + proto.ListFields()) + + def testRepeatedListExtensions(self): + proto = unittest_pb2.TestAllExtensions() + proto.Extensions[unittest_pb2.repeated_fixed32_extension].append(1) + proto.Extensions[unittest_pb2.repeated_int32_extension ].append(5) + proto.Extensions[unittest_pb2.repeated_int32_extension ].append(11) + proto.Extensions[unittest_pb2.repeated_string_extension ].append('foo') + proto.Extensions[unittest_pb2.repeated_string_extension ].append('bar') + proto.Extensions[unittest_pb2.repeated_string_extension ].append('baz') + proto.Extensions[unittest_pb2.optional_int32_extension ] = 21 + self.assertEqual( + [ (unittest_pb2.optional_int32_extension , 21), + (unittest_pb2.repeated_int32_extension , [5, 11]), + (unittest_pb2.repeated_fixed32_extension, [1]), + (unittest_pb2.repeated_string_extension , ['foo', 'bar', 'baz']) ], + proto.ListFields()) + + def testListFieldsAndExtensions(self): + proto = unittest_pb2.TestFieldOrderings() + test_util.SetAllFieldsAndExtensions(proto) + unittest_pb2.my_extension_int + self.assertEqual( + [ (proto.DESCRIPTOR.fields_by_name['my_int' ], 1), + (unittest_pb2.my_extension_int , 23), + (proto.DESCRIPTOR.fields_by_name['my_string'], 'foo'), + (unittest_pb2.my_extension_string , 'bar'), + (proto.DESCRIPTOR.fields_by_name['my_float' ], 1.0) ], + proto.ListFields()) + + def testDefaultValues(self): + proto = unittest_pb2.TestAllTypes() + self.assertEqual(0, proto.optional_int32) + self.assertEqual(0, proto.optional_int64) + self.assertEqual(0, proto.optional_uint32) + self.assertEqual(0, proto.optional_uint64) + self.assertEqual(0, proto.optional_sint32) + self.assertEqual(0, proto.optional_sint64) + self.assertEqual(0, proto.optional_fixed32) + self.assertEqual(0, proto.optional_fixed64) + self.assertEqual(0, proto.optional_sfixed32) + self.assertEqual(0, proto.optional_sfixed64) + self.assertEqual(0.0, proto.optional_float) + self.assertEqual(0.0, proto.optional_double) + self.assertEqual(False, proto.optional_bool) + self.assertEqual('', proto.optional_string) + self.assertEqual('', proto.optional_bytes) + + self.assertEqual(41, proto.default_int32) + self.assertEqual(42, proto.default_int64) + self.assertEqual(43, proto.default_uint32) + self.assertEqual(44, proto.default_uint64) + self.assertEqual(-45, proto.default_sint32) + self.assertEqual(46, proto.default_sint64) + self.assertEqual(47, proto.default_fixed32) + self.assertEqual(48, proto.default_fixed64) + self.assertEqual(49, proto.default_sfixed32) + self.assertEqual(-50, proto.default_sfixed64) + self.assertEqual(51.5, proto.default_float) + self.assertEqual(52e3, proto.default_double) + self.assertEqual(True, proto.default_bool) + self.assertEqual('hello', proto.default_string) + self.assertEqual('world', proto.default_bytes) + self.assertEqual(unittest_pb2.TestAllTypes.BAR, proto.default_nested_enum) + self.assertEqual(unittest_pb2.FOREIGN_BAR, proto.default_foreign_enum) + self.assertEqual(unittest_import_pb2.IMPORT_BAR, + proto.default_import_enum) + + proto = unittest_pb2.TestExtremeDefaultValues() + self.assertEqual(u'\u1234', proto.utf8_string) + + def testHasFieldWithUnknownFieldName(self): + proto = unittest_pb2.TestAllTypes() + self.assertRaises(ValueError, proto.HasField, 'nonexistent_field') + + def testClearFieldWithUnknownFieldName(self): + proto = unittest_pb2.TestAllTypes() + self.assertRaises(ValueError, proto.ClearField, 'nonexistent_field') + + def testDisallowedAssignments(self): + # It's illegal to assign values directly to repeated fields + # or to nonrepeated composite fields. Ensure that this fails. + proto = unittest_pb2.TestAllTypes() + # Repeated fields. + self.assertRaises(AttributeError, setattr, proto, 'repeated_int32', 10) + # Lists shouldn't work, either. + self.assertRaises(AttributeError, setattr, proto, 'repeated_int32', [10]) + # Composite fields. + self.assertRaises(AttributeError, setattr, proto, + 'optional_nested_message', 23) + # Assignment to a repeated nested message field without specifying + # the index in the array of nested messages. + self.assertRaises(AttributeError, setattr, proto.repeated_nested_message, + 'bb', 34) + # Assignment to an attribute of a repeated field. + self.assertRaises(AttributeError, setattr, proto.repeated_float, + 'some_attribute', 34) + # proto.nonexistent_field = 23 should fail as well. + self.assertRaises(AttributeError, setattr, proto, 'nonexistent_field', 23) + + # TODO(robinson): Add type-safety check for enums. + def testSingleScalarTypeSafety(self): + proto = unittest_pb2.TestAllTypes() + self.assertRaises(TypeError, setattr, proto, 'optional_int32', 1.1) + self.assertRaises(TypeError, setattr, proto, 'optional_int32', 'foo') + self.assertRaises(TypeError, setattr, proto, 'optional_string', 10) + self.assertRaises(TypeError, setattr, proto, 'optional_bytes', 10) + + def testSingleScalarBoundsChecking(self): + def TestMinAndMaxIntegers(field_name, expected_min, expected_max): + pb = unittest_pb2.TestAllTypes() + setattr(pb, field_name, expected_min) + setattr(pb, field_name, expected_max) + self.assertRaises(ValueError, setattr, pb, field_name, expected_min - 1) + self.assertRaises(ValueError, setattr, pb, field_name, expected_max + 1) + + TestMinAndMaxIntegers('optional_int32', -(1 << 31), (1 << 31) - 1) + TestMinAndMaxIntegers('optional_uint32', 0, 0xffffffff) + TestMinAndMaxIntegers('optional_int64', -(1 << 63), (1 << 63) - 1) + TestMinAndMaxIntegers('optional_uint64', 0, 0xffffffffffffffff) + TestMinAndMaxIntegers('optional_nested_enum', -(1 << 31), (1 << 31) - 1) + + def testRepeatedScalarTypeSafety(self): + proto = unittest_pb2.TestAllTypes() + self.assertRaises(TypeError, proto.repeated_int32.append, 1.1) + self.assertRaises(TypeError, proto.repeated_int32.append, 'foo') + self.assertRaises(TypeError, proto.repeated_string, 10) + self.assertRaises(TypeError, proto.repeated_bytes, 10) + + proto.repeated_int32.append(10) + proto.repeated_int32[0] = 23 + self.assertRaises(IndexError, proto.repeated_int32.__setitem__, 500, 23) + self.assertRaises(TypeError, proto.repeated_int32.__setitem__, 0, 'abc') + + def testSingleScalarGettersAndSetters(self): + proto = unittest_pb2.TestAllTypes() + self.assertEqual(0, proto.optional_int32) + proto.optional_int32 = 1 + self.assertEqual(1, proto.optional_int32) + # TODO(robinson): Test all other scalar field types. + + def testSingleScalarClearField(self): + proto = unittest_pb2.TestAllTypes() + # Should be allowed to clear something that's not there (a no-op). + proto.ClearField('optional_int32') + proto.optional_int32 = 1 + self.assertTrue(proto.HasField('optional_int32')) + proto.ClearField('optional_int32') + self.assertEqual(0, proto.optional_int32) + self.assertTrue(not proto.HasField('optional_int32')) + # TODO(robinson): Test all other scalar field types. + + def testEnums(self): + proto = unittest_pb2.TestAllTypes() + self.assertEqual(1, proto.FOO) + self.assertEqual(1, unittest_pb2.TestAllTypes.FOO) + self.assertEqual(2, proto.BAR) + self.assertEqual(2, unittest_pb2.TestAllTypes.BAR) + self.assertEqual(3, proto.BAZ) + self.assertEqual(3, unittest_pb2.TestAllTypes.BAZ) + + def testRepeatedScalars(self): + proto = unittest_pb2.TestAllTypes() + + self.assertTrue(not proto.repeated_int32) + self.assertEqual(0, len(proto.repeated_int32)) + proto.repeated_int32.append(5) + proto.repeated_int32.append(10) + proto.repeated_int32.append(15) + self.assertTrue(proto.repeated_int32) + self.assertEqual(3, len(proto.repeated_int32)) + + self.assertEqual([5, 10, 15], proto.repeated_int32) + + # Test single retrieval. + self.assertEqual(5, proto.repeated_int32[0]) + self.assertEqual(15, proto.repeated_int32[-1]) + # Test out-of-bounds indices. + self.assertRaises(IndexError, proto.repeated_int32.__getitem__, 1234) + self.assertRaises(IndexError, proto.repeated_int32.__getitem__, -1234) + # Test incorrect types passed to __getitem__. + self.assertRaises(TypeError, proto.repeated_int32.__getitem__, 'foo') + self.assertRaises(TypeError, proto.repeated_int32.__getitem__, None) + + # Test single assignment. + proto.repeated_int32[1] = 20 + self.assertEqual([5, 20, 15], proto.repeated_int32) + + # Test insertion. + proto.repeated_int32.insert(1, 25) + self.assertEqual([5, 25, 20, 15], proto.repeated_int32) + + # Test slice retrieval. + proto.repeated_int32.append(30) + self.assertEqual([25, 20, 15], proto.repeated_int32[1:4]) + self.assertEqual([5, 25, 20, 15, 30], proto.repeated_int32[:]) + + # Test slice assignment with an iterator + proto.repeated_int32[1:4] = (i for i in xrange(3)) + self.assertEqual([5, 0, 1, 2, 30], proto.repeated_int32) + + # Test slice assignment. + proto.repeated_int32[1:4] = [35, 40, 45] + self.assertEqual([5, 35, 40, 45, 30], proto.repeated_int32) + + # Test that we can use the field as an iterator. + result = [] + for i in proto.repeated_int32: + result.append(i) + self.assertEqual([5, 35, 40, 45, 30], result) + + # Test single deletion. + del proto.repeated_int32[2] + self.assertEqual([5, 35, 45, 30], proto.repeated_int32) + + # Test slice deletion. + del proto.repeated_int32[2:] + self.assertEqual([5, 35], proto.repeated_int32) + + # Test clearing. + proto.ClearField('repeated_int32') + self.assertTrue(not proto.repeated_int32) + self.assertEqual(0, len(proto.repeated_int32)) + + def testRepeatedScalarsRemove(self): + proto = unittest_pb2.TestAllTypes() + + self.assertTrue(not proto.repeated_int32) + self.assertEqual(0, len(proto.repeated_int32)) + proto.repeated_int32.append(5) + proto.repeated_int32.append(10) + proto.repeated_int32.append(5) + proto.repeated_int32.append(5) + + self.assertEqual(4, len(proto.repeated_int32)) + proto.repeated_int32.remove(5) + self.assertEqual(3, len(proto.repeated_int32)) + self.assertEqual(10, proto.repeated_int32[0]) + self.assertEqual(5, proto.repeated_int32[1]) + self.assertEqual(5, proto.repeated_int32[2]) + + proto.repeated_int32.remove(5) + self.assertEqual(2, len(proto.repeated_int32)) + self.assertEqual(10, proto.repeated_int32[0]) + self.assertEqual(5, proto.repeated_int32[1]) + + proto.repeated_int32.remove(10) + self.assertEqual(1, len(proto.repeated_int32)) + self.assertEqual(5, proto.repeated_int32[0]) + + # Remove a non-existent element. + self.assertRaises(ValueError, proto.repeated_int32.remove, 123) + + def testRepeatedComposites(self): + proto = unittest_pb2.TestAllTypes() + self.assertTrue(not proto.repeated_nested_message) + self.assertEqual(0, len(proto.repeated_nested_message)) + m0 = proto.repeated_nested_message.add() + m1 = proto.repeated_nested_message.add() + self.assertTrue(proto.repeated_nested_message) + self.assertEqual(2, len(proto.repeated_nested_message)) + self.assertIs([m0, m1], proto.repeated_nested_message) + self.assertTrue(isinstance(m0, unittest_pb2.TestAllTypes.NestedMessage)) + + # Test out-of-bounds indices. + self.assertRaises(IndexError, proto.repeated_nested_message.__getitem__, + 1234) + self.assertRaises(IndexError, proto.repeated_nested_message.__getitem__, + -1234) + + # Test incorrect types passed to __getitem__. + self.assertRaises(TypeError, proto.repeated_nested_message.__getitem__, + 'foo') + self.assertRaises(TypeError, proto.repeated_nested_message.__getitem__, + None) + + # Test slice retrieval. + m2 = proto.repeated_nested_message.add() + m3 = proto.repeated_nested_message.add() + m4 = proto.repeated_nested_message.add() + self.assertIs([m1, m2, m3], proto.repeated_nested_message[1:4]) + self.assertIs([m0, m1, m2, m3, m4], proto.repeated_nested_message[:]) + + # Test that we can use the field as an iterator. + result = [] + for i in proto.repeated_nested_message: + result.append(i) + self.assertIs([m0, m1, m2, m3, m4], result) + + # Test single deletion. + del proto.repeated_nested_message[2] + self.assertIs([m0, m1, m3, m4], proto.repeated_nested_message) + + # Test slice deletion. + del proto.repeated_nested_message[2:] + self.assertIs([m0, m1], proto.repeated_nested_message) + + # Test clearing. + proto.ClearField('repeated_nested_message') + self.assertTrue(not proto.repeated_nested_message) + self.assertEqual(0, len(proto.repeated_nested_message)) + + def testHandWrittenReflection(self): + # TODO(robinson): We probably need a better way to specify + # protocol types by hand. But then again, this isn't something + # we expect many people to do. Hmm. + FieldDescriptor = descriptor.FieldDescriptor + foo_field_descriptor = FieldDescriptor( + name='foo_field', full_name='MyProto.foo_field', + index=0, number=1, type=FieldDescriptor.TYPE_INT64, + cpp_type=FieldDescriptor.CPPTYPE_INT64, + label=FieldDescriptor.LABEL_OPTIONAL, default_value=0, + containing_type=None, message_type=None, enum_type=None, + is_extension=False, extension_scope=None, + options=descriptor_pb2.FieldOptions()) + mydescriptor = descriptor.Descriptor( + name='MyProto', full_name='MyProto', filename='ignored', + containing_type=None, nested_types=[], enum_types=[], + fields=[foo_field_descriptor], extensions=[], + options=descriptor_pb2.MessageOptions()) + class MyProtoClass(message.Message): + DESCRIPTOR = mydescriptor + __metaclass__ = reflection.GeneratedProtocolMessageType + myproto_instance = MyProtoClass() + self.assertEqual(0, myproto_instance.foo_field) + self.assertTrue(not myproto_instance.HasField('foo_field')) + myproto_instance.foo_field = 23 + self.assertEqual(23, myproto_instance.foo_field) + self.assertTrue(myproto_instance.HasField('foo_field')) + + def testTopLevelExtensionsForOptionalScalar(self): + extendee_proto = unittest_pb2.TestAllExtensions() + extension = unittest_pb2.optional_int32_extension + self.assertTrue(not extendee_proto.HasExtension(extension)) + self.assertEqual(0, extendee_proto.Extensions[extension]) + # As with normal scalar fields, just doing a read doesn't actually set the + # "has" bit. + self.assertTrue(not extendee_proto.HasExtension(extension)) + # Actually set the thing. + extendee_proto.Extensions[extension] = 23 + self.assertEqual(23, extendee_proto.Extensions[extension]) + self.assertTrue(extendee_proto.HasExtension(extension)) + # Ensure that clearing works as well. + extendee_proto.ClearExtension(extension) + self.assertEqual(0, extendee_proto.Extensions[extension]) + self.assertTrue(not extendee_proto.HasExtension(extension)) + + def testTopLevelExtensionsForRepeatedScalar(self): + extendee_proto = unittest_pb2.TestAllExtensions() + extension = unittest_pb2.repeated_string_extension + self.assertEqual(0, len(extendee_proto.Extensions[extension])) + extendee_proto.Extensions[extension].append('foo') + self.assertEqual(['foo'], extendee_proto.Extensions[extension]) + string_list = extendee_proto.Extensions[extension] + extendee_proto.ClearExtension(extension) + self.assertEqual(0, len(extendee_proto.Extensions[extension])) + self.assertTrue(string_list is not extendee_proto.Extensions[extension]) + # Shouldn't be allowed to do Extensions[extension] = 'a' + self.assertRaises(TypeError, operator.setitem, extendee_proto.Extensions, + extension, 'a') + + def testTopLevelExtensionsForOptionalMessage(self): + extendee_proto = unittest_pb2.TestAllExtensions() + extension = unittest_pb2.optional_foreign_message_extension + self.assertTrue(not extendee_proto.HasExtension(extension)) + self.assertEqual(0, extendee_proto.Extensions[extension].c) + # As with normal (non-extension) fields, merely reading from the + # thing shouldn't set the "has" bit. + self.assertTrue(not extendee_proto.HasExtension(extension)) + extendee_proto.Extensions[extension].c = 23 + self.assertEqual(23, extendee_proto.Extensions[extension].c) + self.assertTrue(extendee_proto.HasExtension(extension)) + # Save a reference here. + foreign_message = extendee_proto.Extensions[extension] + extendee_proto.ClearExtension(extension) + self.assertTrue(foreign_message is not extendee_proto.Extensions[extension]) + # Setting a field on foreign_message now shouldn't set + # any "has" bits on extendee_proto. + foreign_message.c = 42 + self.assertEqual(42, foreign_message.c) + self.assertTrue(foreign_message.HasField('c')) + self.assertTrue(not extendee_proto.HasExtension(extension)) + # Shouldn't be allowed to do Extensions[extension] = 'a' + self.assertRaises(TypeError, operator.setitem, extendee_proto.Extensions, + extension, 'a') + + def testTopLevelExtensionsForRepeatedMessage(self): + extendee_proto = unittest_pb2.TestAllExtensions() + extension = unittest_pb2.repeatedgroup_extension + self.assertEqual(0, len(extendee_proto.Extensions[extension])) + group = extendee_proto.Extensions[extension].add() + group.a = 23 + self.assertEqual(23, extendee_proto.Extensions[extension][0].a) + group.a = 42 + self.assertEqual(42, extendee_proto.Extensions[extension][0].a) + group_list = extendee_proto.Extensions[extension] + extendee_proto.ClearExtension(extension) + self.assertEqual(0, len(extendee_proto.Extensions[extension])) + self.assertTrue(group_list is not extendee_proto.Extensions[extension]) + # Shouldn't be allowed to do Extensions[extension] = 'a' + self.assertRaises(TypeError, operator.setitem, extendee_proto.Extensions, + extension, 'a') + + def testNestedExtensions(self): + extendee_proto = unittest_pb2.TestAllExtensions() + extension = unittest_pb2.TestRequired.single + + # We just test the non-repeated case. + self.assertTrue(not extendee_proto.HasExtension(extension)) + required = extendee_proto.Extensions[extension] + self.assertEqual(0, required.a) + self.assertTrue(not extendee_proto.HasExtension(extension)) + required.a = 23 + self.assertEqual(23, extendee_proto.Extensions[extension].a) + self.assertTrue(extendee_proto.HasExtension(extension)) + extendee_proto.ClearExtension(extension) + self.assertTrue(required is not extendee_proto.Extensions[extension]) + self.assertTrue(not extendee_proto.HasExtension(extension)) + + # If message A directly contains message B, and + # a.HasField('b') is currently False, then mutating any + # extension in B should change a.HasField('b') to True + # (and so on up the object tree). + def testHasBitsForAncestorsOfExtendedMessage(self): + # Optional scalar extension. + toplevel = more_extensions_pb2.TopLevelMessage() + self.assertTrue(not toplevel.HasField('submessage')) + self.assertEqual(0, toplevel.submessage.Extensions[ + more_extensions_pb2.optional_int_extension]) + self.assertTrue(not toplevel.HasField('submessage')) + toplevel.submessage.Extensions[ + more_extensions_pb2.optional_int_extension] = 23 + self.assertEqual(23, toplevel.submessage.Extensions[ + more_extensions_pb2.optional_int_extension]) + self.assertTrue(toplevel.HasField('submessage')) + + # Repeated scalar extension. + toplevel = more_extensions_pb2.TopLevelMessage() + self.assertTrue(not toplevel.HasField('submessage')) + self.assertEqual([], toplevel.submessage.Extensions[ + more_extensions_pb2.repeated_int_extension]) + self.assertTrue(not toplevel.HasField('submessage')) + toplevel.submessage.Extensions[ + more_extensions_pb2.repeated_int_extension].append(23) + self.assertEqual([23], toplevel.submessage.Extensions[ + more_extensions_pb2.repeated_int_extension]) + self.assertTrue(toplevel.HasField('submessage')) + + # Optional message extension. + toplevel = more_extensions_pb2.TopLevelMessage() + self.assertTrue(not toplevel.HasField('submessage')) + self.assertEqual(0, toplevel.submessage.Extensions[ + more_extensions_pb2.optional_message_extension].foreign_message_int) + self.assertTrue(not toplevel.HasField('submessage')) + toplevel.submessage.Extensions[ + more_extensions_pb2.optional_message_extension].foreign_message_int = 23 + self.assertEqual(23, toplevel.submessage.Extensions[ + more_extensions_pb2.optional_message_extension].foreign_message_int) + self.assertTrue(toplevel.HasField('submessage')) + + # Repeated message extension. + toplevel = more_extensions_pb2.TopLevelMessage() + self.assertTrue(not toplevel.HasField('submessage')) + self.assertEqual(0, len(toplevel.submessage.Extensions[ + more_extensions_pb2.repeated_message_extension])) + self.assertTrue(not toplevel.HasField('submessage')) + foreign = toplevel.submessage.Extensions[ + more_extensions_pb2.repeated_message_extension].add() + self.assertTrue(foreign is toplevel.submessage.Extensions[ + more_extensions_pb2.repeated_message_extension][0]) + self.assertTrue(toplevel.HasField('submessage')) + + def testDisconnectionAfterClearingEmptyMessage(self): + toplevel = more_extensions_pb2.TopLevelMessage() + extendee_proto = toplevel.submessage + extension = more_extensions_pb2.optional_message_extension + extension_proto = extendee_proto.Extensions[extension] + extendee_proto.ClearExtension(extension) + extension_proto.foreign_message_int = 23 + + self.assertTrue(not toplevel.HasField('submessage')) + self.assertTrue(extension_proto is not extendee_proto.Extensions[extension]) + + def testExtensionFailureModes(self): + extendee_proto = unittest_pb2.TestAllExtensions() + + # Try non-extension-handle arguments to HasExtension, + # ClearExtension(), and Extensions[]... + self.assertRaises(KeyError, extendee_proto.HasExtension, 1234) + self.assertRaises(KeyError, extendee_proto.ClearExtension, 1234) + self.assertRaises(KeyError, extendee_proto.Extensions.__getitem__, 1234) + self.assertRaises(KeyError, extendee_proto.Extensions.__setitem__, 1234, 5) + + # Try something that *is* an extension handle, just not for + # this message... + unknown_handle = more_extensions_pb2.optional_int_extension + self.assertRaises(KeyError, extendee_proto.HasExtension, + unknown_handle) + self.assertRaises(KeyError, extendee_proto.ClearExtension, + unknown_handle) + self.assertRaises(KeyError, extendee_proto.Extensions.__getitem__, + unknown_handle) + self.assertRaises(KeyError, extendee_proto.Extensions.__setitem__, + unknown_handle, 5) + + # Try call HasExtension() with a valid handle, but for a + # *repeated* field. (Just as with non-extension repeated + # fields, Has*() isn't supported for extension repeated fields). + self.assertRaises(KeyError, extendee_proto.HasExtension, + unittest_pb2.repeated_string_extension) + + def testStaticParseFrom(self): + proto1 = unittest_pb2.TestAllTypes() + test_util.SetAllFields(proto1) + + string1 = proto1.SerializeToString() + proto2 = unittest_pb2.TestAllTypes.FromString(string1) + + # Messages should be equal. + self.assertEqual(proto2, proto1) + + def testMergeFromSingularField(self): + # Test merge with just a singular field. + proto1 = unittest_pb2.TestAllTypes() + proto1.optional_int32 = 1 + + proto2 = unittest_pb2.TestAllTypes() + # This shouldn't get overwritten. + proto2.optional_string = 'value' + + proto2.MergeFrom(proto1) + self.assertEqual(1, proto2.optional_int32) + self.assertEqual('value', proto2.optional_string) + + def testMergeFromRepeatedField(self): + # Test merge with just a repeated field. + proto1 = unittest_pb2.TestAllTypes() + proto1.repeated_int32.append(1) + proto1.repeated_int32.append(2) + + proto2 = unittest_pb2.TestAllTypes() + proto2.repeated_int32.append(0) + proto2.MergeFrom(proto1) + + self.assertEqual(0, proto2.repeated_int32[0]) + self.assertEqual(1, proto2.repeated_int32[1]) + self.assertEqual(2, proto2.repeated_int32[2]) + + def testMergeFromOptionalGroup(self): + # Test merge with an optional group. + proto1 = unittest_pb2.TestAllTypes() + proto1.optionalgroup.a = 12 + proto2 = unittest_pb2.TestAllTypes() + proto2.MergeFrom(proto1) + self.assertEqual(12, proto2.optionalgroup.a) + + def testMergeFromRepeatedNestedMessage(self): + # Test merge with a repeated nested message. + proto1 = unittest_pb2.TestAllTypes() + m = proto1.repeated_nested_message.add() + m.bb = 123 + m = proto1.repeated_nested_message.add() + m.bb = 321 + + proto2 = unittest_pb2.TestAllTypes() + m = proto2.repeated_nested_message.add() + m.bb = 999 + proto2.MergeFrom(proto1) + self.assertEqual(999, proto2.repeated_nested_message[0].bb) + self.assertEqual(123, proto2.repeated_nested_message[1].bb) + self.assertEqual(321, proto2.repeated_nested_message[2].bb) + + def testMergeFromAllFields(self): + # With all fields set. + proto1 = unittest_pb2.TestAllTypes() + test_util.SetAllFields(proto1) + proto2 = unittest_pb2.TestAllTypes() + proto2.MergeFrom(proto1) + + # Messages should be equal. + self.assertEqual(proto2, proto1) + + # Serialized string should be equal too. + string1 = proto1.SerializeToString() + string2 = proto2.SerializeToString() + self.assertEqual(string1, string2) + + def testMergeFromExtensionsSingular(self): + proto1 = unittest_pb2.TestAllExtensions() + proto1.Extensions[unittest_pb2.optional_int32_extension] = 1 + + proto2 = unittest_pb2.TestAllExtensions() + proto2.MergeFrom(proto1) + self.assertEqual( + 1, proto2.Extensions[unittest_pb2.optional_int32_extension]) + + def testMergeFromExtensionsRepeated(self): + proto1 = unittest_pb2.TestAllExtensions() + proto1.Extensions[unittest_pb2.repeated_int32_extension].append(1) + proto1.Extensions[unittest_pb2.repeated_int32_extension].append(2) + + proto2 = unittest_pb2.TestAllExtensions() + proto2.Extensions[unittest_pb2.repeated_int32_extension].append(0) + proto2.MergeFrom(proto1) + self.assertEqual( + 3, len(proto2.Extensions[unittest_pb2.repeated_int32_extension])) + self.assertEqual( + 0, proto2.Extensions[unittest_pb2.repeated_int32_extension][0]) + self.assertEqual( + 1, proto2.Extensions[unittest_pb2.repeated_int32_extension][1]) + self.assertEqual( + 2, proto2.Extensions[unittest_pb2.repeated_int32_extension][2]) + + def testMergeFromExtensionsNestedMessage(self): + proto1 = unittest_pb2.TestAllExtensions() + ext1 = proto1.Extensions[ + unittest_pb2.repeated_nested_message_extension] + m = ext1.add() + m.bb = 222 + m = ext1.add() + m.bb = 333 + + proto2 = unittest_pb2.TestAllExtensions() + ext2 = proto2.Extensions[ + unittest_pb2.repeated_nested_message_extension] + m = ext2.add() + m.bb = 111 + + proto2.MergeFrom(proto1) + ext2 = proto2.Extensions[ + unittest_pb2.repeated_nested_message_extension] + self.assertEqual(3, len(ext2)) + self.assertEqual(111, ext2[0].bb) + self.assertEqual(222, ext2[1].bb) + self.assertEqual(333, ext2[2].bb) + + def testCopyFromSingularField(self): + # Test copy with just a singular field. + proto1 = unittest_pb2.TestAllTypes() + proto1.optional_int32 = 1 + proto1.optional_string = 'important-text' + + proto2 = unittest_pb2.TestAllTypes() + proto2.optional_string = 'value' + + proto2.CopyFrom(proto1) + self.assertEqual(1, proto2.optional_int32) + self.assertEqual('important-text', proto2.optional_string) + + def testCopyFromRepeatedField(self): + # Test copy with a repeated field. + proto1 = unittest_pb2.TestAllTypes() + proto1.repeated_int32.append(1) + proto1.repeated_int32.append(2) + + proto2 = unittest_pb2.TestAllTypes() + proto2.repeated_int32.append(0) + proto2.CopyFrom(proto1) + + self.assertEqual(1, proto2.repeated_int32[0]) + self.assertEqual(2, proto2.repeated_int32[1]) + + def testCopyFromAllFields(self): + # With all fields set. + proto1 = unittest_pb2.TestAllTypes() + test_util.SetAllFields(proto1) + proto2 = unittest_pb2.TestAllTypes() + proto2.CopyFrom(proto1) + + # Messages should be equal. + self.assertEqual(proto2, proto1) + + # Serialized string should be equal too. + string1 = proto1.SerializeToString() + string2 = proto2.SerializeToString() + self.assertEqual(string1, string2) + + def testCopyFromSelf(self): + proto1 = unittest_pb2.TestAllTypes() + proto1.repeated_int32.append(1) + proto1.optional_int32 = 2 + proto1.optional_string = 'important-text' + + proto1.CopyFrom(proto1) + self.assertEqual(1, proto1.repeated_int32[0]) + self.assertEqual(2, proto1.optional_int32) + self.assertEqual('important-text', proto1.optional_string) + + def testClear(self): + proto = unittest_pb2.TestAllTypes() + test_util.SetAllFields(proto) + # Clear the message. + proto.Clear() + self.assertEquals(proto.ByteSize(), 0) + empty_proto = unittest_pb2.TestAllTypes() + self.assertEquals(proto, empty_proto) + + # Test if extensions which were set are cleared. + proto = unittest_pb2.TestAllExtensions() + test_util.SetAllExtensions(proto) + # Clear the message. + proto.Clear() + self.assertEquals(proto.ByteSize(), 0) + empty_proto = unittest_pb2.TestAllExtensions() + self.assertEquals(proto, empty_proto) + + def testIsInitialized(self): + # Trivial cases - all optional fields and extensions. + proto = unittest_pb2.TestAllTypes() + self.assertTrue(proto.IsInitialized()) + proto = unittest_pb2.TestAllExtensions() + self.assertTrue(proto.IsInitialized()) + + # The case of uninitialized required fields. + proto = unittest_pb2.TestRequired() + self.assertFalse(proto.IsInitialized()) + proto.a = proto.b = proto.c = 2 + self.assertTrue(proto.IsInitialized()) + + # The case of uninitialized submessage. + proto = unittest_pb2.TestRequiredForeign() + self.assertTrue(proto.IsInitialized()) + proto.optional_message.a = 1 + self.assertFalse(proto.IsInitialized()) + proto.optional_message.b = 0 + proto.optional_message.c = 0 + self.assertTrue(proto.IsInitialized()) + + # Uninitialized repeated submessage. + message1 = proto.repeated_message.add() + self.assertFalse(proto.IsInitialized()) + message1.a = message1.b = message1.c = 0 + self.assertTrue(proto.IsInitialized()) + + # Uninitialized repeated group in an extension. + proto = unittest_pb2.TestAllExtensions() + extension = unittest_pb2.TestRequired.multi + message1 = proto.Extensions[extension].add() + message2 = proto.Extensions[extension].add() + self.assertFalse(proto.IsInitialized()) + message1.a = 1 + message1.b = 1 + message1.c = 1 + self.assertFalse(proto.IsInitialized()) + message2.a = 2 + message2.b = 2 + message2.c = 2 + self.assertTrue(proto.IsInitialized()) + + # Uninitialized nonrepeated message in an extension. + proto = unittest_pb2.TestAllExtensions() + extension = unittest_pb2.TestRequired.single + proto.Extensions[extension].a = 1 + self.assertFalse(proto.IsInitialized()) + proto.Extensions[extension].b = 2 + proto.Extensions[extension].c = 3 + self.assertTrue(proto.IsInitialized()) + + def testStringUTF8Encoding(self): + proto = unittest_pb2.TestAllTypes() + + # Assignment of a unicode object to a field of type 'bytes' is not allowed. + self.assertRaises(TypeError, + setattr, proto, 'optional_bytes', u'unicode object') + + # Check that the default value is of python's 'unicode' type. + self.assertEqual(type(proto.optional_string), unicode) + + proto.optional_string = unicode('Testing') + self.assertEqual(proto.optional_string, str('Testing')) + + # Assign a value of type 'str' which can be encoded in UTF-8. + proto.optional_string = str('Testing') + self.assertEqual(proto.optional_string, unicode('Testing')) + + # Values of type 'str' are also accepted as long as they can be encoded in + # UTF-8. + self.assertEqual(type(proto.optional_string), str) + + # Try to assign a 'str' value which contains bytes that aren't 7-bit ASCII. + self.assertRaises(ValueError, + setattr, proto, 'optional_string', str('a\x80a')) + # Assign a 'str' object which contains a UTF-8 encoded string. + self.assertRaises(ValueError, + setattr, proto, 'optional_string', 'Тест') + # No exception thrown. + proto.optional_string = 'abc' + + def testStringUTF8Serialization(self): + proto = unittest_mset_pb2.TestMessageSet() + extension_message = unittest_mset_pb2.TestMessageSetExtension2 + extension = extension_message.message_set_extension + + test_utf8 = u'Тест' + test_utf8_bytes = test_utf8.encode('utf-8') + + # 'Test' in another language, using UTF-8 charset. + proto.Extensions[extension].str = test_utf8 + + # Serialize using the MessageSet wire format (this is specified in the + # .proto file). + serialized = proto.SerializeToString() + + # Check byte size. + self.assertEqual(proto.ByteSize(), len(serialized)) + + raw = unittest_mset_pb2.RawMessageSet() + raw.MergeFromString(serialized) + + message2 = unittest_mset_pb2.TestMessageSetExtension2() + + self.assertEqual(1, len(raw.item)) + # Check that the type_id is the same as the tag ID in the .proto file. + self.assertEqual(raw.item[0].type_id, 1547769) + + # Check the actually bytes on the wire. + self.assertTrue( + raw.item[0].message.endswith(test_utf8_bytes)) + message2.MergeFromString(raw.item[0].message) + + self.assertEqual(type(message2.str), unicode) + self.assertEqual(message2.str, test_utf8) + + # How about if the bytes on the wire aren't a valid UTF-8 encoded string. + bytes = raw.item[0].message.replace( + test_utf8_bytes, len(test_utf8_bytes) * '\xff') + self.assertRaises(UnicodeDecodeError, message2.MergeFromString, bytes) + + +# Since we had so many tests for protocol buffer equality, we broke these out +# into separate TestCase classes. + + +class TestAllTypesEqualityTest(unittest.TestCase): + + def setUp(self): + self.first_proto = unittest_pb2.TestAllTypes() + self.second_proto = unittest_pb2.TestAllTypes() + + def testSelfEquality(self): + self.assertEqual(self.first_proto, self.first_proto) + + def testEmptyProtosEqual(self): + self.assertEqual(self.first_proto, self.second_proto) + + +class FullProtosEqualityTest(unittest.TestCase): + + """Equality tests using completely-full protos as a starting point.""" + + def setUp(self): + self.first_proto = unittest_pb2.TestAllTypes() + self.second_proto = unittest_pb2.TestAllTypes() + test_util.SetAllFields(self.first_proto) + test_util.SetAllFields(self.second_proto) + + def testNoneNotEqual(self): + self.assertNotEqual(self.first_proto, None) + self.assertNotEqual(None, self.second_proto) + + def testNotEqualToOtherMessage(self): + third_proto = unittest_pb2.TestRequired() + self.assertNotEqual(self.first_proto, third_proto) + self.assertNotEqual(third_proto, self.second_proto) + + def testAllFieldsFilledEquality(self): + self.assertEqual(self.first_proto, self.second_proto) + + def testNonRepeatedScalar(self): + # Nonrepeated scalar field change should cause inequality. + self.first_proto.optional_int32 += 1 + self.assertNotEqual(self.first_proto, self.second_proto) + # ...as should clearing a field. + self.first_proto.ClearField('optional_int32') + self.assertNotEqual(self.first_proto, self.second_proto) + + def testNonRepeatedComposite(self): + # Change a nonrepeated composite field. + self.first_proto.optional_nested_message.bb += 1 + self.assertNotEqual(self.first_proto, self.second_proto) + self.first_proto.optional_nested_message.bb -= 1 + self.assertEqual(self.first_proto, self.second_proto) + # Clear a field in the nested message. + self.first_proto.optional_nested_message.ClearField('bb') + self.assertNotEqual(self.first_proto, self.second_proto) + self.first_proto.optional_nested_message.bb = ( + self.second_proto.optional_nested_message.bb) + self.assertEqual(self.first_proto, self.second_proto) + # Remove the nested message entirely. + self.first_proto.ClearField('optional_nested_message') + self.assertNotEqual(self.first_proto, self.second_proto) + + def testRepeatedScalar(self): + # Change a repeated scalar field. + self.first_proto.repeated_int32.append(5) + self.assertNotEqual(self.first_proto, self.second_proto) + self.first_proto.ClearField('repeated_int32') + self.assertNotEqual(self.first_proto, self.second_proto) + + def testRepeatedComposite(self): + # Change value within a repeated composite field. + self.first_proto.repeated_nested_message[0].bb += 1 + self.assertNotEqual(self.first_proto, self.second_proto) + self.first_proto.repeated_nested_message[0].bb -= 1 + self.assertEqual(self.first_proto, self.second_proto) + # Add a value to a repeated composite field. + self.first_proto.repeated_nested_message.add() + self.assertNotEqual(self.first_proto, self.second_proto) + self.second_proto.repeated_nested_message.add() + self.assertEqual(self.first_proto, self.second_proto) + + def testNonRepeatedScalarHasBits(self): + # Ensure that we test "has" bits as well as value for + # nonrepeated scalar field. + self.first_proto.ClearField('optional_int32') + self.second_proto.optional_int32 = 0 + self.assertNotEqual(self.first_proto, self.second_proto) + + def testNonRepeatedCompositeHasBits(self): + # Ensure that we test "has" bits as well as value for + # nonrepeated composite field. + self.first_proto.ClearField('optional_nested_message') + self.second_proto.optional_nested_message.ClearField('bb') + self.assertNotEqual(self.first_proto, self.second_proto) + # TODO(robinson): Replace next two lines with method + # to set the "has" bit without changing the value, + # if/when such a method exists. + self.first_proto.optional_nested_message.bb = 0 + self.first_proto.optional_nested_message.ClearField('bb') + self.assertEqual(self.first_proto, self.second_proto) + + +class ExtensionEqualityTest(unittest.TestCase): + + def testExtensionEquality(self): + first_proto = unittest_pb2.TestAllExtensions() + second_proto = unittest_pb2.TestAllExtensions() + self.assertEqual(first_proto, second_proto) + test_util.SetAllExtensions(first_proto) + self.assertNotEqual(first_proto, second_proto) + test_util.SetAllExtensions(second_proto) + self.assertEqual(first_proto, second_proto) + + # Ensure that we check value equality. + first_proto.Extensions[unittest_pb2.optional_int32_extension] += 1 + self.assertNotEqual(first_proto, second_proto) + first_proto.Extensions[unittest_pb2.optional_int32_extension] -= 1 + self.assertEqual(first_proto, second_proto) + + # Ensure that we also look at "has" bits. + first_proto.ClearExtension(unittest_pb2.optional_int32_extension) + second_proto.Extensions[unittest_pb2.optional_int32_extension] = 0 + self.assertNotEqual(first_proto, second_proto) + first_proto.Extensions[unittest_pb2.optional_int32_extension] = 0 + self.assertEqual(first_proto, second_proto) + + # Ensure that differences in cached values + # don't matter if "has" bits are both false. + first_proto = unittest_pb2.TestAllExtensions() + second_proto = unittest_pb2.TestAllExtensions() + self.assertEqual( + 0, first_proto.Extensions[unittest_pb2.optional_int32_extension]) + self.assertEqual(first_proto, second_proto) + + +class MutualRecursionEqualityTest(unittest.TestCase): + + def testEqualityWithMutualRecursion(self): + first_proto = unittest_pb2.TestMutualRecursionA() + second_proto = unittest_pb2.TestMutualRecursionA() + self.assertEqual(first_proto, second_proto) + first_proto.bb.a.bb.optional_int32 = 23 + self.assertNotEqual(first_proto, second_proto) + second_proto.bb.a.bb.optional_int32 = 23 + self.assertEqual(first_proto, second_proto) + + +class ByteSizeTest(unittest.TestCase): + + def setUp(self): + self.proto = unittest_pb2.TestAllTypes() + self.extended_proto = more_extensions_pb2.ExtendedMessage() + self.packed_proto = unittest_pb2.TestPackedTypes() + self.packed_extended_proto = unittest_pb2.TestPackedExtensions() + + def Size(self): + return self.proto.ByteSize() + + def testEmptyMessage(self): + self.assertEqual(0, self.proto.ByteSize()) + + def testVarints(self): + def Test(i, expected_varint_size): + self.proto.Clear() + self.proto.optional_int64 = i + # Add one to the varint size for the tag info + # for tag 1. + self.assertEqual(expected_varint_size + 1, self.Size()) + Test(0, 1) + Test(1, 1) + for i, num_bytes in zip(range(7, 63, 7), range(1, 10000)): + Test((1 << i) - 1, num_bytes) + Test(-1, 10) + Test(-2, 10) + Test(-(1 << 63), 10) + + def testStrings(self): + self.proto.optional_string = '' + # Need one byte for tag info (tag #14), and one byte for length. + self.assertEqual(2, self.Size()) + + self.proto.optional_string = 'abc' + # Need one byte for tag info (tag #14), and one byte for length. + self.assertEqual(2 + len(self.proto.optional_string), self.Size()) + + self.proto.optional_string = 'x' * 128 + # Need one byte for tag info (tag #14), and TWO bytes for length. + self.assertEqual(3 + len(self.proto.optional_string), self.Size()) + + def testOtherNumerics(self): + self.proto.optional_fixed32 = 1234 + # One byte for tag and 4 bytes for fixed32. + self.assertEqual(5, self.Size()) + self.proto = unittest_pb2.TestAllTypes() + + self.proto.optional_fixed64 = 1234 + # One byte for tag and 8 bytes for fixed64. + self.assertEqual(9, self.Size()) + self.proto = unittest_pb2.TestAllTypes() + + self.proto.optional_float = 1.234 + # One byte for tag and 4 bytes for float. + self.assertEqual(5, self.Size()) + self.proto = unittest_pb2.TestAllTypes() + + self.proto.optional_double = 1.234 + # One byte for tag and 8 bytes for float. + self.assertEqual(9, self.Size()) + self.proto = unittest_pb2.TestAllTypes() + + self.proto.optional_sint32 = 64 + # One byte for tag and 2 bytes for zig-zag-encoded 64. + self.assertEqual(3, self.Size()) + self.proto = unittest_pb2.TestAllTypes() + + def testComposites(self): + # 3 bytes. + self.proto.optional_nested_message.bb = (1 << 14) + # Plus one byte for bb tag. + # Plus 1 byte for optional_nested_message serialized size. + # Plus two bytes for optional_nested_message tag. + self.assertEqual(3 + 1 + 1 + 2, self.Size()) + + def testGroups(self): + # 4 bytes. + self.proto.optionalgroup.a = (1 << 21) + # Plus two bytes for |a| tag. + # Plus 2 * two bytes for START_GROUP and END_GROUP tags. + self.assertEqual(4 + 2 + 2*2, self.Size()) + + def testRepeatedScalars(self): + self.proto.repeated_int32.append(10) # 1 byte. + self.proto.repeated_int32.append(128) # 2 bytes. + # Also need 2 bytes for each entry for tag. + self.assertEqual(1 + 2 + 2*2, self.Size()) + + def testRepeatedScalarsExtend(self): + self.proto.repeated_int32.extend([10, 128]) # 3 bytes. + # Also need 2 bytes for each entry for tag. + self.assertEqual(1 + 2 + 2*2, self.Size()) + + def testRepeatedScalarsRemove(self): + self.proto.repeated_int32.append(10) # 1 byte. + self.proto.repeated_int32.append(128) # 2 bytes. + # Also need 2 bytes for each entry for tag. + self.assertEqual(1 + 2 + 2*2, self.Size()) + self.proto.repeated_int32.remove(128) + self.assertEqual(1 + 2, self.Size()) + + def testRepeatedComposites(self): + # Empty message. 2 bytes tag plus 1 byte length. + foreign_message_0 = self.proto.repeated_nested_message.add() + # 2 bytes tag plus 1 byte length plus 1 byte bb tag 1 byte int. + foreign_message_1 = self.proto.repeated_nested_message.add() + foreign_message_1.bb = 7 + self.assertEqual(2 + 1 + 2 + 1 + 1 + 1, self.Size()) + + def testRepeatedCompositesDelete(self): + # Empty message. 2 bytes tag plus 1 byte length. + foreign_message_0 = self.proto.repeated_nested_message.add() + # 2 bytes tag plus 1 byte length plus 1 byte bb tag 1 byte int. + foreign_message_1 = self.proto.repeated_nested_message.add() + foreign_message_1.bb = 9 + self.assertEqual(2 + 1 + 2 + 1 + 1 + 1, self.Size()) + + # 2 bytes tag plus 1 byte length plus 1 byte bb tag 1 byte int. + del self.proto.repeated_nested_message[0] + self.assertEqual(2 + 1 + 1 + 1, self.Size()) + + # Now add a new message. + foreign_message_2 = self.proto.repeated_nested_message.add() + foreign_message_2.bb = 12 + + # 2 bytes tag plus 1 byte length plus 1 byte bb tag 1 byte int. + # 2 bytes tag plus 1 byte length plus 1 byte bb tag 1 byte int. + self.assertEqual(2 + 1 + 1 + 1 + 2 + 1 + 1 + 1, self.Size()) + + # 2 bytes tag plus 1 byte length plus 1 byte bb tag 1 byte int. + del self.proto.repeated_nested_message[1] + self.assertEqual(2 + 1 + 1 + 1, self.Size()) + + del self.proto.repeated_nested_message[0] + self.assertEqual(0, self.Size()) + + def testRepeatedGroups(self): + # 2-byte START_GROUP plus 2-byte END_GROUP. + group_0 = self.proto.repeatedgroup.add() + # 2-byte START_GROUP plus 2-byte |a| tag + 1-byte |a| + # plus 2-byte END_GROUP. + group_1 = self.proto.repeatedgroup.add() + group_1.a = 7 + self.assertEqual(2 + 2 + 2 + 2 + 1 + 2, self.Size()) + + def testExtensions(self): + proto = unittest_pb2.TestAllExtensions() + self.assertEqual(0, proto.ByteSize()) + extension = unittest_pb2.optional_int32_extension # Field #1, 1 byte. + proto.Extensions[extension] = 23 + # 1 byte for tag, 1 byte for value. + self.assertEqual(2, proto.ByteSize()) + + def testCacheInvalidationForNonrepeatedScalar(self): + # Test non-extension. + self.proto.optional_int32 = 1 + self.assertEqual(2, self.proto.ByteSize()) + self.proto.optional_int32 = 128 + self.assertEqual(3, self.proto.ByteSize()) + self.proto.ClearField('optional_int32') + self.assertEqual(0, self.proto.ByteSize()) + + # Test within extension. + extension = more_extensions_pb2.optional_int_extension + self.extended_proto.Extensions[extension] = 1 + self.assertEqual(2, self.extended_proto.ByteSize()) + self.extended_proto.Extensions[extension] = 128 + self.assertEqual(3, self.extended_proto.ByteSize()) + self.extended_proto.ClearExtension(extension) + self.assertEqual(0, self.extended_proto.ByteSize()) + + def testCacheInvalidationForRepeatedScalar(self): + # Test non-extension. + self.proto.repeated_int32.append(1) + self.assertEqual(3, self.proto.ByteSize()) + self.proto.repeated_int32.append(1) + self.assertEqual(6, self.proto.ByteSize()) + self.proto.repeated_int32[1] = 128 + self.assertEqual(7, self.proto.ByteSize()) + self.proto.ClearField('repeated_int32') + self.assertEqual(0, self.proto.ByteSize()) + + # Test within extension. + extension = more_extensions_pb2.repeated_int_extension + repeated = self.extended_proto.Extensions[extension] + repeated.append(1) + self.assertEqual(2, self.extended_proto.ByteSize()) + repeated.append(1) + self.assertEqual(4, self.extended_proto.ByteSize()) + repeated[1] = 128 + self.assertEqual(5, self.extended_proto.ByteSize()) + self.extended_proto.ClearExtension(extension) + self.assertEqual(0, self.extended_proto.ByteSize()) + + def testCacheInvalidationForNonrepeatedMessage(self): + # Test non-extension. + self.proto.optional_foreign_message.c = 1 + self.assertEqual(5, self.proto.ByteSize()) + self.proto.optional_foreign_message.c = 128 + self.assertEqual(6, self.proto.ByteSize()) + self.proto.optional_foreign_message.ClearField('c') + self.assertEqual(3, self.proto.ByteSize()) + self.proto.ClearField('optional_foreign_message') + self.assertEqual(0, self.proto.ByteSize()) + child = self.proto.optional_foreign_message + self.proto.ClearField('optional_foreign_message') + child.c = 128 + self.assertEqual(0, self.proto.ByteSize()) + + # Test within extension. + extension = more_extensions_pb2.optional_message_extension + child = self.extended_proto.Extensions[extension] + self.assertEqual(0, self.extended_proto.ByteSize()) + child.foreign_message_int = 1 + self.assertEqual(4, self.extended_proto.ByteSize()) + child.foreign_message_int = 128 + self.assertEqual(5, self.extended_proto.ByteSize()) + self.extended_proto.ClearExtension(extension) + self.assertEqual(0, self.extended_proto.ByteSize()) + + def testCacheInvalidationForRepeatedMessage(self): + # Test non-extension. + child0 = self.proto.repeated_foreign_message.add() + self.assertEqual(3, self.proto.ByteSize()) + self.proto.repeated_foreign_message.add() + self.assertEqual(6, self.proto.ByteSize()) + child0.c = 1 + self.assertEqual(8, self.proto.ByteSize()) + self.proto.ClearField('repeated_foreign_message') + self.assertEqual(0, self.proto.ByteSize()) + + # Test within extension. + extension = more_extensions_pb2.repeated_message_extension + child_list = self.extended_proto.Extensions[extension] + child0 = child_list.add() + self.assertEqual(2, self.extended_proto.ByteSize()) + child_list.add() + self.assertEqual(4, self.extended_proto.ByteSize()) + child0.foreign_message_int = 1 + self.assertEqual(6, self.extended_proto.ByteSize()) + child0.ClearField('foreign_message_int') + self.assertEqual(4, self.extended_proto.ByteSize()) + self.extended_proto.ClearExtension(extension) + self.assertEqual(0, self.extended_proto.ByteSize()) + + def testPackedRepeatedScalars(self): + self.assertEqual(0, self.packed_proto.ByteSize()) + + self.packed_proto.packed_int32.append(10) # 1 byte. + self.packed_proto.packed_int32.append(128) # 2 bytes. + # The tag is 2 bytes (the field number is 90), and the varint + # storing the length is 1 byte. + int_size = 1 + 2 + 3 + self.assertEqual(int_size, self.packed_proto.ByteSize()) + + self.packed_proto.packed_double.append(4.2) # 8 bytes + self.packed_proto.packed_double.append(3.25) # 8 bytes + # 2 more tag bytes, 1 more length byte. + double_size = 8 + 8 + 3 + self.assertEqual(int_size+double_size, self.packed_proto.ByteSize()) + + self.packed_proto.ClearField('packed_int32') + self.assertEqual(double_size, self.packed_proto.ByteSize()) + + def testPackedExtensions(self): + self.assertEqual(0, self.packed_extended_proto.ByteSize()) + extension = self.packed_extended_proto.Extensions[ + unittest_pb2.packed_fixed32_extension] + extension.extend([1, 2, 3, 4]) # 16 bytes + # Tag is 3 bytes. + self.assertEqual(19, self.packed_extended_proto.ByteSize()) + + +# TODO(robinson): We need cross-language serialization consistency tests. +# Issues to be sure to cover include: +# * Handling of unrecognized tags ("uninterpreted_bytes"). +# * Handling of MessageSets. +# * Consistent ordering of tags in the wire format, +# including ordering between extensions and non-extension +# fields. +# * Consistent serialization of negative numbers, especially +# negative int32s. +# * Handling of empty submessages (with and without "has" +# bits set). + +class SerializationTest(unittest.TestCase): + + def testSerializeEmtpyMessage(self): + first_proto = unittest_pb2.TestAllTypes() + second_proto = unittest_pb2.TestAllTypes() + serialized = first_proto.SerializeToString() + self.assertEqual(first_proto.ByteSize(), len(serialized)) + second_proto.MergeFromString(serialized) + self.assertEqual(first_proto, second_proto) + + def testSerializeAllFields(self): + first_proto = unittest_pb2.TestAllTypes() + second_proto = unittest_pb2.TestAllTypes() + test_util.SetAllFields(first_proto) + serialized = first_proto.SerializeToString() + self.assertEqual(first_proto.ByteSize(), len(serialized)) + second_proto.MergeFromString(serialized) + self.assertEqual(first_proto, second_proto) + + def testSerializeAllExtensions(self): + first_proto = unittest_pb2.TestAllExtensions() + second_proto = unittest_pb2.TestAllExtensions() + test_util.SetAllExtensions(first_proto) + serialized = first_proto.SerializeToString() + second_proto.MergeFromString(serialized) + self.assertEqual(first_proto, second_proto) + + def testCanonicalSerializationOrder(self): + proto = more_messages_pb2.OutOfOrderFields() + # These are also their tag numbers. Even though we're setting these in + # reverse-tag order AND they're listed in reverse tag-order in the .proto + # file, they should nonetheless be serialized in tag order. + proto.optional_sint32 = 5 + proto.Extensions[more_messages_pb2.optional_uint64] = 4 + proto.optional_uint32 = 3 + proto.Extensions[more_messages_pb2.optional_int64] = 2 + proto.optional_int32 = 1 + serialized = proto.SerializeToString() + self.assertEqual(proto.ByteSize(), len(serialized)) + d = decoder.Decoder(serialized) + ReadTag = d.ReadFieldNumberAndWireType + self.assertEqual((1, wire_format.WIRETYPE_VARINT), ReadTag()) + self.assertEqual(1, d.ReadInt32()) + self.assertEqual((2, wire_format.WIRETYPE_VARINT), ReadTag()) + self.assertEqual(2, d.ReadInt64()) + self.assertEqual((3, wire_format.WIRETYPE_VARINT), ReadTag()) + self.assertEqual(3, d.ReadUInt32()) + self.assertEqual((4, wire_format.WIRETYPE_VARINT), ReadTag()) + self.assertEqual(4, d.ReadUInt64()) + self.assertEqual((5, wire_format.WIRETYPE_VARINT), ReadTag()) + self.assertEqual(5, d.ReadSInt32()) + + def testCanonicalSerializationOrderSameAsCpp(self): + # Copy of the same test we use for C++. + proto = unittest_pb2.TestFieldOrderings() + test_util.SetAllFieldsAndExtensions(proto) + serialized = proto.SerializeToString() + test_util.ExpectAllFieldsAndExtensionsInOrder(serialized) + + def testMergeFromStringWhenFieldsAlreadySet(self): + first_proto = unittest_pb2.TestAllTypes() + first_proto.repeated_string.append('foobar') + first_proto.optional_int32 = 23 + first_proto.optional_nested_message.bb = 42 + serialized = first_proto.SerializeToString() + + second_proto = unittest_pb2.TestAllTypes() + second_proto.repeated_string.append('baz') + second_proto.optional_int32 = 100 + second_proto.optional_nested_message.bb = 999 + + second_proto.MergeFromString(serialized) + # Ensure that we append to repeated fields. + self.assertEqual(['baz', 'foobar'], list(second_proto.repeated_string)) + # Ensure that we overwrite nonrepeatd scalars. + self.assertEqual(23, second_proto.optional_int32) + # Ensure that we recursively call MergeFromString() on + # submessages. + self.assertEqual(42, second_proto.optional_nested_message.bb) + + def testMessageSetWireFormat(self): + proto = unittest_mset_pb2.TestMessageSet() + extension_message1 = unittest_mset_pb2.TestMessageSetExtension1 + extension_message2 = unittest_mset_pb2.TestMessageSetExtension2 + extension1 = extension_message1.message_set_extension + extension2 = extension_message2.message_set_extension + proto.Extensions[extension1].i = 123 + proto.Extensions[extension2].str = 'foo' + + # Serialize using the MessageSet wire format (this is specified in the + # .proto file). + serialized = proto.SerializeToString() + + raw = unittest_mset_pb2.RawMessageSet() + self.assertEqual(False, + raw.DESCRIPTOR.GetOptions().message_set_wire_format) + raw.MergeFromString(serialized) + self.assertEqual(2, len(raw.item)) + + message1 = unittest_mset_pb2.TestMessageSetExtension1() + message1.MergeFromString(raw.item[0].message) + self.assertEqual(123, message1.i) + + message2 = unittest_mset_pb2.TestMessageSetExtension2() + message2.MergeFromString(raw.item[1].message) + self.assertEqual('foo', message2.str) + + # Deserialize using the MessageSet wire format. + proto2 = unittest_mset_pb2.TestMessageSet() + proto2.MergeFromString(serialized) + self.assertEqual(123, proto2.Extensions[extension1].i) + self.assertEqual('foo', proto2.Extensions[extension2].str) + + # Check byte size. + self.assertEqual(proto2.ByteSize(), len(serialized)) + self.assertEqual(proto.ByteSize(), len(serialized)) + + def testMessageSetWireFormatUnknownExtension(self): + # Create a message using the message set wire format with an unknown + # message. + raw = unittest_mset_pb2.RawMessageSet() + + # Add an item. + item = raw.item.add() + item.type_id = 1545008 + extension_message1 = unittest_mset_pb2.TestMessageSetExtension1 + message1 = unittest_mset_pb2.TestMessageSetExtension1() + message1.i = 12345 + item.message = message1.SerializeToString() + + # Add a second, unknown extension. + item = raw.item.add() + item.type_id = 1545009 + extension_message1 = unittest_mset_pb2.TestMessageSetExtension1 + message1 = unittest_mset_pb2.TestMessageSetExtension1() + message1.i = 12346 + item.message = message1.SerializeToString() + + # Add another unknown extension. + item = raw.item.add() + item.type_id = 1545010 + message1 = unittest_mset_pb2.TestMessageSetExtension2() + message1.str = 'foo' + item.message = message1.SerializeToString() + + serialized = raw.SerializeToString() + + # Parse message using the message set wire format. + proto = unittest_mset_pb2.TestMessageSet() + proto.MergeFromString(serialized) + + # Check that the message parsed well. + extension_message1 = unittest_mset_pb2.TestMessageSetExtension1 + extension1 = extension_message1.message_set_extension + self.assertEquals(12345, proto.Extensions[extension1].i) + + def testUnknownFields(self): + proto = unittest_pb2.TestAllTypes() + test_util.SetAllFields(proto) + + serialized = proto.SerializeToString() + + # The empty message should be parsable with all of the fields + # unknown. + proto2 = unittest_pb2.TestEmptyMessage() + + # Parsing this message should succeed. + proto2.MergeFromString(serialized) + + # Now test with a int64 field set. + proto = unittest_pb2.TestAllTypes() + proto.optional_int64 = 0x0fffffffffffffff + serialized = proto.SerializeToString() + # The empty message should be parsable with all of the fields + # unknown. + proto2 = unittest_pb2.TestEmptyMessage() + # Parsing this message should succeed. + proto2.MergeFromString(serialized) + + def _CheckRaises(self, exc_class, callable_obj, exception): + """This method checks if the excpetion type and message are as expected.""" + try: + callable_obj() + except exc_class, ex: + # Check if the exception message is the right one. + self.assertEqual(exception, str(ex)) + return + else: + raise self.failureException('%s not raised' % str(exc_class)) + + def testSerializeUninitialized(self): + proto = unittest_pb2.TestRequired() + self._CheckRaises( + message.EncodeError, + proto.SerializeToString, + 'Required field protobuf_unittest.TestRequired.a is not set.') + # Shouldn't raise exceptions. + partial = proto.SerializePartialToString() + + proto.a = 1 + self._CheckRaises( + message.EncodeError, + proto.SerializeToString, + 'Required field protobuf_unittest.TestRequired.b is not set.') + # Shouldn't raise exceptions. + partial = proto.SerializePartialToString() + + proto.b = 2 + self._CheckRaises( + message.EncodeError, + proto.SerializeToString, + 'Required field protobuf_unittest.TestRequired.c is not set.') + # Shouldn't raise exceptions. + partial = proto.SerializePartialToString() + + proto.c = 3 + serialized = proto.SerializeToString() + # Shouldn't raise exceptions. + partial = proto.SerializePartialToString() + + proto2 = unittest_pb2.TestRequired() + proto2.MergeFromString(serialized) + self.assertEqual(1, proto2.a) + self.assertEqual(2, proto2.b) + self.assertEqual(3, proto2.c) + proto2.ParseFromString(partial) + self.assertEqual(1, proto2.a) + self.assertEqual(2, proto2.b) + self.assertEqual(3, proto2.c) + + def testSerializeAllPackedFields(self): + first_proto = unittest_pb2.TestPackedTypes() + second_proto = unittest_pb2.TestPackedTypes() + test_util.SetAllPackedFields(first_proto) + serialized = first_proto.SerializeToString() + self.assertEqual(first_proto.ByteSize(), len(serialized)) + bytes_read = second_proto.MergeFromString(serialized) + self.assertEqual(second_proto.ByteSize(), bytes_read) + self.assertEqual(first_proto, second_proto) + + def testSerializeAllPackedExtensions(self): + first_proto = unittest_pb2.TestPackedExtensions() + second_proto = unittest_pb2.TestPackedExtensions() + test_util.SetAllPackedExtensions(first_proto) + serialized = first_proto.SerializeToString() + bytes_read = second_proto.MergeFromString(serialized) + self.assertEqual(second_proto.ByteSize(), bytes_read) + self.assertEqual(first_proto, second_proto) + + def testMergePackedFromStringWhenSomeFieldsAlreadySet(self): + first_proto = unittest_pb2.TestPackedTypes() + first_proto.packed_int32.extend([1, 2]) + first_proto.packed_double.append(3.0) + serialized = first_proto.SerializeToString() + + second_proto = unittest_pb2.TestPackedTypes() + second_proto.packed_int32.append(3) + second_proto.packed_double.extend([1.0, 2.0]) + second_proto.packed_sint32.append(4) + + second_proto.MergeFromString(serialized) + self.assertEqual([3, 1, 2], second_proto.packed_int32) + self.assertEqual([1.0, 2.0, 3.0], second_proto.packed_double) + self.assertEqual([4], second_proto.packed_sint32) + + def testPackedFieldsWireFormat(self): + proto = unittest_pb2.TestPackedTypes() + proto.packed_int32.extend([1, 2, 150, 3]) # 1 + 1 + 2 + 1 bytes + proto.packed_double.extend([1.0, 1000.0]) # 8 + 8 bytes + proto.packed_float.append(2.0) # 4 bytes, will be before double + serialized = proto.SerializeToString() + self.assertEqual(proto.ByteSize(), len(serialized)) + d = decoder.Decoder(serialized) + ReadTag = d.ReadFieldNumberAndWireType + self.assertEqual((90, wire_format.WIRETYPE_LENGTH_DELIMITED), ReadTag()) + self.assertEqual(1+1+1+2, d.ReadInt32()) + self.assertEqual(1, d.ReadInt32()) + self.assertEqual(2, d.ReadInt32()) + self.assertEqual(150, d.ReadInt32()) + self.assertEqual(3, d.ReadInt32()) + self.assertEqual((100, wire_format.WIRETYPE_LENGTH_DELIMITED), ReadTag()) + self.assertEqual(4, d.ReadInt32()) + self.assertEqual(2.0, d.ReadFloat()) + self.assertEqual((101, wire_format.WIRETYPE_LENGTH_DELIMITED), ReadTag()) + self.assertEqual(8+8, d.ReadInt32()) + self.assertEqual(1.0, d.ReadDouble()) + self.assertEqual(1000.0, d.ReadDouble()) + self.assertTrue(d.EndOfStream()) + + def testFieldNumbers(self): + proto = unittest_pb2.TestAllTypes() + self.assertEqual(unittest_pb2.TestAllTypes.NestedMessage.BB_FIELD_NUMBER, 1) + self.assertEqual(unittest_pb2.TestAllTypes.OPTIONAL_INT32_FIELD_NUMBER, 1) + self.assertEqual(unittest_pb2.TestAllTypes.OPTIONALGROUP_FIELD_NUMBER, 16) + self.assertEqual( + unittest_pb2.TestAllTypes.OPTIONAL_NESTED_MESSAGE_FIELD_NUMBER, 18) + self.assertEqual( + unittest_pb2.TestAllTypes.OPTIONAL_NESTED_ENUM_FIELD_NUMBER, 21) + self.assertEqual(unittest_pb2.TestAllTypes.REPEATED_INT32_FIELD_NUMBER, 31) + self.assertEqual(unittest_pb2.TestAllTypes.REPEATEDGROUP_FIELD_NUMBER, 46) + self.assertEqual( + unittest_pb2.TestAllTypes.REPEATED_NESTED_MESSAGE_FIELD_NUMBER, 48) + self.assertEqual( + unittest_pb2.TestAllTypes.REPEATED_NESTED_ENUM_FIELD_NUMBER, 51) + + def testExtensionFieldNumbers(self): + self.assertEqual(unittest_pb2.TestRequired.single.number, 1000) + self.assertEqual(unittest_pb2.TestRequired.SINGLE_FIELD_NUMBER, 1000) + self.assertEqual(unittest_pb2.TestRequired.multi.number, 1001) + self.assertEqual(unittest_pb2.TestRequired.MULTI_FIELD_NUMBER, 1001) + self.assertEqual(unittest_pb2.optional_int32_extension.number, 1) + self.assertEqual(unittest_pb2.OPTIONAL_INT32_EXTENSION_FIELD_NUMBER, 1) + self.assertEqual(unittest_pb2.optionalgroup_extension.number, 16) + self.assertEqual(unittest_pb2.OPTIONALGROUP_EXTENSION_FIELD_NUMBER, 16) + self.assertEqual(unittest_pb2.optional_nested_message_extension.number, 18) + self.assertEqual( + unittest_pb2.OPTIONAL_NESTED_MESSAGE_EXTENSION_FIELD_NUMBER, 18) + self.assertEqual(unittest_pb2.optional_nested_enum_extension.number, 21) + self.assertEqual(unittest_pb2.OPTIONAL_NESTED_ENUM_EXTENSION_FIELD_NUMBER, + 21) + self.assertEqual(unittest_pb2.repeated_int32_extension.number, 31) + self.assertEqual(unittest_pb2.REPEATED_INT32_EXTENSION_FIELD_NUMBER, 31) + self.assertEqual(unittest_pb2.repeatedgroup_extension.number, 46) + self.assertEqual(unittest_pb2.REPEATEDGROUP_EXTENSION_FIELD_NUMBER, 46) + self.assertEqual(unittest_pb2.repeated_nested_message_extension.number, 48) + self.assertEqual( + unittest_pb2.REPEATED_NESTED_MESSAGE_EXTENSION_FIELD_NUMBER, 48) + self.assertEqual(unittest_pb2.repeated_nested_enum_extension.number, 51) + self.assertEqual(unittest_pb2.REPEATED_NESTED_ENUM_EXTENSION_FIELD_NUMBER, + 51) + + def testInitKwargs(self): + proto = unittest_pb2.TestAllTypes( + optional_int32=1, + optional_string='foo', + optional_bool=True, + optional_bytes='bar', + optional_nested_message=unittest_pb2.TestAllTypes.NestedMessage(bb=1), + optional_foreign_message=unittest_pb2.ForeignMessage(c=1), + optional_nested_enum=unittest_pb2.TestAllTypes.FOO, + optional_foreign_enum=unittest_pb2.FOREIGN_FOO, + repeated_int32=[1, 2, 3]) + self.assertTrue(proto.IsInitialized()) + self.assertTrue(proto.HasField('optional_int32')) + self.assertTrue(proto.HasField('optional_string')) + self.assertTrue(proto.HasField('optional_bool')) + self.assertTrue(proto.HasField('optional_bytes')) + self.assertTrue(proto.HasField('optional_nested_message')) + self.assertTrue(proto.HasField('optional_foreign_message')) + self.assertTrue(proto.HasField('optional_nested_enum')) + self.assertTrue(proto.HasField('optional_foreign_enum')) + self.assertEqual(1, proto.optional_int32) + self.assertEqual('foo', proto.optional_string) + self.assertEqual(True, proto.optional_bool) + self.assertEqual('bar', proto.optional_bytes) + self.assertEqual(1, proto.optional_nested_message.bb) + self.assertEqual(1, proto.optional_foreign_message.c) + self.assertEqual(unittest_pb2.TestAllTypes.FOO, + proto.optional_nested_enum) + self.assertEqual(unittest_pb2.FOREIGN_FOO, proto.optional_foreign_enum) + self.assertEqual([1, 2, 3], proto.repeated_int32) + + def testInitArgsUnknownFieldName(self): + def InitalizeEmptyMessageWithExtraKeywordArg(): + unused_proto = unittest_pb2.TestEmptyMessage(unknown='unknown') + self._CheckRaises(ValueError, + InitalizeEmptyMessageWithExtraKeywordArg, + 'Protocol message has no "unknown" field.') + + def testInitRequiredKwargs(self): + proto = unittest_pb2.TestRequired(a=1, b=1, c=1) + self.assertTrue(proto.IsInitialized()) + self.assertTrue(proto.HasField('a')) + self.assertTrue(proto.HasField('b')) + self.assertTrue(proto.HasField('c')) + self.assertTrue(not proto.HasField('dummy2')) + self.assertEqual(1, proto.a) + self.assertEqual(1, proto.b) + self.assertEqual(1, proto.c) + + def testInitRequiredForeignKwargs(self): + proto = unittest_pb2.TestRequiredForeign( + optional_message=unittest_pb2.TestRequired(a=1, b=1, c=1)) + self.assertTrue(proto.IsInitialized()) + self.assertTrue(proto.HasField('optional_message')) + self.assertTrue(proto.optional_message.IsInitialized()) + self.assertTrue(proto.optional_message.HasField('a')) + self.assertTrue(proto.optional_message.HasField('b')) + self.assertTrue(proto.optional_message.HasField('c')) + self.assertTrue(not proto.optional_message.HasField('dummy2')) + self.assertEqual(unittest_pb2.TestRequired(a=1, b=1, c=1), + proto.optional_message) + self.assertEqual(1, proto.optional_message.a) + self.assertEqual(1, proto.optional_message.b) + self.assertEqual(1, proto.optional_message.c) + + def testInitRepeatedKwargs(self): + proto = unittest_pb2.TestAllTypes(repeated_int32=[1, 2, 3]) + self.assertTrue(proto.IsInitialized()) + self.assertEqual(1, proto.repeated_int32[0]) + self.assertEqual(2, proto.repeated_int32[1]) + self.assertEqual(3, proto.repeated_int32[2]) + + +class OptionsTest(unittest.TestCase): + + def testMessageOptions(self): + proto = unittest_mset_pb2.TestMessageSet() + self.assertEqual(True, + proto.DESCRIPTOR.GetOptions().message_set_wire_format) + proto = unittest_pb2.TestAllTypes() + self.assertEqual(False, + proto.DESCRIPTOR.GetOptions().message_set_wire_format) + + def testPackedOptions(self): + proto = unittest_pb2.TestAllTypes() + proto.optional_int32 = 1 + proto.optional_double = 3.0 + for field_descriptor, _ in proto.ListFields(): + self.assertEqual(False, field_descriptor.GetOptions().packed) + + proto = unittest_pb2.TestPackedTypes() + proto.packed_int32.append(1) + proto.packed_double.append(3.0) + for field_descriptor, _ in proto.ListFields(): + self.assertEqual(True, field_descriptor.GetOptions().packed) + self.assertEqual(reflection._FieldDescriptor.LABEL_REPEATED, + field_descriptor.label) + + +class UtilityTest(unittest.TestCase): + + def testImergeSorted(self): + ImergeSorted = reflection._ImergeSorted + # Various types of emptiness. + self.assertEqual([], list(ImergeSorted())) + self.assertEqual([], list(ImergeSorted([]))) + self.assertEqual([], list(ImergeSorted([], []))) + + # One nonempty list. + self.assertEqual([1, 2, 3], list(ImergeSorted([1, 2, 3]))) + self.assertEqual([1, 2, 3], list(ImergeSorted([1, 2, 3], []))) + self.assertEqual([1, 2, 3], list(ImergeSorted([], [1, 2, 3]))) + + # Merging some nonempty lists together. + self.assertEqual([1, 2, 3], list(ImergeSorted([1, 3], [2]))) + self.assertEqual([1, 2, 3], list(ImergeSorted([1], [3], [2]))) + self.assertEqual([1, 2, 3], list(ImergeSorted([1], [3], [2], []))) + + # Elements repeated across component iterators. + self.assertEqual([1, 2, 2, 3, 3], + list(ImergeSorted([1, 2], [3], [2, 3]))) + + # Elements repeated within an iterator. + self.assertEqual([1, 2, 2, 3, 3], + list(ImergeSorted([1, 2, 2], [3], [3]))) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/google/protobuf/internal/service_reflection_test.py b/python/google/protobuf/internal/service_reflection_test.py new file mode 100755 index 0000000..e04f825 --- /dev/null +++ b/python/google/protobuf/internal/service_reflection_test.py @@ -0,0 +1,136 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Tests for google.protobuf.internal.service_reflection.""" + +__author__ = 'petar@google.com (Petar Petrov)' + +import unittest +from google.protobuf import unittest_pb2 +from google.protobuf import service_reflection +from google.protobuf import service + + +class FooUnitTest(unittest.TestCase): + + def testService(self): + class MockRpcChannel(service.RpcChannel): + def CallMethod(self, method, controller, request, response, callback): + self.method = method + self.controller = controller + self.request = request + callback(response) + + class MockRpcController(service.RpcController): + def SetFailed(self, msg): + self.failure_message = msg + + self.callback_response = None + + class MyService(unittest_pb2.TestService): + pass + + self.callback_response = None + + def MyCallback(response): + self.callback_response = response + + rpc_controller = MockRpcController() + channel = MockRpcChannel() + srvc = MyService() + srvc.Foo(rpc_controller, unittest_pb2.FooRequest(), MyCallback) + self.assertEqual('Method Foo not implemented.', + rpc_controller.failure_message) + self.assertEqual(None, self.callback_response) + + rpc_controller.failure_message = None + + service_descriptor = unittest_pb2.TestService.GetDescriptor() + srvc.CallMethod(service_descriptor.methods[1], rpc_controller, + unittest_pb2.BarRequest(), MyCallback) + self.assertEqual('Method Bar not implemented.', + rpc_controller.failure_message) + self.assertEqual(None, self.callback_response) + + class MyServiceImpl(unittest_pb2.TestService): + def Foo(self, rpc_controller, request, done): + self.foo_called = True + def Bar(self, rpc_controller, request, done): + self.bar_called = True + + srvc = MyServiceImpl() + rpc_controller.failure_message = None + srvc.Foo(rpc_controller, unittest_pb2.FooRequest(), MyCallback) + self.assertEqual(None, rpc_controller.failure_message) + self.assertEqual(True, srvc.foo_called) + + rpc_controller.failure_message = None + srvc.CallMethod(service_descriptor.methods[1], rpc_controller, + unittest_pb2.BarRequest(), MyCallback) + self.assertEqual(None, rpc_controller.failure_message) + self.assertEqual(True, srvc.bar_called) + + def testServiceStub(self): + class MockRpcChannel(service.RpcChannel): + def CallMethod(self, method, controller, request, + response_class, callback): + self.method = method + self.controller = controller + self.request = request + callback(response_class()) + + self.callback_response = None + + def MyCallback(response): + self.callback_response = response + + channel = MockRpcChannel() + stub = unittest_pb2.TestService_Stub(channel) + rpc_controller = 'controller' + request = 'request' + + # GetDescriptor now static, still works as instance method for compatability + self.assertEqual(unittest_pb2.TestService_Stub.GetDescriptor(), + stub.GetDescriptor()) + + # Invoke method. + stub.Foo(rpc_controller, request, MyCallback) + + self.assertTrue(isinstance(self.callback_response, + unittest_pb2.FooResponse)) + self.assertEqual(request, channel.request) + self.assertEqual(rpc_controller, channel.controller) + self.assertEqual(stub.GetDescriptor().methods[0], channel.method) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/google/protobuf/internal/test_util.py b/python/google/protobuf/internal/test_util.py new file mode 100755 index 0000000..1a0da55 --- /dev/null +++ b/python/google/protobuf/internal/test_util.py @@ -0,0 +1,612 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Utilities for Python proto2 tests. + +This is intentionally modeled on C++ code in +//net/proto2/internal/test_util.*. +""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import os.path + +import unittest +from google.protobuf import unittest_import_pb2 +from google.protobuf import unittest_pb2 + + +def SetAllFields(message): + """Sets every field in the message to a unique value. + + Args: + message: A unittest_pb2.TestAllTypes instance. + """ + + # + # Optional fields. + # + + message.optional_int32 = 101 + message.optional_int64 = 102 + message.optional_uint32 = 103 + message.optional_uint64 = 104 + message.optional_sint32 = 105 + message.optional_sint64 = 106 + message.optional_fixed32 = 107 + message.optional_fixed64 = 108 + message.optional_sfixed32 = 109 + message.optional_sfixed64 = 110 + message.optional_float = 111 + message.optional_double = 112 + message.optional_bool = True + # TODO(robinson): Firmly spec out and test how + # protos interact with unicode. One specific example: + # what happens if we change the literal below to + # u'115'? What *should* happen? Still some discussion + # to finish with Kenton about bytes vs. strings + # and forcing everything to be utf8. :-/ + message.optional_string = '115' + message.optional_bytes = '116' + + message.optionalgroup.a = 117 + message.optional_nested_message.bb = 118 + message.optional_foreign_message.c = 119 + message.optional_import_message.d = 120 + + message.optional_nested_enum = unittest_pb2.TestAllTypes.BAZ + message.optional_foreign_enum = unittest_pb2.FOREIGN_BAZ + message.optional_import_enum = unittest_import_pb2.IMPORT_BAZ + + message.optional_string_piece = '124' + message.optional_cord = '125' + + # + # Repeated fields. + # + + message.repeated_int32.append(201) + message.repeated_int64.append(202) + message.repeated_uint32.append(203) + message.repeated_uint64.append(204) + message.repeated_sint32.append(205) + message.repeated_sint64.append(206) + message.repeated_fixed32.append(207) + message.repeated_fixed64.append(208) + message.repeated_sfixed32.append(209) + message.repeated_sfixed64.append(210) + message.repeated_float.append(211) + message.repeated_double.append(212) + message.repeated_bool.append(True) + message.repeated_string.append('215') + message.repeated_bytes.append('216') + + message.repeatedgroup.add().a = 217 + message.repeated_nested_message.add().bb = 218 + message.repeated_foreign_message.add().c = 219 + message.repeated_import_message.add().d = 220 + + message.repeated_nested_enum.append(unittest_pb2.TestAllTypes.BAR) + message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAR) + message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAR) + + message.repeated_string_piece.append('224') + message.repeated_cord.append('225') + + # Add a second one of each field. + message.repeated_int32.append(301) + message.repeated_int64.append(302) + message.repeated_uint32.append(303) + message.repeated_uint64.append(304) + message.repeated_sint32.append(305) + message.repeated_sint64.append(306) + message.repeated_fixed32.append(307) + message.repeated_fixed64.append(308) + message.repeated_sfixed32.append(309) + message.repeated_sfixed64.append(310) + message.repeated_float.append(311) + message.repeated_double.append(312) + message.repeated_bool.append(False) + message.repeated_string.append('315') + message.repeated_bytes.append('316') + + message.repeatedgroup.add().a = 317 + message.repeated_nested_message.add().bb = 318 + message.repeated_foreign_message.add().c = 319 + message.repeated_import_message.add().d = 320 + + message.repeated_nested_enum.append(unittest_pb2.TestAllTypes.BAZ) + message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAZ) + message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAZ) + + message.repeated_string_piece.append('324') + message.repeated_cord.append('325') + + # + # Fields that have defaults. + # + + message.default_int32 = 401 + message.default_int64 = 402 + message.default_uint32 = 403 + message.default_uint64 = 404 + message.default_sint32 = 405 + message.default_sint64 = 406 + message.default_fixed32 = 407 + message.default_fixed64 = 408 + message.default_sfixed32 = 409 + message.default_sfixed64 = 410 + message.default_float = 411 + message.default_double = 412 + message.default_bool = False + message.default_string = '415' + message.default_bytes = '416' + + message.default_nested_enum = unittest_pb2.TestAllTypes.FOO + message.default_foreign_enum = unittest_pb2.FOREIGN_FOO + message.default_import_enum = unittest_import_pb2.IMPORT_FOO + + message.default_string_piece = '424' + message.default_cord = '425' + + +def SetAllExtensions(message): + """Sets every extension in the message to a unique value. + + Args: + message: A unittest_pb2.TestAllExtensions instance. + """ + + extensions = message.Extensions + pb2 = unittest_pb2 + import_pb2 = unittest_import_pb2 + + # + # Optional fields. + # + + extensions[pb2.optional_int32_extension] = 101 + extensions[pb2.optional_int64_extension] = 102 + extensions[pb2.optional_uint32_extension] = 103 + extensions[pb2.optional_uint64_extension] = 104 + extensions[pb2.optional_sint32_extension] = 105 + extensions[pb2.optional_sint64_extension] = 106 + extensions[pb2.optional_fixed32_extension] = 107 + extensions[pb2.optional_fixed64_extension] = 108 + extensions[pb2.optional_sfixed32_extension] = 109 + extensions[pb2.optional_sfixed64_extension] = 110 + extensions[pb2.optional_float_extension] = 111 + extensions[pb2.optional_double_extension] = 112 + extensions[pb2.optional_bool_extension] = True + extensions[pb2.optional_string_extension] = '115' + extensions[pb2.optional_bytes_extension] = '116' + + extensions[pb2.optionalgroup_extension].a = 117 + extensions[pb2.optional_nested_message_extension].bb = 118 + extensions[pb2.optional_foreign_message_extension].c = 119 + extensions[pb2.optional_import_message_extension].d = 120 + + extensions[pb2.optional_nested_enum_extension] = pb2.TestAllTypes.BAZ + extensions[pb2.optional_nested_enum_extension] = pb2.TestAllTypes.BAZ + extensions[pb2.optional_foreign_enum_extension] = pb2.FOREIGN_BAZ + extensions[pb2.optional_import_enum_extension] = import_pb2.IMPORT_BAZ + + extensions[pb2.optional_string_piece_extension] = '124' + extensions[pb2.optional_cord_extension] = '125' + + # + # Repeated fields. + # + + extensions[pb2.repeated_int32_extension].append(201) + extensions[pb2.repeated_int64_extension].append(202) + extensions[pb2.repeated_uint32_extension].append(203) + extensions[pb2.repeated_uint64_extension].append(204) + extensions[pb2.repeated_sint32_extension].append(205) + extensions[pb2.repeated_sint64_extension].append(206) + extensions[pb2.repeated_fixed32_extension].append(207) + extensions[pb2.repeated_fixed64_extension].append(208) + extensions[pb2.repeated_sfixed32_extension].append(209) + extensions[pb2.repeated_sfixed64_extension].append(210) + extensions[pb2.repeated_float_extension].append(211) + extensions[pb2.repeated_double_extension].append(212) + extensions[pb2.repeated_bool_extension].append(True) + extensions[pb2.repeated_string_extension].append('215') + extensions[pb2.repeated_bytes_extension].append('216') + + extensions[pb2.repeatedgroup_extension].add().a = 217 + extensions[pb2.repeated_nested_message_extension].add().bb = 218 + extensions[pb2.repeated_foreign_message_extension].add().c = 219 + extensions[pb2.repeated_import_message_extension].add().d = 220 + + extensions[pb2.repeated_nested_enum_extension].append(pb2.TestAllTypes.BAR) + extensions[pb2.repeated_foreign_enum_extension].append(pb2.FOREIGN_BAR) + extensions[pb2.repeated_import_enum_extension].append(import_pb2.IMPORT_BAR) + + extensions[pb2.repeated_string_piece_extension].append('224') + extensions[pb2.repeated_cord_extension].append('225') + + # Append a second one of each field. + extensions[pb2.repeated_int32_extension].append(301) + extensions[pb2.repeated_int64_extension].append(302) + extensions[pb2.repeated_uint32_extension].append(303) + extensions[pb2.repeated_uint64_extension].append(304) + extensions[pb2.repeated_sint32_extension].append(305) + extensions[pb2.repeated_sint64_extension].append(306) + extensions[pb2.repeated_fixed32_extension].append(307) + extensions[pb2.repeated_fixed64_extension].append(308) + extensions[pb2.repeated_sfixed32_extension].append(309) + extensions[pb2.repeated_sfixed64_extension].append(310) + extensions[pb2.repeated_float_extension].append(311) + extensions[pb2.repeated_double_extension].append(312) + extensions[pb2.repeated_bool_extension].append(False) + extensions[pb2.repeated_string_extension].append('315') + extensions[pb2.repeated_bytes_extension].append('316') + + extensions[pb2.repeatedgroup_extension].add().a = 317 + extensions[pb2.repeated_nested_message_extension].add().bb = 318 + extensions[pb2.repeated_foreign_message_extension].add().c = 319 + extensions[pb2.repeated_import_message_extension].add().d = 320 + + extensions[pb2.repeated_nested_enum_extension].append(pb2.TestAllTypes.BAZ) + extensions[pb2.repeated_foreign_enum_extension].append(pb2.FOREIGN_BAZ) + extensions[pb2.repeated_import_enum_extension].append(import_pb2.IMPORT_BAZ) + + extensions[pb2.repeated_string_piece_extension].append('324') + extensions[pb2.repeated_cord_extension].append('325') + + # + # Fields with defaults. + # + + extensions[pb2.default_int32_extension] = 401 + extensions[pb2.default_int64_extension] = 402 + extensions[pb2.default_uint32_extension] = 403 + extensions[pb2.default_uint64_extension] = 404 + extensions[pb2.default_sint32_extension] = 405 + extensions[pb2.default_sint64_extension] = 406 + extensions[pb2.default_fixed32_extension] = 407 + extensions[pb2.default_fixed64_extension] = 408 + extensions[pb2.default_sfixed32_extension] = 409 + extensions[pb2.default_sfixed64_extension] = 410 + extensions[pb2.default_float_extension] = 411 + extensions[pb2.default_double_extension] = 412 + extensions[pb2.default_bool_extension] = False + extensions[pb2.default_string_extension] = '415' + extensions[pb2.default_bytes_extension] = '416' + + extensions[pb2.default_nested_enum_extension] = pb2.TestAllTypes.FOO + extensions[pb2.default_foreign_enum_extension] = pb2.FOREIGN_FOO + extensions[pb2.default_import_enum_extension] = import_pb2.IMPORT_FOO + + extensions[pb2.default_string_piece_extension] = '424' + extensions[pb2.default_cord_extension] = '425' + + +def SetAllFieldsAndExtensions(message): + """Sets every field and extension in the message to a unique value. + + Args: + message: A unittest_pb2.TestAllExtensions message. + """ + message.my_int = 1 + message.my_string = 'foo' + message.my_float = 1.0 + message.Extensions[unittest_pb2.my_extension_int] = 23 + message.Extensions[unittest_pb2.my_extension_string] = 'bar' + + +def ExpectAllFieldsAndExtensionsInOrder(serialized): + """Ensures that serialized is the serialization we expect for a message + filled with SetAllFieldsAndExtensions(). (Specifically, ensures that the + serialization is in canonical, tag-number order). + """ + my_extension_int = unittest_pb2.my_extension_int + my_extension_string = unittest_pb2.my_extension_string + expected_strings = [] + message = unittest_pb2.TestFieldOrderings() + message.my_int = 1 # Field 1. + expected_strings.append(message.SerializeToString()) + message.Clear() + message.Extensions[my_extension_int] = 23 # Field 5. + expected_strings.append(message.SerializeToString()) + message.Clear() + message.my_string = 'foo' # Field 11. + expected_strings.append(message.SerializeToString()) + message.Clear() + message.Extensions[my_extension_string] = 'bar' # Field 50. + expected_strings.append(message.SerializeToString()) + message.Clear() + message.my_float = 1.0 + expected_strings.append(message.SerializeToString()) + message.Clear() + expected = ''.join(expected_strings) + + if expected != serialized: + raise ValueError('Expected %r, found %r' % (expected, serialized)) + + +class GoldenMessageTestCase(unittest.TestCase): + """This adds methods to TestCase useful for verifying our Golden Message.""" + + def ExpectAllFieldsSet(self, message): + """Check all fields for correct values have after Set*Fields() is called.""" + self.assertTrue(message.HasField('optional_int32')) + self.assertTrue(message.HasField('optional_int64')) + self.assertTrue(message.HasField('optional_uint32')) + self.assertTrue(message.HasField('optional_uint64')) + self.assertTrue(message.HasField('optional_sint32')) + self.assertTrue(message.HasField('optional_sint64')) + self.assertTrue(message.HasField('optional_fixed32')) + self.assertTrue(message.HasField('optional_fixed64')) + self.assertTrue(message.HasField('optional_sfixed32')) + self.assertTrue(message.HasField('optional_sfixed64')) + self.assertTrue(message.HasField('optional_float')) + self.assertTrue(message.HasField('optional_double')) + self.assertTrue(message.HasField('optional_bool')) + self.assertTrue(message.HasField('optional_string')) + self.assertTrue(message.HasField('optional_bytes')) + + self.assertTrue(message.HasField('optionalgroup')) + self.assertTrue(message.HasField('optional_nested_message')) + self.assertTrue(message.HasField('optional_foreign_message')) + self.assertTrue(message.HasField('optional_import_message')) + + self.assertTrue(message.optionalgroup.HasField('a')) + self.assertTrue(message.optional_nested_message.HasField('bb')) + self.assertTrue(message.optional_foreign_message.HasField('c')) + self.assertTrue(message.optional_import_message.HasField('d')) + + self.assertTrue(message.HasField('optional_nested_enum')) + self.assertTrue(message.HasField('optional_foreign_enum')) + self.assertTrue(message.HasField('optional_import_enum')) + + self.assertTrue(message.HasField('optional_string_piece')) + self.assertTrue(message.HasField('optional_cord')) + + self.assertEqual(101, message.optional_int32) + self.assertEqual(102, message.optional_int64) + self.assertEqual(103, message.optional_uint32) + self.assertEqual(104, message.optional_uint64) + self.assertEqual(105, message.optional_sint32) + self.assertEqual(106, message.optional_sint64) + self.assertEqual(107, message.optional_fixed32) + self.assertEqual(108, message.optional_fixed64) + self.assertEqual(109, message.optional_sfixed32) + self.assertEqual(110, message.optional_sfixed64) + self.assertEqual(111, message.optional_float) + self.assertEqual(112, message.optional_double) + self.assertEqual(True, message.optional_bool) + self.assertEqual('115', message.optional_string) + self.assertEqual('116', message.optional_bytes) + + self.assertEqual(117, message.optionalgroup.a); + self.assertEqual(118, message.optional_nested_message.bb) + self.assertEqual(119, message.optional_foreign_message.c) + self.assertEqual(120, message.optional_import_message.d) + + self.assertEqual(unittest_pb2.TestAllTypes.BAZ, + message.optional_nested_enum) + self.assertEqual(unittest_pb2.FOREIGN_BAZ, message.optional_foreign_enum) + self.assertEqual(unittest_import_pb2.IMPORT_BAZ, + message.optional_import_enum) + + # ----------------------------------------------------------------- + + self.assertEqual(2, len(message.repeated_int32)) + self.assertEqual(2, len(message.repeated_int64)) + self.assertEqual(2, len(message.repeated_uint32)) + self.assertEqual(2, len(message.repeated_uint64)) + self.assertEqual(2, len(message.repeated_sint32)) + self.assertEqual(2, len(message.repeated_sint64)) + self.assertEqual(2, len(message.repeated_fixed32)) + self.assertEqual(2, len(message.repeated_fixed64)) + self.assertEqual(2, len(message.repeated_sfixed32)) + self.assertEqual(2, len(message.repeated_sfixed64)) + self.assertEqual(2, len(message.repeated_float)) + self.assertEqual(2, len(message.repeated_double)) + self.assertEqual(2, len(message.repeated_bool)) + self.assertEqual(2, len(message.repeated_string)) + self.assertEqual(2, len(message.repeated_bytes)) + + self.assertEqual(2, len(message.repeatedgroup)) + self.assertEqual(2, len(message.repeated_nested_message)) + self.assertEqual(2, len(message.repeated_foreign_message)) + self.assertEqual(2, len(message.repeated_import_message)) + self.assertEqual(2, len(message.repeated_nested_enum)) + self.assertEqual(2, len(message.repeated_foreign_enum)) + self.assertEqual(2, len(message.repeated_import_enum)) + + self.assertEqual(2, len(message.repeated_string_piece)) + self.assertEqual(2, len(message.repeated_cord)) + + self.assertEqual(201, message.repeated_int32[0]) + self.assertEqual(202, message.repeated_int64[0]) + self.assertEqual(203, message.repeated_uint32[0]) + self.assertEqual(204, message.repeated_uint64[0]) + self.assertEqual(205, message.repeated_sint32[0]) + self.assertEqual(206, message.repeated_sint64[0]) + self.assertEqual(207, message.repeated_fixed32[0]) + self.assertEqual(208, message.repeated_fixed64[0]) + self.assertEqual(209, message.repeated_sfixed32[0]) + self.assertEqual(210, message.repeated_sfixed64[0]) + self.assertEqual(211, message.repeated_float[0]) + self.assertEqual(212, message.repeated_double[0]) + self.assertEqual(True, message.repeated_bool[0]) + self.assertEqual('215', message.repeated_string[0]) + self.assertEqual('216', message.repeated_bytes[0]) + + self.assertEqual(217, message.repeatedgroup[0].a) + self.assertEqual(218, message.repeated_nested_message[0].bb) + self.assertEqual(219, message.repeated_foreign_message[0].c) + self.assertEqual(220, message.repeated_import_message[0].d) + + self.assertEqual(unittest_pb2.TestAllTypes.BAR, + message.repeated_nested_enum[0]) + self.assertEqual(unittest_pb2.FOREIGN_BAR, + message.repeated_foreign_enum[0]) + self.assertEqual(unittest_import_pb2.IMPORT_BAR, + message.repeated_import_enum[0]) + + self.assertEqual(301, message.repeated_int32[1]) + self.assertEqual(302, message.repeated_int64[1]) + self.assertEqual(303, message.repeated_uint32[1]) + self.assertEqual(304, message.repeated_uint64[1]) + self.assertEqual(305, message.repeated_sint32[1]) + self.assertEqual(306, message.repeated_sint64[1]) + self.assertEqual(307, message.repeated_fixed32[1]) + self.assertEqual(308, message.repeated_fixed64[1]) + self.assertEqual(309, message.repeated_sfixed32[1]) + self.assertEqual(310, message.repeated_sfixed64[1]) + self.assertEqual(311, message.repeated_float[1]) + self.assertEqual(312, message.repeated_double[1]) + self.assertEqual(False, message.repeated_bool[1]) + self.assertEqual('315', message.repeated_string[1]) + self.assertEqual('316', message.repeated_bytes[1]) + + self.assertEqual(317, message.repeatedgroup[1].a) + self.assertEqual(318, message.repeated_nested_message[1].bb) + self.assertEqual(319, message.repeated_foreign_message[1].c) + self.assertEqual(320, message.repeated_import_message[1].d) + + self.assertEqual(unittest_pb2.TestAllTypes.BAZ, + message.repeated_nested_enum[1]) + self.assertEqual(unittest_pb2.FOREIGN_BAZ, + message.repeated_foreign_enum[1]) + self.assertEqual(unittest_import_pb2.IMPORT_BAZ, + message.repeated_import_enum[1]) + + # ----------------------------------------------------------------- + + self.assertTrue(message.HasField('default_int32')) + self.assertTrue(message.HasField('default_int64')) + self.assertTrue(message.HasField('default_uint32')) + self.assertTrue(message.HasField('default_uint64')) + self.assertTrue(message.HasField('default_sint32')) + self.assertTrue(message.HasField('default_sint64')) + self.assertTrue(message.HasField('default_fixed32')) + self.assertTrue(message.HasField('default_fixed64')) + self.assertTrue(message.HasField('default_sfixed32')) + self.assertTrue(message.HasField('default_sfixed64')) + self.assertTrue(message.HasField('default_float')) + self.assertTrue(message.HasField('default_double')) + self.assertTrue(message.HasField('default_bool')) + self.assertTrue(message.HasField('default_string')) + self.assertTrue(message.HasField('default_bytes')) + + self.assertTrue(message.HasField('default_nested_enum')) + self.assertTrue(message.HasField('default_foreign_enum')) + self.assertTrue(message.HasField('default_import_enum')) + + self.assertEqual(401, message.default_int32) + self.assertEqual(402, message.default_int64) + self.assertEqual(403, message.default_uint32) + self.assertEqual(404, message.default_uint64) + self.assertEqual(405, message.default_sint32) + self.assertEqual(406, message.default_sint64) + self.assertEqual(407, message.default_fixed32) + self.assertEqual(408, message.default_fixed64) + self.assertEqual(409, message.default_sfixed32) + self.assertEqual(410, message.default_sfixed64) + self.assertEqual(411, message.default_float) + self.assertEqual(412, message.default_double) + self.assertEqual(False, message.default_bool) + self.assertEqual('415', message.default_string) + self.assertEqual('416', message.default_bytes) + + self.assertEqual(unittest_pb2.TestAllTypes.FOO, message.default_nested_enum) + self.assertEqual(unittest_pb2.FOREIGN_FOO, message.default_foreign_enum) + self.assertEqual(unittest_import_pb2.IMPORT_FOO, + message.default_import_enum) + +def GoldenFile(filename): + """Finds the given golden file and returns a file object representing it.""" + + # Search up the directory tree looking for the C++ protobuf source code. + path = '.' + while os.path.exists(path): + if os.path.exists(os.path.join(path, 'src/google/protobuf')): + # Found it. Load the golden file from the testdata directory. + full_path = os.path.join(path, 'src/google/protobuf/testdata', filename) + return open(full_path, 'rb') + path = os.path.join(path, '..') + + raise RuntimeError( + 'Could not find golden files. This test must be run from within the ' + 'protobuf source package so that it can read test data files from the ' + 'C++ source tree.') + + +def SetAllPackedFields(message): + """Sets every field in the message to a unique value. + + Args: + message: A unittest_pb2.TestPackedTypes instance. + """ + message.packed_int32.extend([101, 102]) + message.packed_int64.extend([103, 104]) + message.packed_uint32.extend([105, 106]) + message.packed_uint64.extend([107, 108]) + message.packed_sint32.extend([109, 110]) + message.packed_sint64.extend([111, 112]) + message.packed_fixed32.extend([113, 114]) + message.packed_fixed64.extend([115, 116]) + message.packed_sfixed32.extend([117, 118]) + message.packed_sfixed64.extend([119, 120]) + message.packed_float.extend([121.0, 122.0]) + message.packed_double.extend([122.0, 123.0]) + message.packed_bool.extend([True, False]) + message.packed_enum.extend([unittest_pb2.FOREIGN_FOO, + unittest_pb2.FOREIGN_BAR]) + + +def SetAllPackedExtensions(message): + """Sets every extension in the message to a unique value. + + Args: + message: A unittest_pb2.TestPackedExtensions instance. + """ + extensions = message.Extensions + pb2 = unittest_pb2 + + extensions[pb2.packed_int32_extension].append(101) + extensions[pb2.packed_int64_extension].append(102) + extensions[pb2.packed_uint32_extension].append(103) + extensions[pb2.packed_uint64_extension].append(104) + extensions[pb2.packed_sint32_extension].append(105) + extensions[pb2.packed_sint64_extension].append(106) + extensions[pb2.packed_fixed32_extension].append(107) + extensions[pb2.packed_fixed64_extension].append(108) + extensions[pb2.packed_sfixed32_extension].append(109) + extensions[pb2.packed_sfixed64_extension].append(110) + extensions[pb2.packed_float_extension].append(111.0) + extensions[pb2.packed_double_extension].append(112.0) + extensions[pb2.packed_bool_extension].append(True) + extensions[pb2.packed_enum_extension].append(pb2.FOREIGN_BAZ) diff --git a/python/google/protobuf/internal/text_format_test.py b/python/google/protobuf/internal/text_format_test.py new file mode 100755 index 0000000..0cf2718 --- /dev/null +++ b/python/google/protobuf/internal/text_format_test.py @@ -0,0 +1,397 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Test for google.protobuf.text_format.""" + +__author__ = 'kenton@google.com (Kenton Varda)' + +import difflib + +import unittest +from google.protobuf import text_format +from google.protobuf.internal import test_util +from google.protobuf import unittest_pb2 +from google.protobuf import unittest_mset_pb2 + + +class TextFormatTest(test_util.GoldenMessageTestCase): + def ReadGolden(self, golden_filename): + f = test_util.GoldenFile(golden_filename) + golden_lines = f.readlines() + f.close() + return golden_lines + + def CompareToGoldenFile(self, text, golden_filename): + golden_lines = self.ReadGolden(golden_filename) + self.CompareToGoldenLines(text, golden_lines) + + def CompareToGoldenText(self, text, golden_text): + self.CompareToGoldenLines(text, golden_text.splitlines(1)) + + def CompareToGoldenLines(self, text, golden_lines): + actual_lines = text.splitlines(1) + self.assertEqual(golden_lines, actual_lines, + "Text doesn't match golden. Diff:\n" + + ''.join(difflib.ndiff(golden_lines, actual_lines))) + + def testPrintAllFields(self): + message = unittest_pb2.TestAllTypes() + test_util.SetAllFields(message) + self.CompareToGoldenFile( + self.RemoveRedundantZeros(text_format.MessageToString(message)), + 'text_format_unittest_data.txt') + + def testPrintAllExtensions(self): + message = unittest_pb2.TestAllExtensions() + test_util.SetAllExtensions(message) + self.CompareToGoldenFile( + self.RemoveRedundantZeros(text_format.MessageToString(message)), + 'text_format_unittest_extensions_data.txt') + + def testPrintMessageSet(self): + message = unittest_mset_pb2.TestMessageSetContainer() + ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension + ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension + message.message_set.Extensions[ext1].i = 23 + message.message_set.Extensions[ext2].str = 'foo' + self.CompareToGoldenText(text_format.MessageToString(message), + 'message_set {\n' + ' [protobuf_unittest.TestMessageSetExtension1] {\n' + ' i: 23\n' + ' }\n' + ' [protobuf_unittest.TestMessageSetExtension2] {\n' + ' str: \"foo\"\n' + ' }\n' + '}\n') + + def testPrintExotic(self): + message = unittest_pb2.TestAllTypes() + message.repeated_int64.append(-9223372036854775808); + message.repeated_uint64.append(18446744073709551615); + message.repeated_double.append(123.456); + message.repeated_double.append(1.23e22); + message.repeated_double.append(1.23e-18); + message.repeated_string.append('\000\001\a\b\f\n\r\t\v\\\'\"'); + self.CompareToGoldenText( + self.RemoveRedundantZeros(text_format.MessageToString(message)), + 'repeated_int64: -9223372036854775808\n' + 'repeated_uint64: 18446744073709551615\n' + 'repeated_double: 123.456\n' + 'repeated_double: 1.23e+22\n' + 'repeated_double: 1.23e-18\n' + 'repeated_string: ' + '\"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\\"\"\n') + + def testMessageToString(self): + message = unittest_pb2.ForeignMessage() + message.c = 123 + self.assertEqual('c: 123\n', str(message)) + + def RemoveRedundantZeros(self, text): + # Some platforms print 1e+5 as 1e+005. This is fine, but we need to remove + # these zeros in order to match the golden file. + return text.replace('e+0','e+').replace('e+0','e+') \ + .replace('e-0','e-').replace('e-0','e-') + + def testMergeGolden(self): + golden_text = '\n'.join(self.ReadGolden('text_format_unittest_data.txt')) + parsed_message = unittest_pb2.TestAllTypes() + text_format.Merge(golden_text, parsed_message) + + message = unittest_pb2.TestAllTypes() + test_util.SetAllFields(message) + self.assertEquals(message, parsed_message) + + def testMergeGoldenExtensions(self): + golden_text = '\n'.join(self.ReadGolden( + 'text_format_unittest_extensions_data.txt')) + parsed_message = unittest_pb2.TestAllExtensions() + text_format.Merge(golden_text, parsed_message) + + message = unittest_pb2.TestAllExtensions() + test_util.SetAllExtensions(message) + self.assertEquals(message, parsed_message) + + def testMergeAllFields(self): + message = unittest_pb2.TestAllTypes() + test_util.SetAllFields(message) + ascii_text = text_format.MessageToString(message) + + parsed_message = unittest_pb2.TestAllTypes() + text_format.Merge(ascii_text, parsed_message) + self.assertEqual(message, parsed_message) + self.ExpectAllFieldsSet(message) + + def testMergeAllExtensions(self): + message = unittest_pb2.TestAllExtensions() + test_util.SetAllExtensions(message) + ascii_text = text_format.MessageToString(message) + + parsed_message = unittest_pb2.TestAllExtensions() + text_format.Merge(ascii_text, parsed_message) + self.assertEqual(message, parsed_message) + + def testMergeMessageSet(self): + message = unittest_pb2.TestAllTypes() + text = ('repeated_uint64: 1\n' + 'repeated_uint64: 2\n') + text_format.Merge(text, message) + self.assertEqual(1, message.repeated_uint64[0]) + self.assertEqual(2, message.repeated_uint64[1]) + + message = unittest_mset_pb2.TestMessageSetContainer() + text = ('message_set {\n' + ' [protobuf_unittest.TestMessageSetExtension1] {\n' + ' i: 23\n' + ' }\n' + ' [protobuf_unittest.TestMessageSetExtension2] {\n' + ' str: \"foo\"\n' + ' }\n' + '}\n') + text_format.Merge(text, message) + ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension + ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension + self.assertEquals(23, message.message_set.Extensions[ext1].i) + self.assertEquals('foo', message.message_set.Extensions[ext2].str) + + def testMergeExotic(self): + message = unittest_pb2.TestAllTypes() + text = ('repeated_int64: -9223372036854775808\n' + 'repeated_uint64: 18446744073709551615\n' + 'repeated_double: 123.456\n' + 'repeated_double: 1.23e+22\n' + 'repeated_double: 1.23e-18\n' + 'repeated_string: \n' + '\"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\\"\"\n') + text_format.Merge(text, message) + + self.assertEqual(-9223372036854775808, message.repeated_int64[0]) + self.assertEqual(18446744073709551615, message.repeated_uint64[0]) + self.assertEqual(123.456, message.repeated_double[0]) + self.assertEqual(1.23e22, message.repeated_double[1]) + self.assertEqual(1.23e-18, message.repeated_double[2]) + self.assertEqual( + '\000\001\a\b\f\n\r\t\v\\\'\"', message.repeated_string[0]) + + def testMergeUnknownField(self): + message = unittest_pb2.TestAllTypes() + text = 'unknown_field: 8\n' + self.assertRaisesWithMessage( + text_format.ParseError, + ('1:1 : Message type "protobuf_unittest.TestAllTypes" has no field named ' + '"unknown_field".'), + text_format.Merge, text, message) + + def testMergeBadExtension(self): + message = unittest_pb2.TestAllTypes() + text = '[unknown_extension]: 8\n' + self.assertRaisesWithMessage( + text_format.ParseError, + '1:2 : Extension "unknown_extension" not registered.', + text_format.Merge, text, message) + + def testMergeGroupNotClosed(self): + message = unittest_pb2.TestAllTypes() + text = 'RepeatedGroup: <' + self.assertRaisesWithMessage( + text_format.ParseError, '1:16 : Expected ">".', + text_format.Merge, text, message) + + text = 'RepeatedGroup: {' + self.assertRaisesWithMessage( + text_format.ParseError, '1:16 : Expected "}".', + text_format.Merge, text, message) + + def testMergeBadEnumValue(self): + message = unittest_pb2.TestAllTypes() + text = 'optional_nested_enum: BARR' + self.assertRaisesWithMessage( + text_format.ParseError, + ('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" ' + 'has no value named BARR.'), + text_format.Merge, text, message) + + message = unittest_pb2.TestAllTypes() + text = 'optional_nested_enum: 100' + self.assertRaisesWithMessage( + text_format.ParseError, + ('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" ' + 'has no value with number 100.'), + text_format.Merge, text, message) + + def assertRaisesWithMessage(self, e_class, e, func, *args, **kwargs): + """Same as assertRaises, but also compares the exception message.""" + if hasattr(e_class, '__name__'): + exc_name = e_class.__name__ + else: + exc_name = str(e_class) + + try: + func(*args, **kwargs) + except e_class, expr: + if str(expr) != e: + msg = '%s raised, but with wrong message: "%s" instead of "%s"' + raise self.failureException(msg % (exc_name, + str(expr).encode('string_escape'), + e.encode('string_escape'))) + return + else: + raise self.failureException('%s not raised' % exc_name) + + +class TokenizerTest(unittest.TestCase): + + def testSimpleTokenCases(self): + text = ('identifier1:"string1"\n \n\n' + 'identifier2 : \n \n123 \n identifier3 :\'string\'\n' + 'identifiER_4 : 1.1e+2 ID5:-0.23 ID6:\'aaaa\\\'bbbb\'\n' + 'ID7 : "aa\\"bb"\n\n\n\n ID8: {A:inf B:-inf C:true D:false}\n' + 'ID9: 22 ID10: -111111111111111111 ID11: -22\n' + 'ID12: 2222222222222222222') + tokenizer = text_format._Tokenizer(text) + methods = [(tokenizer.ConsumeIdentifier, 'identifier1'), + ':', + (tokenizer.ConsumeString, 'string1'), + (tokenizer.ConsumeIdentifier, 'identifier2'), + ':', + (tokenizer.ConsumeInt32, 123), + (tokenizer.ConsumeIdentifier, 'identifier3'), + ':', + (tokenizer.ConsumeString, 'string'), + (tokenizer.ConsumeIdentifier, 'identifiER_4'), + ':', + (tokenizer.ConsumeFloat, 1.1e+2), + (tokenizer.ConsumeIdentifier, 'ID5'), + ':', + (tokenizer.ConsumeFloat, -0.23), + (tokenizer.ConsumeIdentifier, 'ID6'), + ':', + (tokenizer.ConsumeString, 'aaaa\'bbbb'), + (tokenizer.ConsumeIdentifier, 'ID7'), + ':', + (tokenizer.ConsumeString, 'aa\"bb'), + (tokenizer.ConsumeIdentifier, 'ID8'), + ':', + '{', + (tokenizer.ConsumeIdentifier, 'A'), + ':', + (tokenizer.ConsumeFloat, float('inf')), + (tokenizer.ConsumeIdentifier, 'B'), + ':', + (tokenizer.ConsumeFloat, float('-inf')), + (tokenizer.ConsumeIdentifier, 'C'), + ':', + (tokenizer.ConsumeBool, True), + (tokenizer.ConsumeIdentifier, 'D'), + ':', + (tokenizer.ConsumeBool, False), + '}', + (tokenizer.ConsumeIdentifier, 'ID9'), + ':', + (tokenizer.ConsumeUint32, 22), + (tokenizer.ConsumeIdentifier, 'ID10'), + ':', + (tokenizer.ConsumeInt64, -111111111111111111), + (tokenizer.ConsumeIdentifier, 'ID11'), + ':', + (tokenizer.ConsumeInt32, -22), + (tokenizer.ConsumeIdentifier, 'ID12'), + ':', + (tokenizer.ConsumeUint64, 2222222222222222222)] + + i = 0 + while not tokenizer.AtEnd(): + m = methods[i] + if type(m) == str: + token = tokenizer.token + self.assertEqual(token, m) + tokenizer.NextToken() + else: + self.assertEqual(m[1], m[0]()) + i += 1 + + def testConsumeIntegers(self): + # This test only tests the failures in the integer parsing methods as well + # as the '0' special cases. + int64_max = (1 << 63) - 1 + uint32_max = (1 << 32) - 1 + text = '-1 %d %d' % (uint32_max + 1, int64_max + 1) + tokenizer = text_format._Tokenizer(text) + self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint32) + self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint64) + self.assertEqual(-1, tokenizer.ConsumeInt32()) + + self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint32) + self.assertRaises(text_format.ParseError, tokenizer.ConsumeInt32) + self.assertEqual(uint32_max + 1, tokenizer.ConsumeInt64()) + + self.assertRaises(text_format.ParseError, tokenizer.ConsumeInt64) + self.assertEqual(int64_max + 1, tokenizer.ConsumeUint64()) + self.assertTrue(tokenizer.AtEnd()) + + text = '-0 -0 0 0' + tokenizer = text_format._Tokenizer(text) + self.assertEqual(0, tokenizer.ConsumeUint32()) + self.assertEqual(0, tokenizer.ConsumeUint64()) + self.assertEqual(0, tokenizer.ConsumeUint32()) + self.assertEqual(0, tokenizer.ConsumeUint64()) + self.assertTrue(tokenizer.AtEnd()) + + def testConsumeByteString(self): + text = '"string1\'' + tokenizer = text_format._Tokenizer(text) + self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) + + text = 'string1"' + tokenizer = text_format._Tokenizer(text) + self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) + + text = '\n"\\xt"' + tokenizer = text_format._Tokenizer(text) + self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) + + text = '\n"\\"' + tokenizer = text_format._Tokenizer(text) + self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) + + text = '\n"\\x"' + tokenizer = text_format._Tokenizer(text) + self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) + + def testConsumeBool(self): + text = 'not-a-bool' + tokenizer = text_format._Tokenizer(text) + self.assertRaises(text_format.ParseError, tokenizer.ConsumeBool) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/google/protobuf/internal/type_checkers.py b/python/google/protobuf/internal/type_checkers.py new file mode 100755 index 0000000..a3bc57f --- /dev/null +++ b/python/google/protobuf/internal/type_checkers.py @@ -0,0 +1,287 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Provides type checking routines. + +This module defines type checking utilities in the forms of dictionaries: + +VALUE_CHECKERS: A dictionary of field types and a value validation object. +TYPE_TO_BYTE_SIZE_FN: A dictionary with field types and a size computing + function. +TYPE_TO_SERIALIZE_METHOD: A dictionary with field types and serialization + function. +FIELD_TYPE_TO_WIRE_TYPE: A dictionary with field typed and their + coresponding wire types. +TYPE_TO_DESERIALIZE_METHOD: A dictionary with field types and deserialization + function. +""" + +__author__ = 'robinson@google.com (Will Robinson)' + +from google.protobuf.internal import decoder +from google.protobuf.internal import encoder +from google.protobuf.internal import wire_format +from google.protobuf import descriptor + +_FieldDescriptor = descriptor.FieldDescriptor + + +def GetTypeChecker(cpp_type, field_type): + """Returns a type checker for a message field of the specified types. + + Args: + cpp_type: C++ type of the field (see descriptor.py). + field_type: Protocol message field type (see descriptor.py). + + Returns: + An instance of TypeChecker which can be used to verify the types + of values assigned to a field of the specified type. + """ + if (cpp_type == _FieldDescriptor.CPPTYPE_STRING and + field_type == _FieldDescriptor.TYPE_STRING): + return UnicodeValueChecker() + return _VALUE_CHECKERS[cpp_type] + + +# None of the typecheckers below make any attempt to guard against people +# subclassing builtin types and doing weird things. We're not trying to +# protect against malicious clients here, just people accidentally shooting +# themselves in the foot in obvious ways. + +class TypeChecker(object): + + """Type checker used to catch type errors as early as possible + when the client is setting scalar fields in protocol messages. + """ + + def __init__(self, *acceptable_types): + self._acceptable_types = acceptable_types + + def CheckValue(self, proposed_value): + if not isinstance(proposed_value, self._acceptable_types): + message = ('%.1024r has type %s, but expected one of: %s' % + (proposed_value, type(proposed_value), self._acceptable_types)) + raise TypeError(message) + + +# IntValueChecker and its subclasses perform integer type-checks +# and bounds-checks. +class IntValueChecker(object): + + """Checker used for integer fields. Performs type-check and range check.""" + + def CheckValue(self, proposed_value): + if not isinstance(proposed_value, (int, long)): + message = ('%.1024r has type %s, but expected one of: %s' % + (proposed_value, type(proposed_value), (int, long))) + raise TypeError(message) + if not self._MIN <= proposed_value <= self._MAX: + raise ValueError('Value out of range: %d' % proposed_value) + + +class UnicodeValueChecker(object): + + """Checker used for string fields.""" + + def CheckValue(self, proposed_value): + if not isinstance(proposed_value, (str, unicode)): + message = ('%.1024r has type %s, but expected one of: %s' % + (proposed_value, type(proposed_value), (str, unicode))) + raise TypeError(message) + + # If the value is of type 'str' make sure that it is in 7-bit ASCII + # encoding. + if isinstance(proposed_value, str): + try: + unicode(proposed_value, 'ascii') + except UnicodeDecodeError: + raise ValueError('%.1024r has type str, but isn\'t in 7-bit ASCII ' + 'encoding. Non-ASCII strings must be converted to ' + 'unicode objects before being added.' % + (proposed_value)) + + +class Int32ValueChecker(IntValueChecker): + # We're sure to use ints instead of longs here since comparison may be more + # efficient. + _MIN = -2147483648 + _MAX = 2147483647 + + +class Uint32ValueChecker(IntValueChecker): + _MIN = 0 + _MAX = (1 << 32) - 1 + + +class Int64ValueChecker(IntValueChecker): + _MIN = -(1 << 63) + _MAX = (1 << 63) - 1 + + +class Uint64ValueChecker(IntValueChecker): + _MIN = 0 + _MAX = (1 << 64) - 1 + + +# Type-checkers for all scalar CPPTYPEs. +_VALUE_CHECKERS = { + _FieldDescriptor.CPPTYPE_INT32: Int32ValueChecker(), + _FieldDescriptor.CPPTYPE_INT64: Int64ValueChecker(), + _FieldDescriptor.CPPTYPE_UINT32: Uint32ValueChecker(), + _FieldDescriptor.CPPTYPE_UINT64: Uint64ValueChecker(), + _FieldDescriptor.CPPTYPE_DOUBLE: TypeChecker( + float, int, long), + _FieldDescriptor.CPPTYPE_FLOAT: TypeChecker( + float, int, long), + _FieldDescriptor.CPPTYPE_BOOL: TypeChecker(bool, int), + _FieldDescriptor.CPPTYPE_ENUM: Int32ValueChecker(), + _FieldDescriptor.CPPTYPE_STRING: TypeChecker(str), + } + + +# Map from field type to a function F, such that F(field_num, value) +# gives the total byte size for a value of the given type. This +# byte size includes tag information and any other additional space +# associated with serializing "value". +TYPE_TO_BYTE_SIZE_FN = { + _FieldDescriptor.TYPE_DOUBLE: wire_format.DoubleByteSize, + _FieldDescriptor.TYPE_FLOAT: wire_format.FloatByteSize, + _FieldDescriptor.TYPE_INT64: wire_format.Int64ByteSize, + _FieldDescriptor.TYPE_UINT64: wire_format.UInt64ByteSize, + _FieldDescriptor.TYPE_INT32: wire_format.Int32ByteSize, + _FieldDescriptor.TYPE_FIXED64: wire_format.Fixed64ByteSize, + _FieldDescriptor.TYPE_FIXED32: wire_format.Fixed32ByteSize, + _FieldDescriptor.TYPE_BOOL: wire_format.BoolByteSize, + _FieldDescriptor.TYPE_STRING: wire_format.StringByteSize, + _FieldDescriptor.TYPE_GROUP: wire_format.GroupByteSize, + _FieldDescriptor.TYPE_MESSAGE: wire_format.MessageByteSize, + _FieldDescriptor.TYPE_BYTES: wire_format.BytesByteSize, + _FieldDescriptor.TYPE_UINT32: wire_format.UInt32ByteSize, + _FieldDescriptor.TYPE_ENUM: wire_format.EnumByteSize, + _FieldDescriptor.TYPE_SFIXED32: wire_format.SFixed32ByteSize, + _FieldDescriptor.TYPE_SFIXED64: wire_format.SFixed64ByteSize, + _FieldDescriptor.TYPE_SINT32: wire_format.SInt32ByteSize, + _FieldDescriptor.TYPE_SINT64: wire_format.SInt64ByteSize + } + + +# Maps from field type to an unbound Encoder method F, such that +# F(encoder, field_number, value) will append the serialization +# of a value of this type to the encoder. +_Encoder = encoder.Encoder +TYPE_TO_SERIALIZE_METHOD = { + _FieldDescriptor.TYPE_DOUBLE: _Encoder.AppendDouble, + _FieldDescriptor.TYPE_FLOAT: _Encoder.AppendFloat, + _FieldDescriptor.TYPE_INT64: _Encoder.AppendInt64, + _FieldDescriptor.TYPE_UINT64: _Encoder.AppendUInt64, + _FieldDescriptor.TYPE_INT32: _Encoder.AppendInt32, + _FieldDescriptor.TYPE_FIXED64: _Encoder.AppendFixed64, + _FieldDescriptor.TYPE_FIXED32: _Encoder.AppendFixed32, + _FieldDescriptor.TYPE_BOOL: _Encoder.AppendBool, + _FieldDescriptor.TYPE_STRING: _Encoder.AppendString, + _FieldDescriptor.TYPE_GROUP: _Encoder.AppendGroup, + _FieldDescriptor.TYPE_MESSAGE: _Encoder.AppendMessage, + _FieldDescriptor.TYPE_BYTES: _Encoder.AppendBytes, + _FieldDescriptor.TYPE_UINT32: _Encoder.AppendUInt32, + _FieldDescriptor.TYPE_ENUM: _Encoder.AppendEnum, + _FieldDescriptor.TYPE_SFIXED32: _Encoder.AppendSFixed32, + _FieldDescriptor.TYPE_SFIXED64: _Encoder.AppendSFixed64, + _FieldDescriptor.TYPE_SINT32: _Encoder.AppendSInt32, + _FieldDescriptor.TYPE_SINT64: _Encoder.AppendSInt64, + } + + +TYPE_TO_NOTAG_SERIALIZE_METHOD = { + _FieldDescriptor.TYPE_DOUBLE: _Encoder.AppendDoubleNoTag, + _FieldDescriptor.TYPE_FLOAT: _Encoder.AppendFloatNoTag, + _FieldDescriptor.TYPE_INT64: _Encoder.AppendInt64NoTag, + _FieldDescriptor.TYPE_UINT64: _Encoder.AppendUInt64NoTag, + _FieldDescriptor.TYPE_INT32: _Encoder.AppendInt32NoTag, + _FieldDescriptor.TYPE_FIXED64: _Encoder.AppendFixed64NoTag, + _FieldDescriptor.TYPE_FIXED32: _Encoder.AppendFixed32NoTag, + _FieldDescriptor.TYPE_BOOL: _Encoder.AppendBoolNoTag, + _FieldDescriptor.TYPE_UINT32: _Encoder.AppendUInt32NoTag, + _FieldDescriptor.TYPE_ENUM: _Encoder.AppendEnumNoTag, + _FieldDescriptor.TYPE_SFIXED32: _Encoder.AppendSFixed32NoTag, + _FieldDescriptor.TYPE_SFIXED64: _Encoder.AppendSFixed64NoTag, + _FieldDescriptor.TYPE_SINT32: _Encoder.AppendSInt32NoTag, + _FieldDescriptor.TYPE_SINT64: _Encoder.AppendSInt64NoTag, + } + +# Maps from field type to expected wiretype. +FIELD_TYPE_TO_WIRE_TYPE = { + _FieldDescriptor.TYPE_DOUBLE: wire_format.WIRETYPE_FIXED64, + _FieldDescriptor.TYPE_FLOAT: wire_format.WIRETYPE_FIXED32, + _FieldDescriptor.TYPE_INT64: wire_format.WIRETYPE_VARINT, + _FieldDescriptor.TYPE_UINT64: wire_format.WIRETYPE_VARINT, + _FieldDescriptor.TYPE_INT32: wire_format.WIRETYPE_VARINT, + _FieldDescriptor.TYPE_FIXED64: wire_format.WIRETYPE_FIXED64, + _FieldDescriptor.TYPE_FIXED32: wire_format.WIRETYPE_FIXED32, + _FieldDescriptor.TYPE_BOOL: wire_format.WIRETYPE_VARINT, + _FieldDescriptor.TYPE_STRING: + wire_format.WIRETYPE_LENGTH_DELIMITED, + _FieldDescriptor.TYPE_GROUP: wire_format.WIRETYPE_START_GROUP, + _FieldDescriptor.TYPE_MESSAGE: + wire_format.WIRETYPE_LENGTH_DELIMITED, + _FieldDescriptor.TYPE_BYTES: + wire_format.WIRETYPE_LENGTH_DELIMITED, + _FieldDescriptor.TYPE_UINT32: wire_format.WIRETYPE_VARINT, + _FieldDescriptor.TYPE_ENUM: wire_format.WIRETYPE_VARINT, + _FieldDescriptor.TYPE_SFIXED32: wire_format.WIRETYPE_FIXED32, + _FieldDescriptor.TYPE_SFIXED64: wire_format.WIRETYPE_FIXED64, + _FieldDescriptor.TYPE_SINT32: wire_format.WIRETYPE_VARINT, + _FieldDescriptor.TYPE_SINT64: wire_format.WIRETYPE_VARINT, + } + + +# Maps from field type to an unbound Decoder method F, +# such that F(decoder) will read a field of the requested type. +# +# Note that Message and Group are intentionally missing here. +# They're handled by _RecursivelyMerge(). +_Decoder = decoder.Decoder +TYPE_TO_DESERIALIZE_METHOD = { + _FieldDescriptor.TYPE_DOUBLE: _Decoder.ReadDouble, + _FieldDescriptor.TYPE_FLOAT: _Decoder.ReadFloat, + _FieldDescriptor.TYPE_INT64: _Decoder.ReadInt64, + _FieldDescriptor.TYPE_UINT64: _Decoder.ReadUInt64, + _FieldDescriptor.TYPE_INT32: _Decoder.ReadInt32, + _FieldDescriptor.TYPE_FIXED64: _Decoder.ReadFixed64, + _FieldDescriptor.TYPE_FIXED32: _Decoder.ReadFixed32, + _FieldDescriptor.TYPE_BOOL: _Decoder.ReadBool, + _FieldDescriptor.TYPE_STRING: _Decoder.ReadString, + _FieldDescriptor.TYPE_BYTES: _Decoder.ReadBytes, + _FieldDescriptor.TYPE_UINT32: _Decoder.ReadUInt32, + _FieldDescriptor.TYPE_ENUM: _Decoder.ReadEnum, + _FieldDescriptor.TYPE_SFIXED32: _Decoder.ReadSFixed32, + _FieldDescriptor.TYPE_SFIXED64: _Decoder.ReadSFixed64, + _FieldDescriptor.TYPE_SINT32: _Decoder.ReadSInt32, + _FieldDescriptor.TYPE_SINT64: _Decoder.ReadSInt64, + } diff --git a/python/google/protobuf/internal/wire_format.py b/python/google/protobuf/internal/wire_format.py new file mode 100755 index 0000000..da6464d --- /dev/null +++ b/python/google/protobuf/internal/wire_format.py @@ -0,0 +1,247 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Constants and static functions to support protocol buffer wire format.""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import struct +from google.protobuf import message + + +TAG_TYPE_BITS = 3 # Number of bits used to hold type info in a proto tag. +_TAG_TYPE_MASK = (1 << TAG_TYPE_BITS) - 1 # 0x7 + +# These numbers identify the wire type of a protocol buffer value. +# We use the least-significant TAG_TYPE_BITS bits of the varint-encoded +# tag-and-type to store one of these WIRETYPE_* constants. +# These values must match WireType enum in //net/proto2/public/wire_format.h. +WIRETYPE_VARINT = 0 +WIRETYPE_FIXED64 = 1 +WIRETYPE_LENGTH_DELIMITED = 2 +WIRETYPE_START_GROUP = 3 +WIRETYPE_END_GROUP = 4 +WIRETYPE_FIXED32 = 5 +_WIRETYPE_MAX = 5 + + +# Bounds for various integer types. +INT32_MAX = int((1 << 31) - 1) +INT32_MIN = int(-(1 << 31)) +UINT32_MAX = (1 << 32) - 1 + +INT64_MAX = (1 << 63) - 1 +INT64_MIN = -(1 << 63) +UINT64_MAX = (1 << 64) - 1 + +# "struct" format strings that will encode/decode the specified formats. +FORMAT_UINT32_LITTLE_ENDIAN = '<I' +FORMAT_UINT64_LITTLE_ENDIAN = '<Q' +FORMAT_FLOAT_LITTLE_ENDIAN = '<f' +FORMAT_DOUBLE_LITTLE_ENDIAN = '<d' + + +# We'll have to provide alternate implementations of AppendLittleEndian*() on +# any architectures where these checks fail. +if struct.calcsize(FORMAT_UINT32_LITTLE_ENDIAN) != 4: + raise AssertionError('Format "I" is not a 32-bit number.') +if struct.calcsize(FORMAT_UINT64_LITTLE_ENDIAN) != 8: + raise AssertionError('Format "Q" is not a 64-bit number.') + + +def PackTag(field_number, wire_type): + """Returns an unsigned 32-bit integer that encodes the field number and + wire type information in standard protocol message wire format. + + Args: + field_number: Expected to be an integer in the range [1, 1 << 29) + wire_type: One of the WIRETYPE_* constants. + """ + if not 0 <= wire_type <= _WIRETYPE_MAX: + raise message.EncodeError('Unknown wire type: %d' % wire_type) + return (field_number << TAG_TYPE_BITS) | wire_type + + +def UnpackTag(tag): + """The inverse of PackTag(). Given an unsigned 32-bit number, + returns a (field_number, wire_type) tuple. + """ + return (tag >> TAG_TYPE_BITS), (tag & _TAG_TYPE_MASK) + + +def ZigZagEncode(value): + """ZigZag Transform: Encodes signed integers so that they can be + effectively used with varint encoding. See wire_format.h for + more details. + """ + if value >= 0: + return value << 1 + return (value << 1) ^ (~0) + + +def ZigZagDecode(value): + """Inverse of ZigZagEncode().""" + if not value & 0x1: + return value >> 1 + return (value >> 1) ^ (~0) + + + +# The *ByteSize() functions below return the number of bytes required to +# serialize "field number + type" information and then serialize the value. + + +def Int32ByteSize(field_number, int32): + return Int64ByteSize(field_number, int32) + + +def Int32ByteSizeNoTag(int32): + return _VarUInt64ByteSizeNoTag(0xffffffffffffffff & int32) + + +def Int64ByteSize(field_number, int64): + # Have to convert to uint before calling UInt64ByteSize(). + return UInt64ByteSize(field_number, 0xffffffffffffffff & int64) + + +def UInt32ByteSize(field_number, uint32): + return UInt64ByteSize(field_number, uint32) + + +def UInt64ByteSize(field_number, uint64): + return TagByteSize(field_number) + _VarUInt64ByteSizeNoTag(uint64) + + +def SInt32ByteSize(field_number, int32): + return UInt32ByteSize(field_number, ZigZagEncode(int32)) + + +def SInt64ByteSize(field_number, int64): + return UInt64ByteSize(field_number, ZigZagEncode(int64)) + + +def Fixed32ByteSize(field_number, fixed32): + return TagByteSize(field_number) + 4 + + +def Fixed64ByteSize(field_number, fixed64): + return TagByteSize(field_number) + 8 + + +def SFixed32ByteSize(field_number, sfixed32): + return TagByteSize(field_number) + 4 + + +def SFixed64ByteSize(field_number, sfixed64): + return TagByteSize(field_number) + 8 + + +def FloatByteSize(field_number, flt): + return TagByteSize(field_number) + 4 + + +def DoubleByteSize(field_number, double): + return TagByteSize(field_number) + 8 + + +def BoolByteSize(field_number, b): + return TagByteSize(field_number) + 1 + + +def EnumByteSize(field_number, enum): + return UInt32ByteSize(field_number, enum) + + +def StringByteSize(field_number, string): + return BytesByteSize(field_number, string.encode('utf-8')) + + +def BytesByteSize(field_number, b): + return (TagByteSize(field_number) + + _VarUInt64ByteSizeNoTag(len(b)) + + len(b)) + + +def GroupByteSize(field_number, message): + return (2 * TagByteSize(field_number) # START and END group. + + message.ByteSize()) + + +def MessageByteSize(field_number, message): + return (TagByteSize(field_number) + + _VarUInt64ByteSizeNoTag(message.ByteSize()) + + message.ByteSize()) + + +def MessageSetItemByteSize(field_number, msg): + # First compute the sizes of the tags. + # There are 2 tags for the beginning and ending of the repeated group, that + # is field number 1, one with field number 2 (type_id) and one with field + # number 3 (message). + total_size = (2 * TagByteSize(1) + TagByteSize(2) + TagByteSize(3)) + + # Add the number of bytes for type_id. + total_size += _VarUInt64ByteSizeNoTag(field_number) + + message_size = msg.ByteSize() + + # The number of bytes for encoding the length of the message. + total_size += _VarUInt64ByteSizeNoTag(message_size) + + # The size of the message. + total_size += message_size + return total_size + + +def TagByteSize(field_number): + """Returns the bytes required to serialize a tag with this field number.""" + # Just pass in type 0, since the type won't affect the tag+type size. + return _VarUInt64ByteSizeNoTag(PackTag(field_number, 0)) + + +# Private helper function for the *ByteSize() functions above. + +def _VarUInt64ByteSizeNoTag(uint64): + """Returns the number of bytes required to serialize a single varint + using boundary value comparisons. (unrolled loop optimization -WPierce) + uint64 must be unsigned. + """ + if uint64 <= 0x7f: return 1 + if uint64 <= 0x3fff: return 2 + if uint64 <= 0x1fffff: return 3 + if uint64 <= 0xfffffff: return 4 + if uint64 <= 0x7ffffffff: return 5 + if uint64 <= 0x3ffffffffff: return 6 + if uint64 <= 0x1ffffffffffff: return 7 + if uint64 <= 0xffffffffffffff: return 8 + if uint64 <= 0x7fffffffffffffff: return 9 + if uint64 > UINT64_MAX: + raise message.EncodeError('Value out of range: %d' % uint64) + return 10 diff --git a/python/google/protobuf/internal/wire_format_test.py b/python/google/protobuf/internal/wire_format_test.py new file mode 100755 index 0000000..7600778 --- /dev/null +++ b/python/google/protobuf/internal/wire_format_test.py @@ -0,0 +1,253 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# http://code.google.com/p/protobuf/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Test for google.protobuf.internal.wire_format.""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import unittest +from google.protobuf import message +from google.protobuf.internal import wire_format + + +class WireFormatTest(unittest.TestCase): + + def testPackTag(self): + field_number = 0xabc + tag_type = 2 + self.assertEqual((field_number << 3) | tag_type, + wire_format.PackTag(field_number, tag_type)) + PackTag = wire_format.PackTag + # Number too high. + self.assertRaises(message.EncodeError, PackTag, field_number, 6) + # Number too low. + self.assertRaises(message.EncodeError, PackTag, field_number, -1) + + def testUnpackTag(self): + # Test field numbers that will require various varint sizes. + for expected_field_number in (1, 15, 16, 2047, 2048): + for expected_wire_type in range(6): # Highest-numbered wiretype is 5. + field_number, wire_type = wire_format.UnpackTag( + wire_format.PackTag(expected_field_number, expected_wire_type)) + self.assertEqual(expected_field_number, field_number) + self.assertEqual(expected_wire_type, wire_type) + + self.assertRaises(TypeError, wire_format.UnpackTag, None) + self.assertRaises(TypeError, wire_format.UnpackTag, 'abc') + self.assertRaises(TypeError, wire_format.UnpackTag, 0.0) + self.assertRaises(TypeError, wire_format.UnpackTag, object()) + + def testZigZagEncode(self): + Z = wire_format.ZigZagEncode + self.assertEqual(0, Z(0)) + self.assertEqual(1, Z(-1)) + self.assertEqual(2, Z(1)) + self.assertEqual(3, Z(-2)) + self.assertEqual(4, Z(2)) + self.assertEqual(0xfffffffe, Z(0x7fffffff)) + self.assertEqual(0xffffffff, Z(-0x80000000)) + self.assertEqual(0xfffffffffffffffe, Z(0x7fffffffffffffff)) + self.assertEqual(0xffffffffffffffff, Z(-0x8000000000000000)) + + self.assertRaises(TypeError, Z, None) + self.assertRaises(TypeError, Z, 'abcd') + self.assertRaises(TypeError, Z, 0.0) + self.assertRaises(TypeError, Z, object()) + + def testZigZagDecode(self): + Z = wire_format.ZigZagDecode + self.assertEqual(0, Z(0)) + self.assertEqual(-1, Z(1)) + self.assertEqual(1, Z(2)) + self.assertEqual(-2, Z(3)) + self.assertEqual(2, Z(4)) + self.assertEqual(0x7fffffff, Z(0xfffffffe)) + self.assertEqual(-0x80000000, Z(0xffffffff)) + self.assertEqual(0x7fffffffffffffff, Z(0xfffffffffffffffe)) + self.assertEqual(-0x8000000000000000, Z(0xffffffffffffffff)) + + self.assertRaises(TypeError, Z, None) + self.assertRaises(TypeError, Z, 'abcd') + self.assertRaises(TypeError, Z, 0.0) + self.assertRaises(TypeError, Z, object()) + + def NumericByteSizeTestHelper(self, byte_size_fn, value, expected_value_size): + # Use field numbers that cause various byte sizes for the tag information. + for field_number, tag_bytes in ((15, 1), (16, 2), (2047, 2), (2048, 3)): + expected_size = expected_value_size + tag_bytes + actual_size = byte_size_fn(field_number, value) + self.assertEqual(expected_size, actual_size, + 'byte_size_fn: %s, field_number: %d, value: %r\n' + 'Expected: %d, Actual: %d'% ( + byte_size_fn, field_number, value, expected_size, actual_size)) + + def testByteSizeFunctions(self): + # Test all numeric *ByteSize() functions. + NUMERIC_ARGS = [ + # Int32ByteSize(). + [wire_format.Int32ByteSize, 0, 1], + [wire_format.Int32ByteSize, 127, 1], + [wire_format.Int32ByteSize, 128, 2], + [wire_format.Int32ByteSize, -1, 10], + # Int64ByteSize(). + [wire_format.Int64ByteSize, 0, 1], + [wire_format.Int64ByteSize, 127, 1], + [wire_format.Int64ByteSize, 128, 2], + [wire_format.Int64ByteSize, -1, 10], + # UInt32ByteSize(). + [wire_format.UInt32ByteSize, 0, 1], + [wire_format.UInt32ByteSize, 127, 1], + [wire_format.UInt32ByteSize, 128, 2], + [wire_format.UInt32ByteSize, wire_format.UINT32_MAX, 5], + # UInt64ByteSize(). + [wire_format.UInt64ByteSize, 0, 1], + [wire_format.UInt64ByteSize, 127, 1], + [wire_format.UInt64ByteSize, 128, 2], + [wire_format.UInt64ByteSize, wire_format.UINT64_MAX, 10], + # SInt32ByteSize(). + [wire_format.SInt32ByteSize, 0, 1], + [wire_format.SInt32ByteSize, -1, 1], + [wire_format.SInt32ByteSize, 1, 1], + [wire_format.SInt32ByteSize, -63, 1], + [wire_format.SInt32ByteSize, 63, 1], + [wire_format.SInt32ByteSize, -64, 1], + [wire_format.SInt32ByteSize, 64, 2], + # SInt64ByteSize(). + [wire_format.SInt64ByteSize, 0, 1], + [wire_format.SInt64ByteSize, -1, 1], + [wire_format.SInt64ByteSize, 1, 1], + [wire_format.SInt64ByteSize, -63, 1], + [wire_format.SInt64ByteSize, 63, 1], + [wire_format.SInt64ByteSize, -64, 1], + [wire_format.SInt64ByteSize, 64, 2], + # Fixed32ByteSize(). + [wire_format.Fixed32ByteSize, 0, 4], + [wire_format.Fixed32ByteSize, wire_format.UINT32_MAX, 4], + # Fixed64ByteSize(). + [wire_format.Fixed64ByteSize, 0, 8], + [wire_format.Fixed64ByteSize, wire_format.UINT64_MAX, 8], + # SFixed32ByteSize(). + [wire_format.SFixed32ByteSize, 0, 4], + [wire_format.SFixed32ByteSize, wire_format.INT32_MIN, 4], + [wire_format.SFixed32ByteSize, wire_format.INT32_MAX, 4], + # SFixed64ByteSize(). + [wire_format.SFixed64ByteSize, 0, 8], + [wire_format.SFixed64ByteSize, wire_format.INT64_MIN, 8], + [wire_format.SFixed64ByteSize, wire_format.INT64_MAX, 8], + # FloatByteSize(). + [wire_format.FloatByteSize, 0.0, 4], + [wire_format.FloatByteSize, 1000000000.0, 4], + [wire_format.FloatByteSize, -1000000000.0, 4], + # DoubleByteSize(). + [wire_format.DoubleByteSize, 0.0, 8], + [wire_format.DoubleByteSize, 1000000000.0, 8], + [wire_format.DoubleByteSize, -1000000000.0, 8], + # BoolByteSize(). + [wire_format.BoolByteSize, False, 1], + [wire_format.BoolByteSize, True, 1], + # EnumByteSize(). + [wire_format.EnumByteSize, 0, 1], + [wire_format.EnumByteSize, 127, 1], + [wire_format.EnumByteSize, 128, 2], + [wire_format.EnumByteSize, wire_format.UINT32_MAX, 5], + ] + for args in NUMERIC_ARGS: + self.NumericByteSizeTestHelper(*args) + + # Test strings and bytes. + for byte_size_fn in (wire_format.StringByteSize, wire_format.BytesByteSize): + # 1 byte for tag, 1 byte for length, 3 bytes for contents. + self.assertEqual(5, byte_size_fn(10, 'abc')) + # 2 bytes for tag, 1 byte for length, 3 bytes for contents. + self.assertEqual(6, byte_size_fn(16, 'abc')) + # 2 bytes for tag, 2 bytes for length, 128 bytes for contents. + self.assertEqual(132, byte_size_fn(16, 'a' * 128)) + + # Test UTF-8 string byte size calculation. + # 1 byte for tag, 1 byte for length, 8 bytes for content. + self.assertEqual(10, wire_format.StringByteSize( + 5, unicode('\xd0\xa2\xd0\xb5\xd1\x81\xd1\x82', 'utf-8'))) + + class MockMessage(object): + def __init__(self, byte_size): + self.byte_size = byte_size + def ByteSize(self): + return self.byte_size + + message_byte_size = 10 + mock_message = MockMessage(byte_size=message_byte_size) + # Test groups. + # (2 * 1) bytes for begin and end tags, plus message_byte_size. + self.assertEqual(2 + message_byte_size, + wire_format.GroupByteSize(1, mock_message)) + # (2 * 2) bytes for begin and end tags, plus message_byte_size. + self.assertEqual(4 + message_byte_size, + wire_format.GroupByteSize(16, mock_message)) + + # Test messages. + # 1 byte for tag, plus 1 byte for length, plus contents. + self.assertEqual(2 + mock_message.byte_size, + wire_format.MessageByteSize(1, mock_message)) + # 2 bytes for tag, plus 1 byte for length, plus contents. + self.assertEqual(3 + mock_message.byte_size, + wire_format.MessageByteSize(16, mock_message)) + # 2 bytes for tag, plus 2 bytes for length, plus contents. + mock_message.byte_size = 128 + self.assertEqual(4 + mock_message.byte_size, + wire_format.MessageByteSize(16, mock_message)) + + + # Test message set item byte size. + # 4 bytes for tags, plus 1 byte for length, plus 1 byte for type_id, + # plus contents. + mock_message.byte_size = 10 + self.assertEqual(mock_message.byte_size + 6, + wire_format.MessageSetItemByteSize(1, mock_message)) + + # 4 bytes for tags, plus 2 bytes for length, plus 1 byte for type_id, + # plus contents. + mock_message.byte_size = 128 + self.assertEqual(mock_message.byte_size + 7, + wire_format.MessageSetItemByteSize(1, mock_message)) + + # 4 bytes for tags, plus 2 bytes for length, plus 2 byte for type_id, + # plus contents. + self.assertEqual(mock_message.byte_size + 8, + wire_format.MessageSetItemByteSize(128, mock_message)) + + # Too-long varint. + self.assertRaises(message.EncodeError, + wire_format.UInt64ByteSize, 1, 1 << 128) + + +if __name__ == '__main__': + unittest.main() |