diff options
Diffstat (limited to 'python/google/protobuf/pyext')
20 files changed, 6183 insertions, 0 deletions
diff --git a/python/google/protobuf/pyext/README b/python/google/protobuf/pyext/README new file mode 100644 index 0000000..6d61cb4 --- /dev/null +++ b/python/google/protobuf/pyext/README @@ -0,0 +1,6 @@ +This is the 'v2' C++ implementation for python proto2. + +It is active when: + +PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp +PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION=2 diff --git a/python/google/protobuf/pyext/__init__.py b/python/google/protobuf/pyext/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/python/google/protobuf/pyext/__init__.py diff --git a/python/google/protobuf/pyext/cpp_message.py b/python/google/protobuf/pyext/cpp_message.py new file mode 100644 index 0000000..dcf34a0 --- /dev/null +++ b/python/google/protobuf/pyext/cpp_message.py @@ -0,0 +1,61 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Protocol message implementation hooks for C++ implementation. + +Contains helper functions used to create protocol message classes from +Descriptor objects at runtime backed by the protocol buffer C++ API. +""" + +__author__ = 'tibell@google.com (Johan Tibell)' + +from google.protobuf.pyext import _message +from google.protobuf import message + + +def NewMessage(bases, message_descriptor, dictionary): + """Creates a new protocol message *class*.""" + new_bases = [] + for base in bases: + if base is message.Message: + # _message.Message must come before message.Message as it + # overrides methods in that class. + new_bases.append(_message.Message) + new_bases.append(base) + return tuple(new_bases) + + +def InitMessage(message_descriptor, cls): + """Constructs a new message instance (called before instance's __init__).""" + + def SubInit(self, **kwargs): + super(cls, self).__init__(message_descriptor, **kwargs) + cls.__init__ = SubInit + cls.AddDescriptors(message_descriptor) diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc new file mode 100644 index 0000000..3f7be73 --- /dev/null +++ b/python/google/protobuf/pyext/descriptor.cc @@ -0,0 +1,357 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: petar@google.com (Petar Petrov) + +#include <Python.h> +#include <string> + +#include <google/protobuf/descriptor.pb.h> +#include <google/protobuf/pyext/descriptor.h> +#include <google/protobuf/pyext/scoped_pyobject_ptr.h> + +#define C(str) const_cast<char*>(str) + +#if PY_MAJOR_VERSION >= 3 + #define PyString_FromStringAndSize PyUnicode_FromStringAndSize + #define PyInt_FromLong PyLong_FromLong + #if PY_VERSION_HEX < 0x03030000 + #error "Python 3.0 - 3.2 are not supported." + #else + #define PyString_AsString(ob) \ + (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AS_STRING(ob)) + #endif +#endif + +namespace google { +namespace protobuf { +namespace python { + + +#ifndef PyVarObject_HEAD_INIT +#define PyVarObject_HEAD_INIT(type, size) PyObject_HEAD_INIT(type) size, +#endif +#ifndef Py_TYPE +#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) +#endif + + +static google::protobuf::DescriptorPool* g_descriptor_pool = NULL; + +namespace cfield_descriptor { + +static void Dealloc(CFieldDescriptor* self) { + Py_CLEAR(self->descriptor_field); + Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); +} + +static PyObject* GetFullName(CFieldDescriptor* self, void *closure) { + return PyString_FromStringAndSize( + self->descriptor->full_name().c_str(), + self->descriptor->full_name().size()); +} + +static PyObject* GetName(CFieldDescriptor *self, void *closure) { + return PyString_FromStringAndSize( + self->descriptor->name().c_str(), + self->descriptor->name().size()); +} + +static PyObject* GetCppType(CFieldDescriptor *self, void *closure) { + return PyInt_FromLong(self->descriptor->cpp_type()); +} + +static PyObject* GetLabel(CFieldDescriptor *self, void *closure) { + return PyInt_FromLong(self->descriptor->label()); +} + +static PyObject* GetID(CFieldDescriptor *self, void *closure) { + return PyLong_FromVoidPtr(self); +} + +static PyGetSetDef Getters[] = { + { C("full_name"), (getter)GetFullName, NULL, "Full name", NULL}, + { C("name"), (getter)GetName, NULL, "last name", NULL}, + { C("cpp_type"), (getter)GetCppType, NULL, "C++ Type", NULL}, + { C("label"), (getter)GetLabel, NULL, "Label", NULL}, + { C("id"), (getter)GetID, NULL, "ID", NULL}, + {NULL} +}; + +} // namespace cfield_descriptor + +PyTypeObject CFieldDescriptor_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + C("google.protobuf.internal." + "_net_proto2___python." + "CFieldDescriptor"), // tp_name + sizeof(CFieldDescriptor), // tp_basicsize + 0, // tp_itemsize + (destructor)cfield_descriptor::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + C("A Field Descriptor"), // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + 0, // tp_methods + 0, // tp_members + cfield_descriptor::Getters, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + PyType_GenericAlloc, // tp_alloc + PyType_GenericNew, // tp_new + PyObject_Del, // tp_free +}; + +namespace cdescriptor_pool { + +static void Dealloc(CDescriptorPool* self) { + Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); +} + +static PyObject* NewCDescriptor( + const google::protobuf::FieldDescriptor* field_descriptor) { + CFieldDescriptor* cfield_descriptor = PyObject_New( + CFieldDescriptor, &CFieldDescriptor_Type); + if (cfield_descriptor == NULL) { + return NULL; + } + cfield_descriptor->descriptor = field_descriptor; + cfield_descriptor->descriptor_field = NULL; + + return reinterpret_cast<PyObject*>(cfield_descriptor); +} + +PyObject* FindFieldByName(CDescriptorPool* self, PyObject* name) { + const char* full_field_name = PyString_AsString(name); + if (full_field_name == NULL) { + return NULL; + } + + const google::protobuf::FieldDescriptor* field_descriptor = NULL; + + field_descriptor = self->pool->FindFieldByName(full_field_name); + + if (field_descriptor == NULL) { + PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s", + full_field_name); + return NULL; + } + + return NewCDescriptor(field_descriptor); +} + +PyObject* FindExtensionByName(CDescriptorPool* self, PyObject* arg) { + const char* full_field_name = PyString_AsString(arg); + if (full_field_name == NULL) { + return NULL; + } + + const google::protobuf::FieldDescriptor* field_descriptor = + self->pool->FindExtensionByName(full_field_name); + if (field_descriptor == NULL) { + PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s", + full_field_name); + return NULL; + } + + return NewCDescriptor(field_descriptor); +} + +static PyMethodDef Methods[] = { + { C("FindFieldByName"), + (PyCFunction)FindFieldByName, + METH_O, + C("Searches for a field descriptor by full name.") }, + { C("FindExtensionByName"), + (PyCFunction)FindExtensionByName, + METH_O, + C("Searches for extension descriptor by full name.") }, + {NULL} +}; + +} // namespace cdescriptor_pool + +PyTypeObject CDescriptorPool_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + C("google.protobuf.internal." + "_net_proto2___python." + "CFieldDescriptor"), // tp_name + sizeof(CDescriptorPool), // tp_basicsize + 0, // tp_itemsize + (destructor)cdescriptor_pool::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + C("A Descriptor Pool"), // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + cdescriptor_pool::Methods, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + PyType_GenericAlloc, // tp_alloc + PyType_GenericNew, // tp_new + PyObject_Del, // tp_free +}; + +google::protobuf::DescriptorPool* GetDescriptorPool() { + if (g_descriptor_pool == NULL) { + g_descriptor_pool = new google::protobuf::DescriptorPool( + google::protobuf::DescriptorPool::generated_pool()); + } + return g_descriptor_pool; +} + +PyObject* Python_NewCDescriptorPool(PyObject* ignored, PyObject* args) { + CDescriptorPool* cdescriptor_pool = PyObject_New( + CDescriptorPool, &CDescriptorPool_Type); + if (cdescriptor_pool == NULL) { + return NULL; + } + cdescriptor_pool->pool = GetDescriptorPool(); + return reinterpret_cast<PyObject*>(cdescriptor_pool); +} + + +// Collects errors that occur during proto file building to allow them to be +// propagated in the python exception instead of only living in ERROR logs. +class BuildFileErrorCollector : public google::protobuf::DescriptorPool::ErrorCollector { + public: + BuildFileErrorCollector() : error_message(""), had_errors(false) {} + + void AddError(const string& filename, const string& element_name, + const Message* descriptor, ErrorLocation location, + const string& message) { + // Replicates the logging behavior that happens in the C++ implementation + // when an error collector is not passed in. + if (!had_errors) { + error_message += + ("Invalid proto descriptor for file \"" + filename + "\":\n"); + } + // As this only happens on failure and will result in the program not + // running at all, no effort is made to optimize this string manipulation. + error_message += (" " + element_name + ": " + message + "\n"); + } + + string error_message; + bool had_errors; +}; + +PyObject* Python_BuildFile(PyObject* ignored, PyObject* arg) { + char* message_type; + Py_ssize_t message_len; + + if (PyBytes_AsStringAndSize(arg, &message_type, &message_len) < 0) { + return NULL; + } + + google::protobuf::FileDescriptorProto file_proto; + if (!file_proto.ParseFromArray(message_type, message_len)) { + PyErr_SetString(PyExc_TypeError, "Couldn't parse file content!"); + return NULL; + } + + if (google::protobuf::DescriptorPool::generated_pool()->FindFileByName( + file_proto.name()) != NULL) { + Py_RETURN_NONE; + } + + BuildFileErrorCollector error_collector; + const google::protobuf::FileDescriptor* descriptor = + GetDescriptorPool()->BuildFileCollectingErrors(file_proto, + &error_collector); + if (descriptor == NULL) { + PyErr_Format(PyExc_TypeError, + "Couldn't build proto file into descriptor pool!\n%s", + error_collector.error_message.c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + +bool InitDescriptor() { + CFieldDescriptor_Type.tp_new = PyType_GenericNew; + if (PyType_Ready(&CFieldDescriptor_Type) < 0) + return false; + + CDescriptorPool_Type.tp_new = PyType_GenericNew; + if (PyType_Ready(&CDescriptorPool_Type) < 0) + return false; + + return true; +} + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/pyext/descriptor.h b/python/google/protobuf/pyext/descriptor.h new file mode 100644 index 0000000..ae7a1b9 --- /dev/null +++ b/python/google/protobuf/pyext/descriptor.h @@ -0,0 +1,96 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: petar@google.com (Petar Petrov) + +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_H__ + +#include <Python.h> +#include <structmember.h> + +#include <google/protobuf/descriptor.h> + +#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) +typedef int Py_ssize_t; +#define PY_SSIZE_T_MAX INT_MAX +#define PY_SSIZE_T_MIN INT_MIN +#endif + +namespace google { +namespace protobuf { +namespace python { + +typedef struct CFieldDescriptor { + PyObject_HEAD + + // The proto2 descriptor that this object represents. + const google::protobuf::FieldDescriptor* descriptor; + + // Reference to the original field object in the Python DESCRIPTOR. + PyObject* descriptor_field; +} CFieldDescriptor; + +typedef struct { + PyObject_HEAD + + const google::protobuf::DescriptorPool* pool; +} CDescriptorPool; + +extern PyTypeObject CFieldDescriptor_Type; + +extern PyTypeObject CDescriptorPool_Type; + +namespace cdescriptor_pool { + +// Looks up a field by name. Returns a CDescriptor corresponding to +// the field on success, or NULL on failure. +// +// Returns a new reference. +PyObject* FindFieldByName(CDescriptorPool* self, PyObject* name); + +// Looks up an extension by name. Returns a CDescriptor corresponding +// to the field on success, or NULL on failure. +// +// Returns a new reference. +PyObject* FindExtensionByName(CDescriptorPool* self, PyObject* arg); + +} // namespace cdescriptor_pool + +PyObject* Python_NewCDescriptorPool(PyObject* ignored, PyObject* args); +PyObject* Python_BuildFile(PyObject* ignored, PyObject* args); +bool InitDescriptor(); +google::protobuf::DescriptorPool* GetDescriptorPool(); + +} // namespace python +} // namespace protobuf + +} // namespace google +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_H__ diff --git a/python/google/protobuf/pyext/descriptor_cpp2_test.py b/python/google/protobuf/pyext/descriptor_cpp2_test.py new file mode 100644 index 0000000..3cf45a2 --- /dev/null +++ b/python/google/protobuf/pyext/descriptor_cpp2_test.py @@ -0,0 +1,58 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Tests for google.protobuf.pyext behavior.""" + +__author__ = 'anuraag@google.com (Anuraag Agrawal)' + +import os +os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp' +os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION'] = '2' + +# We must set the implementation version above before the google3 imports. +# pylint: disable=g-import-not-at-top +from google.apputils import basetest +from google.protobuf.internal import api_implementation +# Run all tests from the original module by putting them in our namespace. +# pylint: disable=wildcard-import +from google.protobuf.internal.descriptor_test import * + + +class ConfirmCppApi2Test(basetest.TestCase): + + def testImplementationSetting(self): + self.assertEqual('cpp', api_implementation.Type()) + self.assertEqual(2, api_implementation.Version()) + + +if __name__ == '__main__': + basetest.main() diff --git a/python/google/protobuf/pyext/extension_dict.cc b/python/google/protobuf/pyext/extension_dict.cc new file mode 100644 index 0000000..3861c79 --- /dev/null +++ b/python/google/protobuf/pyext/extension_dict.cc @@ -0,0 +1,338 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: anuraag@google.com (Anuraag Agrawal) +// Author: tibell@google.com (Johan Tibell) + +#include <google/protobuf/pyext/extension_dict.h> + +#include <google/protobuf/stubs/common.h> +#include <google/protobuf/descriptor.h> +#include <google/protobuf/dynamic_message.h> +#include <google/protobuf/message.h> +#include <google/protobuf/pyext/descriptor.h> +#include <google/protobuf/pyext/message.h> +#include <google/protobuf/pyext/repeated_composite_container.h> +#include <google/protobuf/pyext/repeated_scalar_container.h> +#include <google/protobuf/pyext/scoped_pyobject_ptr.h> +#include <google/protobuf/stubs/shared_ptr.h> + +namespace google { +namespace protobuf { +namespace python { + +extern google::protobuf::DynamicMessageFactory* global_message_factory; + +namespace extension_dict { + +// TODO(tibell): Always use self->message for clarity, just like in +// RepeatedCompositeContainer. +static google::protobuf::Message* GetMessage(ExtensionDict* self) { + if (self->parent != NULL) { + return self->parent->message; + } else { + return self->message; + } +} + +CFieldDescriptor* InternalGetCDescriptorFromExtension(PyObject* extension) { + PyObject* cdescriptor = PyObject_GetAttrString(extension, "_cdescriptor"); + if (cdescriptor == NULL) { + PyErr_SetString(PyExc_KeyError, "Unregistered extension."); + return NULL; + } + if (!PyObject_TypeCheck(cdescriptor, &CFieldDescriptor_Type)) { + PyErr_SetString(PyExc_TypeError, "Not a CFieldDescriptor"); + Py_DECREF(cdescriptor); + return NULL; + } + CFieldDescriptor* descriptor = + reinterpret_cast<CFieldDescriptor*>(cdescriptor); + return descriptor; +} + +PyObject* len(ExtensionDict* self) { +#if PY_MAJOR_VERSION >= 3 + return PyLong_FromLong(PyDict_Size(self->values)); +#else + return PyInt_FromLong(PyDict_Size(self->values)); +#endif +} + +// TODO(tibell): Use VisitCompositeField. +int ReleaseExtension(ExtensionDict* self, + PyObject* extension, + const google::protobuf::FieldDescriptor* descriptor) { + if (descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) { + if (descriptor->cpp_type() == + google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + if (repeated_composite_container::Release( + reinterpret_cast<RepeatedCompositeContainer*>( + extension)) < 0) { + return -1; + } + } else { + if (repeated_scalar_container::Release( + reinterpret_cast<RepeatedScalarContainer*>( + extension)) < 0) { + return -1; + } + } + } else if (descriptor->cpp_type() == + google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + if (cmessage::ReleaseSubMessage( + GetMessage(self), descriptor, + reinterpret_cast<CMessage*>(extension)) < 0) { + return -1; + } + } + + return 0; +} + +PyObject* subscript(ExtensionDict* self, PyObject* key) { + CFieldDescriptor* cdescriptor = InternalGetCDescriptorFromExtension( + key); + if (cdescriptor == NULL) { + return NULL; + } + ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor)); + const google::protobuf::FieldDescriptor* descriptor = cdescriptor->descriptor; + if (descriptor == NULL) { + return NULL; + } + if (descriptor->label() != FieldDescriptor::LABEL_REPEATED && + descriptor->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) { + return cmessage::InternalGetScalar(self->parent, descriptor); + } + + PyObject* value = PyDict_GetItem(self->values, key); + if (value != NULL) { + Py_INCREF(value); + return value; + } + + if (descriptor->label() != FieldDescriptor::LABEL_REPEATED && + descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + PyObject* sub_message = cmessage::InternalGetSubMessage( + self->parent, cdescriptor); + if (sub_message == NULL) { + return NULL; + } + PyDict_SetItem(self->values, key, sub_message); + return sub_message; + } + + if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) { + if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + // COPIED + PyObject* py_container = PyObject_CallObject( + reinterpret_cast<PyObject*>(&RepeatedCompositeContainer_Type), + NULL); + if (py_container == NULL) { + return NULL; + } + RepeatedCompositeContainer* container = + reinterpret_cast<RepeatedCompositeContainer*>(py_container); + PyObject* field = cdescriptor->descriptor_field; + PyObject* message_type = PyObject_GetAttrString(field, "message_type"); + PyObject* concrete_class = PyObject_GetAttrString(message_type, + "_concrete_class"); + container->owner = self->owner; + container->parent = self->parent; + container->message = self->parent->message; + container->parent_field = cdescriptor; + container->subclass_init = concrete_class; + Py_DECREF(message_type); + PyDict_SetItem(self->values, key, py_container); + return py_container; + } else { + // COPIED + ScopedPyObjectPtr init_args(PyTuple_Pack(2, self->parent, cdescriptor)); + PyObject* py_container = PyObject_CallObject( + reinterpret_cast<PyObject*>(&RepeatedScalarContainer_Type), + init_args); + if (py_container == NULL) { + return NULL; + } + PyDict_SetItem(self->values, key, py_container); + return py_container; + } + } + PyErr_SetString(PyExc_ValueError, "control reached unexpected line"); + return NULL; +} + +int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) { + CFieldDescriptor* cdescriptor = InternalGetCDescriptorFromExtension( + key); + if (cdescriptor == NULL) { + return -1; + } + ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor)); + const google::protobuf::FieldDescriptor* descriptor = cdescriptor->descriptor; + if (descriptor->label() != FieldDescriptor::LABEL_OPTIONAL || + descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + PyErr_SetString(PyExc_TypeError, "Extension is repeated and/or composite " + "type"); + return -1; + } + cmessage::AssureWritable(self->parent); + if (cmessage::InternalSetScalar(self->parent, descriptor, value) < 0) { + return -1; + } + // TODO(tibell): We shouldn't write scalars to the cache. + PyDict_SetItem(self->values, key, value); + return 0; +} + +PyObject* ClearExtension(ExtensionDict* self, PyObject* extension) { + CFieldDescriptor* cdescriptor = InternalGetCDescriptorFromExtension( + extension); + if (cdescriptor == NULL) { + return NULL; + } + ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor)); + PyObject* value = PyDict_GetItem(self->values, extension); + if (value != NULL) { + if (ReleaseExtension(self, value, cdescriptor->descriptor) < 0) { + return NULL; + } + } + if (cmessage::ClearFieldByDescriptor(self->parent, + cdescriptor->descriptor) == NULL) { + return NULL; + } + if (PyDict_DelItem(self->values, extension) < 0) { + PyErr_Clear(); + } + Py_RETURN_NONE; +} + +PyObject* HasExtension(ExtensionDict* self, PyObject* extension) { + CFieldDescriptor* cdescriptor = InternalGetCDescriptorFromExtension( + extension); + if (cdescriptor == NULL) { + return NULL; + } + ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor)); + PyObject* result = cmessage::HasFieldByDescriptor( + self->parent, cdescriptor->descriptor); + return result; +} + +PyObject* _FindExtensionByName(ExtensionDict* self, PyObject* name) { + ScopedPyObjectPtr extensions_by_name(PyObject_GetAttrString( + reinterpret_cast<PyObject*>(self->parent), "_extensions_by_name")); + if (extensions_by_name == NULL) { + return NULL; + } + PyObject* result = PyDict_GetItem(extensions_by_name, name); + if (result == NULL) { + Py_RETURN_NONE; + } else { + Py_INCREF(result); + return result; + } +} + +int init(ExtensionDict* self, PyObject* args, PyObject* kwargs) { + self->parent = NULL; + self->message = NULL; + self->values = PyDict_New(); + return 0; +} + +void dealloc(ExtensionDict* self) { + Py_CLEAR(self->values); + self->owner.reset(); + Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); +} + +static PyMappingMethods MpMethods = { + (lenfunc)len, /* mp_length */ + (binaryfunc)subscript, /* mp_subscript */ + (objobjargproc)ass_subscript,/* mp_ass_subscript */ +}; + +#define EDMETHOD(name, args, doc) { #name, (PyCFunction)name, args, doc } +static PyMethodDef Methods[] = { + EDMETHOD(ClearExtension, METH_O, "Clears an extension from the object."), + EDMETHOD(HasExtension, METH_O, "Checks if the object has an extension."), + EDMETHOD(_FindExtensionByName, METH_O, + "Finds an extension by name."), + { NULL, NULL } +}; + +} // namespace extension_dict + +PyTypeObject ExtensionDict_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "google.protobuf.internal." + "cpp._message.ExtensionDict", // tp_name + sizeof(ExtensionDict), // tp_basicsize + 0, // tp_itemsize + (destructor)extension_dict::dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + &extension_dict::MpMethods, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "An extension dict", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + extension_dict::Methods, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + (initproc)extension_dict::init, // tp_init +}; + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/pyext/extension_dict.h b/python/google/protobuf/pyext/extension_dict.h new file mode 100644 index 0000000..13c874a --- /dev/null +++ b/python/google/protobuf/pyext/extension_dict.h @@ -0,0 +1,123 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: anuraag@google.com (Anuraag Agrawal) +// Author: tibell@google.com (Johan Tibell) + +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_EXTENSION_DICT_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_EXTENSION_DICT_H__ + +#include <Python.h> + +#include <memory> +#ifndef _SHARED_PTR_H +#include <google/protobuf/stubs/shared_ptr.h> +#endif + + +namespace google { +namespace protobuf { + +class Message; +class FieldDescriptor; + +using internal::shared_ptr; + +namespace python { + +struct CMessage; +struct CFieldDescriptor; + +typedef struct ExtensionDict { + PyObject_HEAD; + shared_ptr<Message> owner; + CMessage* parent; + Message* message; + PyObject* values; +} ExtensionDict; + +extern PyTypeObject ExtensionDict_Type; + +namespace extension_dict { + +// Gets the _cdescriptor reference to a CFieldDescriptor object given a +// python descriptor object. +// +// Returns a new reference. +CFieldDescriptor* InternalGetCDescriptorFromExtension(PyObject* extension); + +// Gets the number of extension values in this ExtensionDict as a python object. +// +// Returns a new reference. +PyObject* len(ExtensionDict* self); + +// Releases extensions referenced outside this dictionary to keep outside +// references alive. +// +// Returns 0 on success, -1 on failure. +int ReleaseExtension(ExtensionDict* self, + PyObject* extension, + const google::protobuf::FieldDescriptor* descriptor); + +// Gets an extension from the dict for the given extension descriptor. +// +// Returns a new reference. +PyObject* subscript(ExtensionDict* self, PyObject* key); + +// Assigns a value to an extension in the dict. Can only be used for singular +// simple types. +// +// Returns 0 on success, -1 on failure. +int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value); + +// Clears an extension from the dict. Will release the extension if there +// is still an external reference left to it. +// +// Returns None on success. +PyObject* ClearExtension(ExtensionDict* self, + PyObject* extension); + +// Checks if the dict has an extension. +// +// Returns a new python boolean reference. +PyObject* HasExtension(ExtensionDict* self, PyObject* extension); + +// Gets an extension from the dict given the extension name as opposed to +// descriptor. +// +// Returns a new reference. +PyObject* _FindExtensionByName(ExtensionDict* self, PyObject* name); + +} // namespace extension_dict +} // namespace python +} // namespace protobuf + +} // namespace google +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_EXTENSION_DICT_H__ diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc new file mode 100644 index 0000000..9fb7083 --- /dev/null +++ b/python/google/protobuf/pyext/message.cc @@ -0,0 +1,2561 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: anuraag@google.com (Anuraag Agrawal) +// Author: tibell@google.com (Johan Tibell) + +#include <google/protobuf/pyext/message.h> + +#include <memory> +#ifndef _SHARED_PTR_H +#include <google/protobuf/stubs/shared_ptr.h> +#endif +#include <string> +#include <vector> + +#ifndef PyVarObject_HEAD_INIT +#define PyVarObject_HEAD_INIT(type, size) PyObject_HEAD_INIT(type) size, +#endif +#ifndef Py_TYPE +#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) +#endif +#include <google/protobuf/descriptor.pb.h> +#include <google/protobuf/stubs/common.h> +#include <google/protobuf/io/coded_stream.h> +#include <google/protobuf/descriptor.h> +#include <google/protobuf/dynamic_message.h> +#include <google/protobuf/message.h> +#include <google/protobuf/text_format.h> +#include <google/protobuf/pyext/descriptor.h> +#include <google/protobuf/pyext/extension_dict.h> +#include <google/protobuf/pyext/repeated_composite_container.h> +#include <google/protobuf/pyext/repeated_scalar_container.h> +#include <google/protobuf/pyext/scoped_pyobject_ptr.h> + +#if PY_MAJOR_VERSION >= 3 + #define PyInt_Check PyLong_Check + #define PyInt_AsLong PyLong_AsLong + #define PyInt_FromLong PyLong_FromLong + #define PyInt_FromSize_t PyLong_FromSize_t + #define PyString_Check PyUnicode_Check + #define PyString_FromString PyUnicode_FromString + #define PyString_FromStringAndSize PyUnicode_FromStringAndSize + #if PY_VERSION_HEX < 0x03030000 + #error "Python 3.0 - 3.2 are not supported." + #else + #define PyString_AsString(ob) \ + (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AS_STRING(ob)) + #endif +#endif + +namespace google { +namespace protobuf { +namespace python { + +// Forward declarations +namespace cmessage { +static PyObject* GetDescriptor(CMessage* self, PyObject* name); +static string GetMessageName(CMessage* self); +int InternalReleaseFieldByDescriptor( + const google::protobuf::FieldDescriptor* field_descriptor, + PyObject* composite_field, + google::protobuf::Message* parent_message); +} // namespace cmessage + +// --------------------------------------------------------------------- +// Visiting the composite children of a CMessage + +struct ChildVisitor { + // Returns 0 on success, -1 on failure. + int VisitRepeatedCompositeContainer(RepeatedCompositeContainer* container) { + return 0; + } + + // Returns 0 on success, -1 on failure. + int VisitRepeatedScalarContainer(RepeatedScalarContainer* container) { + return 0; + } + + // Returns 0 on success, -1 on failure. + int VisitCMessage(CMessage* cmessage, + const google::protobuf::FieldDescriptor* field_descriptor) { + return 0; + } +}; + +// Apply a function to a composite field. Does nothing if child is of +// non-composite type. +template<class Visitor> +static int VisitCompositeField(const FieldDescriptor* descriptor, + PyObject* child, + Visitor visitor) { + if (descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) { + if (descriptor->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + RepeatedCompositeContainer* container = + reinterpret_cast<RepeatedCompositeContainer*>(child); + if (visitor.VisitRepeatedCompositeContainer(container) == -1) + return -1; + } else { + RepeatedScalarContainer* container = + reinterpret_cast<RepeatedScalarContainer*>(child); + if (visitor.VisitRepeatedScalarContainer(container) == -1) + return -1; + } + } else if (descriptor->cpp_type() == + google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + CMessage* cmsg = reinterpret_cast<CMessage*>(child); + if (visitor.VisitCMessage(cmsg, descriptor) == -1) + return -1; + } + // The ExtensionDict might contain non-composite fields, which we + // skip here. + return 0; +} + +// Visit each composite field and extension field of this CMessage. +// Returns -1 on error and 0 on success. +template<class Visitor> +int ForEachCompositeField(CMessage* self, Visitor visitor) { + Py_ssize_t pos = 0; + PyObject* key; + PyObject* field; + + // Visit normal fields. + while (PyDict_Next(self->composite_fields, &pos, &key, &field)) { + PyObject* cdescriptor = cmessage::GetDescriptor(self, key); + if (cdescriptor != NULL) { + const google::protobuf::FieldDescriptor* descriptor = + reinterpret_cast<CFieldDescriptor*>(cdescriptor)->descriptor; + if (VisitCompositeField(descriptor, field, visitor) == -1) + return -1; + } + } + + // Visit extension fields. + if (self->extensions != NULL) { + while (PyDict_Next(self->extensions->values, &pos, &key, &field)) { + CFieldDescriptor* cdescriptor = + extension_dict::InternalGetCDescriptorFromExtension(key); + if (cdescriptor == NULL) + return -1; + if (VisitCompositeField(cdescriptor->descriptor, field, visitor) == -1) + return -1; + } + } + + return 0; +} + +// --------------------------------------------------------------------- + +// Constants used for integer type range checking. +PyObject* kPythonZero; +PyObject* kint32min_py; +PyObject* kint32max_py; +PyObject* kuint32max_py; +PyObject* kint64min_py; +PyObject* kint64max_py; +PyObject* kuint64max_py; + +PyObject* EnumTypeWrapper_class; +PyObject* EncodeError_class; +PyObject* DecodeError_class; +PyObject* PickleError_class; + +// Constant PyString values used for GetAttr/GetItem. +static PyObject* kDESCRIPTOR; +static PyObject* k__descriptors; +static PyObject* kfull_name; +static PyObject* kname; +static PyObject* kmessage_type; +static PyObject* kis_extendable; +static PyObject* kextensions_by_name; +static PyObject* k_extensions_by_name; +static PyObject* k_extensions_by_number; +static PyObject* k_concrete_class; +static PyObject* kfields_by_name; + +static CDescriptorPool* descriptor_pool; + +/* Is 64bit */ +void FormatTypeError(PyObject* arg, char* expected_types) { + PyObject* repr = PyObject_Repr(arg); + if (repr) { + PyErr_Format(PyExc_TypeError, + "%.100s has type %.100s, but expected one of: %s", + PyString_AsString(repr), + Py_TYPE(arg)->tp_name, + expected_types); + Py_DECREF(repr); + } +} + +template<class T> +bool CheckAndGetInteger( + PyObject* arg, T* value, PyObject* min, PyObject* max) { + bool is_long = PyLong_Check(arg); +#if PY_MAJOR_VERSION < 3 + if (!PyInt_Check(arg) && !is_long) { + FormatTypeError(arg, "int, long"); + return false; + } + if (PyObject_Compare(min, arg) > 0 || PyObject_Compare(max, arg) < 0) { +#else + if (!is_long) { + FormatTypeError(arg, "int"); + return false; + } + if (PyObject_RichCompareBool(min, arg, Py_LE) != 1 || + PyObject_RichCompareBool(max, arg, Py_GE) != 1) { +#endif + PyObject *s = PyObject_Str(arg); + if (s) { + PyErr_Format(PyExc_ValueError, + "Value out of range: %s", + PyString_AsString(s)); + Py_DECREF(s); + } + return false; + } +#if PY_MAJOR_VERSION < 3 + if (!is_long) { + *value = static_cast<T>(PyInt_AsLong(arg)); + } else // NOLINT +#endif + { + if (min == kPythonZero) { + *value = static_cast<T>(PyLong_AsUnsignedLongLong(arg)); + } else { + *value = static_cast<T>(PyLong_AsLongLong(arg)); + } + } + return true; +} + +// These are referenced by repeated_scalar_container, and must +// be explicitly instantiated. +template bool CheckAndGetInteger<int32>( + PyObject*, int32*, PyObject*, PyObject*); +template bool CheckAndGetInteger<int64>( + PyObject*, int64*, PyObject*, PyObject*); +template bool CheckAndGetInteger<uint32>( + PyObject*, uint32*, PyObject*, PyObject*); +template bool CheckAndGetInteger<uint64>( + PyObject*, uint64*, PyObject*, PyObject*); + +bool CheckAndGetDouble(PyObject* arg, double* value) { + if (!PyInt_Check(arg) && !PyLong_Check(arg) && + !PyFloat_Check(arg)) { + FormatTypeError(arg, "int, long, float"); + return false; + } + *value = PyFloat_AsDouble(arg); + return true; +} + +bool CheckAndGetFloat(PyObject* arg, float* value) { + double double_value; + if (!CheckAndGetDouble(arg, &double_value)) { + return false; + } + *value = static_cast<float>(double_value); + return true; +} + +bool CheckAndGetBool(PyObject* arg, bool* value) { + if (!PyInt_Check(arg) && !PyBool_Check(arg) && !PyLong_Check(arg)) { + FormatTypeError(arg, "int, long, bool"); + return false; + } + *value = static_cast<bool>(PyInt_AsLong(arg)); + return true; +} + +bool CheckAndSetString( + PyObject* arg, google::protobuf::Message* message, + const google::protobuf::FieldDescriptor* descriptor, + const google::protobuf::Reflection* reflection, + bool append, + int index) { + GOOGLE_DCHECK(descriptor->type() == google::protobuf::FieldDescriptor::TYPE_STRING || + descriptor->type() == google::protobuf::FieldDescriptor::TYPE_BYTES); + if (descriptor->type() == google::protobuf::FieldDescriptor::TYPE_STRING) { + if (!PyBytes_Check(arg) && !PyUnicode_Check(arg)) { + FormatTypeError(arg, "bytes, unicode"); + return false; + } + + if (PyBytes_Check(arg)) { + PyObject* unicode = PyUnicode_FromEncodedObject(arg, "ascii", NULL); + if (unicode == NULL) { + PyObject* repr = PyObject_Repr(arg); + PyErr_Format(PyExc_ValueError, + "%s has type str, but isn't in 7-bit ASCII " + "encoding. Non-ASCII strings must be converted to " + "unicode objects before being added.", + PyString_AsString(repr)); + Py_DECREF(repr); + return false; + } else { + Py_DECREF(unicode); + } + } + } else if (!PyBytes_Check(arg)) { + FormatTypeError(arg, "bytes"); + return false; + } + + PyObject* encoded_string = NULL; + if (descriptor->type() == google::protobuf::FieldDescriptor::TYPE_STRING) { + if (PyBytes_Check(arg)) { +#if PY_MAJOR_VERSION < 3 + encoded_string = PyString_AsEncodedObject(arg, "utf-8", NULL); +#else + encoded_string = arg; // Already encoded. + Py_INCREF(encoded_string); +#endif + } else { + encoded_string = PyUnicode_AsEncodedObject(arg, "utf-8", NULL); + } + } else { + // In this case field type is "bytes". + encoded_string = arg; + Py_INCREF(encoded_string); + } + + if (encoded_string == NULL) { + return false; + } + + char* value; + Py_ssize_t value_len; + if (PyBytes_AsStringAndSize(encoded_string, &value, &value_len) < 0) { + Py_DECREF(encoded_string); + return false; + } + + string value_string(value, value_len); + if (append) { + reflection->AddString(message, descriptor, value_string); + } else if (index < 0) { + reflection->SetString(message, descriptor, value_string); + } else { + reflection->SetRepeatedString(message, descriptor, index, value_string); + } + Py_DECREF(encoded_string); + return true; +} + +PyObject* ToStringObject( + const google::protobuf::FieldDescriptor* descriptor, string value) { + if (descriptor->type() != google::protobuf::FieldDescriptor::TYPE_STRING) { + return PyBytes_FromStringAndSize(value.c_str(), value.length()); + } + + PyObject* result = PyUnicode_DecodeUTF8(value.c_str(), value.length(), NULL); + // If the string can't be decoded in UTF-8, just return a string object that + // contains the raw bytes. This can't happen if the value was assigned using + // the members of the Python message object, but can happen if the values were + // parsed from the wire (binary). + if (result == NULL) { + PyErr_Clear(); + result = PyBytes_FromStringAndSize(value.c_str(), value.length()); + } + return result; +} + +google::protobuf::DynamicMessageFactory* global_message_factory; + +namespace cmessage { + +static int MaybeReleaseOverlappingOneofField( + CMessage* cmessage, + const google::protobuf::FieldDescriptor* field) { +#ifdef GOOGLE_PROTOBUF_HAS_ONEOF + google::protobuf::Message* message = cmessage->message; + const google::protobuf::Reflection* reflection = message->GetReflection(); + if (!field->containing_oneof() || + !reflection->HasOneof(*message, field->containing_oneof()) || + reflection->HasField(*message, field)) { + // No other field in this oneof, no need to release. + return 0; + } + + const OneofDescriptor* oneof = field->containing_oneof(); + const FieldDescriptor* existing_field = + reflection->GetOneofFieldDescriptor(*message, oneof); + if (existing_field->cpp_type() != google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + // Non-message fields don't need to be released. + return 0; + } + const char* field_name = existing_field->name().c_str(); + PyObject* child_message = PyDict_GetItemString( + cmessage->composite_fields, field_name); + if (child_message == NULL) { + // No python reference to this field so no need to release. + return 0; + } + + if (InternalReleaseFieldByDescriptor( + existing_field, child_message, message) < 0) { + return -1; + } + return PyDict_DelItemString(cmessage->composite_fields, field_name); +#else + return 0; +#endif +} + +// --------------------------------------------------------------------- +// Making a message writable + +static google::protobuf::Message* GetMutableMessage( + CMessage* parent, + const google::protobuf::FieldDescriptor* parent_field) { + google::protobuf::Message* parent_message = parent->message; + const google::protobuf::Reflection* reflection = parent_message->GetReflection(); + if (MaybeReleaseOverlappingOneofField(parent, parent_field) < 0) { + return NULL; + } + return reflection->MutableMessage( + parent_message, parent_field, global_message_factory); +} + +struct FixupMessageReference : public ChildVisitor { + // message must outlive this object. + explicit FixupMessageReference(google::protobuf::Message* message) : + message_(message) {} + + int VisitRepeatedCompositeContainer(RepeatedCompositeContainer* container) { + container->message = message_; + return 0; + } + + int VisitRepeatedScalarContainer(RepeatedScalarContainer* container) { + container->message = message_; + return 0; + } + + private: + google::protobuf::Message* message_; +}; + +int AssureWritable(CMessage* self) { + if (self == NULL || !self->read_only) { + return 0; + } + + if (self->parent == NULL) { + // If parent is NULL but we are trying to modify a read-only message, this + // is a reference to a constant default instance that needs to be replaced + // with a mutable top-level message. + const Message* prototype = global_message_factory->GetPrototype( + self->message->GetDescriptor()); + self->message = prototype->New(); + self->owner.reset(self->message); + } else { + // Otherwise, we need a mutable child message. + if (AssureWritable(self->parent) == -1) + return -1; + + // Make self->message writable. + google::protobuf::Message* parent_message = self->parent->message; + google::protobuf::Message* mutable_message = GetMutableMessage( + self->parent, + self->parent_field->descriptor); + if (mutable_message == NULL) { + return -1; + } + self->message = mutable_message; + } + self->read_only = false; + + // When a CMessage is made writable its Message pointer is updated + // to point to a new mutable Message. When that happens we need to + // update any references to the old, read-only CMessage. There are + // three places such references occur: RepeatedScalarContainer, + // RepeatedCompositeContainer, and ExtensionDict. + if (self->extensions != NULL) + self->extensions->message = self->message; + if (ForEachCompositeField(self, FixupMessageReference(self->message)) == -1) + return -1; + + return 0; +} + +// --- Globals: + +static PyObject* GetDescriptor(CMessage* self, PyObject* name) { + PyObject* descriptors = + PyDict_GetItem(Py_TYPE(self)->tp_dict, k__descriptors); + if (descriptors == NULL) { + PyErr_SetString(PyExc_TypeError, "No __descriptors"); + return NULL; + } + + return PyDict_GetItem(descriptors, name); +} + +static const google::protobuf::Message* CreateMessage(const char* message_type) { + string message_name(message_type); + const google::protobuf::Descriptor* descriptor = + GetDescriptorPool()->FindMessageTypeByName(message_name); + if (descriptor == NULL) { + PyErr_SetString(PyExc_TypeError, message_type); + return NULL; + } + return global_message_factory->GetPrototype(descriptor); +} + +// If cmessage_list is not NULL, this function releases values into the +// container CMessages instead of just removing. Repeated composite container +// needs to do this to make sure CMessages stay alive if they're still +// referenced after deletion. Repeated scalar container doesn't need to worry. +int InternalDeleteRepeatedField( + google::protobuf::Message* message, + const google::protobuf::FieldDescriptor* field_descriptor, + PyObject* slice, + PyObject* cmessage_list) { + Py_ssize_t length, from, to, step, slice_length; + const google::protobuf::Reflection* reflection = message->GetReflection(); + int min, max; + length = reflection->FieldSize(*message, field_descriptor); + + if (PyInt_Check(slice) || PyLong_Check(slice)) { + from = to = PyLong_AsLong(slice); + if (from < 0) { + from = to = length + from; + } + step = 1; + min = max = from; + + // Range check. + if (from < 0 || from >= length) { + PyErr_Format(PyExc_IndexError, "list assignment index out of range"); + return -1; + } + } else if (PySlice_Check(slice)) { + from = to = step = slice_length = 0; + PySlice_GetIndicesEx( +#if PY_MAJOR_VERSION < 3 + reinterpret_cast<PySliceObject*>(slice), +#else + slice, +#endif + length, &from, &to, &step, &slice_length); + if (from < to) { + min = from; + max = to - 1; + } else { + min = to + 1; + max = from; + } + } else { + PyErr_SetString(PyExc_TypeError, "list indices must be integers"); + return -1; + } + + Py_ssize_t i = from; + std::vector<bool> to_delete(length, false); + while (i >= min && i <= max) { + to_delete[i] = true; + i += step; + } + + to = 0; + for (i = 0; i < length; ++i) { + if (!to_delete[i]) { + if (i != to) { + reflection->SwapElements(message, field_descriptor, i, to); + if (cmessage_list != NULL) { + // If a list of cmessages is passed in (i.e. from a repeated + // composite container), swap those as well to correspond to the + // swaps in the underlying message so they're in the right order + // when we start releasing. + PyObject* tmp = PyList_GET_ITEM(cmessage_list, i); + PyList_SET_ITEM(cmessage_list, i, + PyList_GET_ITEM(cmessage_list, to)); + PyList_SET_ITEM(cmessage_list, to, tmp); + } + } + ++to; + } + } + + while (i > to) { + if (cmessage_list == NULL) { + reflection->RemoveLast(message, field_descriptor); + } else { + CMessage* last_cmessage = reinterpret_cast<CMessage*>( + PyList_GET_ITEM(cmessage_list, PyList_GET_SIZE(cmessage_list) - 1)); + repeated_composite_container::ReleaseLastTo( + field_descriptor, message, last_cmessage); + if (PySequence_DelItem(cmessage_list, -1) < 0) { + return -1; + } + } + --i; + } + + return 0; +} + +int InitAttributes(CMessage* self, PyObject* arg, PyObject* kwargs) { + ScopedPyObjectPtr descriptor; + if (arg == NULL) { + descriptor.reset( + PyObject_GetAttr(reinterpret_cast<PyObject*>(self), kDESCRIPTOR)); + if (descriptor == NULL) { + return NULL; + } + } else { + descriptor.reset(arg); + descriptor.inc(); + } + ScopedPyObjectPtr is_extendable(PyObject_GetAttr(descriptor, kis_extendable)); + if (is_extendable == NULL) { + return NULL; + } + int retcode = PyObject_IsTrue(is_extendable); + if (retcode == -1) { + return NULL; + } + if (retcode) { + PyObject* py_extension_dict = PyObject_CallObject( + reinterpret_cast<PyObject*>(&ExtensionDict_Type), NULL); + if (py_extension_dict == NULL) { + return NULL; + } + ExtensionDict* extension_dict = reinterpret_cast<ExtensionDict*>( + py_extension_dict); + extension_dict->parent = self; + extension_dict->message = self->message; + self->extensions = extension_dict; + } + + if (kwargs == NULL) { + return 0; + } + + Py_ssize_t pos = 0; + PyObject* name; + PyObject* value; + while (PyDict_Next(kwargs, &pos, &name, &value)) { + if (!PyString_Check(name)) { + PyErr_SetString(PyExc_ValueError, "Field name must be a string"); + return -1; + } + PyObject* py_cdescriptor = GetDescriptor(self, name); + if (py_cdescriptor == NULL) { + PyErr_Format(PyExc_ValueError, "Protocol message has no \"%s\" field.", + PyString_AsString(name)); + return -1; + } + const google::protobuf::FieldDescriptor* descriptor = + reinterpret_cast<CFieldDescriptor*>(py_cdescriptor)->descriptor; + if (descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) { + ScopedPyObjectPtr container(GetAttr(self, name)); + if (container == NULL) { + return -1; + } + if (descriptor->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + if (repeated_composite_container::Extend( + reinterpret_cast<RepeatedCompositeContainer*>(container.get()), + value) + == NULL) { + return -1; + } + } else { + if (repeated_scalar_container::Extend( + reinterpret_cast<RepeatedScalarContainer*>(container.get()), + value) == + NULL) { + return -1; + } + } + } else if (descriptor->cpp_type() == + google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + ScopedPyObjectPtr message(GetAttr(self, name)); + if (message == NULL) { + return -1; + } + if (MergeFrom(reinterpret_cast<CMessage*>(message.get()), + value) == NULL) { + return -1; + } + } else { + if (SetAttr(self, name, value) < 0) { + return -1; + } + } + } + return 0; +} + +static PyObject* New(PyTypeObject* type, PyObject* args, PyObject* kwargs) { + CMessage* self = reinterpret_cast<CMessage*>(type->tp_alloc(type, 0)); + if (self == NULL) { + return NULL; + } + + self->message = NULL; + self->parent = NULL; + self->parent_field = NULL; + self->read_only = false; + self->extensions = NULL; + + self->composite_fields = PyDict_New(); + if (self->composite_fields == NULL) { + return NULL; + } + return reinterpret_cast<PyObject*>(self); +} + +PyObject* NewEmpty(PyObject* type) { + return New(reinterpret_cast<PyTypeObject*>(type), NULL, NULL); +} + +static int Init(CMessage* self, PyObject* args, PyObject* kwargs) { + if (kwargs == NULL) { + // TODO(anuraag): Set error + return -1; + } + + PyObject* descriptor = PyTuple_GetItem(args, 0); + if (descriptor == NULL || PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_ValueError, "args must contain one arg: descriptor"); + return -1; + } + + ScopedPyObjectPtr py_message_type(PyObject_GetAttr(descriptor, kfull_name)); + if (py_message_type == NULL) { + return -1; + } + + const char* message_type = PyString_AsString(py_message_type.get()); + const google::protobuf::Message* message = CreateMessage(message_type); + if (message == NULL) { + return -1; + } + + self->message = message->New(); + self->owner.reset(self->message); + + if (InitAttributes(self, descriptor, kwargs) < 0) { + return -1; + } + return 0; +} + +// --------------------------------------------------------------------- +// Deallocating a CMessage +// +// Deallocating a CMessage requires that we clear any weak references +// from children to the message being deallocated. + +// Clear the weak reference from the child to the parent. +struct ClearWeakReferences : public ChildVisitor { + int VisitRepeatedCompositeContainer(RepeatedCompositeContainer* container) { + container->parent = NULL; + // The elements in the container have the same parent as the + // container itself, so NULL out that pointer as well. + const Py_ssize_t n = PyList_GET_SIZE(container->child_messages); + for (Py_ssize_t i = 0; i < n; ++i) { + CMessage* child_cmessage = reinterpret_cast<CMessage*>( + PyList_GET_ITEM(container->child_messages, i)); + child_cmessage->parent = NULL; + } + return 0; + } + + int VisitRepeatedScalarContainer(RepeatedScalarContainer* container) { + container->parent = NULL; + return 0; + } + + int VisitCMessage(CMessage* cmessage, + const google::protobuf::FieldDescriptor* field_descriptor) { + cmessage->parent = NULL; + return 0; + } +}; + +static void Dealloc(CMessage* self) { + // Null out all weak references from children to this message. + GOOGLE_CHECK_EQ(0, ForEachCompositeField(self, ClearWeakReferences())); + + Py_CLEAR(self->extensions); + Py_CLEAR(self->composite_fields); + self->owner.reset(); + Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); +} + +// --------------------------------------------------------------------- + + +PyObject* IsInitialized(CMessage* self, PyObject* args) { + PyObject* errors = NULL; + if (PyArg_ParseTuple(args, "|O", &errors) < 0) { + return NULL; + } + if (self->message->IsInitialized()) { + Py_RETURN_TRUE; + } + if (errors != NULL) { + ScopedPyObjectPtr initialization_errors( + FindInitializationErrors(self)); + if (initialization_errors == NULL) { + return NULL; + } + ScopedPyObjectPtr extend_name(PyString_FromString("extend")); + if (extend_name == NULL) { + return NULL; + } + ScopedPyObjectPtr result(PyObject_CallMethodObjArgs( + errors, + extend_name.get(), + initialization_errors.get(), + NULL)); + if (result == NULL) { + return NULL; + } + } + Py_RETURN_FALSE; +} + +PyObject* HasFieldByDescriptor( + CMessage* self, const google::protobuf::FieldDescriptor* field_descriptor) { + google::protobuf::Message* message = self->message; + if (!FIELD_BELONGS_TO_MESSAGE(field_descriptor, message)) { + PyErr_SetString(PyExc_KeyError, + "Field does not belong to message!"); + return NULL; + } + if (field_descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) { + PyErr_SetString(PyExc_KeyError, + "Field is repeated. A singular method is required."); + return NULL; + } + bool has_field = + message->GetReflection()->HasField(*message, field_descriptor); + return PyBool_FromLong(has_field ? 1 : 0); +} + +const google::protobuf::FieldDescriptor* FindFieldWithOneofs( + const google::protobuf::Message* message, const char* field_name, bool* in_oneof) { + const google::protobuf::Descriptor* descriptor = message->GetDescriptor(); + const google::protobuf::FieldDescriptor* field_descriptor = + descriptor->FindFieldByName(field_name); + if (field_descriptor == NULL) { + const google::protobuf::OneofDescriptor* oneof_desc = + message->GetDescriptor()->FindOneofByName(field_name); + if (oneof_desc == NULL) { + *in_oneof = false; + return NULL; + } else { + *in_oneof = true; + return message->GetReflection()->GetOneofFieldDescriptor( + *message, oneof_desc); + } + } + return field_descriptor; +} + +PyObject* HasField(CMessage* self, PyObject* arg) { +#if PY_MAJOR_VERSION < 3 + char* field_name; + if (PyString_AsStringAndSize(arg, &field_name, NULL) < 0) { +#else + char* field_name = PyUnicode_AsUTF8(arg); + if (!field_name) { +#endif + return NULL; + } + + google::protobuf::Message* message = self->message; + const google::protobuf::Descriptor* descriptor = message->GetDescriptor(); + bool is_in_oneof; + const google::protobuf::FieldDescriptor* field_descriptor = + FindFieldWithOneofs(message, field_name, &is_in_oneof); + if (field_descriptor == NULL) { + if (!is_in_oneof) { + PyErr_Format(PyExc_ValueError, "Unknown field %s.", field_name); + return NULL; + } else { + Py_RETURN_FALSE; + } + } + + if (field_descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) { + PyErr_Format(PyExc_ValueError, + "Protocol message has no singular \"%s\" field.", field_name); + return NULL; + } + + bool has_field = + message->GetReflection()->HasField(*message, field_descriptor); + if (!has_field && field_descriptor->cpp_type() == + google::protobuf::FieldDescriptor::CPPTYPE_ENUM) { + // We may have an invalid enum value stored in the UnknownFieldSet and need + // to check presence in there as well. + const google::protobuf::UnknownFieldSet& unknown_field_set = + message->GetReflection()->GetUnknownFields(*message); + for (int i = 0; i < unknown_field_set.field_count(); ++i) { + if (unknown_field_set.field(i).number() == field_descriptor->number()) { + Py_RETURN_TRUE; + } + } + Py_RETURN_FALSE; + } + return PyBool_FromLong(has_field ? 1 : 0); +} + +PyObject* ClearExtension(CMessage* self, PyObject* arg) { + if (self->extensions != NULL) { + return extension_dict::ClearExtension(self->extensions, arg); + } + PyErr_SetString(PyExc_TypeError, "Message is not extendable"); + return NULL; +} + +PyObject* HasExtension(CMessage* self, PyObject* arg) { + if (self->extensions != NULL) { + return extension_dict::HasExtension(self->extensions, arg); + } + PyErr_SetString(PyExc_TypeError, "Message is not extendable"); + return NULL; +} + +// --------------------------------------------------------------------- +// Releasing messages +// +// The Python API's ClearField() and Clear() methods behave +// differently than their C++ counterparts. While the C++ versions +// clears the children the Python versions detaches the children, +// without touching their content. This impedance mismatch causes +// some complexity in the implementation, which is captured in this +// section. +// +// When a CMessage field is cleared we need to: +// +// * Release the Message used as the backing store for the CMessage +// from its parent. +// +// * Change the owner field of the released CMessage and all of its +// children to point to the newly released Message. +// +// * Clear the weak references from the released CMessage to the +// parent. +// +// When a RepeatedCompositeContainer field is cleared we need to: +// +// * Release all the Message used as the backing store for the +// CMessages stored in the container. +// +// * Change the owner field of all the released CMessage and all of +// their children to point to the newly released Messages. +// +// * Clear the weak references from the released container to the +// parent. + +struct SetOwnerVisitor : public ChildVisitor { + // new_owner must outlive this object. + explicit SetOwnerVisitor(const shared_ptr<Message>& new_owner) + : new_owner_(new_owner) {} + + int VisitRepeatedCompositeContainer(RepeatedCompositeContainer* container) { + repeated_composite_container::SetOwner(container, new_owner_); + return 0; + } + + int VisitRepeatedScalarContainer(RepeatedScalarContainer* container) { + repeated_scalar_container::SetOwner(container, new_owner_); + return 0; + } + + int VisitCMessage(CMessage* cmessage, + const google::protobuf::FieldDescriptor* field_descriptor) { + return SetOwner(cmessage, new_owner_); + } + + private: + const shared_ptr<Message>& new_owner_; +}; + +// Change the owner of this CMessage and all its children, recursively. +int SetOwner(CMessage* self, const shared_ptr<Message>& new_owner) { + self->owner = new_owner; + if (ForEachCompositeField(self, SetOwnerVisitor(new_owner)) == -1) + return -1; + return 0; +} + +// Releases the message specified by 'field' and returns the +// pointer. If the field does not exist a new message is created using +// 'descriptor'. The caller takes ownership of the returned pointer. +Message* ReleaseMessage(google::protobuf::Message* message, + const google::protobuf::Descriptor* descriptor, + const google::protobuf::FieldDescriptor* field_descriptor) { + Message* released_message = message->GetReflection()->ReleaseMessage( + message, field_descriptor, global_message_factory); + // ReleaseMessage will return NULL which differs from + // child_cmessage->message, if the field does not exist. In this case, + // the latter points to the default instance via a const_cast<>, so we + // have to reset it to a new mutable object since we are taking ownership. + if (released_message == NULL) { + const Message* prototype = global_message_factory->GetPrototype( + descriptor); + GOOGLE_DCHECK(prototype != NULL); + released_message = prototype->New(); + } + + return released_message; +} + +int ReleaseSubMessage(google::protobuf::Message* message, + const google::protobuf::FieldDescriptor* field_descriptor, + CMessage* child_cmessage) { + // Release the Message + shared_ptr<Message> released_message(ReleaseMessage( + message, child_cmessage->message->GetDescriptor(), field_descriptor)); + child_cmessage->message = released_message.get(); + child_cmessage->owner.swap(released_message); + child_cmessage->parent = NULL; + child_cmessage->parent_field = NULL; + child_cmessage->read_only = false; + return ForEachCompositeField(child_cmessage, + SetOwnerVisitor(child_cmessage->owner)); +} + +struct ReleaseChild : public ChildVisitor { + // message must outlive this object. + explicit ReleaseChild(google::protobuf::Message* parent_message) : + parent_message_(parent_message) {} + + int VisitRepeatedCompositeContainer(RepeatedCompositeContainer* container) { + return repeated_composite_container::Release( + reinterpret_cast<RepeatedCompositeContainer*>(container)); + } + + int VisitRepeatedScalarContainer(RepeatedScalarContainer* container) { + return repeated_scalar_container::Release( + reinterpret_cast<RepeatedScalarContainer*>(container)); + } + + int VisitCMessage(CMessage* cmessage, + const google::protobuf::FieldDescriptor* field_descriptor) { + return ReleaseSubMessage(parent_message_, field_descriptor, + reinterpret_cast<CMessage*>(cmessage)); + } + + google::protobuf::Message* parent_message_; +}; + +int InternalReleaseFieldByDescriptor( + const google::protobuf::FieldDescriptor* field_descriptor, + PyObject* composite_field, + google::protobuf::Message* parent_message) { + return VisitCompositeField( + field_descriptor, + composite_field, + ReleaseChild(parent_message)); +} + +int InternalReleaseField(CMessage* self, PyObject* composite_field, + PyObject* name) { + PyObject* cdescriptor = GetDescriptor(self, name); + if (cdescriptor != NULL) { + const google::protobuf::FieldDescriptor* descriptor = + reinterpret_cast<CFieldDescriptor*>(cdescriptor)->descriptor; + return InternalReleaseFieldByDescriptor( + descriptor, composite_field, self->message); + } + + return 0; +} + +PyObject* ClearFieldByDescriptor( + CMessage* self, + const google::protobuf::FieldDescriptor* descriptor) { + if (!FIELD_BELONGS_TO_MESSAGE(descriptor, self->message)) { + PyErr_SetString(PyExc_KeyError, + "Field does not belong to message!"); + return NULL; + } + AssureWritable(self); + self->message->GetReflection()->ClearField(self->message, descriptor); + Py_RETURN_NONE; +} + +PyObject* ClearField(CMessage* self, PyObject* arg) { + char* field_name; + if (!PyString_Check(arg)) { + PyErr_SetString(PyExc_TypeError, "field name must be a string"); + return NULL; + } +#if PY_MAJOR_VERSION < 3 + if (PyString_AsStringAndSize(arg, &field_name, NULL) < 0) { + return NULL; + } +#else + field_name = PyUnicode_AsUTF8(arg); +#endif + AssureWritable(self); + google::protobuf::Message* message = self->message; + const google::protobuf::Descriptor* descriptor = message->GetDescriptor(); + ScopedPyObjectPtr arg_in_oneof; + bool is_in_oneof; + const google::protobuf::FieldDescriptor* field_descriptor = + FindFieldWithOneofs(message, field_name, &is_in_oneof); + if (field_descriptor == NULL) { + if (!is_in_oneof) { + PyErr_Format(PyExc_ValueError, + "Protocol message has no \"%s\" field.", field_name); + return NULL; + } else { + Py_RETURN_NONE; + } + } else if (is_in_oneof) { + arg_in_oneof.reset(PyString_FromString(field_descriptor->name().c_str())); + arg = arg_in_oneof.get(); + } + + PyObject* composite_field = PyDict_GetItem(self->composite_fields, + arg); + + // Only release the field if there's a possibility that there are + // references to it. + if (composite_field != NULL) { + if (InternalReleaseField(self, composite_field, arg) < 0) { + return NULL; + } + PyDict_DelItem(self->composite_fields, arg); + } + message->GetReflection()->ClearField(message, field_descriptor); + if (field_descriptor->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_ENUM) { + google::protobuf::UnknownFieldSet* unknown_field_set = + message->GetReflection()->MutableUnknownFields(message); + unknown_field_set->DeleteByNumber(field_descriptor->number()); + } + + Py_RETURN_NONE; +} + +PyObject* Clear(CMessage* self) { + AssureWritable(self); + if (ForEachCompositeField(self, ReleaseChild(self->message)) == -1) + return NULL; + + // The old ExtensionDict still aliases this CMessage, but all its + // fields have been released. + if (self->extensions != NULL) { + Py_CLEAR(self->extensions); + PyObject* py_extension_dict = PyObject_CallObject( + reinterpret_cast<PyObject*>(&ExtensionDict_Type), NULL); + if (py_extension_dict == NULL) { + return NULL; + } + ExtensionDict* extension_dict = reinterpret_cast<ExtensionDict*>( + py_extension_dict); + extension_dict->parent = self; + extension_dict->message = self->message; + self->extensions = extension_dict; + } + PyDict_Clear(self->composite_fields); + self->message->Clear(); + Py_RETURN_NONE; +} + +// --------------------------------------------------------------------- + +static string GetMessageName(CMessage* self) { + if (self->parent_field != NULL) { + return self->parent_field->descriptor->full_name(); + } else { + return self->message->GetDescriptor()->full_name(); + } +} + +static PyObject* SerializeToString(CMessage* self, PyObject* args) { + if (!self->message->IsInitialized()) { + ScopedPyObjectPtr errors(FindInitializationErrors(self)); + if (errors == NULL) { + return NULL; + } + ScopedPyObjectPtr comma(PyString_FromString(",")); + if (comma == NULL) { + return NULL; + } + ScopedPyObjectPtr joined( + PyObject_CallMethod(comma.get(), "join", "O", errors.get())); + if (joined == NULL) { + return NULL; + } + PyErr_Format(EncodeError_class, "Message %s is missing required fields: %s", + GetMessageName(self).c_str(), PyString_AsString(joined.get())); + return NULL; + } + int size = self->message->ByteSize(); + if (size <= 0) { + return PyBytes_FromString(""); + } + PyObject* result = PyBytes_FromStringAndSize(NULL, size); + if (result == NULL) { + return NULL; + } + char* buffer = PyBytes_AS_STRING(result); + self->message->SerializeWithCachedSizesToArray( + reinterpret_cast<uint8*>(buffer)); + return result; +} + +static PyObject* SerializePartialToString(CMessage* self) { + string contents; + self->message->SerializePartialToString(&contents); + return PyBytes_FromStringAndSize(contents.c_str(), contents.size()); +} + +// Formats proto fields for ascii dumps using python formatting functions where +// appropriate. +class PythonFieldValuePrinter : public google::protobuf::TextFormat::FieldValuePrinter { + public: + PythonFieldValuePrinter() : float_holder_(PyFloat_FromDouble(0)) {} + + // Python has some differences from C++ when printing floating point numbers. + // + // 1) Trailing .0 is always printed. + // 2) Outputted is rounded to 12 digits. + // + // We override floating point printing with the C-API function for printing + // Python floats to ensure consistency. + string PrintFloat(float value) const { return PrintDouble(value); } + string PrintDouble(double value) const { + reinterpret_cast<PyFloatObject*>(float_holder_.get())->ob_fval = value; + ScopedPyObjectPtr s(PyObject_Str(float_holder_.get())); + if (s == NULL) return string(); +#if PY_MAJOR_VERSION < 3 + char *cstr = PyBytes_AS_STRING(static_cast<PyObject*>(s)); +#else + char *cstr = PyUnicode_AsUTF8(s); +#endif + return string(cstr); + } + + private: + // Holder for a python float object which we use to allow us to use + // the Python API for printing doubles. We initialize once and then + // directly modify it for every float printed to save on allocations + // and refcounting. + ScopedPyObjectPtr float_holder_; +}; + +static PyObject* ToStr(CMessage* self) { + google::protobuf::TextFormat::Printer printer; + // Passes ownership + printer.SetDefaultFieldValuePrinter(new PythonFieldValuePrinter()); + printer.SetHideUnknownFields(true); + string output; + if (!printer.PrintToString(*self->message, &output)) { + PyErr_SetString(PyExc_ValueError, "Unable to convert message to str"); + return NULL; + } + return PyString_FromString(output.c_str()); +} + +PyObject* MergeFrom(CMessage* self, PyObject* arg) { + CMessage* other_message; + if (!PyObject_TypeCheck(reinterpret_cast<PyObject *>(arg), &CMessage_Type)) { + PyErr_SetString(PyExc_TypeError, "Must be a message"); + return NULL; + } + + other_message = reinterpret_cast<CMessage*>(arg); + if (other_message->message->GetDescriptor() != + self->message->GetDescriptor()) { + PyErr_Format(PyExc_TypeError, + "Tried to merge from a message with a different type. " + "to: %s, from: %s", + self->message->GetDescriptor()->full_name().c_str(), + other_message->message->GetDescriptor()->full_name().c_str()); + return NULL; + } + AssureWritable(self); + + // TODO(tibell): Message::MergeFrom might turn some child Messages + // into mutable messages, invalidating the message field in the + // corresponding CMessages. We should run a FixupMessageReferences + // pass here. + + self->message->MergeFrom(*other_message->message); + Py_RETURN_NONE; +} + +static PyObject* CopyFrom(CMessage* self, PyObject* arg) { + CMessage* other_message; + if (!PyObject_TypeCheck(reinterpret_cast<PyObject *>(arg), &CMessage_Type)) { + PyErr_SetString(PyExc_TypeError, "Must be a message"); + return NULL; + } + + other_message = reinterpret_cast<CMessage*>(arg); + + if (self == other_message) { + Py_RETURN_NONE; + } + + if (other_message->message->GetDescriptor() != + self->message->GetDescriptor()) { + PyErr_Format(PyExc_TypeError, + "Tried to copy from a message with a different type. " + "to: %s, from: %s", + self->message->GetDescriptor()->full_name().c_str(), + other_message->message->GetDescriptor()->full_name().c_str()); + return NULL; + } + + AssureWritable(self); + + // CopyFrom on the message will not clean up self->composite_fields, + // which can leave us in an inconsistent state, so clear it out here. + Clear(self); + + self->message->CopyFrom(*other_message->message); + + Py_RETURN_NONE; +} + +static PyObject* MergeFromString(CMessage* self, PyObject* arg) { + const void* data; + Py_ssize_t data_length; + if (PyObject_AsReadBuffer(arg, &data, &data_length) < 0) { + return NULL; + } + + AssureWritable(self); + google::protobuf::io::CodedInputStream input( + reinterpret_cast<const uint8*>(data), data_length); + input.SetExtensionRegistry(GetDescriptorPool(), global_message_factory); + bool success = self->message->MergePartialFromCodedStream(&input); + if (success) { + return PyInt_FromLong(input.CurrentPosition()); + } else { + PyErr_Format(DecodeError_class, "Error parsing message"); + return NULL; + } +} + +static PyObject* ParseFromString(CMessage* self, PyObject* arg) { + if (Clear(self) == NULL) { + return NULL; + } + return MergeFromString(self, arg); +} + +static PyObject* ByteSize(CMessage* self, PyObject* args) { + return PyLong_FromLong(self->message->ByteSize()); +} + +static PyObject* RegisterExtension(PyObject* cls, + PyObject* extension_handle) { + ScopedPyObjectPtr message_descriptor(PyObject_GetAttr(cls, kDESCRIPTOR)); + if (message_descriptor == NULL) { + return NULL; + } + if (PyObject_SetAttrString(extension_handle, "containing_type", + message_descriptor) < 0) { + return NULL; + } + ScopedPyObjectPtr extensions_by_name( + PyObject_GetAttr(cls, k_extensions_by_name)); + if (extensions_by_name == NULL) { + PyErr_SetString(PyExc_TypeError, "no extensions_by_name on class"); + return NULL; + } + ScopedPyObjectPtr full_name(PyObject_GetAttr(extension_handle, kfull_name)); + if (full_name == NULL) { + return NULL; + } + if (PyDict_SetItem(extensions_by_name, full_name, extension_handle) < 0) { + return NULL; + } + + // Also store a mapping from extension number to implementing class. + ScopedPyObjectPtr extensions_by_number( + PyObject_GetAttr(cls, k_extensions_by_number)); + if (extensions_by_number == NULL) { + PyErr_SetString(PyExc_TypeError, "no extensions_by_number on class"); + return NULL; + } + ScopedPyObjectPtr number(PyObject_GetAttrString(extension_handle, "number")); + if (number == NULL) { + return NULL; + } + if (PyDict_SetItem(extensions_by_number, number, extension_handle) < 0) { + return NULL; + } + + CFieldDescriptor* cdescriptor = + extension_dict::InternalGetCDescriptorFromExtension(extension_handle); + ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor)); + if (cdescriptor == NULL) { + return NULL; + } + Py_INCREF(extension_handle); + cdescriptor->descriptor_field = extension_handle; + const google::protobuf::FieldDescriptor* descriptor = cdescriptor->descriptor; + // Check if it's a message set + if (descriptor->is_extension() && + descriptor->containing_type()->options().message_set_wire_format() && + descriptor->type() == google::protobuf::FieldDescriptor::TYPE_MESSAGE && + descriptor->message_type() == descriptor->extension_scope() && + descriptor->label() == google::protobuf::FieldDescriptor::LABEL_OPTIONAL) { + ScopedPyObjectPtr message_name(PyString_FromStringAndSize( + descriptor->message_type()->full_name().c_str(), + descriptor->message_type()->full_name().size())); + if (message_name == NULL) { + return NULL; + } + PyDict_SetItem(extensions_by_name, message_name, extension_handle); + } + + Py_RETURN_NONE; +} + +static PyObject* SetInParent(CMessage* self, PyObject* args) { + AssureWritable(self); + Py_RETURN_NONE; +} + +static PyObject* WhichOneof(CMessage* self, PyObject* arg) { + char* oneof_name; + if (!PyString_Check(arg)) { + PyErr_SetString(PyExc_TypeError, "field name must be a string"); + return NULL; + } + oneof_name = PyString_AsString(arg); + if (oneof_name == NULL) { + return NULL; + } + const google::protobuf::OneofDescriptor* oneof_desc = + self->message->GetDescriptor()->FindOneofByName(oneof_name); + if (oneof_desc == NULL) { + PyErr_Format(PyExc_ValueError, + "Protocol message has no oneof \"%s\" field.", oneof_name); + return NULL; + } + const google::protobuf::FieldDescriptor* field_in_oneof = + self->message->GetReflection()->GetOneofFieldDescriptor( + *self->message, oneof_desc); + if (field_in_oneof == NULL) { + Py_RETURN_NONE; + } else { + return PyString_FromString(field_in_oneof->name().c_str()); + } +} + +static PyObject* ListFields(CMessage* self) { + vector<const google::protobuf::FieldDescriptor*> fields; + self->message->GetReflection()->ListFields(*self->message, &fields); + + PyObject* descriptor = PyDict_GetItem(Py_TYPE(self)->tp_dict, kDESCRIPTOR); + if (descriptor == NULL) { + return NULL; + } + ScopedPyObjectPtr fields_by_name( + PyObject_GetAttr(descriptor, kfields_by_name)); + if (fields_by_name == NULL) { + return NULL; + } + ScopedPyObjectPtr extensions_by_name(PyObject_GetAttr( + reinterpret_cast<PyObject*>(Py_TYPE(self)), k_extensions_by_name)); + if (extensions_by_name == NULL) { + PyErr_SetString(PyExc_ValueError, "no extensionsbyname"); + return NULL; + } + // Normally, the list will be exactly the size of the fields. + PyObject* all_fields = PyList_New(fields.size()); + if (all_fields == NULL) { + return NULL; + } + + // When there are unknown extensions, the py list will *not* contain + // the field information. Thus the actual size of the py list will be + // smaller than the size of fields. Set the actual size at the end. + Py_ssize_t actual_size = 0; + for (Py_ssize_t i = 0; i < fields.size(); ++i) { + ScopedPyObjectPtr t(PyTuple_New(2)); + if (t == NULL) { + Py_DECREF(all_fields); + return NULL; + } + + if (fields[i]->is_extension()) { + const string& field_name = fields[i]->full_name(); + PyObject* extension_field = PyDict_GetItemString(extensions_by_name, + field_name.c_str()); + if (extension_field == NULL) { + // If we couldn't fetch extension_field, it means the module that + // defines this extension has not been explicitly imported in Python + // code, and the extension hasn't been registered. There's nothing much + // we can do about this, so just skip it in the output to match the + // behavior of the python implementation. + continue; + } + PyObject* extensions = reinterpret_cast<PyObject*>(self->extensions); + if (extensions == NULL) { + Py_DECREF(all_fields); + return NULL; + } + // 'extension' reference later stolen by PyTuple_SET_ITEM. + PyObject* extension = PyObject_GetItem(extensions, extension_field); + if (extension == NULL) { + Py_DECREF(all_fields); + return NULL; + } + Py_INCREF(extension_field); + PyTuple_SET_ITEM(t.get(), 0, extension_field); + // Steals reference to 'extension' + PyTuple_SET_ITEM(t.get(), 1, extension); + } else { + const string& field_name = fields[i]->name(); + ScopedPyObjectPtr py_field_name(PyString_FromStringAndSize( + field_name.c_str(), field_name.length())); + if (py_field_name == NULL) { + PyErr_SetString(PyExc_ValueError, "bad string"); + Py_DECREF(all_fields); + return NULL; + } + PyObject* field_descriptor = + PyDict_GetItem(fields_by_name, py_field_name); + if (field_descriptor == NULL) { + Py_DECREF(all_fields); + return NULL; + } + + PyObject* field_value = GetAttr(self, py_field_name); + if (field_value == NULL) { + PyErr_SetObject(PyExc_ValueError, py_field_name); + Py_DECREF(all_fields); + return NULL; + } + Py_INCREF(field_descriptor); + PyTuple_SET_ITEM(t.get(), 0, field_descriptor); + PyTuple_SET_ITEM(t.get(), 1, field_value); + } + PyList_SET_ITEM(all_fields, actual_size, t.release()); + ++actual_size; + } + Py_SIZE(all_fields) = actual_size; + return all_fields; +} + +PyObject* FindInitializationErrors(CMessage* self) { + google::protobuf::Message* message = self->message; + vector<string> errors; + message->FindInitializationErrors(&errors); + + PyObject* error_list = PyList_New(errors.size()); + if (error_list == NULL) { + return NULL; + } + for (Py_ssize_t i = 0; i < errors.size(); ++i) { + const string& error = errors[i]; + PyObject* error_string = PyString_FromStringAndSize( + error.c_str(), error.length()); + if (error_string == NULL) { + Py_DECREF(error_list); + return NULL; + } + PyList_SET_ITEM(error_list, i, error_string); + } + return error_list; +} + +static PyObject* RichCompare(CMessage* self, PyObject* other, int opid) { + if (!PyObject_TypeCheck(other, &CMessage_Type)) { + if (opid == Py_EQ) { + Py_RETURN_FALSE; + } else if (opid == Py_NE) { + Py_RETURN_TRUE; + } + } + if (opid == Py_EQ || opid == Py_NE) { + ScopedPyObjectPtr self_fields(ListFields(self)); + ScopedPyObjectPtr other_fields(ListFields( + reinterpret_cast<CMessage*>(other))); + return PyObject_RichCompare(self_fields, other_fields, opid); + } else { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } +} + +PyObject* InternalGetScalar( + CMessage* self, + const google::protobuf::FieldDescriptor* field_descriptor) { + google::protobuf::Message* message = self->message; + const google::protobuf::Reflection* reflection = message->GetReflection(); + + if (!FIELD_BELONGS_TO_MESSAGE(field_descriptor, message)) { + PyErr_SetString( + PyExc_KeyError, "Field does not belong to message!"); + return NULL; + } + + PyObject* result = NULL; + switch (field_descriptor->cpp_type()) { + case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { + int32 value = reflection->GetInt32(*message, field_descriptor); + result = PyInt_FromLong(value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { + int64 value = reflection->GetInt64(*message, field_descriptor); + result = PyLong_FromLongLong(value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { + uint32 value = reflection->GetUInt32(*message, field_descriptor); + result = PyInt_FromSize_t(value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { + uint64 value = reflection->GetUInt64(*message, field_descriptor); + result = PyLong_FromUnsignedLongLong(value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { + float value = reflection->GetFloat(*message, field_descriptor); + result = PyFloat_FromDouble(value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { + double value = reflection->GetDouble(*message, field_descriptor); + result = PyFloat_FromDouble(value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { + bool value = reflection->GetBool(*message, field_descriptor); + result = PyBool_FromLong(value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { + string value = reflection->GetString(*message, field_descriptor); + result = ToStringObject(field_descriptor, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { + if (!message->GetReflection()->HasField(*message, field_descriptor)) { + // Look for the value in the unknown fields. + google::protobuf::UnknownFieldSet* unknown_field_set = + message->GetReflection()->MutableUnknownFields(message); + for (int i = 0; i < unknown_field_set->field_count(); ++i) { + if (unknown_field_set->field(i).number() == + field_descriptor->number()) { + result = PyInt_FromLong(unknown_field_set->field(i).varint()); + break; + } + } + } + + if (result == NULL) { + const google::protobuf::EnumValueDescriptor* enum_value = + message->GetReflection()->GetEnum(*message, field_descriptor); + result = PyInt_FromLong(enum_value->number()); + } + break; + } + default: + PyErr_Format( + PyExc_SystemError, "Getting a value from a field of unknown type %d", + field_descriptor->cpp_type()); + } + + return result; +} + +PyObject* InternalGetSubMessage(CMessage* self, + CFieldDescriptor* cfield_descriptor) { + PyObject* field = cfield_descriptor->descriptor_field; + ScopedPyObjectPtr message_type(PyObject_GetAttr(field, kmessage_type)); + if (message_type == NULL) { + return NULL; + } + ScopedPyObjectPtr concrete_class( + PyObject_GetAttr(message_type, k_concrete_class)); + if (concrete_class == NULL) { + return NULL; + } + PyObject* py_cmsg = cmessage::NewEmpty(concrete_class); + if (py_cmsg == NULL) { + return NULL; + } + if (!PyObject_TypeCheck(py_cmsg, &CMessage_Type)) { + PyErr_SetString(PyExc_TypeError, "Not a CMessage!"); + } + CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg); + + const google::protobuf::FieldDescriptor* field_descriptor = + cfield_descriptor->descriptor; + const google::protobuf::Reflection* reflection = self->message->GetReflection(); + const google::protobuf::Message& sub_message = reflection->GetMessage( + *self->message, field_descriptor, global_message_factory); + cmsg->owner = self->owner; + cmsg->parent = self; + cmsg->parent_field = cfield_descriptor; + cmsg->read_only = !reflection->HasField(*self->message, field_descriptor); + cmsg->message = const_cast<google::protobuf::Message*>(&sub_message); + + if (InitAttributes(cmsg, NULL, NULL) < 0) { + Py_DECREF(py_cmsg); + return NULL; + } + return py_cmsg; +} + +int InternalSetScalar( + CMessage* self, + const google::protobuf::FieldDescriptor* field_descriptor, + PyObject* arg) { + google::protobuf::Message* message = self->message; + const google::protobuf::Reflection* reflection = message->GetReflection(); + + if (!FIELD_BELONGS_TO_MESSAGE(field_descriptor, message)) { + PyErr_SetString( + PyExc_KeyError, "Field does not belong to message!"); + return -1; + } + + if (MaybeReleaseOverlappingOneofField(self, field_descriptor) < 0) { + return -1; + } + + switch (field_descriptor->cpp_type()) { + case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { + GOOGLE_CHECK_GET_INT32(arg, value, -1); + reflection->SetInt32(message, field_descriptor, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { + GOOGLE_CHECK_GET_INT64(arg, value, -1); + reflection->SetInt64(message, field_descriptor, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { + GOOGLE_CHECK_GET_UINT32(arg, value, -1); + reflection->SetUInt32(message, field_descriptor, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { + GOOGLE_CHECK_GET_UINT64(arg, value, -1); + reflection->SetUInt64(message, field_descriptor, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { + GOOGLE_CHECK_GET_FLOAT(arg, value, -1); + reflection->SetFloat(message, field_descriptor, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { + GOOGLE_CHECK_GET_DOUBLE(arg, value, -1); + reflection->SetDouble(message, field_descriptor, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { + GOOGLE_CHECK_GET_BOOL(arg, value, -1); + reflection->SetBool(message, field_descriptor, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { + if (!CheckAndSetString( + arg, message, field_descriptor, reflection, false, -1)) { + return -1; + } + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { + GOOGLE_CHECK_GET_INT32(arg, value, -1); + const google::protobuf::EnumDescriptor* enum_descriptor = + field_descriptor->enum_type(); + const google::protobuf::EnumValueDescriptor* enum_value = + enum_descriptor->FindValueByNumber(value); + if (enum_value != NULL) { + reflection->SetEnum(message, field_descriptor, enum_value); + } else { + PyErr_Format(PyExc_ValueError, "Unknown enum value: %d", value); + return -1; + } + break; + } + default: + PyErr_Format( + PyExc_SystemError, "Setting value to a field of unknown type %d", + field_descriptor->cpp_type()); + return -1; + } + + return 0; +} + +PyObject* FromString(PyTypeObject* cls, PyObject* serialized) { + PyObject* py_cmsg = PyObject_CallObject( + reinterpret_cast<PyObject*>(cls), NULL); + if (py_cmsg == NULL) { + return NULL; + } + CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg); + + ScopedPyObjectPtr py_length(MergeFromString(cmsg, serialized)); + if (py_length == NULL) { + Py_DECREF(py_cmsg); + return NULL; + } + + if (InitAttributes(cmsg, NULL, NULL) < 0) { + Py_DECREF(py_cmsg); + return NULL; + } + return py_cmsg; +} + +static PyObject* AddDescriptors(PyTypeObject* cls, + PyObject* descriptor) { + if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls), + k_extensions_by_name, PyDict_New()) < 0) { + return NULL; + } + if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls), + k_extensions_by_number, PyDict_New()) < 0) { + return NULL; + } + + ScopedPyObjectPtr field_descriptors(PyDict_New()); + + ScopedPyObjectPtr fields(PyObject_GetAttrString(descriptor, "fields")); + if (fields == NULL) { + return NULL; + } + + ScopedPyObjectPtr _NUMBER_string(PyString_FromString("_FIELD_NUMBER")); + if (_NUMBER_string == NULL) { + return NULL; + } + + const Py_ssize_t fields_size = PyList_GET_SIZE(fields.get()); + for (int i = 0; i < fields_size; ++i) { + PyObject* field = PyList_GET_ITEM(fields.get(), i); + ScopedPyObjectPtr field_name(PyObject_GetAttr(field, kname)); + ScopedPyObjectPtr full_field_name(PyObject_GetAttr(field, kfull_name)); + if (field_name == NULL || full_field_name == NULL) { + PyErr_SetString(PyExc_TypeError, "Name is null"); + return NULL; + } + + PyObject* field_descriptor = + cdescriptor_pool::FindFieldByName(descriptor_pool, full_field_name); + if (field_descriptor == NULL) { + PyErr_SetString(PyExc_TypeError, "Couldn't find field"); + return NULL; + } + Py_INCREF(field); + CFieldDescriptor* cfield_descriptor = reinterpret_cast<CFieldDescriptor*>( + field_descriptor); + cfield_descriptor->descriptor_field = field; + if (PyDict_SetItem(field_descriptors, field_name, field_descriptor) < 0) { + return NULL; + } + + // The FieldDescriptor's name field might either be of type bytes or + // of type unicode, depending on whether the FieldDescriptor was + // parsed from a serialized message or read from the + // <message>_pb2.py module. + ScopedPyObjectPtr field_name_upcased( + PyObject_CallMethod(field_name, "upper", NULL)); + if (field_name_upcased == NULL) { + return NULL; + } + + ScopedPyObjectPtr field_number_name(PyObject_CallMethod( + field_name_upcased, "__add__", "(O)", _NUMBER_string.get())); + if (field_number_name == NULL) { + return NULL; + } + + ScopedPyObjectPtr number(PyInt_FromLong( + cfield_descriptor->descriptor->number())); + if (number == NULL) { + return NULL; + } + if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls), + field_number_name, number) == -1) { + return NULL; + } + } + + PyDict_SetItem(cls->tp_dict, k__descriptors, field_descriptors); + + // Enum Values + ScopedPyObjectPtr enum_types(PyObject_GetAttrString(descriptor, + "enum_types")); + if (enum_types == NULL) { + return NULL; + } + ScopedPyObjectPtr type_iter(PyObject_GetIter(enum_types)); + if (type_iter == NULL) { + return NULL; + } + ScopedPyObjectPtr enum_type; + while ((enum_type.reset(PyIter_Next(type_iter))) != NULL) { + ScopedPyObjectPtr wrapped(PyObject_CallFunctionObjArgs( + EnumTypeWrapper_class, enum_type.get(), NULL)); + if (wrapped == NULL) { + return NULL; + } + ScopedPyObjectPtr enum_name(PyObject_GetAttr(enum_type, kname)); + if (enum_name == NULL) { + return NULL; + } + if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls), + enum_name, wrapped) == -1) { + return NULL; + } + + ScopedPyObjectPtr enum_values(PyObject_GetAttrString(enum_type, "values")); + if (enum_values == NULL) { + return NULL; + } + ScopedPyObjectPtr values_iter(PyObject_GetIter(enum_values)); + if (values_iter == NULL) { + return NULL; + } + ScopedPyObjectPtr enum_value; + while ((enum_value.reset(PyIter_Next(values_iter))) != NULL) { + ScopedPyObjectPtr value_name(PyObject_GetAttr(enum_value, kname)); + if (value_name == NULL) { + return NULL; + } + ScopedPyObjectPtr value_number(PyObject_GetAttrString(enum_value, + "number")); + if (value_number == NULL) { + return NULL; + } + if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls), + value_name, value_number) == -1) { + return NULL; + } + } + if (PyErr_Occurred()) { // If PyIter_Next failed + return NULL; + } + } + if (PyErr_Occurred()) { // If PyIter_Next failed + return NULL; + } + + ScopedPyObjectPtr extension_dict( + PyObject_GetAttr(descriptor, kextensions_by_name)); + if (extension_dict == NULL || !PyDict_Check(extension_dict)) { + PyErr_SetString(PyExc_TypeError, "extensions_by_name not a dict"); + return NULL; + } + Py_ssize_t pos = 0; + PyObject* extension_name; + PyObject* extension_field; + + while (PyDict_Next(extension_dict, &pos, &extension_name, &extension_field)) { + if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls), + extension_name, extension_field) == -1) { + return NULL; + } + ScopedPyObjectPtr py_cfield_descriptor( + PyObject_GetAttrString(extension_field, "_cdescriptor")); + if (py_cfield_descriptor == NULL) { + return NULL; + } + CFieldDescriptor* cfield_descriptor = + reinterpret_cast<CFieldDescriptor*>(py_cfield_descriptor.get()); + Py_INCREF(extension_field); + cfield_descriptor->descriptor_field = extension_field; + + ScopedPyObjectPtr field_name_upcased( + PyObject_CallMethod(extension_name, "upper", NULL)); + if (field_name_upcased == NULL) { + return NULL; + } + ScopedPyObjectPtr field_number_name(PyObject_CallMethod( + field_name_upcased, "__add__", "(O)", _NUMBER_string.get())); + if (field_number_name == NULL) { + return NULL; + } + ScopedPyObjectPtr number(PyInt_FromLong( + cfield_descriptor->descriptor->number())); + if (number == NULL) { + return NULL; + } + if (PyObject_SetAttr(reinterpret_cast<PyObject*>(cls), + field_number_name, PyInt_FromLong( + cfield_descriptor->descriptor->number())) == -1) { + return NULL; + } + } + + Py_RETURN_NONE; +} + +PyObject* DeepCopy(CMessage* self, PyObject* arg) { + PyObject* clone = PyObject_CallObject( + reinterpret_cast<PyObject*>(Py_TYPE(self)), NULL); + if (clone == NULL) { + return NULL; + } + if (!PyObject_TypeCheck(clone, &CMessage_Type)) { + Py_DECREF(clone); + return NULL; + } + if (InitAttributes(reinterpret_cast<CMessage*>(clone), NULL, NULL) < 0) { + Py_DECREF(clone); + return NULL; + } + if (MergeFrom(reinterpret_cast<CMessage*>(clone), + reinterpret_cast<PyObject*>(self)) == NULL) { + Py_DECREF(clone); + return NULL; + } + return clone; +} + +PyObject* ToUnicode(CMessage* self) { + // Lazy import to prevent circular dependencies + ScopedPyObjectPtr text_format( + PyImport_ImportModule("google.protobuf.text_format")); + if (text_format == NULL) { + return NULL; + } + ScopedPyObjectPtr method_name(PyString_FromString("MessageToString")); + if (method_name == NULL) { + return NULL; + } + Py_INCREF(Py_True); + ScopedPyObjectPtr encoded(PyObject_CallMethodObjArgs(text_format, method_name, + self, Py_True, NULL)); + Py_DECREF(Py_True); + if (encoded == NULL) { + return NULL; + } +#if PY_MAJOR_VERSION < 3 + PyObject* decoded = PyString_AsDecodedObject(encoded, "utf-8", NULL); +#else + PyObject* decoded = PyUnicode_FromEncodedObject(encoded, "utf-8", NULL); +#endif + if (decoded == NULL) { + return NULL; + } + return decoded; +} + +PyObject* Reduce(CMessage* self) { + ScopedPyObjectPtr constructor(reinterpret_cast<PyObject*>(Py_TYPE(self))); + constructor.inc(); + ScopedPyObjectPtr args(PyTuple_New(0)); + if (args == NULL) { + return NULL; + } + ScopedPyObjectPtr state(PyDict_New()); + if (state == NULL) { + return NULL; + } + ScopedPyObjectPtr serialized(SerializePartialToString(self)); + if (serialized == NULL) { + return NULL; + } + if (PyDict_SetItemString(state, "serialized", serialized) < 0) { + return NULL; + } + return Py_BuildValue("OOO", constructor.get(), args.get(), state.get()); +} + +PyObject* SetState(CMessage* self, PyObject* state) { + if (!PyDict_Check(state)) { + PyErr_SetString(PyExc_TypeError, "state not a dict"); + return NULL; + } + PyObject* serialized = PyDict_GetItemString(state, "serialized"); + if (serialized == NULL) { + return NULL; + } + if (ParseFromString(self, serialized) == NULL) { + return NULL; + } + Py_RETURN_NONE; +} + +// CMessage static methods: +PyObject* _GetFieldDescriptor(PyObject* unused, PyObject* arg) { + return cdescriptor_pool::FindFieldByName(descriptor_pool, arg); +} + +PyObject* _GetExtensionDescriptor(PyObject* unused, PyObject* arg) { + return cdescriptor_pool::FindExtensionByName(descriptor_pool, arg); +} + +static PyMemberDef Members[] = { + {"Extensions", T_OBJECT_EX, offsetof(CMessage, extensions), 0, + "Extension dict"}, + {NULL} +}; + +static PyMethodDef Methods[] = { + { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS, + "Makes a deep copy of the class." }, + { "__reduce__", (PyCFunction)Reduce, METH_NOARGS, + "Outputs picklable representation of the message." }, + { "__setstate__", (PyCFunction)SetState, METH_O, + "Inputs picklable representation of the message." }, + { "__unicode__", (PyCFunction)ToUnicode, METH_NOARGS, + "Outputs a unicode representation of the message." }, + { "AddDescriptors", (PyCFunction)AddDescriptors, METH_O | METH_CLASS, + "Adds field descriptors to the class" }, + { "ByteSize", (PyCFunction)ByteSize, METH_NOARGS, + "Returns the size of the message in bytes." }, + { "Clear", (PyCFunction)Clear, METH_NOARGS, + "Clears the message." }, + { "ClearExtension", (PyCFunction)ClearExtension, METH_O, + "Clears a message field." }, + { "ClearField", (PyCFunction)ClearField, METH_O, + "Clears a message field." }, + { "CopyFrom", (PyCFunction)CopyFrom, METH_O, + "Copies a protocol message into the current message." }, + { "FindInitializationErrors", (PyCFunction)FindInitializationErrors, + METH_NOARGS, + "Finds unset required fields." }, + { "FromString", (PyCFunction)FromString, METH_O | METH_CLASS, + "Creates new method instance from given serialized data." }, + { "HasExtension", (PyCFunction)HasExtension, METH_O, + "Checks if a message field is set." }, + { "HasField", (PyCFunction)HasField, METH_O, + "Checks if a message field is set." }, + { "IsInitialized", (PyCFunction)IsInitialized, METH_VARARGS, + "Checks if all required fields of a protocol message are set." }, + { "ListFields", (PyCFunction)ListFields, METH_NOARGS, + "Lists all set fields of a message." }, + { "MergeFrom", (PyCFunction)MergeFrom, METH_O, + "Merges a protocol message into the current message." }, + { "MergeFromString", (PyCFunction)MergeFromString, METH_O, + "Merges a serialized message into the current message." }, + { "ParseFromString", (PyCFunction)ParseFromString, METH_O, + "Parses a serialized message into the current message." }, + { "RegisterExtension", (PyCFunction)RegisterExtension, METH_O | METH_CLASS, + "Registers an extension with the current message." }, + { "SerializePartialToString", (PyCFunction)SerializePartialToString, + METH_NOARGS, + "Serializes the message to a string, even if it isn't initialized." }, + { "SerializeToString", (PyCFunction)SerializeToString, METH_NOARGS, + "Serializes the message to a string, only for initialized messages." }, + { "SetInParent", (PyCFunction)SetInParent, METH_NOARGS, + "Sets the has bit of the given field in its parent message." }, + { "WhichOneof", (PyCFunction)WhichOneof, METH_O, + "Returns the name of the field set inside a oneof, " + "or None if no field is set." }, + + // Static Methods. + { "_BuildFile", (PyCFunction)Python_BuildFile, METH_O | METH_STATIC, + "Registers a new protocol buffer file in the global C++ descriptor pool." }, + { "_GetFieldDescriptor", (PyCFunction)_GetFieldDescriptor, + METH_O | METH_STATIC, "Finds a field descriptor in the message pool." }, + { "_GetExtensionDescriptor", (PyCFunction)_GetExtensionDescriptor, + METH_O | METH_STATIC, + "Finds a extension descriptor in the message pool." }, + { NULL, NULL} +}; + +PyObject* GetAttr(CMessage* self, PyObject* name) { + PyObject* value = PyDict_GetItem(self->composite_fields, name); + if (value != NULL) { + Py_INCREF(value); + return value; + } + + PyObject* descriptor = GetDescriptor(self, name); + if (descriptor != NULL) { + CFieldDescriptor* cdescriptor = + reinterpret_cast<CFieldDescriptor*>(descriptor); + const google::protobuf::FieldDescriptor* field_descriptor = cdescriptor->descriptor; + if (field_descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) { + if (field_descriptor->cpp_type() == + google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + PyObject* py_container = PyObject_CallObject( + reinterpret_cast<PyObject*>(&RepeatedCompositeContainer_Type), + NULL); + if (py_container == NULL) { + return NULL; + } + RepeatedCompositeContainer* container = + reinterpret_cast<RepeatedCompositeContainer*>(py_container); + PyObject* field = cdescriptor->descriptor_field; + PyObject* message_type = PyObject_GetAttr(field, kmessage_type); + if (message_type == NULL) { + return NULL; + } + PyObject* concrete_class = + PyObject_GetAttr(message_type, k_concrete_class); + if (concrete_class == NULL) { + return NULL; + } + container->parent = self; + container->parent_field = cdescriptor; + container->message = self->message; + container->owner = self->owner; + container->subclass_init = concrete_class; + Py_DECREF(message_type); + if (PyDict_SetItem(self->composite_fields, name, py_container) < 0) { + Py_DECREF(py_container); + return NULL; + } + return py_container; + } else { + ScopedPyObjectPtr init_args(PyTuple_Pack(2, self, cdescriptor)); + PyObject* py_container = PyObject_CallObject( + reinterpret_cast<PyObject*>(&RepeatedScalarContainer_Type), + init_args); + if (py_container == NULL) { + return NULL; + } + if (PyDict_SetItem(self->composite_fields, name, py_container) < 0) { + Py_DECREF(py_container); + return NULL; + } + return py_container; + } + } else { + if (field_descriptor->cpp_type() == + google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + PyObject* sub_message = InternalGetSubMessage(self, cdescriptor); + if (PyDict_SetItem(self->composite_fields, name, sub_message) < 0) { + Py_DECREF(sub_message); + return NULL; + } + return sub_message; + } else { + return InternalGetScalar(self, field_descriptor); + } + } + } + + return CMessage_Type.tp_base->tp_getattro(reinterpret_cast<PyObject*>(self), + name); +} + +int SetAttr(CMessage* self, PyObject* name, PyObject* value) { + if (PyDict_Contains(self->composite_fields, name)) { + PyErr_SetString(PyExc_TypeError, "Can't set composite field"); + return -1; + } + + PyObject* descriptor = GetDescriptor(self, name); + if (descriptor != NULL) { + AssureWritable(self); + CFieldDescriptor* cdescriptor = + reinterpret_cast<CFieldDescriptor*>(descriptor); + const google::protobuf::FieldDescriptor* field_descriptor = cdescriptor->descriptor; + if (field_descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) { + PyErr_Format(PyExc_AttributeError, "Assignment not allowed to repeated " + "field \"%s\" in protocol message object.", + field_descriptor->name().c_str()); + return -1; + } else { + if (field_descriptor->cpp_type() == + google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + PyErr_Format(PyExc_AttributeError, "Assignment not allowed to " + "field \"%s\" in protocol message object.", + field_descriptor->name().c_str()); + return -1; + } else { + return InternalSetScalar(self, field_descriptor, value); + } + } + } + + PyErr_Format(PyExc_AttributeError, "Assignment not allowed"); + return -1; +} + +} // namespace cmessage + +PyTypeObject CMessage_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "google.protobuf.internal." + "cpp._message.CMessage", // tp_name + sizeof(CMessage), // tp_basicsize + 0, // tp_itemsize + (destructor)cmessage::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + (reprfunc)cmessage::ToStr, // tp_str + (getattrofunc)cmessage::GetAttr, // tp_getattro + (setattrofunc)cmessage::SetAttr, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, // tp_flags + "A ProtocolMessage", // tp_doc + 0, // tp_traverse + 0, // tp_clear + (richcmpfunc)cmessage::RichCompare, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + cmessage::Methods, // tp_methods + cmessage::Members, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + (initproc)cmessage::Init, // tp_init + 0, // tp_alloc + cmessage::New, // tp_new +}; + +// --- Exposing the C proto living inside Python proto to C code: + +const Message* (*GetCProtoInsidePyProtoPtr)(PyObject* msg); +Message* (*MutableCProtoInsidePyProtoPtr)(PyObject* msg); + +static const google::protobuf::Message* GetCProtoInsidePyProtoImpl(PyObject* msg) { + if (!PyObject_TypeCheck(msg, &CMessage_Type)) { + return NULL; + } + CMessage* cmsg = reinterpret_cast<CMessage*>(msg); + return cmsg->message; +} + +static google::protobuf::Message* MutableCProtoInsidePyProtoImpl(PyObject* msg) { + if (!PyObject_TypeCheck(msg, &CMessage_Type)) { + return NULL; + } + CMessage* cmsg = reinterpret_cast<CMessage*>(msg); + if (PyDict_Size(cmsg->composite_fields) != 0 || + (cmsg->extensions != NULL && + PyDict_Size(cmsg->extensions->values) != 0)) { + // There is currently no way of accurately syncing arbitrary changes to + // the underlying C++ message back to the CMessage (e.g. removed repeated + // composite containers). We only allow direct mutation of the underlying + // C++ message if there is no child data in the CMessage. + return NULL; + } + cmessage::AssureWritable(cmsg); + return cmsg->message; +} + +static const char module_docstring[] = +"python-proto2 is a module that can be used to enhance proto2 Python API\n" +"performance.\n" +"\n" +"It provides access to the protocol buffers C++ reflection API that\n" +"implements the basic protocol buffer functions."; + +void InitGlobals() { + // TODO(gps): Check all return values in this function for NULL and propagate + // the error (MemoryError) on up to result in an import failure. These should + // also be freed and reset to NULL during finalization. + kPythonZero = PyInt_FromLong(0); + kint32min_py = PyInt_FromLong(kint32min); + kint32max_py = PyInt_FromLong(kint32max); + kuint32max_py = PyLong_FromLongLong(kuint32max); + kint64min_py = PyLong_FromLongLong(kint64min); + kint64max_py = PyLong_FromLongLong(kint64max); + kuint64max_py = PyLong_FromUnsignedLongLong(kuint64max); + + kDESCRIPTOR = PyString_FromString("DESCRIPTOR"); + k__descriptors = PyString_FromString("__descriptors"); + kfull_name = PyString_FromString("full_name"); + kis_extendable = PyString_FromString("is_extendable"); + kextensions_by_name = PyString_FromString("extensions_by_name"); + k_extensions_by_name = PyString_FromString("_extensions_by_name"); + k_extensions_by_number = PyString_FromString("_extensions_by_number"); + k_concrete_class = PyString_FromString("_concrete_class"); + kmessage_type = PyString_FromString("message_type"); + kname = PyString_FromString("name"); + kfields_by_name = PyString_FromString("fields_by_name"); + + global_message_factory = new DynamicMessageFactory(GetDescriptorPool()); + global_message_factory->SetDelegateToGeneratedFactory(true); + + descriptor_pool = reinterpret_cast<google::protobuf::python::CDescriptorPool*>( + Python_NewCDescriptorPool(NULL, NULL)); +} + +bool InitProto2MessageModule(PyObject *m) { + InitGlobals(); + + google::protobuf::python::CMessage_Type.tp_hash = PyObject_HashNotImplemented; + if (PyType_Ready(&google::protobuf::python::CMessage_Type) < 0) { + return false; + } + + // All three of these are actually set elsewhere, directly onto the child + // protocol buffer message class, but set them here as well to document that + // subclasses need to set these. + PyDict_SetItem(google::protobuf::python::CMessage_Type.tp_dict, kDESCRIPTOR, Py_None); + PyDict_SetItem(google::protobuf::python::CMessage_Type.tp_dict, + k_extensions_by_name, Py_None); + PyDict_SetItem(google::protobuf::python::CMessage_Type.tp_dict, + k_extensions_by_number, Py_None); + + PyModule_AddObject(m, "Message", reinterpret_cast<PyObject*>( + &google::protobuf::python::CMessage_Type)); + + google::protobuf::python::RepeatedScalarContainer_Type.tp_new = PyType_GenericNew; + google::protobuf::python::RepeatedScalarContainer_Type.tp_hash = + PyObject_HashNotImplemented; + if (PyType_Ready(&google::protobuf::python::RepeatedScalarContainer_Type) < 0) { + return false; + } + + PyModule_AddObject(m, "RepeatedScalarContainer", + reinterpret_cast<PyObject*>( + &google::protobuf::python::RepeatedScalarContainer_Type)); + + google::protobuf::python::RepeatedCompositeContainer_Type.tp_new = PyType_GenericNew; + google::protobuf::python::RepeatedCompositeContainer_Type.tp_hash = + PyObject_HashNotImplemented; + if (PyType_Ready(&google::protobuf::python::RepeatedCompositeContainer_Type) < 0) { + return false; + } + + PyModule_AddObject( + m, "RepeatedCompositeContainer", + reinterpret_cast<PyObject*>( + &google::protobuf::python::RepeatedCompositeContainer_Type)); + + google::protobuf::python::ExtensionDict_Type.tp_new = PyType_GenericNew; + google::protobuf::python::ExtensionDict_Type.tp_hash = PyObject_HashNotImplemented; + if (PyType_Ready(&google::protobuf::python::ExtensionDict_Type) < 0) { + return false; + } + + PyModule_AddObject( + m, "ExtensionDict", + reinterpret_cast<PyObject*>(&google::protobuf::python::ExtensionDict_Type)); + + if (!google::protobuf::python::InitDescriptor()) { + return false; + } + + PyObject* enum_type_wrapper = PyImport_ImportModule( + "google.protobuf.internal.enum_type_wrapper"); + if (enum_type_wrapper == NULL) { + return false; + } + google::protobuf::python::EnumTypeWrapper_class = + PyObject_GetAttrString(enum_type_wrapper, "EnumTypeWrapper"); + Py_DECREF(enum_type_wrapper); + + PyObject* message_module = PyImport_ImportModule( + "google.protobuf.message"); + if (message_module == NULL) { + return false; + } + google::protobuf::python::EncodeError_class = PyObject_GetAttrString(message_module, + "EncodeError"); + google::protobuf::python::DecodeError_class = PyObject_GetAttrString(message_module, + "DecodeError"); + Py_DECREF(message_module); + + PyObject* pickle_module = PyImport_ImportModule("pickle"); + if (pickle_module == NULL) { + return false; + } + google::protobuf::python::PickleError_class = PyObject_GetAttrString(pickle_module, + "PickleError"); + Py_DECREF(pickle_module); + + // Override {Get,Mutable}CProtoInsidePyProto. + google::protobuf::python::GetCProtoInsidePyProtoPtr = + google::protobuf::python::GetCProtoInsidePyProtoImpl; + google::protobuf::python::MutableCProtoInsidePyProtoPtr = + google::protobuf::python::MutableCProtoInsidePyProtoImpl; + + return true; +} + +} // namespace python +} // namespace protobuf + + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef _module = { + PyModuleDef_HEAD_INIT, + "_message", + google::protobuf::python::module_docstring, + -1, + NULL, + NULL, + NULL, + NULL, + NULL +}; +#define INITFUNC PyInit__message +#define INITFUNC_ERRORVAL NULL +#else // Python 2 +#define INITFUNC init_message +#define INITFUNC_ERRORVAL +#endif + +extern "C" { + PyMODINIT_FUNC INITFUNC(void) { + PyObject* m; +#if PY_MAJOR_VERSION >= 3 + m = PyModule_Create(&_module); +#else + m = Py_InitModule3("_message", NULL, google::protobuf::python::module_docstring); +#endif + if (m == NULL) { + return INITFUNC_ERRORVAL; + } + + if (!google::protobuf::python::InitProto2MessageModule(m)) { + Py_DECREF(m); + return INITFUNC_ERRORVAL; + } + +#if PY_MAJOR_VERSION >= 3 + return m; +#endif + } +} +} // namespace google diff --git a/python/google/protobuf/pyext/message.h b/python/google/protobuf/pyext/message.h new file mode 100644 index 0000000..9f4978f --- /dev/null +++ b/python/google/protobuf/pyext/message.h @@ -0,0 +1,305 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: anuraag@google.com (Anuraag Agrawal) +// Author: tibell@google.com (Johan Tibell) + +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_H__ + +#include <Python.h> + +#include <memory> +#ifndef _SHARED_PTR_H +#include <google/protobuf/stubs/shared_ptr.h> +#endif +#include <string> + + +namespace google { +namespace protobuf { + +class Message; +class Reflection; +class FieldDescriptor; + +using internal::shared_ptr; + +namespace python { + +struct CFieldDescriptor; +struct ExtensionDict; + +typedef struct CMessage { + PyObject_HEAD; + + // This is the top-level C++ Message object that owns the whole + // proto tree. Every Python CMessage holds a reference to it in + // order to keep it alive as long as there's a Python object that + // references any part of the tree. + shared_ptr<Message> owner; + + // Weak reference to a parent CMessage object. This is NULL for any top-level + // message and is set for any child message (i.e. a child submessage or a + // part of a repeated composite field). + // + // Used to make sure all ancestors are also mutable when first modifying + // a child submessage (in other words, turning a default message instance + // into a mutable one). + // + // If a submessage is released (becomes a new top-level message), this field + // MUST be set to NULL. The parent may get deallocated and further attempts + // to use this pointer will result in a crash. + struct CMessage* parent; + + // Weak reference to the parent's descriptor that describes this submessage. + // Used together with the parent's message when making a default message + // instance mutable. + // TODO(anuraag): With a bit of work on the Python/C++ layer, it should be + // possible to make this a direct pointer to a C++ FieldDescriptor, this would + // be easier if this implementation replaces upstream. + CFieldDescriptor* parent_field; + + // Pointer to the C++ Message object for this CMessage. The + // CMessage does not own this pointer. + Message* message; + + // Indicates this submessage is pointing to a default instance of a message. + // Submessages are always first created as read only messages and are then + // made writable, at which point this field is set to false. + bool read_only; + + // A reference to a Python dictionary containing CMessage, + // RepeatedCompositeContainer, and RepeatedScalarContainer + // objects. Used as a cache to make sure we don't have to make a + // Python wrapper for the C++ Message objects on every access, or + // deal with the synchronization nightmare that could create. + PyObject* composite_fields; + + // A reference to the dictionary containing the message's extensions. + // Similar to composite_fields, acting as a cache, but also contains the + // required extension dict logic. + ExtensionDict* extensions; +} CMessage; + +extern PyTypeObject CMessage_Type; + +namespace cmessage { + +// Create a new empty message that can be populated by the parent. +PyObject* NewEmpty(PyObject* type); + +// Release a submessage from its proto tree, making it a new top-level messgae. +// A new message will be created if this is a read-only default instance. +// +// Corresponds to reflection api method ReleaseMessage. +int ReleaseSubMessage(google::protobuf::Message* message, + const google::protobuf::FieldDescriptor* field_descriptor, + CMessage* child_cmessage); + +// Initializes a new CMessage instance for a submessage. Only called once per +// submessage as the result is cached in composite_fields. +// +// Corresponds to reflection api method GetMessage. +PyObject* InternalGetSubMessage(CMessage* self, + CFieldDescriptor* cfield_descriptor); + +// Deletes a range of C++ submessages in a repeated field (following a +// removal in a RepeatedCompositeContainer). +// +// Releases messages to the provided cmessage_list if it is not NULL rather +// than just removing them from the underlying proto. This cmessage_list must +// have a CMessage for each underlying submessage. The CMessages refered to +// by slice will be removed from cmessage_list by this function. +// +// Corresponds to reflection api method RemoveLast. +int InternalDeleteRepeatedField(google::protobuf::Message* message, + const google::protobuf::FieldDescriptor* field_descriptor, + PyObject* slice, PyObject* cmessage_list); + +// Sets the specified scalar value to the message. +int InternalSetScalar(CMessage* self, + const google::protobuf::FieldDescriptor* field_descriptor, + PyObject* value); + +// Retrieves the specified scalar value from the message. +// +// Returns a new python reference. +PyObject* InternalGetScalar(CMessage* self, + const google::protobuf::FieldDescriptor* field_descriptor); + +// Clears the message, removing all contained data. Extension dictionary and +// submessages are released first if there are remaining external references. +// +// Corresponds to message api method Clear. +PyObject* Clear(CMessage* self); + +// Clears the data described by the given descriptor. Used to clear extensions +// (which don't have names). Extension release is handled by ExtensionDict +// class, not this function. +// TODO(anuraag): Try to make this discrepancy in release semantics with +// ClearField less confusing. +// +// Corresponds to reflection api method ClearField. +PyObject* ClearFieldByDescriptor( + CMessage* self, + const google::protobuf::FieldDescriptor* descriptor); + +// Clears the data for the given field name. The message is released if there +// are any external references. +// +// Corresponds to reflection api method ClearField. +PyObject* ClearField(CMessage* self, PyObject* arg); + +// Checks if the message has the field described by the descriptor. Used for +// extensions (which have no name). +// +// Corresponds to reflection api method HasField +PyObject* HasFieldByDescriptor( + CMessage* self, const google::protobuf::FieldDescriptor* field_descriptor); + +// Checks if the message has the named field. +// +// Corresponds to reflection api method HasField. +PyObject* HasField(CMessage* self, PyObject* arg); + +// Initializes constants/enum values on a message. This is called by +// RepeatedCompositeContainer and ExtensionDict after calling the constructor. +// TODO(anuraag): Make it always called from within the constructor since it can +int InitAttributes(CMessage* self, PyObject* descriptor, PyObject* kwargs); + +PyObject* MergeFrom(CMessage* self, PyObject* arg); + +// Retrieves an attribute named 'name' from CMessage 'self'. Returns +// the attribute value on success, or NULL on failure. +// +// Returns a new reference. +PyObject* GetAttr(CMessage* self, PyObject* name); + +// Set the value of the attribute named 'name', for CMessage 'self', +// to the value 'value'. Returns -1 on failure. +int SetAttr(CMessage* self, PyObject* name, PyObject* value); + +PyObject* FindInitializationErrors(CMessage* self); + +// Set the owner field of self and any children of self, recursively. +// Used when self is being released and thus has a new owner (the +// released Message.) +int SetOwner(CMessage* self, const shared_ptr<Message>& new_owner); + +int AssureWritable(CMessage* self); + +} // namespace cmessage + +/* Is 64bit */ +#define IS_64BIT (SIZEOF_LONG == 8) + +#define FIELD_BELONGS_TO_MESSAGE(field_descriptor, message) \ + ((message)->GetDescriptor() == (field_descriptor)->containing_type()) + +#define FIELD_IS_REPEATED(field_descriptor) \ + ((field_descriptor)->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) + +#define GOOGLE_CHECK_GET_INT32(arg, value, err) \ + int32 value; \ + if (!CheckAndGetInteger(arg, &value, kint32min_py, kint32max_py)) { \ + return err; \ + } + +#define GOOGLE_CHECK_GET_INT64(arg, value, err) \ + int64 value; \ + if (!CheckAndGetInteger(arg, &value, kint64min_py, kint64max_py)) { \ + return err; \ + } + +#define GOOGLE_CHECK_GET_UINT32(arg, value, err) \ + uint32 value; \ + if (!CheckAndGetInteger(arg, &value, kPythonZero, kuint32max_py)) { \ + return err; \ + } + +#define GOOGLE_CHECK_GET_UINT64(arg, value, err) \ + uint64 value; \ + if (!CheckAndGetInteger(arg, &value, kPythonZero, kuint64max_py)) { \ + return err; \ + } + +#define GOOGLE_CHECK_GET_FLOAT(arg, value, err) \ + float value; \ + if (!CheckAndGetFloat(arg, &value)) { \ + return err; \ + } \ + +#define GOOGLE_CHECK_GET_DOUBLE(arg, value, err) \ + double value; \ + if (!CheckAndGetDouble(arg, &value)) { \ + return err; \ + } + +#define GOOGLE_CHECK_GET_BOOL(arg, value, err) \ + bool value; \ + if (!CheckAndGetBool(arg, &value)) { \ + return err; \ + } + + +extern PyObject* kPythonZero; +extern PyObject* kint32min_py; +extern PyObject* kint32max_py; +extern PyObject* kuint32max_py; +extern PyObject* kint64min_py; +extern PyObject* kint64max_py; +extern PyObject* kuint64max_py; + +#define C(str) const_cast<char*>(str) + +void FormatTypeError(PyObject* arg, char* expected_types); +template<class T> +bool CheckAndGetInteger( + PyObject* arg, T* value, PyObject* min, PyObject* max); +bool CheckAndGetDouble(PyObject* arg, double* value); +bool CheckAndGetFloat(PyObject* arg, float* value); +bool CheckAndGetBool(PyObject* arg, bool* value); +bool CheckAndSetString( + PyObject* arg, google::protobuf::Message* message, + const google::protobuf::FieldDescriptor* descriptor, + const google::protobuf::Reflection* reflection, + bool append, + int index); +PyObject* ToStringObject( + const google::protobuf::FieldDescriptor* descriptor, string value); + +extern PyObject* PickleError_class; + +} // namespace python +} // namespace protobuf + +} // namespace google +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_H__ diff --git a/python/google/protobuf/pyext/message_factory_cpp2_test.py b/python/google/protobuf/pyext/message_factory_cpp2_test.py new file mode 100644 index 0000000..32ab4f8 --- /dev/null +++ b/python/google/protobuf/pyext/message_factory_cpp2_test.py @@ -0,0 +1,56 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Tests for google.protobuf.message_factory.""" + +import os +os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp' +os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION'] = '2' + +# We must set the implementation version above before the google3 imports. +# pylint: disable=g-import-not-at-top +from google.apputils import basetest +from google.protobuf.internal import api_implementation +# Run all tests from the original module by putting them in our namespace. +# pylint: disable=wildcard-import +from google.protobuf.internal.message_factory_test import * + + +class ConfirmCppApi2Test(basetest.TestCase): + + def testImplementationSetting(self): + self.assertEqual('cpp', api_implementation.Type()) + self.assertEqual(2, api_implementation.Version()) + + +if __name__ == '__main__': + basetest.main() diff --git a/python/google/protobuf/pyext/proto2_api_test.proto b/python/google/protobuf/pyext/proto2_api_test.proto new file mode 100644 index 0000000..72c31b1 --- /dev/null +++ b/python/google/protobuf/pyext/proto2_api_test.proto @@ -0,0 +1,38 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import "google/protobuf/internal/cpp/proto1_api_test.proto"; + +package google.protobuf.python.internal; + +message TestNestedProto1APIMessage { + optional int32 a = 1; + optional TestMessage.NestedMessage b = 2; +} diff --git a/python/google/protobuf/pyext/python.proto b/python/google/protobuf/pyext/python.proto new file mode 100644 index 0000000..d47d402 --- /dev/null +++ b/python/google/protobuf/pyext/python.proto @@ -0,0 +1,66 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: tibell@google.com (Johan Tibell) +// +// These message definitions are used to exercises known corner cases +// in the C++ implementation of the Python API. + + +package google.protobuf.python.internal; + +// Protos optimized for SPEED use a strict superset of the generated code +// of equivalent ones optimized for CODE_SIZE, so we should optimize all our +// tests for speed unless explicitly testing code size optimization. +option optimize_for = SPEED; + +message TestAllTypes { + message NestedMessage { + optional int32 bb = 1; + optional ForeignMessage cc = 2; + } + + repeated NestedMessage repeated_nested_message = 1; + optional NestedMessage optional_nested_message = 2; + optional int32 optional_int32 = 3; +} + +message ForeignMessage { + optional int32 c = 1; + repeated int32 d = 2; +} + +message TestAllExtensions { + extensions 1 to max; +} + +extend TestAllExtensions { + optional TestAllTypes.NestedMessage optional_nested_message_extension = 1; +} diff --git a/python/google/protobuf/pyext/python_protobuf.h b/python/google/protobuf/pyext/python_protobuf.h new file mode 100644 index 0000000..beb6e46 --- /dev/null +++ b/python/google/protobuf/pyext/python_protobuf.h @@ -0,0 +1,57 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: qrczak@google.com (Marcin Kowalczyk) +// +// This module exposes the C proto inside the given Python proto, in +// case the Python proto is implemented with a C proto. + +#ifndef GOOGLE_PROTOBUF_PYTHON_PYTHON_PROTOBUF_H__ +#define GOOGLE_PROTOBUF_PYTHON_PYTHON_PROTOBUF_H__ + +#include <Python.h> + +namespace google { +namespace protobuf { + +class Message; + +namespace python { + +// Return the pointer to the C proto inside the given Python proto, +// or NULL when this is not a Python proto implemented with a C proto. +const Message* GetCProtoInsidePyProto(PyObject* msg); +Message* MutableCProtoInsidePyProto(PyObject* msg); + +} // namespace python +} // namespace protobuf + +} // namespace google +#endif // GOOGLE_PROTOBUF_PYTHON_PYTHON_PROTOBUF_H__ diff --git a/python/google/protobuf/pyext/reflection_cpp2_generated_test.py b/python/google/protobuf/pyext/reflection_cpp2_generated_test.py new file mode 100755 index 0000000..552efd4 --- /dev/null +++ b/python/google/protobuf/pyext/reflection_cpp2_generated_test.py @@ -0,0 +1,94 @@ +#! /usr/bin/python +# -*- coding: utf-8 -*- +# +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unittest for reflection.py, which tests the generated C++ implementation.""" + +__author__ = 'jasonh@google.com (Jason Hsueh)' + +import os +os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp' +os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION'] = '2' + +from google.apputils import basetest +from google.protobuf.internal import api_implementation +from google.protobuf.internal import more_extensions_dynamic_pb2 +from google.protobuf.internal import more_extensions_pb2 +from google.protobuf.internal.reflection_test import * + + +class ReflectionCppTest(basetest.TestCase): + def testImplementationSetting(self): + self.assertEqual('cpp', api_implementation.Type()) + self.assertEqual(2, api_implementation.Version()) + + def testExtensionOfGeneratedTypeInDynamicFile(self): + """Tests that a file built dynamically can extend a generated C++ type. + + The C++ implementation uses a DescriptorPool that has the generated + DescriptorPool as an underlay. Typically, a type can only find + extensions in its own pool. With the python C-extension, the generated C++ + extendee may be available, but not the extension. This tests that the + C-extension implements the correct special handling to make such extensions + available. + """ + pb1 = more_extensions_pb2.ExtendedMessage() + # Test that basic accessors work. + self.assertFalse( + pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_int32_extension)) + self.assertFalse( + pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_message_extension)) + pb1.Extensions[more_extensions_dynamic_pb2.dynamic_int32_extension] = 17 + pb1.Extensions[more_extensions_dynamic_pb2.dynamic_message_extension].a = 24 + self.assertTrue( + pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_int32_extension)) + self.assertTrue( + pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_message_extension)) + + # Now serialize the data and parse to a new message. + pb2 = more_extensions_pb2.ExtendedMessage() + pb2.MergeFromString(pb1.SerializeToString()) + + self.assertTrue( + pb2.HasExtension(more_extensions_dynamic_pb2.dynamic_int32_extension)) + self.assertTrue( + pb2.HasExtension(more_extensions_dynamic_pb2.dynamic_message_extension)) + self.assertEqual( + 17, pb2.Extensions[more_extensions_dynamic_pb2.dynamic_int32_extension]) + self.assertEqual( + 24, + pb2.Extensions[more_extensions_dynamic_pb2.dynamic_message_extension].a) + + + +if __name__ == '__main__': + basetest.main() diff --git a/python/google/protobuf/pyext/repeated_composite_container.cc b/python/google/protobuf/pyext/repeated_composite_container.cc new file mode 100644 index 0000000..5c05b3d --- /dev/null +++ b/python/google/protobuf/pyext/repeated_composite_container.cc @@ -0,0 +1,763 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: anuraag@google.com (Anuraag Agrawal) +// Author: tibell@google.com (Johan Tibell) + +#include <google/protobuf/pyext/repeated_composite_container.h> + +#include <memory> +#ifndef _SHARED_PTR_H +#include <google/protobuf/stubs/shared_ptr.h> +#endif + +#include <google/protobuf/stubs/common.h> +#include <google/protobuf/descriptor.h> +#include <google/protobuf/dynamic_message.h> +#include <google/protobuf/message.h> +#include <google/protobuf/pyext/descriptor.h> +#include <google/protobuf/pyext/message.h> +#include <google/protobuf/pyext/scoped_pyobject_ptr.h> + +#if PY_MAJOR_VERSION >= 3 + #define PyInt_Check PyLong_Check + #define PyInt_AsLong PyLong_AsLong + #define PyInt_FromLong PyLong_FromLong +#endif + +namespace google { +namespace protobuf { +namespace python { + +extern google::protobuf::DynamicMessageFactory* global_message_factory; + +namespace repeated_composite_container { + +// TODO(tibell): We might also want to check: +// GOOGLE_CHECK_NOTNULL((self)->owner.get()); +#define GOOGLE_CHECK_ATTACHED(self) \ + do { \ + GOOGLE_CHECK_NOTNULL((self)->message); \ + GOOGLE_CHECK_NOTNULL((self)->parent_field); \ + } while (0); + +#define GOOGLE_CHECK_RELEASED(self) \ + do { \ + GOOGLE_CHECK((self)->owner.get() == NULL); \ + GOOGLE_CHECK((self)->message == NULL); \ + GOOGLE_CHECK((self)->parent_field == NULL); \ + GOOGLE_CHECK((self)->parent == NULL); \ + } while (0); + +// Returns a new reference. +static PyObject* GetKey(PyObject* x) { + // Just the identity function. + Py_INCREF(x); + return x; +} + +#define GET_KEY(keyfunc, value) \ + ((keyfunc) == NULL ? \ + GetKey((value)) : \ + PyObject_CallFunctionObjArgs((keyfunc), (value), NULL)) + +// Converts a comparison function that returns -1, 0, or 1 into a +// less-than predicate. +// +// Returns -1 on error, 1 if x < y, 0 if x >= y. +static int islt(PyObject *x, PyObject *y, PyObject *compare) { + if (compare == NULL) + return PyObject_RichCompareBool(x, y, Py_LT); + + ScopedPyObjectPtr res(PyObject_CallFunctionObjArgs(compare, x, y, NULL)); + if (res == NULL) + return -1; + if (!PyInt_Check(res)) { + PyErr_Format(PyExc_TypeError, + "comparison function must return int, not %.200s", + Py_TYPE(res)->tp_name); + return -1; + } + return PyInt_AsLong(res) < 0; +} + +// Copied from uarrsort.c but swaps memcpy swaps with protobuf/python swaps +// TODO(anuraag): Is there a better way to do this then reinventing the wheel? +static int InternalQuickSort(RepeatedCompositeContainer* self, + Py_ssize_t start, + Py_ssize_t limit, + PyObject* cmp, + PyObject* keyfunc) { + if (limit - start <= 1) + return 0; // Nothing to sort. + + GOOGLE_CHECK_ATTACHED(self); + + google::protobuf::Message* message = self->message; + const google::protobuf::Reflection* reflection = message->GetReflection(); + const google::protobuf::FieldDescriptor* descriptor = self->parent_field->descriptor; + Py_ssize_t left; + Py_ssize_t right; + + PyObject* children = self->child_messages; + + do { + left = start; + right = limit; + ScopedPyObjectPtr mid( + GET_KEY(keyfunc, PyList_GET_ITEM(children, (start + limit) / 2))); + do { + ScopedPyObjectPtr key(GET_KEY(keyfunc, PyList_GET_ITEM(children, left))); + int is_lt = islt(key, mid, cmp); + if (is_lt == -1) + return -1; + /* array[left]<x */ + while (is_lt) { + ++left; + ScopedPyObjectPtr key(GET_KEY(keyfunc, + PyList_GET_ITEM(children, left))); + is_lt = islt(key, mid, cmp); + if (is_lt == -1) + return -1; + } + key.reset(GET_KEY(keyfunc, PyList_GET_ITEM(children, right - 1))); + is_lt = islt(mid, key, cmp); + if (is_lt == -1) + return -1; + while (is_lt) { + --right; + ScopedPyObjectPtr key(GET_KEY(keyfunc, + PyList_GET_ITEM(children, right - 1))); + is_lt = islt(mid, key, cmp); + if (is_lt == -1) + return -1; + } + if (left < right) { + --right; + if (left < right) { + reflection->SwapElements(message, descriptor, left, right); + PyObject* tmp = PyList_GET_ITEM(children, left); + PyList_SET_ITEM(children, left, PyList_GET_ITEM(children, right)); + PyList_SET_ITEM(children, right, tmp); + } + ++left; + } + } while (left < right); + + if ((right - start) < (limit - left)) { + /* sort [start..right[ */ + if (start < (right - 1)) { + InternalQuickSort(self, start, right, cmp, keyfunc); + } + + /* sort [left..limit[ */ + start = left; + } else { + /* sort [left..limit[ */ + if (left < (limit - 1)) { + InternalQuickSort(self, left, limit, cmp, keyfunc); + } + + /* sort [start..right[ */ + limit = right; + } + } while (start < (limit - 1)); + + return 0; +} + +#undef GET_KEY + +// --------------------------------------------------------------------- +// len() + +static Py_ssize_t Length(RepeatedCompositeContainer* self) { + google::protobuf::Message* message = self->message; + if (message != NULL) { + return message->GetReflection()->FieldSize(*message, + self->parent_field->descriptor); + } else { + // The container has been released (i.e. by a call to Clear() or + // ClearField() on the parent) and thus there's no message. + return PyList_GET_SIZE(self->child_messages); + } +} + +// Returns 0 if successful; returns -1 and sets an exception if +// unsuccessful. +static int UpdateChildMessages(RepeatedCompositeContainer* self) { + if (self->message == NULL) + return 0; + + // A MergeFrom on a parent message could have caused extra messages to be + // added in the underlying protobuf so add them to our list. They can never + // be removed in such a way so there's no need to worry about that. + Py_ssize_t message_length = Length(self); + Py_ssize_t child_length = PyList_GET_SIZE(self->child_messages); + google::protobuf::Message* message = self->message; + const google::protobuf::Reflection* reflection = message->GetReflection(); + for (Py_ssize_t i = child_length; i < message_length; ++i) { + const Message& sub_message = reflection->GetRepeatedMessage( + *(self->message), self->parent_field->descriptor, i); + ScopedPyObjectPtr py_cmsg(cmessage::NewEmpty(self->subclass_init)); + if (py_cmsg == NULL) { + return -1; + } + CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg.get()); + cmsg->owner = self->owner; + cmsg->message = const_cast<google::protobuf::Message*>(&sub_message); + cmsg->parent = self->parent; + if (cmessage::InitAttributes(cmsg, NULL, NULL) < 0) { + return -1; + } + PyList_Append(self->child_messages, py_cmsg); + } + return 0; +} + +// --------------------------------------------------------------------- +// add() + +static PyObject* AddToAttached(RepeatedCompositeContainer* self, + PyObject* args, + PyObject* kwargs) { + GOOGLE_CHECK_ATTACHED(self); + + if (UpdateChildMessages(self) < 0) { + return NULL; + } + if (cmessage::AssureWritable(self->parent) == -1) + return NULL; + google::protobuf::Message* message = self->message; + google::protobuf::Message* sub_message = + message->GetReflection()->AddMessage(message, + self->parent_field->descriptor); + PyObject* py_cmsg = cmessage::NewEmpty(self->subclass_init); + if (py_cmsg == NULL) { + return NULL; + } + CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg); + + cmsg->owner = self->owner; + cmsg->message = sub_message; + cmsg->parent = self->parent; + // cmessage::InitAttributes must be called after cmsg->message has + // been set. + if (cmessage::InitAttributes(cmsg, NULL, kwargs) < 0) { + Py_DECREF(py_cmsg); + return NULL; + } + PyList_Append(self->child_messages, py_cmsg); + return py_cmsg; +} + +static PyObject* AddToReleased(RepeatedCompositeContainer* self, + PyObject* args, + PyObject* kwargs) { + GOOGLE_CHECK_RELEASED(self); + + // Create the CMessage + PyObject* py_cmsg = PyObject_CallObject(self->subclass_init, NULL); + if (py_cmsg == NULL) + return NULL; + CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg); + if (cmessage::InitAttributes(cmsg, NULL, kwargs) < 0) { + Py_DECREF(py_cmsg); + return NULL; + } + + // The Message got created by the call to subclass_init above and + // it set self->owner to the newly allocated message. + + PyList_Append(self->child_messages, py_cmsg); + return py_cmsg; +} + +PyObject* Add(RepeatedCompositeContainer* self, + PyObject* args, + PyObject* kwargs) { + if (self->message == NULL) + return AddToReleased(self, args, kwargs); + else + return AddToAttached(self, args, kwargs); +} + +// --------------------------------------------------------------------- +// extend() + +PyObject* Extend(RepeatedCompositeContainer* self, PyObject* value) { + cmessage::AssureWritable(self->parent); + if (UpdateChildMessages(self) < 0) { + return NULL; + } + ScopedPyObjectPtr iter(PyObject_GetIter(value)); + if (iter == NULL) { + PyErr_SetString(PyExc_TypeError, "Value must be iterable"); + return NULL; + } + ScopedPyObjectPtr next; + while ((next.reset(PyIter_Next(iter))) != NULL) { + if (!PyObject_TypeCheck(next, &CMessage_Type)) { + PyErr_SetString(PyExc_TypeError, "Not a cmessage"); + return NULL; + } + ScopedPyObjectPtr new_message(Add(self, NULL, NULL)); + if (new_message == NULL) { + return NULL; + } + CMessage* new_cmessage = reinterpret_cast<CMessage*>(new_message.get()); + if (cmessage::MergeFrom(new_cmessage, next) == NULL) { + return NULL; + } + } + if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; +} + +PyObject* MergeFrom(RepeatedCompositeContainer* self, PyObject* other) { + if (UpdateChildMessages(self) < 0) { + return NULL; + } + return Extend(self, other); +} + +PyObject* Subscript(RepeatedCompositeContainer* self, PyObject* slice) { + if (UpdateChildMessages(self) < 0) { + return NULL; + } + Py_ssize_t from; + Py_ssize_t to; + Py_ssize_t step; + Py_ssize_t length = Length(self); + Py_ssize_t slicelength; + if (PySlice_Check(slice)) { +#if PY_MAJOR_VERSION >= 3 + if (PySlice_GetIndicesEx(slice, +#else + if (PySlice_GetIndicesEx(reinterpret_cast<PySliceObject*>(slice), +#endif + length, &from, &to, &step, &slicelength) == -1) { + return NULL; + } + return PyList_GetSlice(self->child_messages, from, to); + } else if (PyInt_Check(slice) || PyLong_Check(slice)) { + from = to = PyLong_AsLong(slice); + if (from < 0) { + from = to = length + from; + } + PyObject* result = PyList_GetItem(self->child_messages, from); + if (result == NULL) { + return NULL; + } + Py_INCREF(result); + return result; + } + PyErr_SetString(PyExc_TypeError, "index must be an integer or slice"); + return NULL; +} + +int AssignSubscript(RepeatedCompositeContainer* self, + PyObject* slice, + PyObject* value) { + if (UpdateChildMessages(self) < 0) { + return -1; + } + if (value != NULL) { + PyErr_SetString(PyExc_TypeError, "does not support assignment"); + return -1; + } + + // Delete from the underlying Message, if any. + if (self->message != NULL) { + if (cmessage::InternalDeleteRepeatedField(self->message, + self->parent_field->descriptor, + slice, + self->child_messages) < 0) { + return -1; + } + } else { + Py_ssize_t from; + Py_ssize_t to; + Py_ssize_t step; + Py_ssize_t length = Length(self); + Py_ssize_t slicelength; + if (PySlice_Check(slice)) { +#if PY_MAJOR_VERSION >= 3 + if (PySlice_GetIndicesEx(slice, +#else + if (PySlice_GetIndicesEx(reinterpret_cast<PySliceObject*>(slice), +#endif + length, &from, &to, &step, &slicelength) == -1) { + return -1; + } + return PySequence_DelSlice(self->child_messages, from, to); + } else if (PyInt_Check(slice) || PyLong_Check(slice)) { + from = to = PyLong_AsLong(slice); + if (from < 0) { + from = to = length + from; + } + return PySequence_DelItem(self->child_messages, from); + } + } + + return 0; +} + +static PyObject* Remove(RepeatedCompositeContainer* self, PyObject* value) { + if (UpdateChildMessages(self) < 0) { + return NULL; + } + Py_ssize_t index = PySequence_Index(self->child_messages, value); + if (index == -1) { + return NULL; + } + ScopedPyObjectPtr py_index(PyLong_FromLong(index)); + if (AssignSubscript(self, py_index, NULL) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject* RichCompare(RepeatedCompositeContainer* self, + PyObject* other, + int opid) { + if (UpdateChildMessages(self) < 0) { + return NULL; + } + if (!PyObject_TypeCheck(other, &RepeatedCompositeContainer_Type)) { + PyErr_SetString(PyExc_TypeError, + "Can only compare repeated composite fields " + "against other repeated composite fields."); + return NULL; + } + if (opid == Py_EQ || opid == Py_NE) { + // TODO(anuraag): Don't make new lists just for this... + ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL)); + if (full_slice == NULL) { + return NULL; + } + ScopedPyObjectPtr list(Subscript(self, full_slice)); + if (list == NULL) { + return NULL; + } + ScopedPyObjectPtr other_list( + Subscript( + reinterpret_cast<RepeatedCompositeContainer*>(other), full_slice)); + if (other_list == NULL) { + return NULL; + } + return PyObject_RichCompare(list, other_list, opid); + } else { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } +} + +// --------------------------------------------------------------------- +// sort() + +static PyObject* SortAttached(RepeatedCompositeContainer* self, + PyObject* args, + PyObject* kwds) { + // Sort the underlying Message array. + PyObject *compare = NULL; + int reverse = 0; + PyObject *keyfunc = NULL; + static char *kwlist[] = {"cmp", "key", "reverse", 0}; + + if (args != NULL) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOi:sort", + kwlist, &compare, &keyfunc, &reverse)) + return NULL; + } + if (compare == Py_None) + compare = NULL; + if (keyfunc == Py_None) + keyfunc = NULL; + + const Py_ssize_t length = Length(self); + if (InternalQuickSort(self, 0, length, compare, keyfunc) < 0) + return NULL; + + // Finally reverse the result if requested. + if (reverse) { + google::protobuf::Message* message = self->message; + const google::protobuf::Reflection* reflection = message->GetReflection(); + const google::protobuf::FieldDescriptor* descriptor = self->parent_field->descriptor; + + // Reverse the Message array. + for (int i = 0; i < length / 2; ++i) + reflection->SwapElements(message, descriptor, i, length - i - 1); + + // Reverse the Python list. + ScopedPyObjectPtr res(PyObject_CallMethod(self->child_messages, + "reverse", NULL)); + if (res == NULL) + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject* SortReleased(RepeatedCompositeContainer* self, + PyObject* args, + PyObject* kwds) { + ScopedPyObjectPtr m(PyObject_GetAttrString(self->child_messages, "sort")); + if (m == NULL) + return NULL; + if (PyObject_Call(m, args, kwds) == NULL) + return NULL; + Py_RETURN_NONE; +} + +static PyObject* Sort(RepeatedCompositeContainer* self, + PyObject* args, + PyObject* kwds) { + // Support the old sort_function argument for backwards + // compatibility. + if (kwds != NULL) { + PyObject* sort_func = PyDict_GetItemString(kwds, "sort_function"); + if (sort_func != NULL) { + // Must set before deleting as sort_func is a borrowed reference + // and kwds might be the only thing keeping it alive. + PyDict_SetItemString(kwds, "cmp", sort_func); + PyDict_DelItemString(kwds, "sort_function"); + } + } + + if (UpdateChildMessages(self) < 0) + return NULL; + if (self->message == NULL) { + return SortReleased(self, args, kwds); + } else { + return SortAttached(self, args, kwds); + } +} + +// --------------------------------------------------------------------- + +static PyObject* Item(RepeatedCompositeContainer* self, Py_ssize_t index) { + if (UpdateChildMessages(self) < 0) { + return NULL; + } + Py_ssize_t length = Length(self); + if (index < 0) { + index = length + index; + } + PyObject* item = PyList_GetItem(self->child_messages, index); + if (item == NULL) { + return NULL; + } + Py_INCREF(item); + return item; +} + +// The caller takes ownership of the returned Message. +Message* ReleaseLast(const FieldDescriptor* field, + const Descriptor* type, + Message* message) { + GOOGLE_CHECK_NOTNULL(field); + GOOGLE_CHECK_NOTNULL(type); + GOOGLE_CHECK_NOTNULL(message); + + Message* released_message = message->GetReflection()->ReleaseLast( + message, field); + // TODO(tibell): Deal with proto1. + + // ReleaseMessage will return NULL which differs from + // child_cmessage->message, if the field does not exist. In this case, + // the latter points to the default instance via a const_cast<>, so we + // have to reset it to a new mutable object since we are taking ownership. + if (released_message == NULL) { + const Message* prototype = global_message_factory->GetPrototype(type); + GOOGLE_CHECK_NOTNULL(prototype); + return prototype->New(); + } else { + return released_message; + } +} + +// Release field of message and transfer the ownership to cmessage. +void ReleaseLastTo(const FieldDescriptor* field, + Message* message, + CMessage* cmessage) { + GOOGLE_CHECK_NOTNULL(field); + GOOGLE_CHECK_NOTNULL(message); + GOOGLE_CHECK_NOTNULL(cmessage); + + shared_ptr<Message> released_message( + ReleaseLast(field, cmessage->message->GetDescriptor(), message)); + cmessage->parent = NULL; + cmessage->parent_field = NULL; + cmessage->message = released_message.get(); + cmessage->read_only = false; + cmessage::SetOwner(cmessage, released_message); +} + +// Called to release a container using +// ClearField('container_field_name') on the parent. +int Release(RepeatedCompositeContainer* self) { + if (UpdateChildMessages(self) < 0) { + PyErr_WriteUnraisable(PyBytes_FromString("Failed to update released " + "messages")); + return -1; + } + + Message* message = self->message; + const FieldDescriptor* field = self->parent_field->descriptor; + + // The reflection API only lets us release the last message in a + // repeated field. Therefore we iterate through the children + // starting with the last one. + const Py_ssize_t size = PyList_GET_SIZE(self->child_messages); + GOOGLE_DCHECK_EQ(size, message->GetReflection()->FieldSize(*message, field)); + for (Py_ssize_t i = size - 1; i >= 0; --i) { + CMessage* child_cmessage = reinterpret_cast<CMessage*>( + PyList_GET_ITEM(self->child_messages, i)); + ReleaseLastTo(field, message, child_cmessage); + } + + // Detach from containing message. + self->parent = NULL; + self->parent_field = NULL; + self->message = NULL; + self->owner.reset(); + + return 0; +} + +int SetOwner(RepeatedCompositeContainer* self, + const shared_ptr<Message>& new_owner) { + GOOGLE_CHECK_ATTACHED(self); + + self->owner = new_owner; + const Py_ssize_t n = PyList_GET_SIZE(self->child_messages); + for (Py_ssize_t i = 0; i < n; ++i) { + PyObject* msg = PyList_GET_ITEM(self->child_messages, i); + if (cmessage::SetOwner(reinterpret_cast<CMessage*>(msg), new_owner) == -1) { + return -1; + } + } + return 0; +} + +static int Init(RepeatedCompositeContainer* self, + PyObject* args, + PyObject* kwargs) { + self->message = NULL; + self->parent = NULL; + self->parent_field = NULL; + self->subclass_init = NULL; + self->child_messages = PyList_New(0); + return 0; +} + +static void Dealloc(RepeatedCompositeContainer* self) { + Py_CLEAR(self->child_messages); + // TODO(tibell): Do we need to call delete on these objects to make + // sure their destructors are called? + self->owner.reset(); + Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); +} + +static PySequenceMethods SqMethods = { + (lenfunc)Length, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + (ssizeargfunc)Item /* sq_item */ +}; + +static PyMappingMethods MpMethods = { + (lenfunc)Length, /* mp_length */ + (binaryfunc)Subscript, /* mp_subscript */ + (objobjargproc)AssignSubscript,/* mp_ass_subscript */ +}; + +static PyMethodDef Methods[] = { + { "add", (PyCFunction) Add, METH_VARARGS | METH_KEYWORDS, + "Adds an object to the repeated container." }, + { "extend", (PyCFunction) Extend, METH_O, + "Adds objects to the repeated container." }, + { "remove", (PyCFunction) Remove, METH_O, + "Removes an object from the repeated container." }, + { "sort", (PyCFunction) Sort, METH_VARARGS | METH_KEYWORDS, + "Sorts the repeated container." }, + { "MergeFrom", (PyCFunction) MergeFrom, METH_O, + "Adds objects to the repeated container." }, + { NULL, NULL } +}; + +} // namespace repeated_composite_container + +PyTypeObject RepeatedCompositeContainer_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "google.protobuf.internal." + "cpp._message.RepeatedCompositeContainer", // tp_name + sizeof(RepeatedCompositeContainer), // tp_basicsize + 0, // tp_itemsize + (destructor)repeated_composite_container::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + &repeated_composite_container::SqMethods, // tp_as_sequence + &repeated_composite_container::MpMethods, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "A Repeated scalar container", // tp_doc + 0, // tp_traverse + 0, // tp_clear + (richcmpfunc)repeated_composite_container::RichCompare, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + repeated_composite_container::Methods, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + (initproc)repeated_composite_container::Init, // tp_init +}; + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/pyext/repeated_composite_container.h b/python/google/protobuf/pyext/repeated_composite_container.h new file mode 100644 index 0000000..898ef5a --- /dev/null +++ b/python/google/protobuf/pyext/repeated_composite_container.h @@ -0,0 +1,172 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: anuraag@google.com (Anuraag Agrawal) +// Author: tibell@google.com (Johan Tibell) + +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_COMPOSITE_CONTAINER_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_COMPOSITE_CONTAINER_H__ + +#include <Python.h> + +#include <memory> +#ifndef _SHARED_PTR_H +#include <google/protobuf/stubs/shared_ptr.h> +#endif +#include <string> +#include <vector> + + +namespace google { +namespace protobuf { + +class FieldDescriptor; +class Message; + +using internal::shared_ptr; + +namespace python { + +struct CMessage; +struct CFieldDescriptor; + +// A RepeatedCompositeContainer can be in one of two states: attached +// or released. +// +// When in the attached state all modifications to the container are +// done both on the 'message' and on the 'child_messages' +// list. In this state all Messages refered to by the children in +// 'child_messages' are owner by the 'owner'. +// +// When in the released state 'message', 'owner', 'parent', and +// 'parent_field' are NULL. +typedef struct RepeatedCompositeContainer { + PyObject_HEAD; + + // This is the top-level C++ Message object that owns the whole + // proto tree. Every Python RepeatedCompositeContainer holds a + // reference to it in order to keep it alive as long as there's a + // Python object that references any part of the tree. + shared_ptr<Message> owner; + + // Weak reference to parent object. May be NULL. Used to make sure + // the parent is writable before modifying the + // RepeatedCompositeContainer. + CMessage* parent; + + // A descriptor used to modify the underlying 'message'. + CFieldDescriptor* parent_field; + + // Pointer to the C++ Message that contains this container. The + // RepeatedCompositeContainer does not own this pointer. + // + // If NULL, this message has been released from its parent (by + // calling Clear() or ClearField() on the parent. + Message* message; + + // A callable that is used to create new child messages. + PyObject* subclass_init; + + // A list of child messages. + PyObject* child_messages; +} RepeatedCompositeContainer; + +extern PyTypeObject RepeatedCompositeContainer_Type; + +namespace repeated_composite_container { + +// Returns the number of items in this repeated composite container. +static Py_ssize_t Length(RepeatedCompositeContainer* self); + +// Appends a new CMessage to the container and returns it. The +// CMessage is initialized using the content of kwargs. +// +// Returns a new reference if successful; returns NULL and sets an +// exception if unsuccessful. +PyObject* Add(RepeatedCompositeContainer* self, + PyObject* args, + PyObject* kwargs); + +// Appends all the CMessages in the input iterator to the container. +// +// Returns None if successful; returns NULL and sets an exception if +// unsuccessful. +PyObject* Extend(RepeatedCompositeContainer* self, PyObject* value); + +// Appends a new message to the container for each message in the +// input iterator, merging each data element in. Equivalent to extend. +// +// Returns None if successful; returns NULL and sets an exception if +// unsuccessful. +PyObject* MergeFrom(RepeatedCompositeContainer* self, PyObject* other); + +// Accesses messages in the container. +// +// Returns a new reference to the message for an integer parameter. +// Returns a new reference to a list of messages for a slice. +PyObject* Subscript(RepeatedCompositeContainer* self, PyObject* slice); + +// Deletes items from the container (cannot be used for assignment). +// +// Returns 0 on success, -1 on failure. +int AssignSubscript(RepeatedCompositeContainer* self, + PyObject* slice, + PyObject* value); + +// Releases the messages in the container to the given message. +// +// Returns 0 on success, -1 on failure. +int ReleaseToMessage(RepeatedCompositeContainer* self, + google::protobuf::Message* new_message); + +// Releases the messages in the container to a new message. +// +// Returns 0 on success, -1 on failure. +int Release(RepeatedCompositeContainer* self); + +// Returns 0 on success, -1 on failure. +int SetOwner(RepeatedCompositeContainer* self, + const shared_ptr<Message>& new_owner); + +// Removes the last element of the repeated message field 'field' on +// the Message 'message', and transfers the ownership of the released +// Message to 'cmessage'. +// +// Corresponds to reflection api method ReleaseMessage. +void ReleaseLastTo(const FieldDescriptor* field, + Message* message, + CMessage* cmessage); + +} // namespace repeated_composite_container +} // namespace python +} // namespace protobuf + +} // namespace google +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_COMPOSITE_CONTAINER_H__ diff --git a/python/google/protobuf/pyext/repeated_scalar_container.cc b/python/google/protobuf/pyext/repeated_scalar_container.cc new file mode 100644 index 0000000..e627d37 --- /dev/null +++ b/python/google/protobuf/pyext/repeated_scalar_container.cc @@ -0,0 +1,825 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: anuraag@google.com (Anuraag Agrawal) +// Author: tibell@google.com (Johan Tibell) + +#include <google/protobuf/pyext/repeated_scalar_container.h> + +#include <memory> +#ifndef _SHARED_PTR_H +#include <google/protobuf/stubs/shared_ptr.h> +#endif + +#include <google/protobuf/stubs/common.h> +#include <google/protobuf/descriptor.h> +#include <google/protobuf/dynamic_message.h> +#include <google/protobuf/message.h> +#include <google/protobuf/pyext/descriptor.h> +#include <google/protobuf/pyext/message.h> +#include <google/protobuf/pyext/scoped_pyobject_ptr.h> + +#if PY_MAJOR_VERSION >= 3 + #define PyInt_FromLong PyLong_FromLong + #if PY_VERSION_HEX < 0x03030000 + #error "Python 3.0 - 3.2 are not supported." + #else + #define PyString_AsString(ob) \ + (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AS_STRING(ob)) + #endif +#endif + +namespace google { +namespace protobuf { +namespace python { + +extern google::protobuf::DynamicMessageFactory* global_message_factory; + +namespace repeated_scalar_container { + +static int InternalAssignRepeatedField( + RepeatedScalarContainer* self, PyObject* list) { + self->message->GetReflection()->ClearField(self->message, + self->parent_field->descriptor); + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(list); ++i) { + PyObject* value = PyList_GET_ITEM(list, i); + if (Append(self, value) == NULL) { + return -1; + } + } + return 0; +} + +static Py_ssize_t Len(RepeatedScalarContainer* self) { + google::protobuf::Message* message = self->message; + return message->GetReflection()->FieldSize(*message, + self->parent_field->descriptor); +} + +static int AssignItem(RepeatedScalarContainer* self, + Py_ssize_t index, + PyObject* arg) { + cmessage::AssureWritable(self->parent); + google::protobuf::Message* message = self->message; + const google::protobuf::FieldDescriptor* field_descriptor = + self->parent_field->descriptor; + if (!FIELD_BELONGS_TO_MESSAGE(field_descriptor, message)) { + PyErr_SetString( + PyExc_KeyError, "Field does not belong to message!"); + return -1; + } + + const google::protobuf::Reflection* reflection = message->GetReflection(); + int field_size = reflection->FieldSize(*message, field_descriptor); + if (index < 0) { + index = field_size + index; + } + if (index < 0 || index >= field_size) { + PyErr_Format(PyExc_IndexError, + "list assignment index (%d) out of range", + static_cast<int>(index)); + return -1; + } + + if (arg == NULL) { + ScopedPyObjectPtr py_index(PyLong_FromLong(index)); + return cmessage::InternalDeleteRepeatedField(message, field_descriptor, + py_index, NULL); + } + + if (PySequence_Check(arg) && !(PyBytes_Check(arg) || PyUnicode_Check(arg))) { + PyErr_SetString(PyExc_TypeError, "Value must be scalar"); + return -1; + } + + switch (field_descriptor->cpp_type()) { + case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { + GOOGLE_CHECK_GET_INT32(arg, value, -1); + reflection->SetRepeatedInt32(message, field_descriptor, index, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { + GOOGLE_CHECK_GET_INT64(arg, value, -1); + reflection->SetRepeatedInt64(message, field_descriptor, index, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { + GOOGLE_CHECK_GET_UINT32(arg, value, -1); + reflection->SetRepeatedUInt32(message, field_descriptor, index, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { + GOOGLE_CHECK_GET_UINT64(arg, value, -1); + reflection->SetRepeatedUInt64(message, field_descriptor, index, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { + GOOGLE_CHECK_GET_FLOAT(arg, value, -1); + reflection->SetRepeatedFloat(message, field_descriptor, index, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { + GOOGLE_CHECK_GET_DOUBLE(arg, value, -1); + reflection->SetRepeatedDouble(message, field_descriptor, index, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { + GOOGLE_CHECK_GET_BOOL(arg, value, -1); + reflection->SetRepeatedBool(message, field_descriptor, index, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { + if (!CheckAndSetString( + arg, message, field_descriptor, reflection, false, index)) { + return -1; + } + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { + GOOGLE_CHECK_GET_INT32(arg, value, -1); + const google::protobuf::EnumDescriptor* enum_descriptor = + field_descriptor->enum_type(); + const google::protobuf::EnumValueDescriptor* enum_value = + enum_descriptor->FindValueByNumber(value); + if (enum_value != NULL) { + reflection->SetRepeatedEnum(message, field_descriptor, index, + enum_value); + } else { + ScopedPyObjectPtr s(PyObject_Str(arg)); + if (s != NULL) { + PyErr_Format(PyExc_ValueError, "Unknown enum value: %s", + PyString_AsString(s.get())); + } + return -1; + } + break; + } + default: + PyErr_Format( + PyExc_SystemError, "Adding value to a field of unknown type %d", + field_descriptor->cpp_type()); + return -1; + } + return 0; +} + +static PyObject* Item(RepeatedScalarContainer* self, Py_ssize_t index) { + google::protobuf::Message* message = self->message; + const google::protobuf::FieldDescriptor* field_descriptor = + self->parent_field->descriptor; + const google::protobuf::Reflection* reflection = message->GetReflection(); + + int field_size = reflection->FieldSize(*message, field_descriptor); + if (index < 0) { + index = field_size + index; + } + if (index < 0 || index >= field_size) { + PyErr_Format(PyExc_IndexError, + "list assignment index (%d) out of range", + static_cast<int>(index)); + return NULL; + } + + PyObject* result = NULL; + switch (field_descriptor->cpp_type()) { + case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { + int32 value = reflection->GetRepeatedInt32( + *message, field_descriptor, index); + result = PyInt_FromLong(value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { + int64 value = reflection->GetRepeatedInt64( + *message, field_descriptor, index); + result = PyLong_FromLongLong(value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { + uint32 value = reflection->GetRepeatedUInt32( + *message, field_descriptor, index); + result = PyLong_FromLongLong(value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { + uint64 value = reflection->GetRepeatedUInt64( + *message, field_descriptor, index); + result = PyLong_FromUnsignedLongLong(value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { + float value = reflection->GetRepeatedFloat( + *message, field_descriptor, index); + result = PyFloat_FromDouble(value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { + double value = reflection->GetRepeatedDouble( + *message, field_descriptor, index); + result = PyFloat_FromDouble(value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { + bool value = reflection->GetRepeatedBool( + *message, field_descriptor, index); + result = PyBool_FromLong(value ? 1 : 0); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { + const google::protobuf::EnumValueDescriptor* enum_value = + message->GetReflection()->GetRepeatedEnum( + *message, field_descriptor, index); + result = PyInt_FromLong(enum_value->number()); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { + string value = reflection->GetRepeatedString( + *message, field_descriptor, index); + result = ToStringObject(field_descriptor, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + PyObject* py_cmsg = PyObject_CallObject(reinterpret_cast<PyObject*>( + &CMessage_Type), NULL); + if (py_cmsg == NULL) { + return NULL; + } + CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg); + const google::protobuf::Message& msg = reflection->GetRepeatedMessage( + *message, field_descriptor, index); + cmsg->owner = self->owner; + cmsg->parent = self->parent; + cmsg->message = const_cast<google::protobuf::Message*>(&msg); + cmsg->read_only = false; + result = reinterpret_cast<PyObject*>(py_cmsg); + break; + } + default: + PyErr_Format( + PyExc_SystemError, + "Getting value from a repeated field of unknown type %d", + field_descriptor->cpp_type()); + } + + return result; +} + +static PyObject* Subscript(RepeatedScalarContainer* self, PyObject* slice) { + Py_ssize_t from; + Py_ssize_t to; + Py_ssize_t step; + Py_ssize_t length; + Py_ssize_t slicelength; + bool return_list = false; +#if PY_MAJOR_VERSION < 3 + if (PyInt_Check(slice)) { + from = to = PyInt_AsLong(slice); + } else // NOLINT +#endif + if (PyLong_Check(slice)) { + from = to = PyLong_AsLong(slice); + } else if (PySlice_Check(slice)) { + length = Len(self); +#if PY_MAJOR_VERSION >= 3 + if (PySlice_GetIndicesEx(slice, +#else + if (PySlice_GetIndicesEx(reinterpret_cast<PySliceObject*>(slice), +#endif + length, &from, &to, &step, &slicelength) == -1) { + return NULL; + } + return_list = true; + } else { + PyErr_SetString(PyExc_TypeError, "list indices must be integers"); + return NULL; + } + + if (!return_list) { + return Item(self, from); + } + + PyObject* list = PyList_New(0); + if (list == NULL) { + return NULL; + } + if (from <= to) { + if (step < 0) { + return list; + } + for (Py_ssize_t index = from; index < to; index += step) { + if (index < 0 || index >= length) { + break; + } + ScopedPyObjectPtr s(Item(self, index)); + PyList_Append(list, s); + } + } else { + if (step > 0) { + return list; + } + for (Py_ssize_t index = from; index > to; index += step) { + if (index < 0 || index >= length) { + break; + } + ScopedPyObjectPtr s(Item(self, index)); + PyList_Append(list, s); + } + } + return list; +} + +PyObject* Append(RepeatedScalarContainer* self, PyObject* item) { + cmessage::AssureWritable(self->parent); + google::protobuf::Message* message = self->message; + const google::protobuf::FieldDescriptor* field_descriptor = + self->parent_field->descriptor; + + if (!FIELD_BELONGS_TO_MESSAGE(field_descriptor, message)) { + PyErr_SetString( + PyExc_KeyError, "Field does not belong to message!"); + return NULL; + } + + const google::protobuf::Reflection* reflection = message->GetReflection(); + switch (field_descriptor->cpp_type()) { + case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { + GOOGLE_CHECK_GET_INT32(item, value, NULL); + reflection->AddInt32(message, field_descriptor, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { + GOOGLE_CHECK_GET_INT64(item, value, NULL); + reflection->AddInt64(message, field_descriptor, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { + GOOGLE_CHECK_GET_UINT32(item, value, NULL); + reflection->AddUInt32(message, field_descriptor, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { + GOOGLE_CHECK_GET_UINT64(item, value, NULL); + reflection->AddUInt64(message, field_descriptor, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { + GOOGLE_CHECK_GET_FLOAT(item, value, NULL); + reflection->AddFloat(message, field_descriptor, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { + GOOGLE_CHECK_GET_DOUBLE(item, value, NULL); + reflection->AddDouble(message, field_descriptor, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { + GOOGLE_CHECK_GET_BOOL(item, value, NULL); + reflection->AddBool(message, field_descriptor, value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { + if (!CheckAndSetString( + item, message, field_descriptor, reflection, true, -1)) { + return NULL; + } + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { + GOOGLE_CHECK_GET_INT32(item, value, NULL); + const google::protobuf::EnumDescriptor* enum_descriptor = + field_descriptor->enum_type(); + const google::protobuf::EnumValueDescriptor* enum_value = + enum_descriptor->FindValueByNumber(value); + if (enum_value != NULL) { + reflection->AddEnum(message, field_descriptor, enum_value); + } else { + ScopedPyObjectPtr s(PyObject_Str(item)); + if (s != NULL) { + PyErr_Format(PyExc_ValueError, "Unknown enum value: %s", + PyString_AsString(s.get())); + } + return NULL; + } + break; + } + default: + PyErr_Format( + PyExc_SystemError, "Adding value to a field of unknown type %d", + field_descriptor->cpp_type()); + return NULL; + } + + Py_RETURN_NONE; +} + +static int AssSubscript(RepeatedScalarContainer* self, + PyObject* slice, + PyObject* value) { + Py_ssize_t from; + Py_ssize_t to; + Py_ssize_t step; + Py_ssize_t length; + Py_ssize_t slicelength; + bool create_list = false; + + cmessage::AssureWritable(self->parent); + google::protobuf::Message* message = self->message; + const google::protobuf::FieldDescriptor* field_descriptor = + self->parent_field->descriptor; + +#if PY_MAJOR_VERSION < 3 + if (PyInt_Check(slice)) { + from = to = PyInt_AsLong(slice); + } else +#endif + if (PyLong_Check(slice)) { + from = to = PyLong_AsLong(slice); + } else if (PySlice_Check(slice)) { + const google::protobuf::Reflection* reflection = message->GetReflection(); + length = reflection->FieldSize(*message, field_descriptor); +#if PY_MAJOR_VERSION >= 3 + if (PySlice_GetIndicesEx(slice, +#else + if (PySlice_GetIndicesEx(reinterpret_cast<PySliceObject*>(slice), +#endif + length, &from, &to, &step, &slicelength) == -1) { + return -1; + } + create_list = true; + } else { + PyErr_SetString(PyExc_TypeError, "list indices must be integers"); + return -1; + } + + if (value == NULL) { + return cmessage::InternalDeleteRepeatedField( + message, field_descriptor, slice, NULL); + } + + if (!create_list) { + return AssignItem(self, from, value); + } + + ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL)); + if (full_slice == NULL) { + return -1; + } + ScopedPyObjectPtr new_list(Subscript(self, full_slice)); + if (new_list == NULL) { + return -1; + } + if (PySequence_SetSlice(new_list, from, to, value) < 0) { + return -1; + } + + return InternalAssignRepeatedField(self, new_list); +} + +PyObject* Extend(RepeatedScalarContainer* self, PyObject* value) { + cmessage::AssureWritable(self->parent); + if (PyObject_Not(value)) { + Py_RETURN_NONE; + } + ScopedPyObjectPtr iter(PyObject_GetIter(value)); + if (iter == NULL) { + PyErr_SetString(PyExc_TypeError, "Value must be iterable"); + return NULL; + } + ScopedPyObjectPtr next; + while ((next.reset(PyIter_Next(iter))) != NULL) { + if (Append(self, next) == NULL) { + return NULL; + } + } + if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject* Insert(RepeatedScalarContainer* self, PyObject* args) { + Py_ssize_t index; + PyObject* value; + if (!PyArg_ParseTuple(args, "lO", &index, &value)) { + return NULL; + } + ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL)); + ScopedPyObjectPtr new_list(Subscript(self, full_slice)); + if (PyList_Insert(new_list, index, value) < 0) { + return NULL; + } + int ret = InternalAssignRepeatedField(self, new_list); + if (ret < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject* Remove(RepeatedScalarContainer* self, PyObject* value) { + Py_ssize_t match_index = -1; + for (Py_ssize_t i = 0; i < Len(self); ++i) { + ScopedPyObjectPtr elem(Item(self, i)); + if (PyObject_RichCompareBool(elem, value, Py_EQ)) { + match_index = i; + break; + } + } + if (match_index == -1) { + PyErr_SetString(PyExc_ValueError, "remove(x): x not in container"); + return NULL; + } + if (AssignItem(self, match_index, NULL) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject* RichCompare(RepeatedScalarContainer* self, + PyObject* other, + int opid) { + if (opid != Py_EQ && opid != Py_NE) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + + // Copy the contents of this repeated scalar container, and other if it is + // also a repeated scalar container, into Python lists so we can delegate + // to the list's compare method. + + ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL)); + if (full_slice == NULL) { + return NULL; + } + + ScopedPyObjectPtr other_list_deleter; + if (PyObject_TypeCheck(other, &RepeatedScalarContainer_Type)) { + other_list_deleter.reset(Subscript( + reinterpret_cast<RepeatedScalarContainer*>(other), full_slice)); + other = other_list_deleter.get(); + } + + ScopedPyObjectPtr list(Subscript(self, full_slice)); + if (list == NULL) { + return NULL; + } + return PyObject_RichCompare(list, other, opid); +} + +PyObject* Reduce(RepeatedScalarContainer* unused_self) { + PyErr_Format( + PickleError_class, + "can't pickle repeated message fields, convert to list first"); + return NULL; +} + +static PyObject* Sort(RepeatedScalarContainer* self, + PyObject* args, + PyObject* kwds) { + // Support the old sort_function argument for backwards + // compatibility. + if (kwds != NULL) { + PyObject* sort_func = PyDict_GetItemString(kwds, "sort_function"); + if (sort_func != NULL) { + // Must set before deleting as sort_func is a borrowed reference + // and kwds might be the only thing keeping it alive. + if (PyDict_SetItemString(kwds, "cmp", sort_func) == -1) + return NULL; + if (PyDict_DelItemString(kwds, "sort_function") == -1) + return NULL; + } + } + + ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL)); + if (full_slice == NULL) { + return NULL; + } + ScopedPyObjectPtr list(Subscript(self, full_slice)); + if (list == NULL) { + return NULL; + } + ScopedPyObjectPtr m(PyObject_GetAttrString(list, "sort")); + if (m == NULL) { + return NULL; + } + ScopedPyObjectPtr res(PyObject_Call(m, args, kwds)); + if (res == NULL) { + return NULL; + } + int ret = InternalAssignRepeatedField(self, list); + if (ret < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static int Init(RepeatedScalarContainer* self, + PyObject* args, + PyObject* kwargs) { + PyObject* py_parent; + PyObject* py_parent_field; + if (!PyArg_UnpackTuple(args, "__init__()", 2, 2, &py_parent, + &py_parent_field)) { + return -1; + } + + if (!PyObject_TypeCheck(py_parent, &CMessage_Type)) { + PyErr_Format(PyExc_TypeError, + "expect %s, but got %s", + CMessage_Type.tp_name, + Py_TYPE(py_parent)->tp_name); + return -1; + } + + if (!PyObject_TypeCheck(py_parent_field, &CFieldDescriptor_Type)) { + PyErr_Format(PyExc_TypeError, + "expect %s, but got %s", + CFieldDescriptor_Type.tp_name, + Py_TYPE(py_parent_field)->tp_name); + return -1; + } + + CMessage* cmessage = reinterpret_cast<CMessage*>(py_parent); + CFieldDescriptor* cdescriptor = reinterpret_cast<CFieldDescriptor*>( + py_parent_field); + + if (!FIELD_BELONGS_TO_MESSAGE(cdescriptor->descriptor, cmessage->message)) { + PyErr_SetString( + PyExc_KeyError, "Field does not belong to message!"); + return -1; + } + + self->message = cmessage->message; + self->parent = cmessage; + self->parent_field = cdescriptor; + self->owner = cmessage->owner; + return 0; +} + +// Initializes the underlying Message object of "to" so it becomes a new parent +// repeated scalar, and copies all the values from "from" to it. A child scalar +// container can be released by passing it as both from and to (e.g. making it +// the recipient of the new parent message and copying the values from itself). +static int InitializeAndCopyToParentContainer( + RepeatedScalarContainer* from, + RepeatedScalarContainer* to) { + ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL)); + if (full_slice == NULL) { + return -1; + } + ScopedPyObjectPtr values(Subscript(from, full_slice)); + if (values == NULL) { + return -1; + } + google::protobuf::Message* new_message = global_message_factory->GetPrototype( + from->message->GetDescriptor())->New(); + to->parent = NULL; + // TODO(anuraag): Document why it's OK to hang on to parent_field, + // even though it's a weak reference. It ought to be enough to + // hold on to the FieldDescriptor only. + to->parent_field = from->parent_field; + to->message = new_message; + to->owner.reset(new_message); + if (InternalAssignRepeatedField(to, values) < 0) { + return -1; + } + return 0; +} + +int Release(RepeatedScalarContainer* self) { + return InitializeAndCopyToParentContainer(self, self); +} + +PyObject* DeepCopy(RepeatedScalarContainer* self, PyObject* arg) { + ScopedPyObjectPtr init_args( + PyTuple_Pack(2, self->parent, self->parent_field)); + PyObject* clone = PyObject_CallObject( + reinterpret_cast<PyObject*>(&RepeatedScalarContainer_Type), init_args); + if (clone == NULL) { + return NULL; + } + if (!PyObject_TypeCheck(clone, &RepeatedScalarContainer_Type)) { + Py_DECREF(clone); + return NULL; + } + if (InitializeAndCopyToParentContainer( + self, reinterpret_cast<RepeatedScalarContainer*>(clone)) < 0) { + Py_DECREF(clone); + return NULL; + } + return clone; +} + +static void Dealloc(RepeatedScalarContainer* self) { + self->owner.reset(); + Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self)); +} + +void SetOwner(RepeatedScalarContainer* self, + const shared_ptr<Message>& new_owner) { + self->owner = new_owner; +} + +static PySequenceMethods SqMethods = { + (lenfunc)Len, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + (ssizeargfunc)Item, /* sq_item */ + 0, /* sq_slice */ + (ssizeobjargproc)AssignItem /* sq_ass_item */ +}; + +static PyMappingMethods MpMethods = { + (lenfunc)Len, /* mp_length */ + (binaryfunc)Subscript, /* mp_subscript */ + (objobjargproc)AssSubscript, /* mp_ass_subscript */ +}; + +static PyMethodDef Methods[] = { + { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS, + "Makes a deep copy of the class." }, + { "__reduce__", (PyCFunction)Reduce, METH_NOARGS, + "Outputs picklable representation of the repeated field." }, + { "append", (PyCFunction)Append, METH_O, + "Appends an object to the repeated container." }, + { "extend", (PyCFunction)Extend, METH_O, + "Appends objects to the repeated container." }, + { "insert", (PyCFunction)Insert, METH_VARARGS, + "Appends objects to the repeated container." }, + { "remove", (PyCFunction)Remove, METH_O, + "Removes an object from the repeated container." }, + { "sort", (PyCFunction)Sort, METH_VARARGS | METH_KEYWORDS, + "Sorts the repeated container."}, + { NULL, NULL } +}; + +} // namespace repeated_scalar_container + +PyTypeObject RepeatedScalarContainer_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "google.protobuf.internal." + "cpp._message.RepeatedScalarContainer", // tp_name + sizeof(RepeatedScalarContainer), // tp_basicsize + 0, // tp_itemsize + (destructor)repeated_scalar_container::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + &repeated_scalar_container::SqMethods, // tp_as_sequence + &repeated_scalar_container::MpMethods, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "A Repeated scalar container", // tp_doc + 0, // tp_traverse + 0, // tp_clear + (richcmpfunc)repeated_scalar_container::RichCompare, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + repeated_scalar_container::Methods, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + (initproc)repeated_scalar_container::Init, // tp_init +}; + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/pyext/repeated_scalar_container.h b/python/google/protobuf/pyext/repeated_scalar_container.h new file mode 100644 index 0000000..69d15d5 --- /dev/null +++ b/python/google/protobuf/pyext/repeated_scalar_container.h @@ -0,0 +1,112 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: anuraag@google.com (Anuraag Agrawal) +// Author: tibell@google.com (Johan Tibell) + +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_SCALAR_CONTAINER_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_SCALAR_CONTAINER_H__ + +#include <Python.h> + +#include <memory> +#ifndef _SHARED_PTR_H +#include <google/protobuf/stubs/shared_ptr.h> +#endif + + +namespace google { +namespace protobuf { + +class Message; + +using internal::shared_ptr; + +namespace python { + +struct CFieldDescriptor; +struct CMessage; + +typedef struct RepeatedScalarContainer { + PyObject_HEAD; + + // This is the top-level C++ Message object that owns the whole + // proto tree. Every Python RepeatedScalarContainer holds a + // reference to it in order to keep it alive as long as there's a + // Python object that references any part of the tree. + shared_ptr<Message> owner; + + // Pointer to the C++ Message that contains this container. The + // RepeatedScalarContainer does not own this pointer. + Message* message; + + // Weak reference to a parent CMessage object (i.e. may be NULL.) + // + // Used to make sure all ancestors are also mutable when first + // modifying the container. + CMessage* parent; + + // Weak reference to the parent's descriptor that describes this + // field. Used together with the parent's message when making a + // default message instance mutable. + CFieldDescriptor* parent_field; +} RepeatedScalarContainer; + +extern PyTypeObject RepeatedScalarContainer_Type; + +namespace repeated_scalar_container { + +// Appends the scalar 'item' to the end of the container 'self'. +// +// Returns None if successful; returns NULL and sets an exception if +// unsuccessful. +PyObject* Append(RepeatedScalarContainer* self, PyObject* item); + +// Releases the messages in the container to a new message. +// +// Returns 0 on success, -1 on failure. +int Release(RepeatedScalarContainer* self); + +// Appends all the elements in the input iterator to the container. +// +// Returns None if successful; returns NULL and sets an exception if +// unsuccessful. +PyObject* Extend(RepeatedScalarContainer* self, PyObject* value); + +// Set the owner field of self and any children of self. +void SetOwner(RepeatedScalarContainer* self, + const shared_ptr<Message>& new_owner); + +} // namespace repeated_scalar_container +} // namespace python +} // namespace protobuf + +} // namespace google +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_SCALAR_CONTAINER_H__ diff --git a/python/google/protobuf/pyext/scoped_pyobject_ptr.h b/python/google/protobuf/pyext/scoped_pyobject_ptr.h new file mode 100644 index 0000000..9f337c3 --- /dev/null +++ b/python/google/protobuf/pyext/scoped_pyobject_ptr.h @@ -0,0 +1,95 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: tibell@google.com (Johan Tibell) + +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_SCOPED_PYOBJECT_PTR_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_SCOPED_PYOBJECT_PTR_H__ + +#include <Python.h> + +namespace google { +class ScopedPyObjectPtr { + public: + // Constructor. Defaults to intializing with NULL. + // There is no way to create an uninitialized ScopedPyObjectPtr. + explicit ScopedPyObjectPtr(PyObject* p = NULL) : ptr_(p) { } + + // Destructor. If there is a PyObject object, delete it. + ~ScopedPyObjectPtr() { + Py_XDECREF(ptr_); + } + + // Reset. Deletes the current owned object, if any. + // Then takes ownership of a new object, if given. + // this->reset(this->get()) works. + PyObject* reset(PyObject* p = NULL) { + if (p != ptr_) { + Py_XDECREF(ptr_); + ptr_ = p; + } + return ptr_; + } + + // Releases ownership of the object. + PyObject* release() { + PyObject* p = ptr_; + ptr_ = NULL; + return p; + } + + operator PyObject*() { return ptr_; } + + PyObject* operator->() const { + assert(ptr_ != NULL); + return ptr_; + } + + PyObject* get() const { return ptr_; } + + Py_ssize_t refcnt() const { return Py_REFCNT(ptr_); } + + void inc() const { Py_INCREF(ptr_); } + + // Comparison operators. + // These return whether a ScopedPyObjectPtr and a raw pointer + // refer to the same object, not just to two different but equal + // objects. + bool operator==(const PyObject* p) const { return ptr_ == p; } + bool operator!=(const PyObject* p) const { return ptr_ != p; } + + private: + PyObject* ptr_; + + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ScopedPyObjectPtr); +}; + +} // namespace google +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_SCOPED_PYOBJECT_PTR_H__ |