protobuf-c/protobuf-c.c: add functionality to merge multiple instances

of the same field on the wire (Fixes #91)
t/generated-code2/test-generated-code2.c: add a test case for merging
messages
t/test-full.proto: expand message definitions to test for merging nested
messages

protobuf-c/protobuf-c.c

@@ -1541,6 +1541,136 @@ max_b128_numbers (size_t len, const uint8_t *data)
   return rv;
 }
 
+/* Merge earlier message into the latter message as follows:
+ * For numeric types and strings, if the same value appears multiple times, the
+ * parser accepts the last value it sees.
+ * For embedded message fields, the parser merges multiple instances of the same
+ * field. That is, all singular scalar fields in the latter instance replace those
+ * in the former, singular embedded messages are merged, and repeated fields are
+ * concatenated.
+ * The earlier message should be freed after calling this function, as some of its
+ * fields may have been reused and changed to their default values during the merge*/
+static protobuf_c_boolean
+merge_messages (ProtobufCMessage *earlier_msg,
+                ProtobufCMessage *latter_msg,
+                ProtobufCAllocator *allocator)
+{
+  unsigned i;
+  const ProtobufCFieldDescriptor *fields = earlier_msg->descriptor->fields;
+  for (i = 0; i < latter_msg->descriptor->n_fields; i++)
+  {
+    if (fields[i].label == PROTOBUF_C_LABEL_REPEATED)
+    {
+      size_t *n_earlier =
+        STRUCT_MEMBER_PTR (size_t, earlier_msg, fields[i].quantifier_offset);
+      uint8_t **p_earlier =
+        STRUCT_MEMBER_PTR (uint8_t *, earlier_msg, fields[i].offset);
+      size_t *n_latter =
+        STRUCT_MEMBER_PTR (size_t, latter_msg, fields[i].quantifier_offset);
+      uint8_t **p_latter =
+        STRUCT_MEMBER_PTR (uint8_t *, latter_msg, fields[i].offset);
+
+      if (*n_earlier > 0)
+      {
+        if (*n_latter > 0)
+        {
+          /* Concatenate the repeated field */
+          size_t el_size = sizeof_elt_in_repeated_array (fields[i].type);
+          uint8_t *new_field;
+          DO_ALLOC (new_field, allocator, (*n_earlier + *n_latter) * el_size, return 0);
+
+          memcpy (new_field, *p_latter, *n_latter * el_size);
+          memcpy (new_field + *n_latter * el_size, *p_earlier, *n_earlier * el_size);
+
+          FREE (allocator, *p_latter);
+          FREE (allocator, *p_earlier);
+          *p_latter = new_field;
+          *n_latter = *n_earlier + *n_latter;
+        }
+        else
+        {
+          /* Zero copy the repeated field from the earlier message */
+          *n_latter = *n_earlier;
+          *p_latter = *p_earlier;
+        }
+        /* Make sure the field does not get double freed */
+        *n_earlier = 0;
+        *p_earlier = 0;
+      }
+    }
+    else if (fields[i].type == PROTOBUF_C_TYPE_MESSAGE)
+    {
+      ProtobufCMessage **em =
+        STRUCT_MEMBER_PTR (ProtobufCMessage *, earlier_msg, fields[i].offset);
+      ProtobufCMessage **lm =
+        STRUCT_MEMBER_PTR (ProtobufCMessage *, latter_msg, fields[i].offset);
+      if (*em != NULL)
+        if (*lm != NULL)
+          merge_messages (*em, *lm, allocator);
+        else
+        {
+          /* Zero copy the optional message */
+          assert (fields[i].label == PROTOBUF_C_LABEL_OPTIONAL);
+          *lm = *em;
+          *em = NULL;
+        }
+    }
+    else if (fields[i].label == PROTOBUF_C_LABEL_OPTIONAL)
+    {
+      size_t el_size = 0;
+      protobuf_c_boolean need_to_merge = 0;
+      void *earlier_elem =
+            STRUCT_MEMBER_P (earlier_msg, fields[i].offset);
+      void *latter_elem =
+            STRUCT_MEMBER_P (latter_msg, fields[i].offset);
+      const void *def_val = fields[i].default_value;
+
+      switch (fields[i].type)
+      {
+        case PROTOBUF_C_TYPE_BYTES:
+          el_size = sizeof (ProtobufCBinaryData);
+          uint8_t *e_data = ((ProtobufCBinaryData *)earlier_elem)->data;
+          uint8_t *l_data = ((ProtobufCBinaryData *)latter_elem)->data;
+          const ProtobufCBinaryData *d_bd = (ProtobufCBinaryData *)def_val;
+
+          need_to_merge = (e_data != NULL && (d_bd != NULL && e_data != d_bd->data))
+            && (l_data == NULL || (d_bd != NULL && l_data == d_bd->data));
+          break;
+        case PROTOBUF_C_TYPE_STRING:
+          el_size = sizeof (char *);
+          char *e_str = *(char **)earlier_elem;
+          char *l_str = *(char **)latter_elem;
+          const char *d_str = def_val;
+
+          need_to_merge = e_str != d_str && l_str == d_str;
+          break;
+        default:
+          el_size = sizeof_elt_in_repeated_array (fields[i].type);
+
+          need_to_merge =
+            STRUCT_MEMBER (protobuf_c_boolean, earlier_msg, fields[i].quantifier_offset)
+            && !STRUCT_MEMBER (protobuf_c_boolean, latter_msg, fields[i].quantifier_offset);
+          break;
+      }
+
+      if (need_to_merge)
+      {
+        memcpy (latter_elem, earlier_elem, el_size);
+        /* Reset the element from the old message to 0 to make sure earlier message
+         * deallocation doesn't corrupt zero-copied data in the new message, earlier
+         * message will be freed after this function is called anyway */
+        memset (earlier_elem, 0, el_size);
+
+        if (fields[i].quantifier_offset != 0)
+        {
+          /* Set the has field, if applicable */
+          STRUCT_MEMBER (protobuf_c_boolean, latter_msg, fields[i].quantifier_offset) = 1;
+          STRUCT_MEMBER (protobuf_c_boolean, earlier_msg, fields[i].quantifier_offset) = 0;
+        }
+      }
+    }
+  }
+}
 
 /* Given a raw slab of packed-repeated values,
    determine the number of elements.
@@ -1798,14 +1928,22 @@ parse_required_member (ScannedMember *scanned_member,
         const ProtobufCMessage *def_mess;
         unsigned pref_len = scanned_member->length_prefix_len;
         def_mess = scanned_member->field->default_value;
-        if (maybe_clear && *pmessage != NULL && *pmessage != def_mess)
-          protobuf_c_message_free_unpacked (*pmessage, allocator);
         subm = protobuf_c_message_unpack (scanned_member->field->descriptor,
                                           allocator,
                                           len - pref_len, data + pref_len);
-        *pmessage = subm;       /* since we freed the message we must clear the field, even if NULL */
+
+        if (maybe_clear && *pmessage != NULL && *pmessage != def_mess)
+        {
+          if (subm != NULL)
+            merge_messages (*pmessage, subm, allocator);
+          /* Delete the previous message */
+          protobuf_c_message_free_unpacked (*pmessage, allocator);
+          *pmessage = NULL;
+        }
         if (subm == NULL)
           return 0;
+
+        *pmessage = subm;
         return 1;
       }
     }

t/generated-code2/test-generated-code2.c

@@ -1406,6 +1406,93 @@ test_optional_default_values (void)
   foo__default_optional_values__free_unpacked (mess2, NULL);
 }
 
+static void
+test_optional_merge (void)
+{
+   Foo__TestMessOptional msg1 = FOO__TEST_MESS_OPTIONAL__INIT;
+   Foo__SubMess sub1 = FOO__SUB_MESS__INIT;
+   Foo__SubMess__SubSubMess subsub1 = FOO__SUB_MESS__SUB_SUB_MESS__INIT;
+
+   msg1.has_test_int32 = 1;
+   msg1.test_int32 = 12345;
+   msg1.has_test_sint32 = 1;
+   msg1.test_sint32 = -12345;
+   msg1.has_test_int64 = 1;
+   msg1.test_int64 = 232;
+   msg1.test_string = "123";
+   msg1.test_message = &sub1;
+   sub1.has_val1 = 1;
+   sub1.val1 = 4;
+   sub1.has_val2 = 1;
+   sub1.val2 = 5;
+   int32_t arr1[] = {0, 1};
+   sub1.n_rep = 2;
+   sub1.rep = arr1;
+   sub1.sub1 = &subsub1;
+   sub1.sub2 = &subsub1;
+   subsub1.n_rep = 2;
+   subsub1.rep = arr1;
+   subsub1.str1 = "test";
+
+   size_t msg_size = foo__test_mess_optional__get_packed_size (&msg1);
+
+   Foo__TestMessOptional msg2 = FOO__TEST_MESS_OPTIONAL__INIT;
+   Foo__SubMess sub2 = FOO__SUB_MESS__INIT;
+   Foo__SubMess__SubSubMess subsub2 = FOO__SUB_MESS__SUB_SUB_MESS__INIT;
+
+   msg2.has_test_int64 = 1;
+   msg2.test_int64 = 2;
+   msg2.has_test_enum = 1;
+   msg2.test_enum = FOO__TEST_ENUM__VALUE128;
+   msg2.test_string = "456";
+   msg2.test_message = &sub2;
+   sub2.has_val2 = 1;
+   sub2.val2 = 666;
+   int32_t arr2[] = {2, 3};
+   sub2.n_rep = 2;
+   sub2.rep = arr2;
+   sub2.sub1 = &subsub2;
+   subsub2.has_val1 = 1;
+   subsub2.val1 = 1;
+
+   msg_size += foo__test_mess_optional__get_packed_size (&msg2);
+
+   uint8_t *packed_buffer = (uint8_t *)malloc (msg_size);
+
+   size_t packed_size = foo__test_mess_optional__pack (&msg1, packed_buffer);
+   packed_size += foo__test_mess_optional__pack (&msg2, packed_buffer + packed_size);
+
+   assert (packed_size == msg_size);
+
+   Foo__TestMessOptional *merged = foo__test_mess_optional__unpack
+      (NULL, msg_size, packed_buffer);
+
+   assert (merged->has_test_int32 && merged->test_int32 == msg1.test_int32);
+   assert (merged->has_test_sint32 && merged->test_sint32 == msg1.test_sint32);
+   assert (merged->has_test_int64 && merged->test_int64 == msg2.test_int64);
+   assert (merged->has_test_enum && merged->test_enum == msg2.test_enum);
+   assert (strcmp (merged->test_string, msg2.test_string) == 0);
+   assert (merged->test_message->has_val1 && merged->test_message->val1 == sub1.val1);
+   assert (merged->test_message->has_val2 && merged->test_message->val2 == sub2.val2);
+   /* Repeated fields should get concatenated */
+   int32_t merged_arr[] = {2, 3, 0, 1};
+   assert (merged->test_message->n_rep == 4 &&
+           memcmp(merged->test_message->rep, merged_arr, sizeof(merged_arr)) == 0);
+
+   assert (merged->test_message->sub1->val1 == subsub2.val1);
+   assert (memcmp(merged->test_message->sub1->bytes1.data,
+            subsub2.bytes1.data, subsub2.bytes1.len) == 0);
+   assert (strcmp(merged->test_message->sub1->str1, subsub1.str1) == 0);
+   assert (merged->test_message->sub1->n_rep == subsub1.n_rep &&
+           memcmp(merged->test_message->sub1->rep, arr1, sizeof(arr1)) == 0);
+
+   assert (!merged->test_message->sub2->has_val1
+         && merged->test_message->sub2->val1 == subsub1.val1);
+
+   free (packed_buffer);
+   foo__test_mess_optional__free_unpacked (merged, NULL);
+}
+
 static struct alloc_data {
   uint32_t alloc_count;
   int32_t allocs_left;
@@ -1583,6 +1670,8 @@ static Test tests[] =
   { "test required default values", test_required_default_values },
   { "test optional default values", test_optional_default_values },
 
+  { "test optional merge", test_optional_merge },
+
   { "test free unpacked", test_alloc_free_all },
   { "test alloc failure", test_alloc_fail },
 };

t/test-full.proto

@@ -2,6 +2,18 @@ package foo;
 
 message SubMess {
   required int32 test = 4;
+
+  optional int32 val1 = 6;
+  optional int32 val2 = 7;
+  repeated int32 rep = 8;
+  message SubSubMess {
+    optional int32 val1 = 1 [default = 100];
+    repeated int32 rep = 4;
+    optional bytes bytes1 = 2 [default = "a \0 char"];
+    optional string str1 = 3 [default = "hello world\n"];
+  }
+  optional SubSubMess sub1 = 9;
+  optional SubSubMess sub2 = 10;
 };
  
 enum TestEnumSmall {