Added support for nested flatbuffers serializing to json (#4391)

* added bool for nested_flatbuffer
setting bool flag nested_flatbuffer according to attributes
setting nested type

* added JSON serialization for nested flatbuffers

* simplified lookup of nested_flatbuffer StructDef

* added nested_flatbuffer parsing workound relying on flexbuffers

* moved nested flatbuffer parsing into its own function for clarity
removed flexbuffers workound to simplify and speed up nested_flatbuffer parsing
added support for 'legacy' nested_flatbuffers, ubyte arrays

* inlined functions for nested parser init/clean since only used once

* whitespace formatting

* changed type of FieldDef.nested_flatbuffer from bool to StructDef*
removed subsequent type lookups

* removed copies of unrequired data when initializing nested parser

* applied changes requested by reviewer

* removed superfluous lookup of nested_flatbuffer field attributes

* renamed camelCased variables to snake_case

include/flatbuffers/idl.h

@@ -232,7 +232,7 @@ struct Definition {
 
 struct FieldDef : public Definition {
   FieldDef() : deprecated(false), required(false), key(false),
-               flexbuffer(false), padding(0) {}
+               flexbuffer(false), nested_flatbuffer(NULL), padding(0) {}
 
   Offset<reflection::Field> Serialize(FlatBufferBuilder *builder, uint16_t id,
                                       const Parser &parser) const;
@@ -245,6 +245,7 @@ struct FieldDef : public Definition {
   bool native_inline;  // Field will be defined inline (instead of as a pointer)
                        // for native tables if field is a struct.
   bool flexbuffer; // This field contains FlexBuffer data.
+  StructDef *nested_flatbuffer; // This field contains nested FlatBuffer data.
   size_t padding;  // Bytes to always pad after this field.
 };
 
@@ -602,6 +603,9 @@ private:
   FLATBUFFERS_CHECKED_ERROR ParseVectorDelimiters(
       size_t &count, ParseVectorDelimitersBody body, void *state);
   FLATBUFFERS_CHECKED_ERROR ParseVector(const Type &type, uoffset_t *ovalue);
+  FLATBUFFERS_CHECKED_ERROR ParseNestedFlatbuffer(Value &val, FieldDef *field,
+                                                  size_t parent_fieldn,
+                                                  const StructDef *parent_struct_def);
   FLATBUFFERS_CHECKED_ERROR ParseMetaData(SymbolTable<Value> *attributes);
   FLATBUFFERS_CHECKED_ERROR TryTypedValue(int dtoken, bool check, Value &e,
                                           BaseType req, bool *destmatch);

src/idl_gen_general.cpp

@@ -1065,28 +1065,24 @@ void GenStruct(StructDef &struct_def, std::string *code_ptr) {
       }
     }
   // generate object accessors if is nested_flatbuffer
-  auto nested = field.attributes.Lookup("nested_flatbuffer");
-  if (nested) {
-    auto nested_qualified_name =
-      parser_.namespaces_.back()->GetFullyQualifiedName(nested->constant);
-    auto nested_type = parser_.structs_.Lookup(nested_qualified_name);
-    auto nested_type_name = WrapInNameSpace(*nested_type);
-    auto nestedMethodName = MakeCamel(field.name, lang_.first_camel_upper)
+  if (field.nested_flatbuffer) {
+    auto nested_type_name = WrapInNameSpace(*field.nested_flatbuffer);
+    auto nested_method_name = MakeCamel(field.name, lang_.first_camel_upper)
       + "As" + nested_type_name;
-    auto getNestedMethodName = nestedMethodName;
+    auto get_nested_method_name = nested_method_name;
     if (lang_.language == IDLOptions::kCSharp) {
-      getNestedMethodName = "Get" + nestedMethodName;
+      get_nested_method_name = "Get" + nested_method_name;
       conditional_cast = "(" + nested_type_name + lang_.optional_suffix + ")";
     }
     if (lang_.language != IDLOptions::kCSharp) {
       code += "  public " + nested_type_name + lang_.optional_suffix + " ";
-      code += nestedMethodName + "() { return ";
-      code += getNestedMethodName + "(new " + nested_type_name + "()); }\n";
+      code += nested_method_name + "() { return ";
+      code += get_nested_method_name + "(new " + nested_type_name + "()); }\n";
     } else {
       obj = "(new " + nested_type_name + "())";
     }
     code += "  public " + nested_type_name + lang_.optional_suffix + " ";
-    code += getNestedMethodName + "(";
+    code += get_nested_method_name + "(";
     if (lang_.language != IDLOptions::kCSharp)
       code += nested_type_name + " obj";
     code += ") { int o = " + lang_.accessor_prefix + "__offset(";

src/idl_gen_text.cpp

@@ -165,6 +165,10 @@ template<typename T> static bool GenField(const FieldDef &fd,
                                             opts, _text);
 }
 
+static bool GenStruct(const StructDef &struct_def, const Table *table,
+						  int indent, const IDLOptions &opts,
+						  std::string *_text);
+
 // Generate text for non-scalar field.
 static bool GenFieldOffset(const FieldDef &fd, const Table *table, bool fixed,
                            int indent, Type *union_type,
@@ -180,6 +184,10 @@ static bool GenFieldOffset(const FieldDef &fd, const Table *table, bool fixed,
     auto root = flexbuffers::GetRoot(vec->data(), vec->size());
     root.ToString(true, opts.strict_json, *_text);
     return true;
+  } else if (fd.nested_flatbuffer) {
+    auto vec = table->GetPointer<const Vector<uint8_t> *>(fd.value.offset);
+    auto root = GetRoot<Table>(vec->data());
+    return GenStruct(*fd.nested_flatbuffer, root, indent, opts, _text);
   } else {
     val = IsStruct(fd.value.type)
       ? table->GetStruct<const void *>(fd.value.offset)

src/idl_parser.cpp

@@ -16,6 +16,7 @@
 
 #include <algorithm>
 #include <list>
+#include <iostream>
 
 #ifdef _WIN32
 #if !defined(_USE_MATH_DEFINES)
@@ -723,6 +724,10 @@ CheckedError Parser::ParseField(StructDef &struct_def) {
     // This will cause an error if the root type of the nested flatbuffer
     // wasn't defined elsewhere.
     LookupCreateStruct(nested->constant);
+
+    // Keep a pointer to StructDef in FieldDef to simplify re-use later
+    auto nested_qualified_name = namespaces_.back()->GetFullyQualifiedName(nested->constant);
+    field->nested_flatbuffer = structs_.Lookup(nested_qualified_name);
   }
 
   if (field->attributes.Lookup("flexbuffer")) {
@@ -941,6 +946,8 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
           builder.Finish();
           auto off = parser->builder_.CreateVector(builder.GetBuffer());
           val.constant = NumToString(off.o);
+        } else if (field->nested_flatbuffer) {
+          ECHECK(parser->ParseNestedFlatbuffer(val, field, fieldn, struct_def_inner));
         } else {
           ECHECK(parser->ParseAnyValue(val, field, fieldn, struct_def_inner));
         }
@@ -1120,6 +1127,38 @@ CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) {
   return NoError();
 }
 
+CheckedError Parser::ParseNestedFlatbuffer(Value &val, FieldDef *field,
+                                          size_t fieldn,
+                                          const StructDef *parent_struct_def) {
+  if (token_ == '[') {// backwards compat for 'legacy' ubyte buffers
+    ECHECK(ParseAnyValue(val, field, fieldn, parent_struct_def));
+  } else {
+    auto cursor_at_value_begin = cursor_;
+    ECHECK(SkipAnyJsonValue());
+    std::string substring(cursor_at_value_begin -1 , cursor_ -1);
+
+    // Create and initialize new parser
+    Parser nested_parser;
+    assert(field->nested_flatbuffer);
+    nested_parser.root_struct_def_ = field->nested_flatbuffer;
+    nested_parser.enums_ = enums_;
+    nested_parser.opts = opts;
+    nested_parser.uses_flexbuffers_ = uses_flexbuffers_;
+
+    // Parse JSON substring into new flatbuffer builder using nested_parser
+    if (!nested_parser.Parse(substring.c_str(), nullptr, nullptr)) {
+      ECHECK(Error(nested_parser.error_));
+    }
+    auto off = builder_.CreateVector(nested_parser.builder_.GetBufferPointer(), nested_parser.builder_.GetSize());
+    val.constant = NumToString(off.o);
+
+    // Clean nested_parser before destruction to avoid deleting the elements in the SymbolTables
+    nested_parser.enums_.dict.clear();
+    nested_parser.enums_.vec.clear();
+  }
+  return NoError();
+}
+
 CheckedError Parser::ParseMetaData(SymbolTable<Value> *attributes) {
   if (Is('(')) {
     NEXT();