Add binary schema reflection (#7932)

* Add binary schema reflection

* remove not-used parameter

* move logic from object API to base API

* forward declare

* remove duplicate code gen that was stompping on the edits

* reduce to just typedef generation

* fixed bazel rules to not stomp

* more bazel fixes to support additional generated files

CMakeLists.txt

@@ -614,7 +614,6 @@ if(FLATBUFFERS_BUILD_TESTS)
   file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/samples" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
 
   # TODO Add (monster_test.fbs monsterdata_test.json)->monsterdata_test.mon
-  compile_flatbuffers_schema_to_cpp(tests/monster_test.fbs)
   compile_flatbuffers_schema_to_binary(tests/monster_test.fbs)
   compile_flatbuffers_schema_to_cpp_opt(tests/namespace_test/namespace_test1.fbs "--no-includes;--gen-compare;--gen-name-strings")
   compile_flatbuffers_schema_to_cpp_opt(tests/namespace_test/namespace_test2.fbs "--no-includes;--gen-compare;--gen-name-strings")

build_defs.bzl

@@ -165,6 +165,7 @@ def flatbuffer_cc_library(
         name,
         srcs,
         srcs_filegroup_name = "",
+        outs = [],
         out_prefix = "",
         deps = [],
         includes = [],
@@ -185,6 +186,7 @@ def flatbuffer_cc_library(
       srcs_filegroup_name: Name of the output filegroup that holds srcs. Pass this
           filegroup into the `includes` parameter of any other
           flatbuffer_cc_library that depends on this one's schemas.
+      outs: Additional outputs expected to be generated by flatc.
       out_prefix: Prepend this path to the front of all generated files. Usually
           is a directory name.
       deps: Optional, list of other flatbuffer_cc_library's to depend on. Cannot be specified
@@ -232,7 +234,7 @@ def flatbuffer_cc_library(
     flatbuffer_library_public(
         name = srcs_lib,
         srcs = srcs,
-        outs = output_headers,
+        outs = outs + output_headers,
         language_flag = "-c",
         out_prefix = out_prefix,
         includes = includes,

src/idl_gen_cpp.cpp

@@ -280,6 +280,16 @@ class CppGenerator : public BaseGenerator {
     if (!opts_.cpp_includes.empty()) { code_ += ""; }
   }
 
+  void GenEmbeddedIncludes() {
+    if (parser_.opts.binary_schema_gen_embed && parser_.root_struct_def_) {
+      const std::string file_path =
+          GeneratedFileName(opts_.include_prefix, file_name_ + "_bfbs", opts_);
+      code_ += "// For access to the binary schema that produced this file.";
+      code_ += "#include \"" + file_path + "\"";
+      code_ += "";
+    }
+  }
+
   std::string EscapeKeyword(const std::string &name) const {
     return keywords_.find(name) == keywords_.end() ? name : name + "_";
   }
@@ -408,6 +418,7 @@ class CppGenerator : public BaseGenerator {
 
     if (opts_.include_dependence_headers) { GenIncludeDependencies(); }
     GenExtraIncludes();
+    GenEmbeddedIncludes();
 
     FLATBUFFERS_ASSERT(!cur_name_space_);
 
@@ -2152,6 +2163,15 @@ class CppGenerator : public BaseGenerator {
     code_ += "";
   }
 
+  // Adds a typedef to the binary schema type so one could get the bfbs based
+  // on the type at runtime.
+  void GenBinarySchemaTypeDef(const StructDef *struct_def) {
+    if (struct_def && opts_.binary_schema_gen_embed) {
+      code_ += "  typedef " + WrapInNameSpace(*struct_def) +
+               "BinarySchema BinarySchema;";
+    }
+  }
+
   void GenNativeTablePost(const StructDef &struct_def) {
     if (opts_.gen_compare) {
       const auto native_name = NativeName(Name(struct_def), &struct_def, opts_);
@@ -2687,6 +2707,8 @@ class CppGenerator : public BaseGenerator {
       code_ += "  typedef {{NATIVE_NAME}} NativeTableType;";
     }
     code_ += "  typedef {{STRUCT_NAME}}Builder Builder;";
+    GenBinarySchemaTypeDef(parser_.root_struct_def_);
+
     if (opts_.g_cpp_std >= cpp::CPP_STD_17) { code_ += "  struct Traits;"; }
     if (opts_.mini_reflect != IDLOptions::kNone) {
       code_ +=

tests/BUILD.bazel

@@ -1,6 +1,6 @@
 load("@aspect_bazel_lib//lib:copy_to_bin.bzl", "copy_to_bin")
 load("@rules_cc//cc:defs.bzl", "cc_test")
-load("//:build_defs.bzl", "flatbuffer_cc_library")
+load("//:build_defs.bzl", "DEFAULT_FLATC_ARGS", "flatbuffer_cc_library")
 
 package(default_visibility = ["//visibility:private"])
 
@@ -160,6 +160,7 @@ cc_library(
     ],
     hdrs = [
         "monster_test.grpc.fb.h",
+        "monster_test_bfbs_generated.h",
         "monster_test_generated.h",
     ],
     includes = ["."],
@@ -182,6 +183,13 @@ flatbuffer_cc_library(
 flatbuffer_cc_library(
     name = "monster_test_cc_fbs",
     srcs = ["monster_test.fbs"],
+    outs = ["monster_test_bfbs_generated.h"],
+    flatc_args = DEFAULT_FLATC_ARGS + [
+        "--bfbs-comments",
+        "--bfbs-builtins",
+        "--bfbs-gen-embed",
+        "--bfbs-filenames tests",
+    ],
     include_paths = ["tests/include_test"],
     visibility = ["//grpc/tests:__subpackages__"],
     deps = [":include_test_fbs"],

tests/monster_test_generated.h

@@ -15,6 +15,9 @@ static_assert(FLATBUFFERS_VERSION_MAJOR == 23 &&
               FLATBUFFERS_VERSION_REVISION == 3,
              "Non-compatible flatbuffers version included");
 
+// For access to the binary schema that produced this file.
+#include "monster_test_bfbs_generated.h"
+
 namespace MyGame {
 
 struct InParentNamespace;
@@ -946,6 +949,7 @@ struct InParentNamespaceT : public ::flatbuffers::NativeTable {
 struct InParentNamespace FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table {
   typedef InParentNamespaceT NativeTableType;
   typedef InParentNamespaceBuilder Builder;
+  typedef MyGame::Example::MonsterBinarySchema BinarySchema;
   static const ::flatbuffers::TypeTable *MiniReflectTypeTable() {
     return InParentNamespaceTypeTable();
   }
@@ -990,6 +994,7 @@ struct MonsterT : public ::flatbuffers::NativeTable {
 struct Monster FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table {
   typedef MonsterT NativeTableType;
   typedef MonsterBuilder Builder;
+  typedef MyGame::Example::MonsterBinarySchema BinarySchema;
   static const ::flatbuffers::TypeTable *MiniReflectTypeTable() {
     return MonsterTypeTable();
   }
@@ -1037,6 +1042,7 @@ struct TestSimpleTableWithEnumT : public ::flatbuffers::NativeTable {
 struct TestSimpleTableWithEnum FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table {
   typedef TestSimpleTableWithEnumT NativeTableType;
   typedef TestSimpleTableWithEnumBuilder Builder;
+  typedef MyGame::Example::MonsterBinarySchema BinarySchema;
   static const ::flatbuffers::TypeTable *MiniReflectTypeTable() {
     return TestSimpleTableWithEnumTypeTable();
   }
@@ -1097,6 +1103,7 @@ struct StatT : public ::flatbuffers::NativeTable {
 struct Stat FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table {
   typedef StatT NativeTableType;
   typedef StatBuilder Builder;
+  typedef MyGame::Example::MonsterBinarySchema BinarySchema;
   static const ::flatbuffers::TypeTable *MiniReflectTypeTable() {
     return StatTypeTable();
   }
@@ -1201,6 +1208,7 @@ struct ReferrableT : public ::flatbuffers::NativeTable {
 struct Referrable FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table {
   typedef ReferrableT NativeTableType;
   typedef ReferrableBuilder Builder;
+  typedef MyGame::Example::MonsterBinarySchema BinarySchema;
   static const ::flatbuffers::TypeTable *MiniReflectTypeTable() {
     return ReferrableTypeTable();
   }
@@ -1327,6 +1335,7 @@ struct MonsterT : public ::flatbuffers::NativeTable {
 struct Monster FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table {
   typedef MonsterT NativeTableType;
   typedef MonsterBuilder Builder;
+  typedef MyGame::Example::MonsterBinarySchema BinarySchema;
   static const ::flatbuffers::TypeTable *MiniReflectTypeTable() {
     return MonsterTypeTable();
   }
@@ -2429,6 +2438,7 @@ struct TypeAliasesT : public ::flatbuffers::NativeTable {
 struct TypeAliases FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table {
   typedef TypeAliasesT NativeTableType;
   typedef TypeAliasesBuilder Builder;
+  typedef MyGame::Example::MonsterBinarySchema BinarySchema;
   static const ::flatbuffers::TypeTable *MiniReflectTypeTable() {
     return TypeAliasesTypeTable();
   }

tests/test.cpp

@@ -25,6 +25,7 @@
 #include "flatbuffers/flatbuffers.h"
 #include "flatbuffers/idl.h"
 #include "flatbuffers/minireflect.h"
+#include "flatbuffers/reflection_generated.h"
 #include "flatbuffers/registry.h"
 #include "flatbuffers/util.h"
 #include "fuzz_test.h"
@@ -912,7 +913,7 @@ void NativeTypeTest() {
 // Guard against -Wunused-function on platforms without file tests.
 #ifndef FLATBUFFERS_NO_FILE_TESTS
 // VS10 does not support typed enums, exclude from tests
-#if !defined(_MSC_VER) || _MSC_VER >= 1700
+#  if !defined(_MSC_VER) || _MSC_VER >= 1700
 void FixedLengthArrayJsonTest(const std::string &tests_data_path, bool binary) {
   // load FlatBuffer schema (.fbs) and JSON from disk
   std::string schemafile;
@@ -1031,7 +1032,7 @@ void FixedLengthArraySpanTest(const std::string &tests_data_path) {
         std::equal(const_d_c.begin(), const_d_c.end(), mutable_d_c.begin()));
   }
   // test little endian array of int32
-#  if FLATBUFFERS_LITTLEENDIAN
+#    if FLATBUFFERS_LITTLEENDIAN
   {
     flatbuffers::span<const int32_t, 2> const_d_a =
         flatbuffers::make_span(*const_nested.a());
@@ -1046,12 +1047,12 @@ void FixedLengthArraySpanTest(const std::string &tests_data_path) {
     TEST_ASSERT(
         std::equal(const_d_a.begin(), const_d_a.end(), mutable_d_a.begin()));
   }
-#  endif
+#    endif
 }
-#else
+#  else
 void FixedLengthArrayJsonTest(bool /*binary*/) {}
 void FixedLengthArraySpanTest() {}
-#endif
+#  endif
 
 void TestEmbeddedBinarySchema(const std::string &tests_data_path) {
   // load JSON from disk
@@ -1101,6 +1102,37 @@ void TestEmbeddedBinarySchema(const std::string &tests_data_path) {
 }
 #endif
 
+template<typename T> void EmbeddedSchemaAccessByType() {
+  // Get the binary schema from the Type itself.
+  // Verify the schema is OK.
+  flatbuffers::Verifier verifierEmbeddedSchema(
+      T::TableType::BinarySchema::data(), T::TableType::BinarySchema::size());
+  TEST_EQ(reflection::VerifySchemaBuffer(verifierEmbeddedSchema), true);
+
+  // Reflect it.
+  auto schema = reflection::GetSchema(T::TableType::BinarySchema::data());
+
+  // This should equal the expected root table.
+  TEST_EQ_STR(schema->root_table()->name()->c_str(), "MyGame.Example.Monster");
+}
+
+void EmbeddedSchemaAccess() {
+  // Get the binary schema for the monster.
+  // Verify the schema is OK.
+  flatbuffers::Verifier verifierEmbeddedSchema(Monster::BinarySchema::data(),
+                                               Monster::BinarySchema::size());
+  TEST_EQ(reflection::VerifySchemaBuffer(verifierEmbeddedSchema), true);
+
+  // Reflect it.
+  auto schema = reflection::GetSchema(Monster::BinarySchema::data());
+
+  // This should equal the expected root table.
+  TEST_EQ_STR(schema->root_table()->name()->c_str(), "MyGame.Example.Monster");
+
+  // Repeat above, but do so through a template parameter:
+  EmbeddedSchemaAccessByType<StatT>();
+}
+
 void NestedVerifierTest() {
   // Create a nested monster.
   flatbuffers::FlatBufferBuilder nested_builder;
@@ -1458,9 +1490,7 @@ void NativeInlineTableVectorTest() {
   TestNativeInlineTableT unpacked;
   root->UnPackTo(&unpacked);
 
-  for (int i = 0; i < 10; ++i) {
+  for (int i = 0; i < 10; ++i) { TEST_ASSERT(unpacked.t[i] == test.t[i]); }
-    TEST_ASSERT(unpacked.t[i] == test.t[i]);
-  }
 
   TEST_ASSERT(unpacked.t == test.t);
 }
@@ -1620,6 +1650,7 @@ int FlatBufferTests(const std::string &tests_data_path) {
   StructKeyInStructTest();
   NestedStructKeyInStructTest();
   FixedSizedStructArrayKeyInStructTest();
+  EmbeddedSchemaAccess();
   return 0;
 }
 }  // namespace