Annotated Flatbuffer Binary (#7174)

* Annotated Flatbuffer Binary

* Various fixes

* Handles old schema

* handle multiple missing fields

* minor edits

* bazel fix, spelling fix, ascii fix

src/flatc.cpp

@@ -19,6 +19,8 @@
 #include <list>
 #include <sstream>
 
+#include "annotated_binary_text_gen.h"
+#include "binary_annotator.h"
 #include "flatbuffers/util.h"
 
 namespace flatbuffers {
@@ -215,6 +217,8 @@ const static FlatCOption options[] = {
     "in JSON, which is unsafe unless checked by a verifier afterwards." },
   { "", "ts-flat-files", "",
     "Only generated one typescript file per .fbs file." },
+  { "", "annotate", "SCHEMA",
+    "Annotate the provided BINARY_FILE with the specified SCHEMA file." },
 };
 
 static void AppendTextWrappedString(std::stringstream &ss, std::string &text,
@@ -297,7 +301,7 @@ std::string FlatCompiler::GetShortUsageString(const char *program_name) const {
     ss << ", ";
   }
   ss.seekp(-2, ss.cur);
-  ss << "]... FILE... [-- FILE...]";
+  ss << "]... FILE... [-- BINARY_FILE...]";
   std::string help = ss.str();
   std::stringstream ss_textwrap;
   AppendTextWrappedString(ss_textwrap, help, 80, 0);
@@ -306,7 +310,8 @@ std::string FlatCompiler::GetShortUsageString(const char *program_name) const {
 
 std::string FlatCompiler::GetUsageString(const char *program_name) const {
   std::stringstream ss;
-  ss << "Usage: " << program_name << " [OPTION]... FILE... [-- FILE...]\n";
+  ss << "Usage: " << program_name
+     << " [OPTION]... FILE... [-- BINARY_FILE...]\n";
   for (size_t i = 0; i < params_.num_generators; ++i) {
     const Generator &g = params_.generators[i];
     AppendOption(ss, g.option, 80, 25);
@@ -320,16 +325,48 @@ std::string FlatCompiler::GetUsageString(const char *program_name) const {
 
   std::string files_description =
       "FILEs may be schemas (must end in .fbs), binary schemas (must end in "
-      ".bfbs) or JSON files (conforming to preceding schema). FILEs after the "
-      "-- must be binary flatbuffer format files. Output files are named using "
-      "the base file name of the input, and written to the current directory "
-      "or the path given by -o. example: " +
+      ".bfbs) or JSON files (conforming to preceding schema). BINARY_FILEs "
+      "after the -- must be binary flatbuffer format files. Output files are "
+      "named using the base file name of the input, and written to the current "
+      "directory or the path given by -o. example: " +
       std::string(program_name) + " -c -b schema1.fbs schema2.fbs data.json";
   AppendTextWrappedString(ss, files_description, 80, 0);
   ss << "\n";
   return ss.str();
 }
 
+void FlatCompiler::AnnotateBinaries(
+    const uint8_t *binary_schema, const uint64_t binary_schema_size,
+    const std::string &schema_filename,
+    const std::vector<std::string> &binary_files) {
+  for (const std::string &filename : binary_files) {
+    std::string binary_contents;
+    if (!flatbuffers::LoadFile(filename.c_str(), true, &binary_contents)) {
+      Warn("unable to load binary file: " + filename);
+      continue;
+    }
+
+    const uint8_t *binary =
+        reinterpret_cast<const uint8_t *>(binary_contents.c_str());
+    const size_t binary_size = binary_contents.size();
+
+    flatbuffers::BinaryAnnotator binary_annotator(binary_schema,
+                                                  binary_schema_size, binary);
+
+    auto annotations = binary_annotator.Annotate();
+
+    // TODO(dbaileychess): Right now we just support a single text-based
+    // output of the annotated binary schema, which we generate here. We
+    // could output the raw annotations instead and have third-party tools
+    // use them to generate their own output.
+    flatbuffers::AnnotatedBinaryTextGenerator text_generator(
+        flatbuffers::AnnotatedBinaryTextGenerator::Options{}, annotations,
+        binary, binary_size);
+
+    text_generator.Generate(filename, schema_filename);
+  }
+}
+
 int FlatCompiler::Compile(int argc, const char **argv) {
   if (params_.generators == nullptr || params_.num_generators == 0) {
     return 0;
@@ -353,6 +390,7 @@ int FlatCompiler::Compile(int argc, const char **argv) {
   std::vector<bool> generator_enabled(params_.num_generators, false);
   size_t binary_files_from = std::numeric_limits<size_t>::max();
   std::string conform_to_schema;
+  std::string annotate_schema;
 
   const char *program_name = argv[0];
 
@@ -554,6 +592,9 @@ int FlatCompiler::Compile(int argc, const char **argv) {
         opts.json_nested_legacy_flatbuffers = true;
       } else if (arg == "--ts-flat-files") {
         opts.ts_flat_file = true;
+      } else if (arg == "--annotate") {
+        if (++argi >= argc) Error("missing path following: " + arg, true);
+        annotate_schema = flatbuffers::PosixPath(argv[argi]);
       } else {
         for (size_t i = 0; i < params_.num_generators; ++i) {
           if (arg == "--" + params_.generators[i].option.long_opt ||
@@ -582,7 +623,8 @@ int FlatCompiler::Compile(int argc, const char **argv) {
   if (opts.proto_mode) {
     if (any_generator)
       Error("cannot generate code directly from .proto files", true);
-  } else if (!any_generator && conform_to_schema.empty()) {
+  } else if (!any_generator && conform_to_schema.empty() &&
+             annotate_schema.empty()) {
     Error("no options: specify at least one generator.", true);
   }
 
@@ -611,6 +653,53 @@ int FlatCompiler::Compile(int argc, const char **argv) {
     }
   }
 
+  if (!annotate_schema.empty()) {
+    const std::string ext = flatbuffers::GetExtension(annotate_schema);
+    if (!(ext == reflection::SchemaExtension() || ext == "fbs")) {
+      Error("Expected a `.bfbs` or `.fbs` schema, got: " + annotate_schema);
+    }
+
+    const bool is_binary_schema = ext == reflection::SchemaExtension();
+
+    std::string schema_contents;
+    if (!flatbuffers::LoadFile(annotate_schema.c_str(),
+                               /*binary=*/is_binary_schema, &schema_contents)) {
+      Error("unable to load schema: " + annotate_schema);
+    }
+
+    const uint8_t *binary_schema = nullptr;
+    uint64_t binary_schema_size = 0;
+
+    IDLOptions binary_opts;
+    binary_opts.lang_to_generate |= flatbuffers::IDLOptions::kBinary;
+    flatbuffers::Parser parser(binary_opts);
+
+    if (is_binary_schema) {
+      binary_schema =
+          reinterpret_cast<const uint8_t *>(schema_contents.c_str());
+      binary_schema_size = schema_contents.size();
+    } else {
+      // If we need to generate the .bfbs file from the provided schema file
+      // (.fbs)
+      ParseFile(parser, annotate_schema, schema_contents, include_directories);
+      parser.Serialize();
+
+      binary_schema = parser.builder_.GetBufferPointer();
+      binary_schema_size = parser.builder_.GetSize();
+    }
+
+    if (binary_schema == nullptr || !binary_schema_size) {
+      Error("could not parse a value binary schema from: " + annotate_schema);
+    }
+
+    // Annotate the provided files with the binary_schema.
+    AnnotateBinaries(binary_schema, binary_schema_size, annotate_schema,
+                     filenames);
+
+    // We don't support doing anything else after annotating a binary.
+    return 0;
+  }
+
   std::unique_ptr<flatbuffers::Parser> parser(new flatbuffers::Parser(opts));
 
   for (auto file_it = filenames.begin(); file_it != filenames.end();