Parsing from proto should keep field ID. (fixes #7645) (#7655)

* Parsing from proto should keep field ID. (fixes #7645)

* Fix failed tests

* Fix windows warning

* Improve attribute generation in proto to fbs

* Check if id is used twice. fix Some clang-format problems

* Test if fake id can solve the test problem

* Validate proto file in proto -> fbs generation.

* Fix error messages

* Ignore id in union

* Add keep proto id for legacy and check gap flag have been added. Reserved id will be checked.

* Add needed flags

* unit tests

* fix fromat problem. fix comments and error messages.

* clear

* More unit tests

* Fix windows build

* Fix include problems

* Fake commit to invoke rebuild

* Fix buzel build

* Fix some issues

* Fix comments, fix return value and sort for android NDK

* Fix return type

* Break down big function

* Place todo

---------

Co-authored-by: Derek Bailey <derekbailey@google.com>

tests/BUILD.bazel

@@ -82,13 +82,22 @@ cc_test(
         ":optional_scalars.json",
         ":optional_scalars_defaults.json",
         ":prototest/imported.proto",
-        ":prototest/test.golden",
+        ":prototest/non-positive-id.proto",
+        ":prototest/test.golden.fbs",
         ":prototest/test.proto",
-        ":prototest/test_include.golden",
-        ":prototest/test_suffix.golden",
-        ":prototest/test_union.golden",
-        ":prototest/test_union_include.golden",
-        ":prototest/test_union_suffix.golden",
+        ":prototest/test_id.golden.fbs",
+        ":prototest/test_include.golden.fbs",
+        ":prototest/test_include_id.golden.fbs",
+        ":prototest/test_suffix.golden.fbs",
+        ":prototest/test_suffix_id.golden.fbs",
+        ":prototest/test_union.golden.fbs",
+        ":prototest/test_union_id.golden.fbs",
+        ":prototest/test_union_include.golden.fbs",
+        ":prototest/test_union_include_id.golden.fbs",
+        ":prototest/test_union_suffix.golden.fbs",
+        ":prototest/test_union_suffix_id.golden.fbs",
+        ":prototest/twice-id.proto",
+        ":prototest/use-reserved-id.proto",
         ":unicode_test.json",
         ":union_vector/union_vector.fbs",
         ":union_vector/union_vector.json",

tests/proto_test.cpp

@@ -1,183 +1,292 @@
 #include "proto_test.h"
 
-#include "flatbuffers/idl.h"
 #include "test_assert.h"
 
 namespace flatbuffers {
 namespace tests {
 
-// Parse a .proto schema, output as .fbs
-void ParseProtoTest(const std::string &tests_data_path) {
-  // load the .proto and the golden file from disk
-  std::string protofile;
-  std::string goldenfile;
-  std::string goldenunionfile;
-  TEST_EQ(
-      flatbuffers::LoadFile((tests_data_path + "prototest/test.proto").c_str(),
-                            false, &protofile),
-      true);
-  TEST_EQ(
-      flatbuffers::LoadFile((tests_data_path + "prototest/test.golden").c_str(),
-                            false, &goldenfile),
-      true);
-  TEST_EQ(flatbuffers::LoadFile(
-              (tests_data_path + "prototest/test_union.golden").c_str(), false,
-              &goldenunionfile),
-          true);
-
-  flatbuffers::IDLOptions opts;
-  opts.include_dependence_headers = false;
-  opts.proto_mode = true;
+void RunTest(const flatbuffers::IDLOptions &opts, const std::string &proto_path,
+             const std::string &proto_file, const std::string &golden_file,
+             const std::string import_proto_file) {
+  const char *include_directories[] = { proto_path.c_str(), nullptr };
 
   // Parse proto.
   flatbuffers::Parser parser(opts);
-  auto protopath = tests_data_path + "prototest/";
-  const char *include_directories[] = { protopath.c_str(), nullptr };
-  TEST_EQ(parser.Parse(protofile.c_str(), include_directories), true);
+  TEST_EQ(parser.Parse(proto_file.c_str(), include_directories), true);
 
   // Generate fbs.
   auto fbs = flatbuffers::GenerateFBS(parser, "test");
 
   // Ensure generated file is parsable.
   flatbuffers::Parser parser2;
+
+  if (!import_proto_file.empty()) {
+    // Generate fbs from import.proto
+    flatbuffers::Parser import_parser(opts);
+    TEST_EQ(import_parser.Parse(import_proto_file.c_str(), include_directories),
+            true);
+    auto import_fbs = flatbuffers::GenerateFBS(import_parser, "test");
+    // Since `imported.fbs` isn't in the filesystem AbsolutePath can't figure it
+    // out by itself. We manually construct it so Parser works.
+    std::string imported_fbs = flatbuffers::PosixPath(
+        flatbuffers::AbsolutePath(proto_path) + "/imported.fbs");
+    TEST_EQ(parser2.Parse(import_fbs.c_str(), include_directories,
+                          imported_fbs.c_str()),
+            true);
+  }
+
   TEST_EQ(parser2.Parse(fbs.c_str(), nullptr), true);
-  TEST_EQ_STR(fbs.c_str(), goldenfile.c_str());
-
-  // Parse proto with --oneof-union option.
-  opts.proto_oneof_union = true;
-  flatbuffers::Parser parser3(opts);
-  TEST_EQ(parser3.Parse(protofile.c_str(), include_directories), true);
-
-  // Generate fbs.
-  auto fbs_union = flatbuffers::GenerateFBS(parser3, "test");
-
-  // Ensure generated file is parsable.
-  flatbuffers::Parser parser4;
-  TEST_EQ(parser4.Parse(fbs_union.c_str(), nullptr), true);
-  TEST_EQ_STR(fbs_union.c_str(), goldenunionfile.c_str());
+  TEST_EQ_STR(fbs.c_str(), golden_file.c_str());
 }
 
-// Parse a .proto schema, output as .fbs
-void ParseProtoTestWithSuffix(const std::string &tests_data_path) {
+void proto_test(const std::string &proto_path, const std::string &proto_file) {
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = false;
+  opts.proto_mode = true;
+
   // load the .proto and the golden file from disk
-  std::string protofile;
-  std::string goldenfile;
-  std::string goldenunionfile;
-  TEST_EQ(
-      flatbuffers::LoadFile((tests_data_path + "prototest/test.proto").c_str(),
-                            false, &protofile),
-      true);
-  TEST_EQ(flatbuffers::LoadFile(
-              (tests_data_path + "prototest/test_suffix.golden").c_str(), false,
-              &goldenfile),
-          true);
-  TEST_EQ(flatbuffers::LoadFile(
-              (tests_data_path + "prototest/test_union_suffix.golden").c_str(),
-              false, &goldenunionfile),
+  std::string golden_file;
+  TEST_EQ(flatbuffers::LoadFile((proto_path + "test.golden.fbs").c_str(), false,
+                                &golden_file),
           true);
 
+  RunTest(opts, proto_path, proto_file, golden_file);
+}
+
+void proto_test_id(const std::string &proto_path,
+                   const std::string &proto_file) {
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = false;
+  opts.proto_mode = true;
+  opts.keep_proto_id = true;
+
+  // load the .proto and the golden file from disk
+  std::string golden_file;
+  TEST_EQ(flatbuffers::LoadFile((proto_path + "test_id.golden.fbs").c_str(),
+                                false, &golden_file),
+          true);
+
+  RunTest(opts, proto_path, proto_file, golden_file);
+}
+
+void proto_test_union(const std::string &proto_path,
+                      const std::string &proto_file) {
+  // Parse proto with --oneof-union option.
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = false;
+  opts.proto_mode = true;
+  opts.proto_oneof_union = true;
+
+  std::string golden_file;
+  TEST_EQ(flatbuffers::LoadFile((proto_path + "test_union.golden.fbs").c_str(),
+                                false, &golden_file),
+          true);
+  RunTest(opts, proto_path, proto_file, golden_file);
+}
+
+void proto_test_union_id(const std::string &proto_path,
+                         const std::string &proto_file) {
+  // Parse proto with --oneof-union option.
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = false;
+  opts.proto_mode = true;
+  opts.proto_oneof_union = true;
+  opts.keep_proto_id = true;
+
+  std::string golden_file;
+  TEST_EQ(
+      flatbuffers::LoadFile((proto_path + "test_union_id.golden.fbs").c_str(),
+                            false, &golden_file),
+      true);
+  RunTest(opts, proto_path, proto_file, golden_file);
+}
+
+void proto_test_union_suffix(const std::string &proto_path,
+                             const std::string &proto_file) {
   flatbuffers::IDLOptions opts;
   opts.include_dependence_headers = false;
   opts.proto_mode = true;
   opts.proto_namespace_suffix = "test_namespace_suffix";
-
-  // Parse proto.
-  flatbuffers::Parser parser(opts);
-  auto protopath = tests_data_path + "prototest/";
-  const char *include_directories[] = { protopath.c_str(), nullptr };
-  TEST_EQ(parser.Parse(protofile.c_str(), include_directories), true);
-
-  // Generate fbs.
-  auto fbs = flatbuffers::GenerateFBS(parser, "test");
-
-  // Ensure generated file is parsable.
-  flatbuffers::Parser parser2;
-  TEST_EQ(parser2.Parse(fbs.c_str(), nullptr), true);
-  TEST_EQ_STR(fbs.c_str(), goldenfile.c_str());
-
-  // Parse proto with --oneof-union option.
   opts.proto_oneof_union = true;
-  flatbuffers::Parser parser3(opts);
-  TEST_EQ(parser3.Parse(protofile.c_str(), include_directories), true);
 
-  // Generate fbs.
-  auto fbs_union = flatbuffers::GenerateFBS(parser3, "test");
-
-  // Ensure generated file is parsable.
-  flatbuffers::Parser parser4;
-  TEST_EQ(parser4.Parse(fbs_union.c_str(), nullptr), true);
-  TEST_EQ_STR(fbs_union.c_str(), goldenunionfile.c_str());
+  std::string golden_file;
+  TEST_EQ(flatbuffers::LoadFile(
+              (proto_path + "test_union_suffix.golden.fbs").c_str(), false,
+              &golden_file),
+          true);
+  RunTest(opts, proto_path, proto_file, golden_file);
 }
 
-// Parse a .proto schema, output as .fbs
-void ParseProtoTestWithIncludes(const std::string &tests_data_path) {
-  // load the .proto and the golden file from disk
-  std::string protofile;
-  std::string goldenfile;
-  std::string goldenunionfile;
-  std::string importprotofile;
-  TEST_EQ(
-      flatbuffers::LoadFile((tests_data_path + "prototest/test.proto").c_str(),
-                            false, &protofile),
-      true);
-  TEST_EQ(flatbuffers::LoadFile(
-              (tests_data_path + "prototest/imported.proto").c_str(), false,
-              &importprotofile),
-          true);
-  TEST_EQ(flatbuffers::LoadFile(
-              (tests_data_path + "prototest/test_include.golden").c_str(),
-              false, &goldenfile),
-          true);
-  TEST_EQ(flatbuffers::LoadFile(
-              (tests_data_path + "prototest/test_union_include.golden").c_str(),
-              false, &goldenunionfile),
-          true);
+void proto_test_union_suffix_id(const std::string &proto_path,
+                                const std::string &proto_file) {
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = false;
+  opts.proto_mode = true;
+  opts.proto_namespace_suffix = "test_namespace_suffix";
+  opts.proto_oneof_union = true;
+  opts.keep_proto_id = true;
 
+  std::string golden_file;
+  TEST_EQ(flatbuffers::LoadFile(
+              (proto_path + "test_union_suffix_id.golden.fbs").c_str(), false,
+              &golden_file),
+          true);
+  RunTest(opts, proto_path, proto_file, golden_file);
+}
+
+void proto_test_include(const std::string &proto_path,
+                        const std::string &proto_file,
+                        const std::string &import_proto_file) {
   flatbuffers::IDLOptions opts;
   opts.include_dependence_headers = true;
   opts.proto_mode = true;
 
-  // Parse proto.
-  flatbuffers::Parser parser(opts);
-  auto protopath = tests_data_path + "prototest/";
-  const char *include_directories[] = { protopath.c_str(), nullptr };
-  TEST_EQ(parser.Parse(protofile.c_str(), include_directories), true);
+  std::string golden_file;
+  TEST_EQ(
+      flatbuffers::LoadFile((proto_path + "test_include.golden.fbs").c_str(),
+                            false, &golden_file),
+      true);
 
-  // Generate fbs.
-  auto fbs = flatbuffers::GenerateFBS(parser, "test");
+  RunTest(opts, proto_path, proto_file, golden_file, import_proto_file);
+}
 
-  // Generate fbs from import.proto
-  flatbuffers::Parser import_parser(opts);
-  TEST_EQ(import_parser.Parse(importprotofile.c_str(), include_directories),
-          true);
-  auto import_fbs = flatbuffers::GenerateFBS(import_parser, "test");
+void proto_test_include_id(const std::string &proto_path,
+                           const std::string &proto_file,
+                           const std::string &import_proto_file) {
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = true;
+  opts.proto_mode = true;
+  opts.keep_proto_id = true;
 
-  // Ensure generated file is parsable.
-  flatbuffers::Parser parser2;
-  // Since `imported.fbs` isn't in the filesystem AbsolutePath can't figure it
-  // out by itself. We manually construct it so Parser works.
-  std::string imported_fbs = flatbuffers::PosixPath(
-      flatbuffers::AbsolutePath(protopath) + "/imported.fbs");
-  TEST_EQ(parser2.Parse(import_fbs.c_str(), include_directories,
-                        imported_fbs.c_str()),
-          true);
-  TEST_EQ(parser2.Parse(fbs.c_str(), nullptr), true);
-  TEST_EQ_STR(fbs.c_str(), goldenfile.c_str());
+  std::string golden_file;
+  TEST_EQ(
+      flatbuffers::LoadFile((proto_path + "test_include_id.golden.fbs").c_str(),
+                            false, &golden_file),
+      true);
 
-  // Parse proto with --oneof-union option.
+  RunTest(opts, proto_path, proto_file, golden_file, import_proto_file);
+}
+
+void proto_test_include_union(const std::string &proto_path,
+                              const std::string &proto_file,
+                              const std::string &import_proto_file) {
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = true;
+  opts.proto_mode = true;
   opts.proto_oneof_union = true;
-  flatbuffers::Parser parser3(opts);
-  TEST_EQ(parser3.Parse(protofile.c_str(), include_directories), true);
 
-  // Generate fbs.
-  auto fbs_union = flatbuffers::GenerateFBS(parser3, "test");
-
-  // Ensure generated file is parsable.
-  flatbuffers::Parser parser4;
-  TEST_EQ(parser4.Parse(import_fbs.c_str(), nullptr, imported_fbs.c_str()),
+  std::string golden_file;
+  TEST_EQ(flatbuffers::LoadFile(
+              (proto_path + "test_union_include.golden.fbs").c_str(), false,
+              &golden_file),
           true);
-  TEST_EQ(parser4.Parse(fbs_union.c_str(), nullptr), true);
-  TEST_EQ_STR(fbs_union.c_str(), goldenunionfile.c_str());
+
+  RunTest(opts, proto_path, proto_file, golden_file, import_proto_file);
+}
+
+void proto_test_include_union_id(const std::string &proto_path,
+                                 const std::string &proto_file,
+                                 const std::string &import_proto_file) {
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = true;
+  opts.proto_mode = true;
+  opts.proto_oneof_union = true;
+  opts.keep_proto_id = true;
+
+  std::string golden_file;
+  TEST_EQ(flatbuffers::LoadFile(
+              (proto_path + "test_union_include_id.golden.fbs").c_str(), false,
+              &golden_file),
+          true);
+
+  RunTest(opts, proto_path, proto_file, golden_file, import_proto_file);
+}
+
+void ParseCorruptedProto(const std::string &proto_path) {
+  const char *include_directories[] = { proto_path.c_str(), nullptr };
+
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = true;
+  opts.proto_mode = true;
+  opts.proto_oneof_union = true;
+
+  std::string proto_file;
+
+  // Parse proto with non positive id.
+  {
+    flatbuffers::Parser parser(opts);
+    TEST_EQ(
+        flatbuffers::LoadFile((proto_path + "non-positive-id.proto").c_str(),
+                              false, &proto_file),
+        true);
+    TEST_EQ(parser.Parse(proto_file.c_str(), include_directories), true);
+    auto fbs = flatbuffers::GenerateFBS(parser, "test");
+    TEST_EQ(fbs.empty(), true);
+  }
+
+  // Parse proto with twice id.
+  {
+    flatbuffers::Parser parser(opts);
+    TEST_EQ(flatbuffers::LoadFile((proto_path + "twice-id.proto").c_str(),
+                                  false, &proto_file),
+            true);
+    TEST_EQ(parser.Parse(proto_file.c_str(), include_directories), true);
+    auto fbs = flatbuffers::GenerateFBS(parser, "test");
+    TEST_EQ(fbs.empty(), true);
+  }
+
+  // Parse proto with using reserved id.
+  {
+    flatbuffers::Parser parser(opts);
+    TEST_EQ(flatbuffers::LoadFile((proto_path + "twice-id.proto").c_str(),
+                                  false, &proto_file),
+            true);
+    TEST_EQ(parser.Parse(proto_file.c_str(), include_directories), true);
+    auto fbs = flatbuffers::GenerateFBS(parser, "test");
+    TEST_EQ(fbs.empty(), true);
+  }
+
+  // Parse proto with error on gap.
+  {
+    opts.proto_id_gap_action = IDLOptions::ProtoIdGapAction::ERROR;
+    flatbuffers::Parser parser(opts);
+    TEST_EQ(flatbuffers::LoadFile((proto_path + "test.proto").c_str(), false,
+                                  &proto_file),
+            true);
+    TEST_EQ(parser.Parse(proto_file.c_str(), include_directories), true);
+    auto fbs = flatbuffers::GenerateFBS(parser, "test");
+    TEST_EQ(fbs.empty(), true);
+  }
+}
+
+// Parse a .proto schema, output as .fbs
+void ParseProtoTest(const std::string &tests_data_path) {
+  auto proto_path = tests_data_path + "prototest/";
+  std::string proto_file;
+  TEST_EQ(
+      flatbuffers::LoadFile((tests_data_path + "prototest/test.proto").c_str(),
+                            false, &proto_file),
+      true);
+
+  std::string import_proto_file;
+  TEST_EQ(flatbuffers::LoadFile(
+              (tests_data_path + "prototest/imported.proto").c_str(), false,
+              &import_proto_file),
+          true);
+
+  proto_test(proto_path, proto_file);
+  proto_test_union(proto_path, proto_file);
+  proto_test_union_suffix(proto_path, proto_file);
+  proto_test_include(proto_path, proto_file, import_proto_file);
+  proto_test_include_union(proto_path, proto_file, import_proto_file);
+
+  proto_test_id(proto_path, proto_file);
+  proto_test_union_id(proto_path, proto_file);
+  proto_test_union_suffix_id(proto_path, proto_file);
+  proto_test_include_id(proto_path, proto_file, import_proto_file);
+  proto_test_include_union_id(proto_path, proto_file, import_proto_file);
+
+  ParseCorruptedProto(proto_path);
 }
 
 void ParseProtoBufAsciiTest() {

tests/proto_test.h

@@ -1,16 +1,30 @@
 #ifndef TESTS_PROTO_TEST_H
 #define TESTS_PROTO_TEST_H
 
+#include "flatbuffers/idl.h"
+
 #include <string>
 
 namespace flatbuffers {
 namespace tests {
 
-void ParseProtoTest(const std::string& tests_data_path);
-void ParseProtoTestWithSuffix(const std::string& tests_data_path);
-void ParseProtoTestWithIncludes(const std::string& tests_data_path);
-void ParseProtoBufAsciiTest();
+void RunTest(const flatbuffers::IDLOptions &opts, const std::string &proto_path, const std::string &proto_file,
+        const std::string &golden_file, const std::string import_proto_file = {});
+void proto_test(const std::string &proto_path, const std::string &proto_file);
+void proto_test_union(const std::string &proto_path, const std::string &proto_file);
+void proto_test_union_suffix(const std::string &proto_path, const std::string &proto_file);
+void proto_test_include(const std::string &proto_path, const std::string &proto_file, const std::string &import_proto_file);
+void proto_test_include_union(const std::string &proto_path, const std::string &proto_file, const std::string &import_proto_file);
 
+void proto_test_id(const std::string &proto_path, const std::string &proto_file);
+void proto_test_union_id(const std::string &proto_path, const std::string &proto_file);
+void proto_test_union_suffix_id(const std::string &proto_path, const std::string &proto_file);
+void proto_test_include_id(const std::string &proto_path, const std::string &proto_file, const std::string &import_proto_file);
+void proto_test_include_union_id(const std::string &proto_path, const std::string &proto_file, const std::string &import_proto_file);
+
+void ParseCorruptedProto(const std::string &proto_path);
+void ParseProtoTest(const std::string& tests_data_path);
+void ParseProtoBufAsciiTest();
 
 }  // namespace tests
 }  // namespace flatbuffers

tests/prototest/GenerateProtoGoldens.sh

@@ -16,9 +16,16 @@
 
 pushd "$(dirname $0)" >/dev/null
 
-./../../flatc --proto test.proto && mv test.fbs test_include.golden
-./../../flatc --proto --gen-all test.proto && mv test.fbs test.golden
-./../../flatc --proto --oneof-union test.proto && mv test.fbs test_union_include.golden
-./../../flatc --proto --gen-all --oneof-union test.proto && mv test.fbs test_union.golden
-./../../flatc --proto --gen-all --proto-namespace-suffix test_namespace_suffix test.proto && mv test.fbs test_suffix.golden
-./../../flatc --proto --gen-all --proto-namespace-suffix test_namespace_suffix --oneof-union test.proto && mv test.fbs test_union_suffix.golden
+./../../flatc --proto test.proto && mv test.fbs test_include.golden.fbs
+./../../flatc --proto --gen-all test.proto && mv test.fbs test.golden.fbs
+./../../flatc --proto --oneof-union test.proto && mv test.fbs test_union_include.golden.fbs
+./../../flatc --proto --gen-all --oneof-union test.proto && mv test.fbs test_union.golden.fbs
+./../../flatc --proto --gen-all --proto-namespace-suffix test_namespace_suffix test.proto && mv test.fbs test_suffix.golden.fbs
+./../../flatc --proto --gen-all --proto-namespace-suffix test_namespace_suffix --oneof-union test.proto && mv test.fbs test_union_suffix.golden.fbs
+
+./../../flatc --proto --keep-proto-id test.proto && mv test.fbs test_include_id.golden.fbs
+./../../flatc --proto --keep-proto-id --gen-all test.proto && mv test.fbs test_id.golden.fbs
+./../../flatc --proto --keep-proto-id --oneof-union test.proto && mv test.fbs test_union_include_id.golden.fbs
+./../../flatc --proto --keep-proto-id --gen-all --oneof-union test.proto && mv test.fbs test_union_id.golden.fbs
+./../../flatc --proto --keep-proto-id --gen-all --proto-namespace-suffix test_namespace_suffix test.proto && mv test.fbs test_suffix_id.golden.fbs
+./../../flatc --proto --keep-proto-id --gen-all --proto-namespace-suffix test_namespace_suffix --oneof-union test.proto && mv test.fbs test_union_suffix_id.golden.fbs

tests/prototest/non-positive-id.proto

@@ -0,0 +1,9 @@
+// Sample .proto file that we can not translate to the corresponding .fbs because it has non-positive ids.
+
+option some_option = is_ignored;
+
+package proto.test;
+
+message ProtoMessage {
+  optional uint64 NonPositiveId = -1;
+}

tests/prototest/test.proto

@@ -24,6 +24,7 @@ message ProtoMessage {
   // Ignored non-doc comment.
   // A nested message declaration, will be moved to top level in .fbs
   message OtherMessage {
+    reserved 2, 9 to 11, 15;
     optional double a = 26;
     /// doc comment for b.
     optional float b = 32 [default = 3.14149];
@@ -40,7 +41,7 @@ message ProtoMessage {
   }
   optional int32 c = 12 [default = 16];
   optional int64 d = 1 [default = 0];
-  optional uint32 p = 1;
+  optional uint32 p = 40;
   optional uint64 e = 2;
   /// doc comment for f.
   optional sint32 f = 3 [default = -1];
@@ -55,7 +56,7 @@ message ProtoMessage {
   /// lines
   required string l = 10;
   optional bytes m = 11;
-  optional OtherMessage n = 12;
+  optional OtherMessage n = 41;
   repeated string o = 14;
   optional ImportedMessage z = 16;
   /// doc comment for r.

tests/prototest/test_id.golden.fbs

@@ -0,0 +1,87 @@
+// Generated from test.proto
+
+namespace proto.test;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test;
+
+table ImportedMessage {
+  a:int (id: 0);
+}
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 12);
+  d:long (id: 1);
+  p:uint (id: 21);
+  e:ulong (id: 2);
+  /// doc comment for f.
+  f:int = -1 (id: 3);
+  g:long (id: 4);
+  h:uint (id: 5);
+  q:ulong (id: 6);
+  i:int (id: 7);
+  j:long (id: 8);
+  /// doc comment for k.
+  k:bool (id: 9);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 10);
+  m:[ubyte] (id: 11);
+  n:proto.test.ProtoMessage_.OtherMessage (id: 22);
+  o:[string] (id: 13);
+  z:proto.test.ImportedMessage (id: 14);
+  /// doc comment for r.
+  r:proto.test.ProtoMessage_.Anonymous0 (id: 0);
+  outer_enum:proto.test.ProtoEnum (id: 15);
+  u:float = +inf (id: 16);
+  v:float = +inf (id: 17);
+  w:float = -inf (id: 18);
+  grades:[proto.test.ProtoMessage_.GradesEntry] (id: 19);
+  other_message_map:[proto.test.ProtoMessage_.OtherMessageMapEntry] (id: 20);
+}
+
+namespace proto.test.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table Anonymous0 {
+  /// doc comment for s.
+  s:proto.test.ImportedMessage (id: 0);
+  /// doc comment for t on 2
+  /// lines.
+  t:proto.test.ProtoMessage_.OtherMessage (id: 1);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/test_include_id.golden.fbs

@@ -0,0 +1,85 @@
+// Generated from test.proto
+
+include "imported.fbs";
+
+namespace proto.test;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test;
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 12);
+  d:long (id: 1);
+  p:uint (id: 21);
+  e:ulong (id: 2);
+  /// doc comment for f.
+  f:int = -1 (id: 3);
+  g:long (id: 4);
+  h:uint (id: 5);
+  q:ulong (id: 6);
+  i:int (id: 7);
+  j:long (id: 8);
+  /// doc comment for k.
+  k:bool (id: 9);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 10);
+  m:[ubyte] (id: 11);
+  n:proto.test.ProtoMessage_.OtherMessage (id: 22);
+  o:[string] (id: 13);
+  z:proto.test.ImportedMessage (id: 14);
+  /// doc comment for r.
+  r:proto.test.ProtoMessage_.Anonymous0 (id: 0);
+  outer_enum:proto.test.ProtoEnum (id: 15);
+  u:float = +inf (id: 16);
+  v:float = +inf (id: 17);
+  w:float = -inf (id: 18);
+  grades:[proto.test.ProtoMessage_.GradesEntry] (id: 19);
+  other_message_map:[proto.test.ProtoMessage_.OtherMessageMapEntry] (id: 20);
+}
+
+namespace proto.test.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table Anonymous0 {
+  /// doc comment for s.
+  s:proto.test.ImportedMessage (id: 0);
+  /// doc comment for t on 2
+  /// lines.
+  t:proto.test.ProtoMessage_.OtherMessage (id: 1);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/test_suffix_id.golden.fbs

@@ -0,0 +1,87 @@
+// Generated from test.proto
+
+namespace proto.test.test_namespace_suffix;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test.test_namespace_suffix;
+
+table ImportedMessage {
+  a:int (id: 0);
+}
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 12);
+  d:long (id: 1);
+  p:uint (id: 21);
+  e:ulong (id: 2);
+  /// doc comment for f.
+  f:int = -1 (id: 3);
+  g:long (id: 4);
+  h:uint (id: 5);
+  q:ulong (id: 6);
+  i:int (id: 7);
+  j:long (id: 8);
+  /// doc comment for k.
+  k:bool (id: 9);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 10);
+  m:[ubyte] (id: 11);
+  n:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage (id: 22);
+  o:[string] (id: 13);
+  z:proto.test.test_namespace_suffix.ImportedMessage (id: 14);
+  /// doc comment for r.
+  r:proto.test.test_namespace_suffix.ProtoMessage_.Anonymous0 (id: 0);
+  outer_enum:proto.test.test_namespace_suffix.ProtoEnum (id: 15);
+  u:float = +inf (id: 16);
+  v:float = +inf (id: 17);
+  w:float = -inf (id: 18);
+  grades:[proto.test.test_namespace_suffix.ProtoMessage_.GradesEntry] (id: 19);
+  other_message_map:[proto.test.test_namespace_suffix.ProtoMessage_.OtherMessageMapEntry] (id: 20);
+}
+
+namespace proto.test.test_namespace_suffix.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table Anonymous0 {
+  /// doc comment for s.
+  s:proto.test.test_namespace_suffix.ImportedMessage (id: 0);
+  /// doc comment for t on 2
+  /// lines.
+  t:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage (id: 1);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/test_union_id.golden.fbs

@@ -0,0 +1,89 @@
+// Generated from test.proto
+
+namespace proto.test;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test.ProtoMessage_;
+
+union RUnion {
+  /// doc comment for s.
+  proto.test.ImportedMessage,
+  /// doc comment for t on 2
+  /// lines.
+  proto.test.ProtoMessage_.OtherMessage,
+}
+
+namespace proto.test;
+
+table ImportedMessage {
+  a:int (id: 0);
+}
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 13);
+  d:long (id: 2);
+  p:uint (id: 22);
+  e:ulong (id: 3);
+  /// doc comment for f.
+  f:int = -1 (id: 4);
+  g:long (id: 5);
+  h:uint (id: 6);
+  q:ulong (id: 7);
+  i:int (id: 8);
+  j:long (id: 9);
+  /// doc comment for k.
+  k:bool (id: 10);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 11);
+  m:[ubyte] (id: 12);
+  n:proto.test.ProtoMessage_.OtherMessage (id: 23);
+  o:[string] (id: 14);
+  z:proto.test.ImportedMessage (id: 15);
+  /// doc comment for r.
+  r:proto.test.ProtoMessage_.RUnion (id: 1);
+  outer_enum:proto.test.ProtoEnum (id: 16);
+  u:float = +inf (id: 17);
+  v:float = +inf (id: 18);
+  w:float = -inf (id: 19);
+  grades:[proto.test.ProtoMessage_.GradesEntry] (id: 20);
+  other_message_map:[proto.test.ProtoMessage_.OtherMessageMapEntry] (id: 21);
+}
+
+namespace proto.test.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/test_union_include_id.golden.fbs

@@ -0,0 +1,87 @@
+// Generated from test.proto
+
+include "imported.fbs";
+
+namespace proto.test;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test.ProtoMessage_;
+
+union RUnion {
+  /// doc comment for s.
+  proto.test.ImportedMessage,
+  /// doc comment for t on 2
+  /// lines.
+  proto.test.ProtoMessage_.OtherMessage,
+}
+
+namespace proto.test;
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 13);
+  d:long (id: 2);
+  p:uint (id: 22);
+  e:ulong (id: 3);
+  /// doc comment for f.
+  f:int = -1 (id: 4);
+  g:long (id: 5);
+  h:uint (id: 6);
+  q:ulong (id: 7);
+  i:int (id: 8);
+  j:long (id: 9);
+  /// doc comment for k.
+  k:bool (id: 10);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 11);
+  m:[ubyte] (id: 12);
+  n:proto.test.ProtoMessage_.OtherMessage (id: 23);
+  o:[string] (id: 14);
+  z:proto.test.ImportedMessage (id: 15);
+  /// doc comment for r.
+  r:proto.test.ProtoMessage_.RUnion (id: 1);
+  outer_enum:proto.test.ProtoEnum (id: 16);
+  u:float = +inf (id: 17);
+  v:float = +inf (id: 18);
+  w:float = -inf (id: 19);
+  grades:[proto.test.ProtoMessage_.GradesEntry] (id: 20);
+  other_message_map:[proto.test.ProtoMessage_.OtherMessageMapEntry] (id: 21);
+}
+
+namespace proto.test.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/test_union_suffix_id.golden.fbs

@@ -0,0 +1,89 @@
+// Generated from test.proto
+
+namespace proto.test.test_namespace_suffix;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test.test_namespace_suffix.ProtoMessage_;
+
+union RUnion {
+  /// doc comment for s.
+  proto.test.test_namespace_suffix.ImportedMessage,
+  /// doc comment for t on 2
+  /// lines.
+  proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage,
+}
+
+namespace proto.test.test_namespace_suffix;
+
+table ImportedMessage {
+  a:int (id: 0);
+}
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 13);
+  d:long (id: 2);
+  p:uint (id: 22);
+  e:ulong (id: 3);
+  /// doc comment for f.
+  f:int = -1 (id: 4);
+  g:long (id: 5);
+  h:uint (id: 6);
+  q:ulong (id: 7);
+  i:int (id: 8);
+  j:long (id: 9);
+  /// doc comment for k.
+  k:bool (id: 10);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 11);
+  m:[ubyte] (id: 12);
+  n:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage (id: 23);
+  o:[string] (id: 14);
+  z:proto.test.test_namespace_suffix.ImportedMessage (id: 15);
+  /// doc comment for r.
+  r:proto.test.test_namespace_suffix.ProtoMessage_.RUnion (id: 1);
+  outer_enum:proto.test.test_namespace_suffix.ProtoEnum (id: 16);
+  u:float = +inf (id: 17);
+  v:float = +inf (id: 18);
+  w:float = -inf (id: 19);
+  grades:[proto.test.test_namespace_suffix.ProtoMessage_.GradesEntry] (id: 20);
+  other_message_map:[proto.test.test_namespace_suffix.ProtoMessage_.OtherMessageMapEntry] (id: 21);
+}
+
+namespace proto.test.test_namespace_suffix.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/twice-id.proto

@@ -0,0 +1,10 @@
+// Sample .proto file that we can not translate to the corresponding .fbs because it has used an id twice
+
+option some_option = is_ignored;
+
+package proto.test;
+
+message ProtoMessage {
+  optional sint32 e = 2;
+  optional uint64 twice = 2;
+}

tests/prototest/use-reserved-id.proto

@@ -0,0 +1,10 @@
+// Sample .proto file that we can not translate to the corresponding .fbs because it has used ids from reserved ids.
+
+option some_option = is_ignored;
+
+package proto.test;
+
+message ProtoMessage {
+  reserved 200, 9 to 11, 1500;
+  optional sint32 reserved_id_usage = 10;
+}

tests/test.cpp

@@ -1542,8 +1542,6 @@ int FlatBufferTests(const std::string &tests_data_path) {
   FixedLengthArrayJsonTest(tests_data_path, true);
   ReflectionTest(tests_data_path, flatbuf.data(), flatbuf.size());
   ParseProtoTest(tests_data_path);
-  ParseProtoTestWithSuffix(tests_data_path);
-  ParseProtoTestWithIncludes(tests_data_path);
   EvolutionTest(tests_data_path);
   UnionDeprecationTest(tests_data_path);
   UnionVectorTest(tests_data_path);