Parsing from proto should keep field ID. (fixes #7645) (#7655)

* Parsing from proto should keep field ID. (fixes #7645)

* Fix failed tests

* Fix windows warning

* Improve attribute generation in proto to fbs

* Check if id is used twice. fix Some clang-format problems

* Test if fake id can solve the test problem

* Validate proto file in proto -> fbs generation.

* Fix error messages

* Ignore id in union

* Add keep proto id for legacy and check gap flag have been added. Reserved id will be checked.

* Add needed flags

* unit tests

* fix fromat problem. fix comments and error messages.

* clear

* More unit tests

* Fix windows build

* Fix include problems

* Fake commit to invoke rebuild

* Fix buzel build

* Fix some issues

* Fix comments, fix return value and sort for android NDK

* Fix return type

* Break down big function

* Place todo

---------

Co-authored-by: Derek Bailey <derekbailey@google.com>

tests/prototest/GenerateProtoGoldens.sh

@@ -16,9 +16,16 @@
 
 pushd "$(dirname $0)" >/dev/null
 
-./../../flatc --proto test.proto && mv test.fbs test_include.golden
-./../../flatc --proto --gen-all test.proto && mv test.fbs test.golden
-./../../flatc --proto --oneof-union test.proto && mv test.fbs test_union_include.golden
-./../../flatc --proto --gen-all --oneof-union test.proto && mv test.fbs test_union.golden
-./../../flatc --proto --gen-all --proto-namespace-suffix test_namespace_suffix test.proto && mv test.fbs test_suffix.golden
-./../../flatc --proto --gen-all --proto-namespace-suffix test_namespace_suffix --oneof-union test.proto && mv test.fbs test_union_suffix.golden
+./../../flatc --proto test.proto && mv test.fbs test_include.golden.fbs
+./../../flatc --proto --gen-all test.proto && mv test.fbs test.golden.fbs
+./../../flatc --proto --oneof-union test.proto && mv test.fbs test_union_include.golden.fbs
+./../../flatc --proto --gen-all --oneof-union test.proto && mv test.fbs test_union.golden.fbs
+./../../flatc --proto --gen-all --proto-namespace-suffix test_namespace_suffix test.proto && mv test.fbs test_suffix.golden.fbs
+./../../flatc --proto --gen-all --proto-namespace-suffix test_namespace_suffix --oneof-union test.proto && mv test.fbs test_union_suffix.golden.fbs
+
+./../../flatc --proto --keep-proto-id test.proto && mv test.fbs test_include_id.golden.fbs
+./../../flatc --proto --keep-proto-id --gen-all test.proto && mv test.fbs test_id.golden.fbs
+./../../flatc --proto --keep-proto-id --oneof-union test.proto && mv test.fbs test_union_include_id.golden.fbs
+./../../flatc --proto --keep-proto-id --gen-all --oneof-union test.proto && mv test.fbs test_union_id.golden.fbs
+./../../flatc --proto --keep-proto-id --gen-all --proto-namespace-suffix test_namespace_suffix test.proto && mv test.fbs test_suffix_id.golden.fbs
+./../../flatc --proto --keep-proto-id --gen-all --proto-namespace-suffix test_namespace_suffix --oneof-union test.proto && mv test.fbs test_union_suffix_id.golden.fbs

tests/prototest/non-positive-id.proto

@@ -0,0 +1,9 @@
+// Sample .proto file that we can not translate to the corresponding .fbs because it has non-positive ids.
+
+option some_option = is_ignored;
+
+package proto.test;
+
+message ProtoMessage {
+  optional uint64 NonPositiveId = -1;
+}

tests/prototest/test.proto

@@ -24,6 +24,7 @@ message ProtoMessage {
   // Ignored non-doc comment.
   // A nested message declaration, will be moved to top level in .fbs
   message OtherMessage {
+    reserved 2, 9 to 11, 15;
     optional double a = 26;
     /// doc comment for b.
     optional float b = 32 [default = 3.14149];
@@ -40,7 +41,7 @@ message ProtoMessage {
   }
   optional int32 c = 12 [default = 16];
   optional int64 d = 1 [default = 0];
-  optional uint32 p = 1;
+  optional uint32 p = 40;
   optional uint64 e = 2;
   /// doc comment for f.
   optional sint32 f = 3 [default = -1];
@@ -55,7 +56,7 @@ message ProtoMessage {
   /// lines
   required string l = 10;
   optional bytes m = 11;
-  optional OtherMessage n = 12;
+  optional OtherMessage n = 41;
   repeated string o = 14;
   optional ImportedMessage z = 16;
   /// doc comment for r.

tests/prototest/test_id.golden.fbs

@@ -0,0 +1,87 @@
+// Generated from test.proto
+
+namespace proto.test;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test;
+
+table ImportedMessage {
+  a:int (id: 0);
+}
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 12);
+  d:long (id: 1);
+  p:uint (id: 21);
+  e:ulong (id: 2);
+  /// doc comment for f.
+  f:int = -1 (id: 3);
+  g:long (id: 4);
+  h:uint (id: 5);
+  q:ulong (id: 6);
+  i:int (id: 7);
+  j:long (id: 8);
+  /// doc comment for k.
+  k:bool (id: 9);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 10);
+  m:[ubyte] (id: 11);
+  n:proto.test.ProtoMessage_.OtherMessage (id: 22);
+  o:[string] (id: 13);
+  z:proto.test.ImportedMessage (id: 14);
+  /// doc comment for r.
+  r:proto.test.ProtoMessage_.Anonymous0 (id: 0);
+  outer_enum:proto.test.ProtoEnum (id: 15);
+  u:float = +inf (id: 16);
+  v:float = +inf (id: 17);
+  w:float = -inf (id: 18);
+  grades:[proto.test.ProtoMessage_.GradesEntry] (id: 19);
+  other_message_map:[proto.test.ProtoMessage_.OtherMessageMapEntry] (id: 20);
+}
+
+namespace proto.test.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table Anonymous0 {
+  /// doc comment for s.
+  s:proto.test.ImportedMessage (id: 0);
+  /// doc comment for t on 2
+  /// lines.
+  t:proto.test.ProtoMessage_.OtherMessage (id: 1);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/test_include_id.golden.fbs

@@ -0,0 +1,85 @@
+// Generated from test.proto
+
+include "imported.fbs";
+
+namespace proto.test;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test;
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 12);
+  d:long (id: 1);
+  p:uint (id: 21);
+  e:ulong (id: 2);
+  /// doc comment for f.
+  f:int = -1 (id: 3);
+  g:long (id: 4);
+  h:uint (id: 5);
+  q:ulong (id: 6);
+  i:int (id: 7);
+  j:long (id: 8);
+  /// doc comment for k.
+  k:bool (id: 9);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 10);
+  m:[ubyte] (id: 11);
+  n:proto.test.ProtoMessage_.OtherMessage (id: 22);
+  o:[string] (id: 13);
+  z:proto.test.ImportedMessage (id: 14);
+  /// doc comment for r.
+  r:proto.test.ProtoMessage_.Anonymous0 (id: 0);
+  outer_enum:proto.test.ProtoEnum (id: 15);
+  u:float = +inf (id: 16);
+  v:float = +inf (id: 17);
+  w:float = -inf (id: 18);
+  grades:[proto.test.ProtoMessage_.GradesEntry] (id: 19);
+  other_message_map:[proto.test.ProtoMessage_.OtherMessageMapEntry] (id: 20);
+}
+
+namespace proto.test.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table Anonymous0 {
+  /// doc comment for s.
+  s:proto.test.ImportedMessage (id: 0);
+  /// doc comment for t on 2
+  /// lines.
+  t:proto.test.ProtoMessage_.OtherMessage (id: 1);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/test_suffix_id.golden.fbs

@@ -0,0 +1,87 @@
+// Generated from test.proto
+
+namespace proto.test.test_namespace_suffix;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test.test_namespace_suffix;
+
+table ImportedMessage {
+  a:int (id: 0);
+}
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 12);
+  d:long (id: 1);
+  p:uint (id: 21);
+  e:ulong (id: 2);
+  /// doc comment for f.
+  f:int = -1 (id: 3);
+  g:long (id: 4);
+  h:uint (id: 5);
+  q:ulong (id: 6);
+  i:int (id: 7);
+  j:long (id: 8);
+  /// doc comment for k.
+  k:bool (id: 9);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 10);
+  m:[ubyte] (id: 11);
+  n:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage (id: 22);
+  o:[string] (id: 13);
+  z:proto.test.test_namespace_suffix.ImportedMessage (id: 14);
+  /// doc comment for r.
+  r:proto.test.test_namespace_suffix.ProtoMessage_.Anonymous0 (id: 0);
+  outer_enum:proto.test.test_namespace_suffix.ProtoEnum (id: 15);
+  u:float = +inf (id: 16);
+  v:float = +inf (id: 17);
+  w:float = -inf (id: 18);
+  grades:[proto.test.test_namespace_suffix.ProtoMessage_.GradesEntry] (id: 19);
+  other_message_map:[proto.test.test_namespace_suffix.ProtoMessage_.OtherMessageMapEntry] (id: 20);
+}
+
+namespace proto.test.test_namespace_suffix.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table Anonymous0 {
+  /// doc comment for s.
+  s:proto.test.test_namespace_suffix.ImportedMessage (id: 0);
+  /// doc comment for t on 2
+  /// lines.
+  t:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage (id: 1);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/test_union_id.golden.fbs

@@ -0,0 +1,89 @@
+// Generated from test.proto
+
+namespace proto.test;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test.ProtoMessage_;
+
+union RUnion {
+  /// doc comment for s.
+  proto.test.ImportedMessage,
+  /// doc comment for t on 2
+  /// lines.
+  proto.test.ProtoMessage_.OtherMessage,
+}
+
+namespace proto.test;
+
+table ImportedMessage {
+  a:int (id: 0);
+}
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 13);
+  d:long (id: 2);
+  p:uint (id: 22);
+  e:ulong (id: 3);
+  /// doc comment for f.
+  f:int = -1 (id: 4);
+  g:long (id: 5);
+  h:uint (id: 6);
+  q:ulong (id: 7);
+  i:int (id: 8);
+  j:long (id: 9);
+  /// doc comment for k.
+  k:bool (id: 10);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 11);
+  m:[ubyte] (id: 12);
+  n:proto.test.ProtoMessage_.OtherMessage (id: 23);
+  o:[string] (id: 14);
+  z:proto.test.ImportedMessage (id: 15);
+  /// doc comment for r.
+  r:proto.test.ProtoMessage_.RUnion (id: 1);
+  outer_enum:proto.test.ProtoEnum (id: 16);
+  u:float = +inf (id: 17);
+  v:float = +inf (id: 18);
+  w:float = -inf (id: 19);
+  grades:[proto.test.ProtoMessage_.GradesEntry] (id: 20);
+  other_message_map:[proto.test.ProtoMessage_.OtherMessageMapEntry] (id: 21);
+}
+
+namespace proto.test.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/test_union_include_id.golden.fbs

@@ -0,0 +1,87 @@
+// Generated from test.proto
+
+include "imported.fbs";
+
+namespace proto.test;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test.ProtoMessage_;
+
+union RUnion {
+  /// doc comment for s.
+  proto.test.ImportedMessage,
+  /// doc comment for t on 2
+  /// lines.
+  proto.test.ProtoMessage_.OtherMessage,
+}
+
+namespace proto.test;
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 13);
+  d:long (id: 2);
+  p:uint (id: 22);
+  e:ulong (id: 3);
+  /// doc comment for f.
+  f:int = -1 (id: 4);
+  g:long (id: 5);
+  h:uint (id: 6);
+  q:ulong (id: 7);
+  i:int (id: 8);
+  j:long (id: 9);
+  /// doc comment for k.
+  k:bool (id: 10);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 11);
+  m:[ubyte] (id: 12);
+  n:proto.test.ProtoMessage_.OtherMessage (id: 23);
+  o:[string] (id: 14);
+  z:proto.test.ImportedMessage (id: 15);
+  /// doc comment for r.
+  r:proto.test.ProtoMessage_.RUnion (id: 1);
+  outer_enum:proto.test.ProtoEnum (id: 16);
+  u:float = +inf (id: 17);
+  v:float = +inf (id: 18);
+  w:float = -inf (id: 19);
+  grades:[proto.test.ProtoMessage_.GradesEntry] (id: 20);
+  other_message_map:[proto.test.ProtoMessage_.OtherMessageMapEntry] (id: 21);
+}
+
+namespace proto.test.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/test_union_suffix_id.golden.fbs

@@ -0,0 +1,89 @@
+// Generated from test.proto
+
+namespace proto.test.test_namespace_suffix;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test.test_namespace_suffix.ProtoMessage_;
+
+union RUnion {
+  /// doc comment for s.
+  proto.test.test_namespace_suffix.ImportedMessage,
+  /// doc comment for t on 2
+  /// lines.
+  proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage,
+}
+
+namespace proto.test.test_namespace_suffix;
+
+table ImportedMessage {
+  a:int (id: 0);
+}
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 13);
+  d:long (id: 2);
+  p:uint (id: 22);
+  e:ulong (id: 3);
+  /// doc comment for f.
+  f:int = -1 (id: 4);
+  g:long (id: 5);
+  h:uint (id: 6);
+  q:ulong (id: 7);
+  i:int (id: 8);
+  j:long (id: 9);
+  /// doc comment for k.
+  k:bool (id: 10);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 11);
+  m:[ubyte] (id: 12);
+  n:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage (id: 23);
+  o:[string] (id: 14);
+  z:proto.test.test_namespace_suffix.ImportedMessage (id: 15);
+  /// doc comment for r.
+  r:proto.test.test_namespace_suffix.ProtoMessage_.RUnion (id: 1);
+  outer_enum:proto.test.test_namespace_suffix.ProtoEnum (id: 16);
+  u:float = +inf (id: 17);
+  v:float = +inf (id: 18);
+  w:float = -inf (id: 19);
+  grades:[proto.test.test_namespace_suffix.ProtoMessage_.GradesEntry] (id: 20);
+  other_message_map:[proto.test.test_namespace_suffix.ProtoMessage_.OtherMessageMapEntry] (id: 21);
+}
+
+namespace proto.test.test_namespace_suffix.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/twice-id.proto

@@ -0,0 +1,10 @@
+// Sample .proto file that we can not translate to the corresponding .fbs because it has used an id twice
+
+option some_option = is_ignored;
+
+package proto.test;
+
+message ProtoMessage {
+  optional sint32 e = 2;
+  optional uint64 twice = 2;
+}

tests/prototest/use-reserved-id.proto

@@ -0,0 +1,10 @@
+// Sample .proto file that we can not translate to the corresponding .fbs because it has used ids from reserved ids.
+
+option some_option = is_ignored;
+
+package proto.test;
+
+message ProtoMessage {
+  reserved 200, 9 to 11, 1500;
+  optional sint32 reserved_id_usage = 10;
+}