From 001adf782dba8e9fa869b15389725b3eab32e51b Mon Sep 17 00:00:00 2001 From: Michael Le Date: Tue, 8 Nov 2022 10:51:24 -0800 Subject: [PATCH] Add support for parsing proto map fields (#7613) * Add support for proto 3 map to fbs gen * Run clang-format * Update proto golden test * Rename variables * Remove iostream * Remove iostream * Run clang format Co-authored-by: Derek Bailey --- include/flatbuffers/idl.h | 5 +-- src/idl_gen_fbs.cpp | 1 + src/idl_parser.cpp | 39 +++++++++++++++++++++++ tests/proto_test.cpp | 2 +- tests/prototest/GenerateProtoGoldens.sh | 24 ++++++++++++++ tests/prototest/test.golden | 12 +++++++ tests/prototest/test.proto | 3 ++ tests/prototest/test_include.golden | 12 +++++++ tests/prototest/test_suffix.golden | 12 +++++++ tests/prototest/test_union.golden | 12 +++++++ tests/prototest/test_union_include.golden | 12 +++++++ tests/prototest/test_union_suffix.golden | 12 +++++++ 12 files changed, 143 insertions(+), 3 deletions(-) create mode 100755 tests/prototest/GenerateProtoGoldens.sh diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 2cd67cd96..1701236bf 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -489,11 +489,11 @@ inline bool IsVector(const Type &type) { return type.base_type == BASE_TYPE_VECTOR; } -inline bool IsVectorOfStruct(const Type& type) { +inline bool IsVectorOfStruct(const Type &type) { return IsVector(type) && IsStruct(type.VectorType()); } -inline bool IsVectorOfTable(const Type& type) { +inline bool IsVectorOfTable(const Type &type) { return IsVector(type) && IsTable(type.VectorType()); } @@ -1031,6 +1031,7 @@ class Parser : public ParserState { FLATBUFFERS_CHECKED_ERROR ParseService(const char *filename); FLATBUFFERS_CHECKED_ERROR ParseProtoFields(StructDef *struct_def, bool isextend, bool inside_oneof); + FLATBUFFERS_CHECKED_ERROR ParseProtoMapField(StructDef *struct_def); FLATBUFFERS_CHECKED_ERROR ParseProtoOption(); FLATBUFFERS_CHECKED_ERROR ParseProtoKey(); FLATBUFFERS_CHECKED_ERROR ParseProtoDecl(); diff --git a/src/idl_gen_fbs.cpp b/src/idl_gen_fbs.cpp index 782557f09..9c58dc4a3 100644 --- a/src/idl_gen_fbs.cpp +++ b/src/idl_gen_fbs.cpp @@ -137,6 +137,7 @@ std::string GenerateFBS(const Parser &parser, const std::string &file_name) { schema += " " + field.name + ":" + GenType(field.value.type); if (field.value.constant != "0") schema += " = " + field.value.constant; if (field.IsRequired()) schema += " (required)"; + if (field.key) schema += " (key)"; schema += ";\n"; } } diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index caa26ba0e..530c7b3c4 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -507,6 +507,8 @@ CheckedError Parser::Next() { case ')': case '[': case ']': + case '<': + case '>': case ',': case ':': case ';': @@ -2896,6 +2898,8 @@ CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend, NEXT(); while (!Is(';')) { NEXT(); } // A variety of formats, just skip. NEXT(); + } else if (IsIdent("map")) { + ECHECK(ParseProtoMapField(struct_def)); } else { std::vector field_comment = doc_comment_; // Parse the qualifier. @@ -3030,6 +3034,41 @@ CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend, return NoError(); } +CheckedError Parser::ParseProtoMapField(StructDef *struct_def) { + NEXT(); + EXPECT('<'); + Type key_type; + ECHECK(ParseType(key_type)); + EXPECT(','); + Type value_type; + ECHECK(ParseType(value_type)); + EXPECT('>'); + auto field_name = attribute_; + NEXT(); + EXPECT('='); + EXPECT(kTokenIntegerConstant); + EXPECT(';'); + + auto entry_table_name = ConvertCase(field_name, Case::kUpperCamel) + "Entry"; + StructDef *entry_table; + ECHECK(StartStruct(entry_table_name, &entry_table)); + entry_table->has_key = true; + FieldDef *key_field; + ECHECK(AddField(*entry_table, "key", key_type, &key_field)); + key_field->key = true; + FieldDef *value_field; + ECHECK(AddField(*entry_table, "value", value_type, &value_field)); + + Type field_type; + field_type.base_type = BASE_TYPE_VECTOR; + field_type.element = BASE_TYPE_STRUCT; + field_type.struct_def = entry_table; + FieldDef *field; + ECHECK(AddField(*struct_def, field_name, field_type, &field)); + + return NoError(); +} + CheckedError Parser::ParseProtoKey() { if (token_ == '(') { NEXT(); diff --git a/tests/proto_test.cpp b/tests/proto_test.cpp index 7281c2cc6..1d1c98ac5 100644 --- a/tests/proto_test.cpp +++ b/tests/proto_test.cpp @@ -201,4 +201,4 @@ void ParseProtoBufAsciiTest() { } } // namespace tests -} // namespace flatbuffers \ No newline at end of file +} // namespace flatbuffers diff --git a/tests/prototest/GenerateProtoGoldens.sh b/tests/prototest/GenerateProtoGoldens.sh new file mode 100755 index 000000000..8cf24f918 --- /dev/null +++ b/tests/prototest/GenerateProtoGoldens.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Copyright 2022 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pushd "$(dirname $0)" >/dev/null + +./../../flatc --proto test.proto && mv test.fbs test_include.golden +./../../flatc --proto --gen-all test.proto && mv test.fbs test.golden +./../../flatc --proto --oneof-union test.proto && mv test.fbs test_union_include.golden +./../../flatc --proto --gen-all --oneof-union test.proto && mv test.fbs test_union.golden +./../../flatc --proto --gen-all --proto-namespace-suffix test_namespace_suffix test.proto && mv test.fbs test_suffix.golden +./../../flatc --proto --gen-all --proto-namespace-suffix test_namespace_suffix --oneof-union test.proto && mv test.fbs test_union_suffix.golden diff --git a/tests/prototest/test.golden b/tests/prototest/test.golden index 949a003d5..4484b4982 100644 --- a/tests/prototest/test.golden +++ b/tests/prototest/test.golden @@ -54,6 +54,8 @@ table ProtoMessage { u:float = +inf; v:float = +inf; w:float = -inf; + grades:[proto.test.ProtoMessage_.GradesEntry]; + other_message_map:[proto.test.ProtoMessage_.OtherMessageMapEntry]; } namespace proto.test.ProtoMessage_; @@ -73,3 +75,13 @@ table Anonymous0 { t:proto.test.ProtoMessage_.OtherMessage; } +table GradesEntry { + key:string (key); + value:float; +} + +table OtherMessageMapEntry { + key:string (key); + value:proto.test.ProtoMessage_.OtherMessage; +} + diff --git a/tests/prototest/test.proto b/tests/prototest/test.proto index 71ec8d51d..98c92f88a 100644 --- a/tests/prototest/test.proto +++ b/tests/prototest/test.proto @@ -71,4 +71,7 @@ message ProtoMessage { optional float u = 34 [default = inf]; optional float v = 35 [default = +inf]; optional float w = 36 [default = -inf]; + + map grades = 37; + map other_message_map = 38; } diff --git a/tests/prototest/test_include.golden b/tests/prototest/test_include.golden index b98de44fb..791f7f7b5 100644 --- a/tests/prototest/test_include.golden +++ b/tests/prototest/test_include.golden @@ -52,6 +52,8 @@ table ProtoMessage { u:float = +inf; v:float = +inf; w:float = -inf; + grades:[proto.test.ProtoMessage_.GradesEntry]; + other_message_map:[proto.test.ProtoMessage_.OtherMessageMapEntry]; } namespace proto.test.ProtoMessage_; @@ -71,3 +73,13 @@ table Anonymous0 { t:proto.test.ProtoMessage_.OtherMessage; } +table GradesEntry { + key:string (key); + value:float; +} + +table OtherMessageMapEntry { + key:string (key); + value:proto.test.ProtoMessage_.OtherMessage; +} + diff --git a/tests/prototest/test_suffix.golden b/tests/prototest/test_suffix.golden index 4ab2146c5..b76acb7fb 100644 --- a/tests/prototest/test_suffix.golden +++ b/tests/prototest/test_suffix.golden @@ -54,6 +54,8 @@ table ProtoMessage { u:float = +inf; v:float = +inf; w:float = -inf; + grades:[proto.test.test_namespace_suffix.ProtoMessage_.GradesEntry]; + other_message_map:[proto.test.test_namespace_suffix.ProtoMessage_.OtherMessageMapEntry]; } namespace proto.test.test_namespace_suffix.ProtoMessage_; @@ -73,3 +75,13 @@ table Anonymous0 { t:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage; } +table GradesEntry { + key:string (key); + value:float; +} + +table OtherMessageMapEntry { + key:string (key); + value:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage; +} + diff --git a/tests/prototest/test_union.golden b/tests/prototest/test_union.golden index 241f3491b..15d365b54 100644 --- a/tests/prototest/test_union.golden +++ b/tests/prototest/test_union.golden @@ -64,6 +64,8 @@ table ProtoMessage { u:float = +inf; v:float = +inf; w:float = -inf; + grades:[proto.test.ProtoMessage_.GradesEntry]; + other_message_map:[proto.test.ProtoMessage_.OtherMessageMapEntry]; } namespace proto.test.ProtoMessage_; @@ -75,3 +77,13 @@ table OtherMessage { foo_bar_baz:proto.test.ProtoMessage_.OtherMessage_.ProtoEnum; } +table GradesEntry { + key:string (key); + value:float; +} + +table OtherMessageMapEntry { + key:string (key); + value:proto.test.ProtoMessage_.OtherMessage; +} + diff --git a/tests/prototest/test_union_include.golden b/tests/prototest/test_union_include.golden index 1fdb2acbc..ce3646356 100644 --- a/tests/prototest/test_union_include.golden +++ b/tests/prototest/test_union_include.golden @@ -62,6 +62,8 @@ table ProtoMessage { u:float = +inf; v:float = +inf; w:float = -inf; + grades:[proto.test.ProtoMessage_.GradesEntry]; + other_message_map:[proto.test.ProtoMessage_.OtherMessageMapEntry]; } namespace proto.test.ProtoMessage_; @@ -73,3 +75,13 @@ table OtherMessage { foo_bar_baz:proto.test.ProtoMessage_.OtherMessage_.ProtoEnum; } +table GradesEntry { + key:string (key); + value:float; +} + +table OtherMessageMapEntry { + key:string (key); + value:proto.test.ProtoMessage_.OtherMessage; +} + diff --git a/tests/prototest/test_union_suffix.golden b/tests/prototest/test_union_suffix.golden index 2278edd2c..0c0c5e586 100644 --- a/tests/prototest/test_union_suffix.golden +++ b/tests/prototest/test_union_suffix.golden @@ -64,6 +64,8 @@ table ProtoMessage { u:float = +inf; v:float = +inf; w:float = -inf; + grades:[proto.test.test_namespace_suffix.ProtoMessage_.GradesEntry]; + other_message_map:[proto.test.test_namespace_suffix.ProtoMessage_.OtherMessageMapEntry]; } namespace proto.test.test_namespace_suffix.ProtoMessage_; @@ -75,3 +77,13 @@ table OtherMessage { foo_bar_baz:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage_.ProtoEnum; } +table GradesEntry { + key:string (key); + value:float; +} + +table OtherMessageMapEntry { + key:string (key); + value:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage; +} +