From 4a34cd70dc1684612f6ad5eebb3a058644990735 Mon Sep 17 00:00:00 2001 From: Khanh Nguyen <44149581+Kn99HN@users.noreply.github.com> Date: Thu, 2 Mar 2023 10:01:44 -0800 Subject: [PATCH] Add Code Generator for idl_gen_fbs to parse .proto files (#7832) * Add code generator for proto files * Update * Add --proto to script * Remove cmt * Move proto parsing logic into else block to share same set up logic for code_generator * Remove IsValidCodeGenerator --- include/flatbuffers/idl.h | 1 + scripts/generate_code.py | 6 +++ src/BUILD.bazel | 1 + src/flatc.cpp | 12 ++---- src/flatc_main.cpp | 6 +++ src/idl_gen_fbs.cpp | 81 +++++++++++++++++++++++++++++++++++---- src/idl_gen_fbs.h | 28 ++++++++++++++ 7 files changed, 119 insertions(+), 16 deletions(-) create mode 100644 src/idl_gen_fbs.h diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 57f83410e..7f71adb76 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -717,6 +717,7 @@ struct IDLOptions { kKotlin = 1 << 15, kSwift = 1 << 16, kNim = 1 << 17, + kProto = 1 << 18, kMAX }; diff --git a/scripts/generate_code.py b/scripts/generate_code.py index 4ee157192..82981d418 100755 --- a/scripts/generate_code.py +++ b/scripts/generate_code.py @@ -105,6 +105,7 @@ PHP_OPTS = ["--php"] DART_OPTS = ["--dart"] PYTHON_OPTS = ["--python"] BINARY_OPTS = ["-b", "--schema", "--bfbs-comments", "--bfbs-builtins"] +PROTO_OPTS = ["--proto"] # Basic Usage @@ -192,6 +193,11 @@ flatc( data="monsterdata_test.json", ) +flatc( + PROTO_OPTS, + schema="prototest/test.proto", +) + # For Rust we currently generate two independent schemas, with namespace_test2 # duplicating the types in namespace_test1 flatc( diff --git a/src/BUILD.bazel b/src/BUILD.bazel index 3f4ba0c7f..66c355d70 100644 --- a/src/BUILD.bazel +++ b/src/BUILD.bazel @@ -11,6 +11,7 @@ cc_library( srcs = [ "code_generators.cpp", "idl_gen_fbs.cpp", + "idl_gen_fbs.h", "idl_gen_text.cpp", "idl_gen_text.h", "idl_parser.cpp", diff --git a/src/flatc.cpp b/src/flatc.cpp index 2da6bc457..699a9b83f 100644 --- a/src/flatc.cpp +++ b/src/flatc.cpp @@ -166,7 +166,6 @@ const static FlatCOption flatc_options[] = { "Allow binaries without file_identifier to be read. This may crash flatc " "given a mismatched schema." }, { "", "size-prefixed", "", "Input binaries are size prefixed buffers." }, - { "", "proto", "", "Input is a .proto, translate to .fbs." }, { "", "proto-namespace-suffix", "SUFFIX", "Add this namespace to any flatbuffers generated from protobufs." }, { "", "oneof-union", "", "Translate .proto oneofs to flatbuffer unions." }, @@ -243,7 +242,7 @@ const static FlatCOption flatc_options[] = { "ts_entry_points." }, { "", "ts-entry-points", "", "Generate entry point typescript per namespace. Implies gen-all." }, - { "", "annotate-sparse-vectors", "", "Don't annotate every vector element."}, + { "", "annotate-sparse-vectors", "", "Don't annotate every vector element." }, { "", "annotate", "SCHEMA", "Annotate the provided BINARY_FILE with the specified SCHEMA file." }, { "", "no-leak-private-annotation", "", @@ -548,8 +547,6 @@ FlatCOptions FlatCompiler::ParseFromCommandLineArguments(int argc, opts.size_prefixed = true; } else if (arg == "--") { // Separator between text and binary inputs. options.binary_files_from = options.filenames.size(); - } else if (arg == "--proto") { - opts.proto_mode = true; } else if (arg == "--proto-namespace-suffix") { if (++argi >= argc) Error("missing namespace suffix" + arg, true); opts.proto_namespace_suffix = argv[argi]; @@ -647,12 +644,13 @@ FlatCOptions FlatCompiler::ParseFromCommandLineArguments(int argc, } else if (arg == "--no-leak-private-annotation") { opts.no_leak_private_annotations = true; } else if (arg == "--annotate-sparse-vectors") { - options.annotate_include_vector_contents = false; + options.annotate_include_vector_contents = false; } else if (arg == "--annotate") { if (++argi >= argc) Error("missing path following: " + arg, true); options.annotate_schema = flatbuffers::PosixPath(argv[argi]); } else { - // Look up if the command line argument refers to a code generator. + if (arg == "--proto") { opts.proto_mode = true; } + auto code_generator_it = code_generators_.find(arg); if (code_generator_it == code_generators_.end()) { Error("unknown commandline argument: " + arg, true); @@ -888,8 +886,6 @@ std::unique_ptr FlatCompiler::GenerateCode(const FlatCOptions &options, Error("root type must be a table"); } - if (opts.proto_mode) GenerateFBS(*parser, options.output_path, filebase); - // We do not want to generate code for the definitions in this file // in any files coming up next. parser->MarkGenerated(); diff --git a/src/flatc_main.cpp b/src/flatc_main.cpp index 52203b135..e4ddb00f3 100644 --- a/src/flatc_main.cpp +++ b/src/flatc_main.cpp @@ -27,6 +27,7 @@ #include "idl_gen_cpp.h" #include "idl_gen_csharp.h" #include "idl_gen_dart.h" +#include "idl_gen_fbs.h" #include "idl_gen_go.h" #include "idl_gen_java.h" #include "idl_gen_json_schema.h" @@ -100,6 +101,11 @@ int main(int argc, const char *argv[]) { "Generate Dart classes for tables/structs" }, flatbuffers::NewDartCodeGenerator()); + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "", "proto", "", + "Input is a .proto, translate to .fbs" }, + flatbuffers::NewFBSCodeGenerator()); + flatc.RegisterCodeGenerator( flatbuffers::FlatCOption{ "g", "go", "", "Generate Go files for tables/structs" }, diff --git a/src/idl_gen_fbs.cpp b/src/idl_gen_fbs.cpp index d8db7d4ff..6a6d0351b 100644 --- a/src/idl_gen_fbs.cpp +++ b/src/idl_gen_fbs.cpp @@ -15,10 +15,13 @@ */ // independent from idl_parser, since this code is not needed for most clients +#include "idl_gen_fbs.h" + #include #include #include +#include "flatbuffers/code_generator.h" #include "flatbuffers/code_generators.h" #include "flatbuffers/flatbuffers.h" #include "flatbuffers/idl.h" @@ -130,29 +133,30 @@ static bool ProtobufIdSanityCheck(const StructDef &struct_def, const auto &fields = struct_def.fields.vec; if (HasNonPositiveFieldId(fields)) { // TODO: Use LogCompilerWarn - fprintf(stderr, - "Field id in struct %s has a non positive number value\n", - struct_def.name.c_str()); + fprintf(stderr, "Field id in struct %s has a non positive number value\n", + struct_def.name.c_str()); return false; } if (HasTwiceUsedId(fields)) { // TODO: Use LogCompilerWarn - fprintf(stderr, "Fields in struct %s have used an id twice\n", struct_def.name.c_str()); + fprintf(stderr, "Fields in struct %s have used an id twice\n", + struct_def.name.c_str()); return false; } if (HasFieldIdFromReservedIds(fields, struct_def.reserved_ids)) { // TODO: Use LogCompilerWarn - fprintf(stderr, - "Fields in struct %s use id from reserved ids\n", struct_def.name.c_str()); + fprintf(stderr, "Fields in struct %s use id from reserved ids\n", + struct_def.name.c_str()); return false; } if (gap_action != IDLOptions::ProtoIdGapAction::NO_OP) { if (HasGapInProtoId(fields)) { // TODO: Use LogCompilerWarn - fprintf(stderr, "Fields in struct %s have gap between ids\n", struct_def.name.c_str()); + fprintf(stderr, "Fields in struct %s have gap between ids\n", + struct_def.name.c_str()); if (gap_action == IDLOptions::ProtoIdGapAction::ERROR) { return false; } } } @@ -199,7 +203,8 @@ static ProtobufToFbsIdMap MapProtoIdsToFieldsId( } } else { // TODO: Use LogCompilerWarn - fprintf(stderr, "Fields id in struct %s is missing\n", struct_def.name.c_str()); + fprintf(stderr, "Fields id in struct %s is missing\n", + struct_def.name.c_str()); return {}; } } @@ -367,4 +372,64 @@ bool GenerateFBS(const Parser &parser, const std::string &path, return SaveFile((path + file_name + ".fbs").c_str(), fbs, false); } +namespace { + +class FBSCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateFBS(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + // Generate code from the provided `buffer` of given `length`. The buffer is a + // serialized reflection.fbs. + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + (void)parser; + (void)path; + (void)filename; + (void)output; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + (void)parser; + (void)path; + (void)filename; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + + bool IsSchemaOnly() const override { return false; } + + bool SupportsBfbsGeneration() const override { return false; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kProto; } + + std::string LanguageName() const override { return "proto"; } +}; + +} // namespace + +std::unique_ptr NewFBSCodeGenerator() { + return std::unique_ptr(new FBSCodeGenerator()); +} + } // namespace flatbuffers diff --git a/src/idl_gen_fbs.h b/src/idl_gen_fbs.h new file mode 100644 index 000000000..7f73d33bc --- /dev/null +++ b/src/idl_gen_fbs.h @@ -0,0 +1,28 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_FBS_H_ +#define FLATBUFFERS_IDL_GEN_FBS_H_ + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +std::unique_ptr NewFBSCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_FBS_H_