BigfootDev/flatbuffers
1
0
Fork 1
mirror of https://github.com/google/flatbuffers.git synced 2026-06-10 07:06:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added .proto parsing and convertion to .fbs.

Browse Source 
Bug: 15777858
Change-Id: Iabef9b8c8044e593bb89510feebdee00d2f1840b
Tested: on Linux and Windows.
This commit is contained in:
Wouter van Oortmerssen
2014-09-26 16:46:30 -07:00
parent 18cf19f876
commit d38b9af243
15 changed files with 461 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
src/flatc.cpp
View File

@@ -60,29 +60,29 @@ struct Generator {
const std::string &path,
const std::string &file_name,
const flatbuffers::GeneratorOptions &opts);
const char *extension;
const char *opt;
const char *name;
flatbuffers::GeneratorOptions::Language lang;
const char *help;
};
const Generator generators[] = {
{ flatbuffers::GenerateBinary, "b", "binary",
{ flatbuffers::GenerateBinary, "-b", "binary",
flatbuffers::GeneratorOptions::kMAX,
"Generate wire format binaries for any data definitions" },
{ flatbuffers::GenerateTextFile, "t", "text",
{ flatbuffers::GenerateTextFile, "-t", "text",
flatbuffers::GeneratorOptions::kMAX,
"Generate text output for any data definitions" },
{ flatbuffers::GenerateCPP, "c", "C++",
{ flatbuffers::GenerateCPP, "-c", "C++",
flatbuffers::GeneratorOptions::kMAX,
"Generate C++ headers for tables/structs" },
{ flatbuffers::GenerateGo, "g", "Go",
{ flatbuffers::GenerateGo, "-g", "Go",
flatbuffers::GeneratorOptions::kMAX,
"Generate Go files for tables/structs" },
{ flatbuffers::GenerateGeneral, "j", "Java",
{ flatbuffers::GenerateGeneral, "-j", "Java",
flatbuffers::GeneratorOptions::kJava,
"Generate Java classes for tables/structs" },
{ flatbuffers::GenerateGeneral, "n", "C#",
{ flatbuffers::GenerateGeneral, "-n", "C#",
flatbuffers::GeneratorOptions::kCSharp,
"Generate C# classes for tables/structs" }
};
@@ -98,31 +98,33 @@ static void Error(const char *err, const char *obj, bool usage,
if (usage) {
printf("usage: %s [OPTION]... FILE... [-- FILE...]\n", program_name);
for (size_t i = 0; i < sizeof(generators) / sizeof(generators[0]); ++i)
printf(" -%s %s.\n", generators[i].extension, generators[i].help);
printf(" -o PATH Prefix PATH to all generated files.\n"
" -I PATH Search for includes in the specified path.\n"
" -S Strict JSON: add quotes to field names.\n"
" -P Don\'t prefix enum values with the enum name in C++.\n"
" -H Generate include statements for included schemas the\n"
" generated file depends on (C++).\n"
"FILEs may depend on declarations in earlier files.\n"
"FILEs after the -- must be binary flatbuffer format files.\n"
"Output files are named using the base file name of the input,"
"and written to the current directory or the path given by -o.\n"
"example: %s -c -b schema1.fbs schema2.fbs data.json\n",
program_name);
printf(" %s %s.\n", generators[i].opt, generators[i].help);
printf(
" -o PATH Prefix PATH to all generated files.\n"
" -I PATH Search for includes in the specified path.\n"
" --strict-json Strict JSON: add quotes to field names.\n"
" --no-prefix Don\'t prefix enum values with the enum type in C++.\n"
" --gen-includes Generate include statements for included schemas the\n"
" generated file depends on (C++).\n"
" --proto Input is a .proto, translate to .fbs.\n"
"FILEs may depend on declarations in earlier files.\n"
"FILEs after the -- must be binary flatbuffer format files.\n"
"Output files are named using the base file name of the input,"
"and written to the current directory or the path given by -o.\n"
"example: %s -c -b schema1.fbs schema2.fbs data.json\n",
program_name);
}
exit(1);
}
int main(int argc, const char *argv[]) {
program_name = argv[0];
flatbuffers::Parser parser;
flatbuffers::GeneratorOptions opts;
std::string output_path;
const size_t num_generators = sizeof(generators) / sizeof(generators[0]);
bool generator_enabled[num_generators] = { false };
bool any_generator = false;
bool proto_mode = false;
std::vector<std::string> filenames;
std::vector<const char *> include_directories;
size_t binary_files_from = std::numeric_limits<size_t>::max();
@@ -131,40 +133,34 @@ int main(int argc, const char *argv[]) {
if (arg[0] == '-') {
if (filenames.size() && arg[1] != '-')
Error("invalid option location", arg, true);
if (strlen(arg) != 2)
Error("invalid commandline argument", arg, true);
switch (arg[1]) {
case 'o':
if (++i >= argc) Error("missing path following", arg, true);
output_path = flatbuffers::ConCatPathFileName(argv[i], "");
break;
case 'I':
if (++i >= argc) Error("missing path following", arg, true);
include_directories.push_back(argv[i]);
break;
case 'S':
opts.strict_json = true;
break;
case 'P':
opts.prefixed_enums = false;
break;
case 'H':
opts.include_dependence_headers = true;
break;
case '-': // Separator between text and binary input files.
binary_files_from = filenames.size();
break;
default:
for (size_t i = 0; i < num_generators; ++i) {
if(!strcmp(arg+1, generators[i].extension)) {
generator_enabled[i] = true;
any_generator = true;
goto found;
}
std::string opt = arg;
if (opt == "-o") {
if (++i >= argc) Error("missing path following", arg, true);
output_path = flatbuffers::ConCatPathFileName(argv[i], "");
} else if(opt == "-I") {
if (++i >= argc) Error("missing path following", arg, true);
include_directories.push_back(argv[i]);
} else if(opt == "--strict-json") {
opts.strict_json = true;
} else if(opt == "--no-prefix") {
opts.prefixed_enums = false;
} else if(opt == "--gen-includes") {
opts.include_dependence_headers = true;
} else if(opt == "--") { // Separator between text and binary inputs.
binary_files_from = filenames.size();
} else if(opt == "--proto") {
proto_mode = true;
any_generator = true;
} else {
for (size_t i = 0; i < num_generators; ++i) {
if(opt == generators[i].opt) {
generator_enabled[i] = true;
any_generator = true;
goto found;
}
Error("unknown commandline argument", arg, true);
found:
break;
}
Error("unknown commandline argument", arg, true);
found:;
}
} else {
filenames.push_back(argv[i]);
@@ -178,6 +174,7 @@ int main(int argc, const char *argv[]) {
"specify one of -c -g -j -t -b etc.", true);
// Now process the files:
flatbuffers::Parser parser(proto_mode);
for (auto file_it = filenames.begin();
file_it != filenames.end();
++file_it) {
@@ -219,6 +216,8 @@ int main(int argc, const char *argv[]) {
}
}
if (proto_mode) GenerateFBS(parser, output_path, filebase, opts);
// We do not want to generate code for the definitions in this file
// in any files coming up next.
parser.MarkGenerated();

16
src/idl_gen_cpp.cpp
View File

@@ -168,7 +168,7 @@ static void GenEnum(EnumDef &enum_def, std::string *code_ptr,
// on the wrong type.
auto signature = "inline bool Verify" + enum_def.name +
"(flatbuffers::Verifier &verifier, " +
"const void *union_obj, uint8_t type)";
"const void *union_obj, " + enum_def.name + " type)";
code += signature + ";\n\n";
code_post += signature + " {\n switch (type) {\n";
for (auto it = enum_def.vals.vec.begin();
@@ -201,7 +201,7 @@ std::string GenUnderlyingCast(const Parser &parser, const FieldDef &field,
// Generate an accessor struct, builder structs & function for a table.
static void GenTable(const Parser &parser, StructDef &struct_def,
std::string *code_ptr) {
const GeneratorOptions &opts, std::string *code_ptr) {
if (struct_def.generated) return;
std::string &code = *code_ptr;
@@ -359,13 +359,13 @@ static void GenTable(const Parser &parser, StructDef &struct_def,
code += ",\n " + GenTypeWire(parser, field.value.type, " ", true);
code += field.name + " = ";
if (field.value.type.enum_def && IsScalar(field.value.type.base_type)) {
auto ed = field.value.type.enum_def->ReverseLookup(
StringToInt(field.value.constant.c_str()), false);
if (ed) {
auto ev = field.value.type.enum_def->ReverseLookup(
static_cast<int>(StringToInt(field.value.constant.c_str())), false);
if (ev) {
code += WrapInNameSpace(parser,
field.value.type.enum_def->defined_namespace,
field.value.type.enum_def->name + "_" +
ed->name);
GenEnumVal(*field.value.type.enum_def, *ev,
opts));
} else {
code += GenUnderlyingCast(parser, field, true, field.value.constant);
}
@@ -561,7 +561,7 @@ std::string GenerateCPP(const Parser &parser,
}
for (auto it = parser.structs_.vec.begin();
it != parser.structs_.vec.end(); ++it) {
if (!(**it).fixed) GenTable(parser, **it, &decl_code);
if (!(**it).fixed) GenTable(parser, **it, opts, &decl_code);
}
// Only output file-level code if there were any declarations.

100
src/idl_gen_fbs.cpp Normal file
View File

@@ -0,0 +1,100 @@
/*
* Copyright 2014 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// independent from idl_parser, since this code is not needed for most clients
#include "flatbuffers/flatbuffers.h"
#include "flatbuffers/idl.h"
#include "flatbuffers/util.h"
namespace flatbuffers {
static std::string GenType(const Type &type) {
switch (type.base_type) {
case BASE_TYPE_STRUCT: return type.struct_def->name;
case BASE_TYPE_UNION: return type.enum_def->name;
case BASE_TYPE_VECTOR: return "[" + GenType(type.VectorType()) + "]";
default: return kTypeNames[type.base_type];
}
}
// Generate a flatbuffer schema from the Parser's internal representation.
std::string GenerateFBS(const Parser &parser, const std::string &file_name,
const GeneratorOptions &opts) {
std::string schema;
schema += "// Generated from " + file_name + ".proto\n\n";
if (opts.include_dependence_headers) {
int num_includes = 0;
for (auto it = parser.included_files_.begin();
it != parser.included_files_.end(); ++it) {
auto basename = flatbuffers::StripPath(
flatbuffers::StripExtension(it->first));
if (basename != file_name) {
schema += "include \"" + basename + ".fbs\";\n";
num_includes++;
}
}
if (num_includes) schema += "\n";
}
schema += "namespace ";
auto name_space = parser.namespaces_.back();
for (auto it = name_space->components.begin();
it != name_space->components.end(); ++it) {
if (it != name_space->components.begin()) schema += ".";
schema += *it;
}
schema += ";\n\n";
// Generate code for all the enum declarations.
for (auto it = parser.enums_.vec.begin();
it != parser.enums_.vec.end(); ++it) {
EnumDef &enum_def = **it;
schema += "enum " + enum_def.name + " : ";
schema += GenType(enum_def.underlying_type) + " {\n";
for (auto it = enum_def.vals.vec.begin();
it != enum_def.vals.vec.end(); ++it) {
auto &ev = **it;
schema += " " + ev.name + " = " + NumToString(ev.value) + ",\n";
}
schema += "}\n\n";
}
// Generate code for all structs/tables.
for (auto it = parser.structs_.vec.begin();
it != parser.structs_.vec.end(); ++it) {
StructDef &struct_def = **it;
schema += "table " + struct_def.name + " {\n";
for (auto it = struct_def.fields.vec.begin();
it != struct_def.fields.vec.end(); ++it) {
auto &field = **it;
schema += " " + field.name + ":" + GenType(field.value.type);
if (field.value.constant != "0") schema += " = " + field.value.constant;
if (field.required) schema += " (required)";
schema += ";\n";
}
schema += "}\n\n";
}
return schema;
}
bool GenerateFBS(const Parser &parser,
const std::string &path,
const std::string &file_name,
const GeneratorOptions &opts) {
return SaveFile((path + file_name + ".fbs").c_str(),
GenerateFBS(parser, file_name, opts), false);
}
} // namespace flatbuffers

191
src/idl_parser.cpp
View File

@@ -280,20 +280,24 @@ void Parser::Expect(int t) {
Next();
}
void Parser::ParseTypeIdent(Type &type) {
auto enum_def = enums_.Lookup(attribute_);
if (enum_def) {
type = enum_def->underlying_type;
if (enum_def->is_union) type.base_type = BASE_TYPE_UNION;
} else {
type.base_type = BASE_TYPE_STRUCT;
type.struct_def = LookupCreateStruct(attribute_);
}
}
// Parse any IDL type.
void Parser::ParseType(Type &type) {
if (token_ >= kTokenBOOL && token_ <= kTokenSTRING) {
type.base_type = static_cast<BaseType>(token_ - kTokenNONE);
} else {
if (token_ == kTokenIdentifier) {
auto enum_def = enums_.Lookup(attribute_);
if (enum_def) {
type = enum_def->underlying_type;
if (enum_def->is_union) type.base_type = BASE_TYPE_UNION;
} else {
type.base_type = BASE_TYPE_STRUCT;
type.struct_def = LookupCreateStruct(attribute_);
}
ParseTypeIdent(type);
} else if (token_ == '[') {
Next();
Type subtype;
@@ -374,7 +378,8 @@ void Parser::ParseField(StructDef &struct_def) {
IsScalar(type.base_type) &&
!struct_def.fixed &&
!type.enum_def->attributes.Lookup("bit_flags") &&
!type.enum_def->ReverseLookup(StringToInt(field.value.constant.c_str())))
!type.enum_def->ReverseLookup(static_cast<int>(
StringToInt(field.value.constant.c_str()))))
Error("enum " + type.enum_def->name +
" does not have a declaration for this field\'s default of " +
field.value.constant);
@@ -717,14 +722,18 @@ void Parser::ParseEnum(bool is_union) {
enum_def.underlying_type.base_type = BASE_TYPE_UTYPE;
enum_def.underlying_type.enum_def = &enum_def;
} else {
// Give specialized error message, since this type spec used to
// be optional in the first FlatBuffers release.
if (!IsNext(':')) Error("must specify the underlying integer type for this"
" enum (e.g. \': short\', which was the default).");
// Specify the integer type underlying this enum.
ParseType(enum_def.underlying_type);
if (!IsInteger(enum_def.underlying_type.base_type))
Error("underlying enum type must be integral");
if (proto_mode_) {
enum_def.underlying_type.base_type = BASE_TYPE_SHORT;
} else {
// Give specialized error message, since this type spec used to
// be optional in the first FlatBuffers release.
if (!IsNext(':')) Error("must specify the underlying integer type for this"
" enum (e.g. \': short\', which was the default).");
// Specify the integer type underlying this enum.
ParseType(enum_def.underlying_type);
if (!IsInteger(enum_def.underlying_type.base_type))
Error("underlying enum type must be integral");
}
// Make this type refer back to the enum it was derived from.
enum_def.underlying_type.enum_def = &enum_def;
}
@@ -752,7 +761,7 @@ void Parser::ParseEnum(bool is_union) {
if (prevsize && enum_def.vals.vec[prevsize - 1]->value >= ev.value)
Error("enum values must be specified in ascending order");
}
} while (IsNext(',') && token_ != '}');
} while (IsNext(proto_mode_ ? ';' : ',') && token_ != '}');
Expect('}');
if (enum_def.attributes.Lookup("bit_flags")) {
for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end();
@@ -765,22 +774,27 @@ void Parser::ParseEnum(bool is_union) {
}
}
void Parser::ParseDecl() {
std::vector<std::string> dc = doc_comment_;
bool fixed = IsNext(kTokenStruct);
if (!fixed) Expect(kTokenTable);
StructDef &Parser::StartStruct() {
std::string name = attribute_;
Expect(kTokenIdentifier);
auto &struct_def = *LookupCreateStruct(name);
if (!struct_def.predecl) Error("datatype already exists: " + name);
struct_def.predecl = false;
struct_def.name = name;
struct_def.doc_comment = dc;
struct_def.fixed = fixed;
// Move this struct to the back of the vector just in case it was predeclared,
// to preserve declartion order.
// to preserve declaration order.
remove(structs_.vec.begin(), structs_.vec.end(), &struct_def);
structs_.vec.back() = &struct_def;
return struct_def;
}
void Parser::ParseDecl() {
std::vector<std::string> dc = doc_comment_;
bool fixed = IsNext(kTokenStruct);
if (!fixed) Expect(kTokenTable);
auto &struct_def = StartStruct();
struct_def.doc_comment = dc;
struct_def.fixed = fixed;
ParseMetaData(struct_def);
struct_def.sortbysize =
struct_def.attributes.Lookup("original_order") == nullptr && !fixed;
@@ -875,6 +889,119 @@ void Parser::MarkGenerated() {
}
}
void Parser::ParseNamespace() {
Next();
auto ns = new Namespace();
namespaces_.push_back(ns);
for (;;) {
ns->components.push_back(attribute_);
Expect(kTokenIdentifier);
if (!IsNext('.')) break;
}
Expect(';');
}
// Best effort parsing of .proto declarations, with the aim to turn them
// in the closest corresponding FlatBuffer equivalent.
// We parse everything as identifiers instead of keywords, since we don't
// want protobuf keywords to become invalid identifiers in FlatBuffers.
void Parser::ParseProtoDecl() {
if (attribute_ == "package") {
// These are identical in syntax to FlatBuffer's namespace decl.
ParseNamespace();
} else if (attribute_ == "message") {
Next();
auto &struct_def = StartStruct();
Expect('{');
while (token_ != '}') {
// Parse the qualifier.
bool required = false;
bool repeated = false;
if (attribute_ == "optional") {
// This is the default.
} else if (attribute_ == "required") {
required = true;
} else if (attribute_ == "repeated") {
repeated = true;
} else {
Error("expecting optional/required/repeated, got: " + attribute_);
}
Type type = ParseTypeFromProtoType();
// Repeated elements get mapped to a vector.
if (repeated) {
type.element = type.base_type;
type.base_type = BASE_TYPE_VECTOR;
}
std::string name = attribute_;
Expect(kTokenIdentifier);
// Parse the field id. Since we're just translating schemas, not
// any kind of binary compatibility, we can safely ignore these, and
// assign our own.
Expect('=');
Expect(kTokenIntegerConstant);
auto &field = AddField(struct_def, name, type);
field.required = required;
// See if there's a default specified.
if (IsNext('[')) {
if (attribute_ != "default") Error("\'default\' expected");
Next();
Expect('=');
field.value.constant = attribute_;
Next();
Expect(']');
}
Expect(';');
}
Next();
} else if (attribute_ == "enum") {
// These are almost the same, just with different terminator:
ParseEnum(false);
} else if (attribute_ == "import") {
Next();
included_files_[attribute_] = true;
Expect(kTokenStringConstant);
Expect(';');
} else if (attribute_ == "option") { // Skip these.
Next();
Expect(kTokenIdentifier);
Expect('=');
Next(); // Any single token.
Expect(';');
} else {
Error("don\'t know how to parse .proto declaration starting with " +
attribute_);
}
}
// Parse a protobuf type, and map it to the corresponding FlatBuffer one.
Type Parser::ParseTypeFromProtoType() {
Expect(kTokenIdentifier);
struct type_lookup { const char *proto_type; BaseType fb_type; };
static type_lookup lookup[] = {
{ "float", BASE_TYPE_FLOAT }, { "double", BASE_TYPE_DOUBLE },
{ "int32", BASE_TYPE_INT }, { "int64", BASE_TYPE_LONG },
{ "uint32", BASE_TYPE_UINT }, { "uint64", BASE_TYPE_ULONG },
{ "sint32", BASE_TYPE_INT }, { "sint64", BASE_TYPE_LONG },
{ "fixed32", BASE_TYPE_UINT }, { "fixed64", BASE_TYPE_ULONG },
{ "sfixed32", BASE_TYPE_INT }, { "sfixed64", BASE_TYPE_LONG },
{ "bool", BASE_TYPE_BOOL },
{ "string", BASE_TYPE_STRING },
{ "bytes", BASE_TYPE_STRING },
{ nullptr, BASE_TYPE_NONE }
};
Type type;
for (auto tl = lookup; tl->proto_type; tl++) {
if (attribute_ == tl->proto_type) {
type.base_type = tl->fb_type;
Next();
return type;
}
}
ParseTypeIdent(type);
Expect(kTokenIdentifier);
return type;
}
bool Parser::Parse(const char *source, const char **include_paths,
const char *source_filename) {
if (source_filename) included_files_[source_filename] = true;
@@ -922,16 +1049,10 @@ bool Parser::Parse(const char *source, const char **include_paths,
}
// Now parse all other kinds of declarations:
while (token_ != kTokenEof) {
if (token_ == kTokenNameSpace) {
Next();
auto ns = new Namespace();
namespaces_.push_back(ns);
for (;;) {
ns->components.push_back(attribute_);
Expect(kTokenIdentifier);
if (!IsNext('.')) break;
}
Expect(';');
if (proto_mode_) {
ParseProtoDecl();
} else if (token_ == kTokenNameSpace) {
ParseNamespace();
} else if (token_ == '{') {
if (!root_struct_def) Error("no root type set to parse json with");
if (builder_.GetSize()) {