From 06fd6d640c4690cf858199b05b445160ac31334b Mon Sep 17 00:00:00 2001 From: Casper Date: Sat, 19 Jun 2021 19:21:33 -0400 Subject: [PATCH] Replace filenames in reflection with filenames+includes. (#6703) * Replace filenames in reflection with filenames+includes. This is needed for some use cases and may be just useful metadata. * deser files_included_per_file_ * check project_root * fix bazel * git clang format Co-authored-by: Casper Neo --- include/flatbuffers/reflection_generated.h | 95 +- reflection/reflection.fbs | 12 +- samples/monster.bfbs | Bin 1920 -> 1912 bytes src/idl_parser.cpp | 41 +- tests/arrays_test.bfbs | Bin 1328 -> 1312 bytes tests/monster_test.bfbs | Bin 13120 -> 13184 bytes tests/monster_test_bfbs_generated.h | 1279 ++++++++++---------- tests/test.cpp | 32 +- 8 files changed, 802 insertions(+), 657 deletions(-) diff --git a/include/flatbuffers/reflection_generated.h b/include/flatbuffers/reflection_generated.h index d0be270bf..93cc86f0d 100644 --- a/include/flatbuffers/reflection_generated.h +++ b/include/flatbuffers/reflection_generated.h @@ -32,6 +32,9 @@ struct RPCCallBuilder; struct Service; struct ServiceBuilder; +struct SchemaFile; +struct SchemaFileBuilder; + struct Schema; struct SchemaBuilder; @@ -1126,6 +1129,84 @@ inline flatbuffers::Offset CreateServiceDirect( declaration_file__); } +/// File specific information. +/// Symbols declared within a file may be recovered by iterating over all +/// symbols and examining the `declaration_file` field. +struct SchemaFile FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef SchemaFileBuilder Builder; + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_FILENAME = 4, + VT_INCLUDED_FILENAMES = 6 + }; + /// Filename, relative to project root. + const flatbuffers::String *filename() const { + return GetPointer(VT_FILENAME); + } + bool KeyCompareLessThan(const SchemaFile *o) const { + return *filename() < *o->filename(); + } + int KeyCompareWithValue(const char *val) const { + return strcmp(filename()->c_str(), val); + } + /// Names of included files, relative to project root. + const flatbuffers::Vector> *included_filenames() const { + return GetPointer> *>(VT_INCLUDED_FILENAMES); + } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyOffsetRequired(verifier, VT_FILENAME) && + verifier.VerifyString(filename()) && + VerifyOffset(verifier, VT_INCLUDED_FILENAMES) && + verifier.VerifyVector(included_filenames()) && + verifier.VerifyVectorOfStrings(included_filenames()) && + verifier.EndTable(); + } +}; + +struct SchemaFileBuilder { + typedef SchemaFile Table; + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_filename(flatbuffers::Offset filename) { + fbb_.AddOffset(SchemaFile::VT_FILENAME, filename); + } + void add_included_filenames(flatbuffers::Offset>> included_filenames) { + fbb_.AddOffset(SchemaFile::VT_INCLUDED_FILENAMES, included_filenames); + } + explicit SchemaFileBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + fbb_.Required(o, SchemaFile::VT_FILENAME); + return o; + } +}; + +inline flatbuffers::Offset CreateSchemaFile( + flatbuffers::FlatBufferBuilder &_fbb, + flatbuffers::Offset filename = 0, + flatbuffers::Offset>> included_filenames = 0) { + SchemaFileBuilder builder_(_fbb); + builder_.add_included_filenames(included_filenames); + builder_.add_filename(filename); + return builder_.Finish(); +} + +inline flatbuffers::Offset CreateSchemaFileDirect( + flatbuffers::FlatBufferBuilder &_fbb, + const char *filename = nullptr, + const std::vector> *included_filenames = nullptr) { + auto filename__ = filename ? _fbb.CreateString(filename) : 0; + auto included_filenames__ = included_filenames ? _fbb.CreateVector>(*included_filenames) : 0; + return reflection::CreateSchemaFile( + _fbb, + filename__, + included_filenames__); +} + struct Schema FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SchemaBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { @@ -1161,8 +1242,8 @@ struct Schema FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { } /// All the files used in this compilation. Files are relative to where /// flatc was invoked. - const flatbuffers::Vector> *fbs_files() const { - return GetPointer> *>(VT_FBS_FILES); + const flatbuffers::Vector> *fbs_files() const { + return GetPointer> *>(VT_FBS_FILES); } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && @@ -1184,7 +1265,7 @@ struct Schema FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { VerifyField(verifier, VT_ADVANCED_FEATURES) && VerifyOffset(verifier, VT_FBS_FILES) && verifier.VerifyVector(fbs_files()) && - verifier.VerifyVectorOfStrings(fbs_files()) && + verifier.VerifyVectorOfTables(fbs_files()) && verifier.EndTable(); } }; @@ -1214,7 +1295,7 @@ struct SchemaBuilder { void add_advanced_features(reflection::AdvancedFeatures advanced_features) { fbb_.AddElement(Schema::VT_ADVANCED_FEATURES, static_cast(advanced_features), 0); } - void add_fbs_files(flatbuffers::Offset>> fbs_files) { + void add_fbs_files(flatbuffers::Offset>> fbs_files) { fbb_.AddOffset(Schema::VT_FBS_FILES, fbs_files); } explicit SchemaBuilder(flatbuffers::FlatBufferBuilder &_fbb) @@ -1239,7 +1320,7 @@ inline flatbuffers::Offset CreateSchema( flatbuffers::Offset root_table = 0, flatbuffers::Offset>> services = 0, reflection::AdvancedFeatures advanced_features = static_cast(0), - flatbuffers::Offset>> fbs_files = 0) { + flatbuffers::Offset>> fbs_files = 0) { SchemaBuilder builder_(_fbb); builder_.add_advanced_features(advanced_features); builder_.add_fbs_files(fbs_files); @@ -1261,13 +1342,13 @@ inline flatbuffers::Offset CreateSchemaDirect( flatbuffers::Offset root_table = 0, std::vector> *services = nullptr, reflection::AdvancedFeatures advanced_features = static_cast(0), - const std::vector> *fbs_files = nullptr) { + std::vector> *fbs_files = nullptr) { auto objects__ = objects ? _fbb.CreateVectorOfSortedTables(objects) : 0; auto enums__ = enums ? _fbb.CreateVectorOfSortedTables(enums) : 0; auto file_ident__ = file_ident ? _fbb.CreateString(file_ident) : 0; auto file_ext__ = file_ext ? _fbb.CreateString(file_ext) : 0; auto services__ = services ? _fbb.CreateVectorOfSortedTables(services) : 0; - auto fbs_files__ = fbs_files ? _fbb.CreateVector>(*fbs_files) : 0; + auto fbs_files__ = fbs_files ? _fbb.CreateVectorOfSortedTables(fbs_files) : 0; return reflection::CreateSchema( _fbb, objects__, diff --git a/reflection/reflection.fbs b/reflection/reflection.fbs index 507247105..ce5f832dc 100644 --- a/reflection/reflection.fbs +++ b/reflection/reflection.fbs @@ -116,6 +116,16 @@ enum AdvancedFeatures : ulong (bit_flags) { DefaultVectorsAndStrings, } +/// File specific information. +/// Symbols declared within a file may be recovered by iterating over all +/// symbols and examining the `declaration_file` field. +table SchemaFile { + /// Filename, relative to project root. + filename:string (required, key); + /// Names of included files, relative to project root. + included_filenames:[string]; +} + table Schema { objects:[Object] (required); // Sorted. enums:[Enum] (required); // Sorted. @@ -126,7 +136,7 @@ table Schema { advanced_features:AdvancedFeatures; /// All the files used in this compilation. Files are relative to where /// flatc was invoked. - fbs_files:[string]; + fbs_files:[SchemaFile]; // Sorted. } root_type Schema; diff --git a/samples/monster.bfbs b/samples/monster.bfbs index 4fa92c1f63d86cf66945fa9924b0286696379d04..c49fa8cd971c7c7d94b24e0991656bf70931f003 100644 GIT binary patch delta 65 zcmZqR|G{S}!@$7cyQW3=TkS0K_Ul3=-YJ#K6Ea Mc_N$eMh6Ra0RGwsn*aa+ diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index 31af81431..720e80323 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -3530,10 +3530,29 @@ void Parser::Serialize() { const std::string *file = (*it)->declaration_file; if (file) files.insert(*file); } - // TODO(caspern): CreateVectorOfSharedStrings - std::vector> file_offsets; - for (auto it = files.begin(); it != files.end(); it++) { - file_offsets.push_back(builder_.CreateSharedString(*it)); + + // Create Schemafiles vector of tables. + flatbuffers::Offset< + flatbuffers::Vector>> + schema_files__; + if (!opts.project_root.empty()) { + std::vector> schema_files; + std::vector> included_files; + for (auto f = files_included_per_file_.begin(); + f != files_included_per_file_.end(); f++) { + const auto filename__ = builder_.CreateSharedString( + RelativeToRootPath(opts.project_root, f->first)); + for (auto i = f->second.begin(); i != f->second.end(); i++) { + included_files.push_back(builder_.CreateSharedString( + RelativeToRootPath(opts.project_root, *i))); + } + const auto included_files__ = builder_.CreateVector(included_files); + included_files.clear(); + + schema_files.push_back( + reflection::CreateSchemaFile(builder_, filename__, included_files__)); + } + schema_files__ = builder_.CreateVectorOfSortedTables(&schema_files); } const auto objs__ = builder_.CreateVectorOfSortedTables(&object_offsets); @@ -3541,11 +3560,11 @@ void Parser::Serialize() { const auto fiid__ = builder_.CreateString(file_identifier_); const auto fext__ = builder_.CreateString(file_extension_); const auto serv__ = builder_.CreateVectorOfSortedTables(&service_offsets); - const auto files__ = builder_.CreateVector(file_offsets); const auto schema_offset = reflection::CreateSchema( builder_, objs__, enum__, fiid__, fext__, (root_struct_def_ ? root_struct_def_->serialized_location : 0), serv__, - static_cast(advanced_features_), files__); + static_cast(advanced_features_), + schema_files__); if (opts.size_prefixed) { builder_.FinishSizePrefixed(schema_offset, reflection::SchemaIdentifier()); } else { @@ -3967,6 +3986,16 @@ bool Parser::Deserialize(const reflection::Schema *schema) { } } advanced_features_ = schema->advanced_features(); + + if (schema->fbs_files()) + for (auto s = schema->fbs_files()->begin(); s != schema->fbs_files()->end(); + ++s) { + for (auto f = s->included_filenames()->begin(); + f != s->included_filenames()->end(); ++f) { + files_included_per_file_[s->filename()->str()].insert(f->str()); + } + } + return true; } diff --git a/tests/arrays_test.bfbs b/tests/arrays_test.bfbs index 64dd12eee045847e87ea6f447b419a381bbcb185..5ab4edc0d90949f177a11d09611a5f03217cd3d4 100644 GIT binary patch delta 66 zcmdnMwSdb~hJk^>$;~O4fkB8tg@J{EgMo)ZfI);oflzP9Kgr` NRKdW&xY6@D3jpPa27&+p delta 82 zcmZ3$wSmi1fq{X+$;~MkNHK^o=rFJ_a4_&N2msj<3smWJ173%jYZrLEi@o diff --git a/tests/monster_test.bfbs b/tests/monster_test.bfbs index 906d8e68230b76285fd0b0f901226b57376e134f..f6ab2cceb5fe7f2afe3aa4a1a9e299cb6abf3217 100644 GIT binary patch delta 276 zcmX?*){xF2!@$7cCF1jsf4Vih2k0OAB51_llwT>-?5Kzs#=nSnSrKaYV0 zNc;Nx1p_GtHX!~36xhJTz;FVnz=xfI!2l>I!O6hD2b5#cVPFsenkA#j!0-Wxzi2Tq ztk7p*SfIzi&;nF*2Pp4hz);W7V8FnTV+hn}#K3R{$Oc(v0mL9jf-H2o`2T+iP#nYt zIp_e89RU>M0ICOx0qtj40~GrKG@}40z5uEgBt8YmegG5)IY|Vl8ssq^1_7`GHGpgZ JhK=`LZt=+DqOyaC5Bnfw8RQS=53CRw!F4Mu}O@CFtZya+GD zVDw;kIPZDS;bgmfy9|J2lFTJE32<;RG<19V_)^%2RE$Jh_=bP#=q?E}gjEdp2rT*R zJZ{ZyO4l~*)%aH6U53Jix(bgTname()->c_str(), "MyGame.Example.Monster"); TEST_EQ_STR(root_table->declaration_file()->c_str(), "//monster_test.fbs"); @@ -935,12 +936,37 @@ void ReflectionTest(uint8_t *flatbuf, size_t length) { ->declaration_file() ->c_str(), "//include_test/sub/include_test2.fbs"); + + // Check scheam filenames and their includes. TEST_EQ(schema.fbs_files()->size(), 3); - TEST_EQ_STR(schema.fbs_files()->Get(0)->c_str(), + + const auto fbs0 = schema.fbs_files()->Get(0); + TEST_EQ_STR(fbs0->filename()->c_str(), "//include_test/include_test1.fbs"); + const auto fbs0_includes = fbs0->included_filenames(); + TEST_EQ(fbs0_includes->size(), 2); + + // TODO(caspern): Should we force or disallow inclusion of self? + TEST_EQ_STR(fbs0_includes->Get(0)->c_str(), "//include_test/include_test1.fbs"); - TEST_EQ_STR(schema.fbs_files()->Get(1)->c_str(), + TEST_EQ_STR(fbs0_includes->Get(1)->c_str(), "//include_test/sub/include_test2.fbs"); - TEST_EQ_STR(schema.fbs_files()->Get(2)->c_str(), "//monster_test.fbs"); + + const auto fbs1 = schema.fbs_files()->Get(1); + TEST_EQ_STR(fbs1->filename()->c_str(), + "//include_test/sub/include_test2.fbs"); + const auto fbs1_includes = fbs1->included_filenames(); + TEST_EQ(fbs1_includes->size(), 2); + TEST_EQ_STR(fbs1_includes->Get(0)->c_str(), + "//include_test/include_test1.fbs"); + TEST_EQ_STR(fbs1_includes->Get(1)->c_str(), + "//include_test/sub/include_test2.fbs"); + + const auto fbs2 = schema.fbs_files()->Get(2); + TEST_EQ_STR(fbs2->filename()->c_str(), "//monster_test.fbs"); + const auto fbs2_includes = fbs2->included_filenames(); + TEST_EQ(fbs2_includes->size(), 1); + TEST_EQ_STR(fbs2_includes->Get(0)->c_str(), + "//include_test/include_test1.fbs"); // Check Root table fields auto fields = root_table->fields();