diff --git a/CMakeLists.txt b/CMakeLists.txt index 439ad544d..95d7a5a95 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -166,6 +166,10 @@ set(FlatBuffers_Compiler_SRCS src/bfbs_gen_lua.h src/bfbs_namer.h include/flatbuffers/code_generators.h + src/binary_annotator.h + src/binary_annotator.cpp + src/annotated_binary_text_gen.h + src/annotated_binary_text_gen.cpp src/bfbs_gen_lua.cpp src/code_generators.cpp grpc/src/compiler/schema_interface.h diff --git a/include/flatbuffers/flatc.h b/include/flatbuffers/flatc.h index 3dba5e141..af4ccae70 100644 --- a/include/flatbuffers/flatc.h +++ b/include/flatbuffers/flatc.h @@ -103,6 +103,11 @@ class FlatCompiler { void Error(const std::string &err, bool usage = true, bool show_exe_name = true) const; + void AnnotateBinaries(const uint8_t *binary_schema, + uint64_t binary_schema_size, + const std::string & schema_filename, + const std::vector &binary_files); + InitParams params_; }; diff --git a/include/flatbuffers/reflection.h b/include/flatbuffers/reflection.h index 8e700f0b7..776a9c258 100644 --- a/include/flatbuffers/reflection.h +++ b/include/flatbuffers/reflection.h @@ -279,6 +279,12 @@ T *GetAnyFieldAddressOf(const Struct &st, const reflection::Field &field) { return reinterpret_cast(st.GetAddressOf(field.offset())); } +// Loop over all the fields of the provided `object` and call `func` on each one +// in increasing order by their field->id(). If `reverse` is true, `func` is +// called in descending order +void ForAllFields(const reflection::Object *object, bool reverse, + std::function func); + // ------------------------- SETTERS ------------------------- // Set any scalar field, if you know its exact type. diff --git a/src/BUILD.bazel b/src/BUILD.bazel index 4a7651bca..b01551dc0 100644 --- a/src/BUILD.bazel +++ b/src/BUILD.bazel @@ -34,10 +34,14 @@ cc_library( cc_library( name = "flatc_library", srcs = [ + "annotated_binary_text_gen.cpp", + "annotated_binary_text_gen.h", "bfbs_gen.h", "bfbs_gen_lua.cpp", "bfbs_gen_lua.h", "bfbs_namer.h", + "binary_annotator.cpp", + "binary_annotator.h", "flatc.cpp", "namer.h", ], diff --git a/src/annotated_binary_text_gen.cpp b/src/annotated_binary_text_gen.cpp new file mode 100644 index 000000000..6f1fa6a74 --- /dev/null +++ b/src/annotated_binary_text_gen.cpp @@ -0,0 +1,320 @@ +#include "annotated_binary_text_gen.h" + +#include "flatbuffers/util.h" + +namespace flatbuffers { +namespace { + +struct OutputConfig { + size_t largest_type_string = 10; + + size_t largest_value_string = 20; + + size_t max_bytes_per_line = 8; + + size_t offset_max_char = 4; + + char delimiter = '|'; +}; + +static std::string ToString(const BinarySectionType type) { + switch (type) { + case BinarySectionType::Header: return "header"; + case BinarySectionType::Table: return "table"; + case BinarySectionType::RootTable: return "root_table"; + case BinarySectionType::VTable: return "vtable"; + case BinarySectionType::Struct: return "struct"; + case BinarySectionType::String: return "string"; + case BinarySectionType::Vector: return "vector"; + case BinarySectionType::Unknown: return "unknown"; + case BinarySectionType::Union: return "union"; + case BinarySectionType::Padding: return "padding"; + default: return "todo"; + } +} + +static bool IsOffset(const BinaryRegionType type) { + return type == BinaryRegionType::UOffset || type == BinaryRegionType::SOffset; +} + +template +std::string ToValueString(const BinaryRegion ®ion, const uint8_t *binary) { + std::string s; + s += "0x"; + const T val = GetScalar(binary + region.offset); + const uint64_t start_index = region.offset + region.length - 1; + for (uint64_t i = 0; i < region.length; ++i) { + s += ToHex(binary[start_index - i]); + } + s += " ("; + s += std::to_string(val); + s += ")"; + return s; +} + +template<> +std::string ToValueString(const BinaryRegion ®ion, + const uint8_t *binary) { + return std::string(reinterpret_cast(binary + region.offset), + static_cast(region.array_length)); +} + +static std::string ToValueString(const BinaryRegion ®ion, + const uint8_t *binary, + const OutputConfig &output_config) { + std::string s; + + if (region.array_length) { + if (region.type == BinaryRegionType::Uint8 || + region.type == BinaryRegionType::Unknown) { + // Interpet each value as a ASCII to aid debugging + for (uint64_t i = 0; i < region.array_length; ++i) { + const uint8_t c = *(binary + region.offset + i); + s += isprint(c) ? toascii(c) : '.'; + } + return s; + } else if (region.type == BinaryRegionType::Char) { + // string value + return ToValueString(region, binary); + } + } + + switch (region.type) { + case BinaryRegionType::Uint32: + return ToValueString(region, binary); + case BinaryRegionType::Int32: return ToValueString(region, binary); + case BinaryRegionType::Uint16: + return ToValueString(region, binary); + case BinaryRegionType::Int16: return ToValueString(region, binary); + case BinaryRegionType::Bool: return ToValueString(region, binary); + case BinaryRegionType::Uint8: return ToValueString(region, binary); + case BinaryRegionType::Char: return ToValueString(region, binary); + case BinaryRegionType::Byte: + case BinaryRegionType::Int8: return ToValueString(region, binary); + case BinaryRegionType::Int64: return ToValueString(region, binary); + case BinaryRegionType::Uint64: + return ToValueString(region, binary); + case BinaryRegionType::Double: return ToValueString(region, binary); + case BinaryRegionType::Float: return ToValueString(region, binary); + + // Handle Offsets separately, incase they add additional details. + case BinaryRegionType::UOffset: + s += ToValueString(region, binary); + break; + case BinaryRegionType::SOffset: + s += ToValueString(region, binary); + break; + case BinaryRegionType::VOffset: + s += ToValueString(region, binary); + break; + + default: break; + } + // If this is an offset type, include the calculated offset location in the + // value. + // TODO(dbaileychess): It might be nicer to put this in the comment field. + if (IsOffset(region.type)) { + s += " Loc: +0x"; + s += ToHex(region.points_to_offset, output_config.offset_max_char); + } + return s; +} + +struct DocContinuation { + // The start column where the value text first starts + size_t value_start_column = 0; + + // The remaining part of the doc to print. + std::string value; +}; + +static std::string GenerateTypeString(const BinaryRegion ®ion) { + return ToString(region.type) + + ((region.array_length) + ? "[" + std::to_string(region.array_length) + "]" + : ""); +} + +static std::string GenerateDocumentation(const BinaryRegion ®ion, + const BinarySection &, + const uint8_t *binary, + DocContinuation &continuation, + const OutputConfig &output_config) { + std::string s; + + // Check if there is a doc continuation that should be prioritized. + if (continuation.value_start_column) { + s += std::string(continuation.value_start_column - 2, ' '); + s += output_config.delimiter; + s += " "; + + s += continuation.value.substr(0, output_config.max_bytes_per_line); + continuation.value = continuation.value.substr( + std::min(output_config.max_bytes_per_line, continuation.value.size())); + return s; + } + + { + std::stringstream ss; + ss << std::setw(output_config.largest_type_string) << std::left; + ss << GenerateTypeString(region); + s += ss.str(); + } + s += " "; + s += output_config.delimiter; + s += " "; + if (region.array_length) { + // Record where the value is first being outputted. + continuation.value_start_column = s.size(); + + // Get the full-length value, which we will chunk below. + const std::string value = ToValueString(region, binary, output_config); + + std::stringstream ss; + ss << std::setw(output_config.largest_value_string) << std::left; + ss << value.substr(0, output_config.max_bytes_per_line); + s += ss.str(); + + continuation.value = + value.substr(std::min(output_config.max_bytes_per_line, value.size())); + } else { + std::stringstream ss; + ss << std::setw(output_config.largest_value_string) << std::left; + ss << ToValueString(region, binary, output_config); + s += ss.str(); + } + + s += " "; + if (!region.comment.empty()) { + s += output_config.delimiter; + s += " "; + s += region.comment; + } + + return s; +} + +static std::string GenerateRegion(const BinaryRegion ®ion, + const BinarySection §ion, + const uint8_t *binary, + const OutputConfig &output_config) { + std::string s; + bool doc_generated = false; + DocContinuation doc_continuation; + for (uint64_t i = 0; i < region.length; ++i) { + if ((i % output_config.max_bytes_per_line) == 0) { + // Start a new line of output + s += '\n'; + s += " "; + s += "+0x"; + s += ToHex(region.offset + i, output_config.offset_max_char); + s += " "; + s += output_config.delimiter; + } + + // Add each byte + s += " "; + s += ToHex(binary[region.offset + i]); + + // Check for end of line or end of region conditions. + if (((i + 1) % output_config.max_bytes_per_line == 0) || + i + 1 == region.length) { + if (i + 1 == region.length) { + // We are out of bytes but haven't the kMaxBytesPerLine, so we need to + // zero those out to align everything globally. + for (uint64_t j = i + 1; (j % output_config.max_bytes_per_line) != 0; + ++j) { + s += " "; + } + } + s += " "; + s += output_config.delimiter; + // This is the end of the first line or its the last byte of the region, + // generate the end-of-line documentation. + if (!doc_generated) { + s += " "; + s += GenerateDocumentation(region, section, binary, doc_continuation, + output_config); + + // If we have a value in the doc continuation, that means the doc is + // being printed on multiple lines. + doc_generated = doc_continuation.value.empty(); + } + } + } + + return s; +} + +static std::string GenerateSection(const BinarySection §ion, + const uint8_t *binary, + const OutputConfig &output_config) { + std::string s; + s += "\n"; + s += ToString(section.type); + if (!section.name.empty()) { s += " (" + section.name + ")"; } + s += ":"; + for (const BinaryRegion ®ion : section.regions) { + s += GenerateRegion(region, section, binary, output_config); + } + return s; +} +} // namespace + +bool AnnotatedBinaryTextGenerator::Generate( + const std::string &filename, const std::string &schema_filename) { + OutputConfig output_config; + output_config.max_bytes_per_line = options_.max_bytes_per_line; + + // Given the length of the binary, we can calculate the maximum number of + // characters to display in the offset hex: (i.e. 2 would lead to 0XFF being + // the max output). + output_config.offset_max_char = + binary_length_ > 0xFFFFFF + ? 8 + : (binary_length_ > 0xFFFF ? 6 : (binary_length_ > 0xFF ? 4 : 2)); + + // Find the largest type string of all the regions in this file, so we can + // align the output nicely. + output_config.largest_type_string = 0; + for (const auto §ion : annotations_) { + for (const auto ®ion : section.second.regions) { + std::string s = GenerateTypeString(region); + if (s.size() > output_config.largest_type_string) { + output_config.largest_type_string = s.size(); + } + + // Don't consider array regions, as they will be split to multiple lines. + if (!region.array_length) { + s = ToValueString(region, binary_, output_config); + if (s.size() > output_config.largest_value_string) { + output_config.largest_value_string = s.size(); + } + } + } + } + + // Generate each of the binary sections + std::string s; + + s += "// Annotated Flatbuffer Binary\n"; + s += "//\n"; + s += "// Schema file: " + schema_filename + "\n"; + s += "// Binary file: " + filename + "\n"; + + for (const auto §ion : annotations_) { + s += GenerateSection(section.second, binary_, output_config); + s += "\n"; + } + + // Modify the output filename. + std::string output_filename = StripExtension(filename); + output_filename += options_.output_postfix; + output_filename += + "." + (options_.output_extension.empty() ? GetExtension(filename) + : options_.output_extension); + + return SaveFile(output_filename.c_str(), s, false); +} + +} // namespace flatbuffers diff --git a/src/annotated_binary_text_gen.h b/src/annotated_binary_text_gen.h new file mode 100644 index 000000000..42b134a0c --- /dev/null +++ b/src/annotated_binary_text_gen.h @@ -0,0 +1,71 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_ANNOTATED_BINARY_TEXT_GEN_H_ +#define FLATBUFFERS_ANNOTATED_BINARY_TEXT_GEN_H_ + +#include +#include +#include + +#include "binary_annotator.h" + +namespace flatbuffers { + +class AnnotatedBinaryTextGenerator { + public: + struct Options { + // The maximum number of raw bytes to print per line in the output. 8 is a + // good default due to the largest type (double) being 8 bytes long. + size_t max_bytes_per_line = 8; + + // The output file postfix, appended between the filename and the extension. + // Example binary1.bin -> binary1_annotated.bin + std::string output_postfix = ""; + + // The output file extension, replacing any extension given. If empty, don't + // change the provided extension. AFB = Annotated Flatbuffer Binary + // + // Example: binary1.bin -> binary1.afb + std::string output_extension = "afb"; + }; + + explicit AnnotatedBinaryTextGenerator( + const Options &options, std::map annotations, + const uint8_t *const binary, const int64_t binary_length) + : annotations_(std::move(annotations)), + binary_(binary), + binary_length_(binary_length), + options_(options) {} + + // Generate the annotated binary for the given `filename`. Returns true if the + // annotated binary was succesfully saved. + bool Generate(const std::string &filename, const std::string &schema_filename); + + private: + const std::map annotations_; + + // The binary data itself. + const uint8_t *binary_; + const int64_t binary_length_; + + // Output configuration + const Options options_; +}; + +} // namespace flatbuffers + +#endif // FLATBUFFERS_ANNOTATED_BINARY_TEXT_GEN_H_ diff --git a/src/binary_annotator.cpp b/src/binary_annotator.cpp new file mode 100644 index 000000000..862936c34 --- /dev/null +++ b/src/binary_annotator.cpp @@ -0,0 +1,769 @@ +#include "binary_annotator.h" + +#include +#include + +#include "flatbuffers/reflection.h" +#include "flatbuffers/verifier.h" + +namespace flatbuffers { +namespace { + +static BinaryRegion MakeBinaryRegion( + const uint64_t offset = 0, const uint64_t length = 0, + const BinaryRegionType type = BinaryRegionType::Unknown, + const uint64_t array_length = 0, const uint64_t points_to_offset = 0, + const std::string &comment = "") { + BinaryRegion region; + region.offset = offset; + region.length = length; + region.type = type; + region.array_length = array_length; + region.points_to_offset = points_to_offset; + region.comment = comment; + return region; +} + +static BinarySection MakeBinarySection( + const std::string &name, const BinarySectionType type, + const std::vector ®ions) { + BinarySection section; + section.name = name; + section.type = type; + section.regions = regions; + return section; +} + +static uint64_t BuildField(const uint64_t offset, + const reflection::Field *field, + std::vector ®ions) { + const uint64_t type_size = GetTypeSize(field->type()->base_type()); + const BinaryRegionType type = GetRegionType(field->type()->base_type()); + regions.emplace_back(MakeBinaryRegion( + offset, type_size, type, 0, 0, + std::string("table field `") + field->name()->c_str() + "` (" + + reflection::EnumNameBaseType(field->type()->base_type()) + ")")); + return offset + type_size; +} + +static uint64_t BuildStructureField(const uint64_t offset, + const reflection::Object *object, + const reflection::Field *field, + std::vector ®ions) { + const uint64_t type_size = GetTypeSize(field->type()->base_type()); + regions.emplace_back(MakeBinaryRegion( + offset, type_size, GetRegionType(field->type()->base_type()), 0, 0, + std::string("struct field `") + object->name()->c_str() + "." + + field->name()->c_str() + "` (" + + reflection::EnumNameBaseType(field->type()->base_type()) + ")")); + return offset + type_size; +} + +static uint64_t BuildArrayField(uint64_t offset, + const reflection::Object *object, + const reflection::Field *field, + const uint16_t array_length, + std::vector ®ions) { + const uint64_t type_size = GetTypeSize(field->type()->element()); + for (uint16_t i = 0; i < array_length; ++i) { + regions.emplace_back(MakeBinaryRegion( + offset, type_size, GetRegionType(field->type()->element()), 0, 0, + std::string("array field `") + object->name()->c_str() + "." + + field->name()->c_str() + "[" + std::to_string(i) + "]` (" + + reflection::EnumNameBaseType(field->type()->element()) + ")")); + offset += type_size; + } + + // The following groups the complete array together which shows up nicely as + // an array, but then we don't show the individual values. So the above method + // treats each field of the array as a separate region. + // regions.emplace_back( + // BinaryRegion{ offset, array_length * type_size, + // GetRegionType(field->type()->element()), array_length, 0, + // std::string("array field '") + object->name()->c_str() + + // "." + field->name()->c_str() + "' value" }); + + return offset; +} + +static bool IsNonZeroRegion(uint64_t offset, uint64_t length, + const uint8_t *binary) { + for (uint64_t i = offset; i < offset + length; ++i) { + if (binary[i] != 0) { return true; } + } + return false; +} + +} // namespace + +std::map BinaryAnnotator::Annotate() { + flatbuffers::Verifier verifier(bfbs_, static_cast(bfbs_length_)); + if (!reflection::VerifySchemaBuffer(verifier)) { return {}; } + + // Make sure we start with a clean slate. + vtables_.clear(); + strings_.clear(); + sections_.clear(); + + // First parse the header region which always start at offset 0. + // The returned offset will point to the root_table location. + const uint64_t root_table_offset = BuildHeader(0); + + // Build the root table, and all else will be referenced from it. + BuildTable(root_table_offset, BinarySectionType::RootTable, + schema_->root_table()); + + // Now that all the sections are built, scan the regions between them and + // insert padding bytes that are implied. + FixMissingSections(); + + return sections_; +} + +uint64_t BinaryAnnotator::BuildHeader(const uint64_t offset) { + std::vector regions; + + // TODO(dbaileychess): sized prefixed value + const uint32_t root_table_offset = GetScalar(offset); + regions.emplace_back(MakeBinaryRegion( + offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0, root_table_offset, + std::string("offset to root table `") + + schema_->root_table()->name()->str() + "`")); + + if (IsNonZeroRegion(offset, 4, binary_)) { + // Check if the file identifier region has non-zero data, and assume its the + // file identifier. Otherwise, it will get filled in with padding later. + regions.emplace_back(MakeBinaryRegion( + offset + sizeof(uint32_t), 4 * sizeof(uint8_t), BinaryRegionType::Char, + 4, 0, std::string("File Identifier"))); + } + + sections_.insert(std::make_pair( + offset, MakeBinarySection("", BinarySectionType::Header, regions))); + return root_table_offset; +} + +void BinaryAnnotator::BuildVTable(uint64_t offset, + const reflection::Object *table) { + const uint64_t vtable_offset = offset; + + // First see if we have used this vtable before, if so skip building it again. + auto it = vtables_.find(vtable_offset); + if (it != vtables_.end()) { return; } + + std::vector regions; + + // Vtables start with the size of the vtable + const uint16_t vtable_size = GetScalar(offset); + regions.emplace_back(MakeBinaryRegion(offset, sizeof(uint16_t), + BinaryRegionType::Uint16, 0, 0, + std::string("size of this vtable"))); + offset += sizeof(uint16_t); + + // Then they have the size of the table they reference. + const uint16_t table_size = GetScalar(offset); + regions.emplace_back( + MakeBinaryRegion(offset, sizeof(uint16_t), BinaryRegionType::Uint16, 0, 0, + std::string("size of referring table"))); + offset += sizeof(uint16_t); + + const uint64_t offset_start = offset; + + // A mapping between field (and its id) to the relative offset (uin16_t) from + // the start of the table. + std::map fields; + + // Counter for determining if the binary has more vtable entries than the + // schema provided. This can occur if the binary was created at a newer schema + // version and is being processed with an older one. + uint16_t fields_processed = 0; + + // Loop over all the fields. + ForAllFields(table, /*reverse=*/false, [&](const reflection::Field *field) { + const uint64_t field_offset = offset_start + field->id() * sizeof(uint16_t); + + if (field_offset >= vtable_offset + vtable_size) { + // This field_offset is too large for this vtable, so it must come from a + // newer schema than the binary was create with or the binary writer did + // not write it. For either case, it is safe to ignore. + + // TODO(dbaileychess): We could show which fields are not set an their + // default values if we want. We just need a way to make it obvious that + // it isn't part of the buffer. + return; + } + + const uint16_t offset_from_table = GetScalar(field_offset); + + VTable::Entry entry; + entry.field = field; + entry.offset_from_table = offset_from_table; + fields.insert(std::make_pair(field->id(), entry)); + + std::string default_label; + if (offset_from_table == 0) { + // Not present, so could be default or be optional. + if (field->required()) { + // If this is a required field, make it known this is an error. + regions.push_back(MakeBinaryRegion( + field_offset, sizeof(uint16_t), BinaryRegionType::VOffset, 0, 0, + std::string("ERROR: required field `") + field->name()->c_str() + + "` (id: " + std::to_string(field->id()) + ") is not present!")); + return; + } else { + // Its an optional field, so get the default value and interpret and + // provided an annotation for it. + if (IsScalar(field->type()->base_type())) { + default_label += " type()->base_type()) + ? std::to_string(field->default_real()) + : std::to_string(field->default_integer()); + default_label += "> ("; + } else { + default_label += " ("; + } + default_label += + reflection::EnumNameBaseType(field->type()->base_type()); + default_label += ")"; + } + } + + regions.push_back(MakeBinaryRegion( + field_offset, sizeof(uint16_t), BinaryRegionType::VOffset, 0, 0, + std::string("offset to field `") + field->name()->c_str() + + "` (id: " + std::to_string(field->id()) + ")" + default_label)); + + fields_processed++; + }); + + // Check if we covered all the expectant fields. If not, we need to add them + // as unknown fields. + const uint16_t expectant_vtable_fields = + (vtable_size - sizeof(uint16_t) - sizeof(uint16_t)) / sizeof(uint16_t); + + for (uint16_t id = fields_processed; id < expectant_vtable_fields; ++id) { + const uint64_t field_offset = offset_start + id * sizeof(uint16_t); + const uint16_t offset_from_table = GetScalar(field_offset); + + VTable::Entry entry; + entry.field = nullptr; // No field to reference. + entry.offset_from_table = offset_from_table; + fields.insert(std::make_pair(id, entry)); + + regions.push_back(MakeBinaryRegion( + field_offset, sizeof(uint16_t), BinaryRegionType::VOffset, 0, 0, + std::string("offset to unknown field (id: " + std::to_string(id) + + ")"))); + } + + sections_[vtable_offset] = MakeBinarySection( + table->name()->str(), BinarySectionType::VTable, std::move(regions)); + + VTable vtable; + vtable.fields = std::move(fields); + vtable.table_size = table_size; + vtable.vtable_size = vtable_size; + + vtables_[vtable_offset] = vtable; +} + +void BinaryAnnotator::BuildTable(uint64_t offset, const BinarySectionType type, + const reflection::Object *table) { + std::vector regions; + const uint64_t table_offset = offset; + + // Tables start with the vtable + const uint64_t vtable_offset = table_offset - GetScalar(offset); + regions.emplace_back( + MakeBinaryRegion(offset, sizeof(int32_t), BinaryRegionType::SOffset, 0, + vtable_offset, std::string("offset to vtable"))); + offset += sizeof(int32_t); + + // Parse the vtable first so we know what the rest of the fields in the table + // are. + BuildVTable(vtable_offset, table); + + const VTable &vtable = vtables_.at(vtable_offset); + + // This is the size and length of this table. + const uint16_t table_size = vtable.table_size; + const uint64_t table_end_offset = table_offset + table_size; + + const uint64_t field_offset_start = offset; + + // We need to iterate over the vtable fields by their offset in the binary, + // not by their IDs. So copy them over to another vector that we can sort on + // the offset_from_table property. + std::vector fields; + for (const auto &vtable_field : vtable.fields) { + fields.push_back(vtable_field.second); + } + + std::stable_sort(fields.begin(), fields.end(), + [](const VTable::Entry &a, const VTable::Entry &b) { + return a.offset_from_table < b.offset_from_table; + }); + + // Iterate over all the fields by order of their offset. + for (size_t i = 0; i < fields.size(); ++i) { + const reflection::Field *field = fields[i].field; + const uint16_t offset_from_table = fields[i].offset_from_table; + + if (offset_from_table == 0) { + // Skip non-present fields. + continue; + } + + // The field offsets are relative to the start of the table. + const uint64_t field_offset = table_offset + offset_from_table; + + // We have a vtable entry for a non-existant field, that means its a binary + // generated by a newer schema than we are currently processing. + if (field == nullptr) { + // Calculate the length of this unknown field. + const uint64_t unknown_field_length = + // Check if there is another unknown field after this one. + ((i + 1 < fields.size()) + ? table_offset + fields[i + 1].offset_from_table + // Otherwise use the known end of the table. + : table_end_offset) - + field_offset; + + std::string hint; + if (unknown_field_length == 4) { + // The field is 4 in length, so it could be an offset? Provide a hint. + hint += " (field_offset)); + hint += ">"; + } + + regions.emplace_back( + MakeBinaryRegion(field_offset, unknown_field_length * sizeof(uint8_t), + BinaryRegionType::Unknown, unknown_field_length, 0, + std::string("Unknown field") + hint)); + continue; + } + + if (IsScalar(field->type()->base_type())) { + // These are the raw values store in the table. + offset = BuildField(field_offset, field, regions); + continue; + } + + switch (field->type()->base_type()) { + case reflection::BaseType::Obj: { + const reflection::Object *next_object = + schema_->objects()->Get(field->type()->index()); + + if (next_object->is_struct()) { + // Structs are stored inline. + offset = BuildStruct(field_offset, regions, next_object); + } else { + const uint64_t next_object_offset = + field_offset + GetScalar(field_offset); + regions.emplace_back(MakeBinaryRegion( + field_offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0, + next_object_offset, + std::string("offset to field `") + field->name()->c_str() + "`")); + offset += sizeof(uint32_t); + + BuildTable(next_object_offset, BinarySectionType::Table, next_object); + } + } break; + + case reflection::BaseType::String: { + const uint64_t string_offset = + field_offset + GetScalar(field_offset); + + regions.emplace_back(MakeBinaryRegion( + field_offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0, + string_offset, + std::string("offset to field `") + field->name()->c_str() + "`")); + + BuildString(string_offset, table, field); + } break; + + case reflection::BaseType::Vector: { + const uint64_t vector_offset = + field_offset + GetScalar(field_offset); + + regions.emplace_back(MakeBinaryRegion( + field_offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0, + vector_offset, + std::string("offset to field `") + field->name()->c_str() + "`")); + + BuildVector(vector_offset, table, field, table_offset, vtable); + } break; + + case reflection::BaseType::Union: { + const uint64_t union_offset = + field_offset + GetScalar(field_offset); + + // The union type field is always one less than the union itself. + const uint16_t union_type_id = field->id() - 1; + + auto vtable_entry = vtable.fields.find(union_type_id); + if (vtable_entry == vtable.fields.end()) { + // TODO(dbaileychess): need to capture this error condition. + break; + } + + const uint64_t type_offset = + table_offset + vtable_entry->second.offset_from_table; + + const uint8_t realized_type = GetScalar(type_offset); + + const std::string enum_type = + BuildUnion(union_offset, realized_type, field); + + regions.emplace_back(MakeBinaryRegion( + field_offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0, + union_offset, + std::string("offset to field `") + field->name()->c_str() + + "` (union of type `" + enum_type + "`)")); + + } break; + + default: break; + } + } + + // Fill in any regions that weren't covered above, as those are padding + // regions. + size_t region_index = 1; + std::vector padding_regions; + uint64_t i = field_offset_start; + while (region_index < regions.size() && i < table_end_offset) { + const uint64_t region_start = regions[region_index].offset; + const uint64_t region_end = region_start + regions[region_index].length; + + if (i < region_start) { + const uint64_t pad_bytes = region_start - i; + // We are at an index that is lower than any region, so pad upto its + // offset. + padding_regions.emplace_back( + MakeBinaryRegion(i, pad_bytes * sizeof(uint8_t), + BinaryRegionType::Uint8, pad_bytes, 0, "padding")); + i = region_end; + region_index++; + } else if (i < region_end) { + i = region_end + 1; + } else { + region_index++; + } + } + + // Handle the case where there is padding after the last known binary + // region. Calculate where we left off towards the expected end of the + // table. + if (i < table_end_offset) { + const uint64_t pad_bytes = table_end_offset - i + 1; + padding_regions.emplace_back( + MakeBinaryRegion(i - 1, pad_bytes * sizeof(uint8_t), + BinaryRegionType::Uint8, pad_bytes, 0, "padding")); + } + + regions.insert(regions.end(), padding_regions.begin(), padding_regions.end()); + + std::stable_sort(regions.begin(), regions.end(), + [&](const BinaryRegion &a, const BinaryRegion &b) { + return a.offset < b.offset; + }); + + sections_.insert(std::make_pair( + table_offset, + MakeBinarySection(table->name()->str(), type, std::move(regions)))); +} + +uint64_t BinaryAnnotator::BuildStruct(uint64_t offset, + std::vector ®ions, + const reflection::Object *object) { + if (!object->is_struct()) { return offset; } + + // Loop over all the fields in increasing order + ForAllFields(object, /*reverse=*/false, [&](const reflection::Field *field) { + if (IsScalar(field->type()->base_type())) { + offset = BuildStructureField(offset, object, field, regions); + + } else if (field->type()->base_type() == reflection::BaseType::Obj) { + // Structs are stored inline, even when nested. + offset = BuildStruct(offset, regions, + schema_->objects()->Get(field->type()->index())); + } else if (field->type()->base_type() == reflection::BaseType::Array) { + // Arrays are just repeated structures. + if (IsScalar(field->type()->element())) { + offset = BuildArrayField(offset, object, field, + field->type()->fixed_length(), regions); + } else { + for (uint16_t i = 0; i < field->type()->fixed_length(); ++i) { + // TODO(dbaileychess): This works, but the comments on the fields lose + // some context. Need to figure a way how to plumb the nested arrays + // comments together that isn't too confusing. + offset = BuildStruct(offset, regions, + schema_->objects()->Get(field->type()->index())); + } + } + } + }); + + return offset; +} + +void BinaryAnnotator::BuildString(uint64_t offset, + const reflection::Object *table, + const reflection::Field *field) { + // Check if we have already generated this string section, and this is a + // shared string instance. + if (strings_.find(offset) != strings_.end()) { return; } + + std::vector regions; + const uint32_t string_length = GetScalar(offset); + + const uint64_t string_soffset = offset; + + regions.emplace_back(MakeBinaryRegion(offset, sizeof(uint32_t), + BinaryRegionType::Uint32, 0, 0, + std::string("length of string"))); + offset += sizeof(uint32_t); + + regions.emplace_back(MakeBinaryRegion(offset, string_length * sizeof(char), + BinaryRegionType::Char, string_length, + 0, "")); + offset += string_length * sizeof(char); + + regions.emplace_back(MakeBinaryRegion(offset, sizeof(char), + BinaryRegionType::Char, 0, 0, + std::string("string terminator"))); + offset += sizeof(char); + + sections_.insert(std::make_pair( + string_soffset, + MakeBinarySection( + std::string(table->name()->c_str()) + "." + field->name()->c_str(), + BinarySectionType::String, std::move(regions)))); + + // Insert into the strings set to find possible instances of shared strings. + strings_.insert(string_soffset); +} + +void BinaryAnnotator::BuildVector(uint64_t offset, + const reflection::Object *table, + const reflection::Field *field, + const uint64_t parent_table_offset, + const VTable &vtable) { + std::vector regions; + const uint32_t vector_length = GetScalar(offset); + + const uint64_t vector_offset = offset; + + regions.emplace_back( + MakeBinaryRegion(offset, sizeof(uint32_t), BinaryRegionType::Uint32, 0, 0, + std::string("length of vector (# items)"))); + offset += sizeof(uint32_t); + + switch (field->type()->element()) { + case reflection::BaseType::Obj: { + const reflection::Object *object = + schema_->objects()->Get(field->type()->index()); + + if (object->is_struct()) { + // Vector of structs + for (size_t i = 0; i < vector_length; ++i) { + // Structs are inline to the vector. + offset = BuildStruct(offset, regions, object); + } + } else { + // Vector of objects + for (size_t i = 0; i < vector_length; ++i) { + // The table offset is relative from the offset location itself. + const uint64_t table_offset = offset + GetScalar(offset); + + regions.emplace_back(MakeBinaryRegion( + offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0, + table_offset, + std::string("offset to table[") + std::to_string(i) + "]")); + + BuildTable(table_offset, BinarySectionType::Table, object); + + offset += sizeof(uint32_t); + } + } + } break; + case reflection::BaseType::String: { + // Vector of strings + for (size_t i = 0; i < vector_length; ++i) { + // The string offset is relative from the offset location itself. + const uint64_t string_offset = offset + GetScalar(offset); + + regions.emplace_back(MakeBinaryRegion( + offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0, + string_offset, + std::string("offset to string[") + std::to_string(i) + "]")); + + BuildString(string_offset, table, field); + + offset += sizeof(uint32_t); + } + } break; + case reflection::BaseType::Union: { + // Vector of unions + // Unions have both their realized type (uint8_t for now) that are + // stored sperately. These are stored in the field->index() - 1 + // location. + const uint16_t union_type_vector_id = field->id() - 1; + + auto vtable_entry = vtable.fields.find(union_type_vector_id); + if (vtable_entry == vtable.fields.end()) { + // TODO(dbaileychess): need to capture this error condition. + break; + } + + const uint64_t union_type_vector_field_offset = + parent_table_offset + vtable_entry->second.offset_from_table; + + // Get the offset to the first type (the + sizeof(uint32_t) is to skip + // over the vector length which we already know) + const uint64_t union_type_vector_data_offset = + union_type_vector_field_offset + + GetScalar(union_type_vector_field_offset) + + sizeof(uint32_t); + + for (size_t i = 0; i < vector_length; ++i) { + // The union offset is relative from the offset location itself. + const uint64_t union_offset = offset + GetScalar(offset); + + const uint8_t realized_type = GetScalar( + union_type_vector_data_offset + i * sizeof(uint8_t)); + + const std::string enum_type = + BuildUnion(union_offset, realized_type, field); + + regions.emplace_back(MakeBinaryRegion( + offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0, + union_offset, + std::string("offset to union[") + std::to_string(i) + "] (`" + + enum_type + "`)")); + + offset += sizeof(uint32_t); + } + } break; + default: { + if (IsScalar(field->type()->element())) { + const BinaryRegionType binary_region_type = + GetRegionType(field->type()->element()); + + const uint64_t type_size = GetTypeSize(field->type()->element()); + + // TODO(dbaileychess): It might be nicer to user the + // BinaryRegion.array_length field to indicate this. + for (size_t i = 0; i < vector_length; ++i) { + regions.emplace_back(MakeBinaryRegion( + offset, type_size, binary_region_type, 0, 0, + std::string("value[") + std::to_string(i) + "]")); + offset += type_size; + } + } + } break; + } + + sections_.insert(std::make_pair( + vector_offset, + MakeBinarySection( + std::string(table->name()->c_str()) + "." + field->name()->c_str(), + BinarySectionType::Vector, std::move(regions)))); +} + +std::string BinaryAnnotator::BuildUnion(uint64_t offset, + const uint8_t realized_type, + const reflection::Field *field) { + const reflection::Enum *next_enum = + schema_->enums()->Get(field->type()->index()); + + const reflection::EnumVal *enum_val = next_enum->values()->Get(realized_type); + + const reflection::Type *union_type = enum_val->union_type(); + + if (union_type->base_type() == reflection::BaseType::Obj) { + const reflection::Object *object = + schema_->objects()->Get(union_type->index()); + + if (object->is_struct()) { + // Union of vectors point to a new Binary section + std::vector regions; + + offset = BuildStruct(offset, regions, object); + + sections_.insert(std::make_pair( + regions[0].offset, + MakeBinarySection(std::string(object->name()->c_str()) + "." + + field->name()->c_str(), + BinarySectionType::Union, std::move(regions)))); + } else { + BuildTable(offset, BinarySectionType::Table, object); + } + } + // TODO(dbaileychess): handle the other union types. + + return enum_val->name()->c_str(); +} + +void BinaryAnnotator::FixMissingSections() { + uint64_t offset = 0; + + std::vector sections_to_insert; + + for (auto ¤t_section : sections_) { + BinarySection §ion = current_section.second; + const uint64_t section_start_offset = current_section.first; + const uint64_t section_end_offset = + section.regions.back().offset + section.regions.back().length; + + if (offset < section_start_offset) { + // We are at an offset that is less then the current section. + const uint64_t pad_bytes = section_start_offset - offset + 1; + + const uint64_t start_offset = offset - 1; + + std::vector regions; + + // Check if the region is all zeros or not, as that can tell us if it is + // padding or not. + if (IsNonZeroRegion(offset - 1, pad_bytes, binary_)) { + // Some of the padding bytes are non-zero, so this might be an unknown + // section of the binary. + regions.emplace_back(MakeBinaryRegion( + start_offset, pad_bytes * sizeof(uint8_t), + BinaryRegionType::Unknown, pad_bytes, 0, + pad_bytes < 8 ? "could be a corrupted padding region (non zero) " + "due to the length < 8 bytes." + : "WARN: nothing refers to this. Check if any " + "`Unkown Field`s point to this.")); + + sections_to_insert.emplace_back( + MakeBinarySection("no known references", BinarySectionType::Unknown, + std::move(regions))); + } else { + // This region is most likely padding. + regions.emplace_back(MakeBinaryRegion( + start_offset, pad_bytes * sizeof(uint8_t), BinaryRegionType::Uint8, + pad_bytes, 0, + // Output a different annotation if the pad bytes exceed what we + // expect to be the maximum padding. + pad_bytes > 7 ? "likely padding but might be an unknown section " + "due to being larger than 7 bytes" + : "padding")); + + sections_to_insert.emplace_back(MakeBinarySection( + "", BinarySectionType::Padding, std::move(regions))); + } + } + offset = section_end_offset + 1; + } + + for (const BinarySection §ion_to_insert : sections_to_insert) { + sections_.insert( + std::make_pair(section_to_insert.regions[0].offset, section_to_insert)); + } +} + +} // namespace flatbuffers \ No newline at end of file diff --git a/src/binary_annotator.h b/src/binary_annotator.h new file mode 100644 index 000000000..5544e94d7 --- /dev/null +++ b/src/binary_annotator.h @@ -0,0 +1,227 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_BINARY_ANNOTATOR_H_ +#define FLATBUFFERS_BINARY_ANNOTATOR_H_ + +#include +#include + +#include "flatbuffers/reflection.h" + +namespace flatbuffers { + +enum class BinaryRegionType { + Unknown = 0, + UOffset = 1, + SOffset = 2, + VOffset = 3, + Bool = 4, + Byte = 5, + Char = 6, + Uint8 = 7, + Int8 = 8, + Uint16 = 9, + Int16 = 10, + Uint32 = 11, + Int32 = 12, + Uint64 = 13, + Int64 = 14, + Float = 15, + Double = 16 +}; + +template static inline T GetScalar(const uint8_t *binary) { + return *reinterpret_cast(binary); +} + +template +static inline std::string ToHex(T i, size_t width = sizeof(T)) { + std::stringstream stream; + stream << std::hex << std::uppercase << std::setfill('0') << std::setw(width) + << i; + return stream.str(); +} + +// Specialized version for uint8_t that don't work well with std::hex. +static inline std::string ToHex(uint8_t i) { + return ToHex(static_cast(i), 2); +} + +struct BinaryRegion { + // Offset into the binary where this region begins. + uint64_t offset = 0; + + // The length of this region in bytes. + uint64_t length = 0; + + // The underlying datatype of this region + BinaryRegionType type = BinaryRegionType::Unknown; + + // If `type` is an array/vector, this is the number of those types this region + // encompasses. + uint64_t array_length = 0; + + // If the is an offset to some other region, this is what it points to. The + // offset is relative to overall binary, not to this region. + uint64_t points_to_offset = 0; + + // The comment on the region. + // TODO(dbaileychess): Consider moving this to a more structure comment field + // so that other generators can parse it easier. + std::string comment; +}; + +enum class BinarySectionType { + Unknown = 0, + Header = 1, + Table = 2, + RootTable = 3, + VTable = 4, + Struct = 5, + String = 6, + Vector = 7, + Union = 8, + Padding = 9, +}; + +// A section of the binary that is grouped together in some logical manner, and +// often is pointed too by some other offset BinaryRegion. Sections include +// `tables`, `vtables`, `strings`, `vectors`, etc.. +struct BinarySection { + // User-specified name of the section, if applicable. + std::string name; + + // The type of this section. + BinarySectionType type = BinarySectionType::Unknown; + + // The binary regions that make up this section, in order of their offsets. + std::vector regions; +}; + +inline static BinaryRegionType GetRegionType(reflection::BaseType base_type) { + switch (base_type) { + case reflection::UType: return BinaryRegionType::Uint8; + case reflection::Bool: return BinaryRegionType::Uint8; + case reflection::Byte: return BinaryRegionType::Uint8; + case reflection::UByte: return BinaryRegionType::Uint8; + case reflection::Short: return BinaryRegionType::Int16; + case reflection::UShort: return BinaryRegionType::Uint16; + case reflection::Int: return BinaryRegionType::Uint32; + case reflection::UInt: return BinaryRegionType::Uint32; + case reflection::Long: return BinaryRegionType::Int64; + case reflection::ULong: return BinaryRegionType::Uint64; + case reflection::Float: return BinaryRegionType::Float; + case reflection::Double: return BinaryRegionType::Double; + default: return BinaryRegionType::Unknown; + } +} + +inline static std::string ToString(const BinaryRegionType type) { + switch (type) { + case BinaryRegionType::UOffset: return "UOffset32"; + case BinaryRegionType::SOffset: return "SOffset32"; + case BinaryRegionType::VOffset: return "VOffset16"; + case BinaryRegionType::Bool: return "bool"; + case BinaryRegionType::Char: return "char"; + case BinaryRegionType::Byte: return "int8_t"; + case BinaryRegionType::Uint8: return "uint8_t"; + case BinaryRegionType::Uint16: return "uint16_t"; + case BinaryRegionType::Uint32: return "uint32_t"; + case BinaryRegionType::Uint64: return "uint64_t"; ; + case BinaryRegionType::Int8: return "int8_t"; + case BinaryRegionType::Int16: return "int16_t"; + case BinaryRegionType::Int32: return "int32_t"; + case BinaryRegionType::Int64: return "int64_t"; + case BinaryRegionType::Double: return "double"; + case BinaryRegionType::Float: return "float"; + case BinaryRegionType::Unknown: return "?uint8_t"; + default: return "todo"; + } +} + +class BinaryAnnotator { + public: + explicit BinaryAnnotator(const uint8_t *const bfbs, const int64_t bfbs_length, + const uint8_t *const binary) + : bfbs_(bfbs), + bfbs_length_(bfbs_length), + schema_(reflection::GetSchema(bfbs)), + binary_(binary) {} + + std::map Annotate(); + + private: + struct VTable { + struct Entry { + const reflection::Field *field = nullptr; + uint16_t offset_from_table = 0; + }; + + // Field ID -> {field def, offset from table} + std::map fields; + + uint16_t vtable_size = 0; + uint16_t table_size = 0; + }; + + uint64_t BuildHeader(uint64_t offset); + + void BuildVTable(uint64_t offset, const reflection::Object *table); + + void BuildTable(uint64_t offset, const BinarySectionType type, + const reflection::Object *table); + + uint64_t BuildStruct(uint64_t offset, std::vector ®ions, + const reflection::Object *structure); + + void BuildString(uint64_t offset, const reflection::Object *table, + const reflection::Field *field); + + void BuildVector(uint64_t offset, const reflection::Object *table, + const reflection::Field *field, uint64_t parent_table_offset, + const VTable &vtable); + + std::string BuildUnion(uint64_t offset, uint8_t realized_type, + const reflection::Field *field); + + void FixMissingSections(); + + template inline T GetScalar(uint64_t offset) { + return *reinterpret_cast(binary_ + offset); + } + + // The schema for the binary file + const uint8_t *bfbs_; + const int64_t bfbs_length_; + const reflection::Schema *schema_; + + // The binary data itself. + const uint8_t *binary_; + + // Map of binary offset to vtables, to dedupe vtables. + std::map vtables_; + + // A set of binary offset to string sections, to dedupe shared strings. + std::set strings_; + + // The annotated binary sections, index by their absolute offset. + std::map sections_; +}; + +} // namespace flatbuffers + +#endif // FLATBUFFERS_BINARY_ANNOTATOR_H_ \ No newline at end of file diff --git a/src/flatc.cpp b/src/flatc.cpp index 3b2ef9f36..ae82b41d5 100644 --- a/src/flatc.cpp +++ b/src/flatc.cpp @@ -19,6 +19,8 @@ #include #include +#include "annotated_binary_text_gen.h" +#include "binary_annotator.h" #include "flatbuffers/util.h" namespace flatbuffers { @@ -215,6 +217,8 @@ const static FlatCOption options[] = { "in JSON, which is unsafe unless checked by a verifier afterwards." }, { "", "ts-flat-files", "", "Only generated one typescript file per .fbs file." }, + { "", "annotate", "SCHEMA", + "Annotate the provided BINARY_FILE with the specified SCHEMA file." }, }; static void AppendTextWrappedString(std::stringstream &ss, std::string &text, @@ -297,7 +301,7 @@ std::string FlatCompiler::GetShortUsageString(const char *program_name) const { ss << ", "; } ss.seekp(-2, ss.cur); - ss << "]... FILE... [-- FILE...]"; + ss << "]... FILE... [-- BINARY_FILE...]"; std::string help = ss.str(); std::stringstream ss_textwrap; AppendTextWrappedString(ss_textwrap, help, 80, 0); @@ -306,7 +310,8 @@ std::string FlatCompiler::GetShortUsageString(const char *program_name) const { std::string FlatCompiler::GetUsageString(const char *program_name) const { std::stringstream ss; - ss << "Usage: " << program_name << " [OPTION]... FILE... [-- FILE...]\n"; + ss << "Usage: " << program_name + << " [OPTION]... FILE... [-- BINARY_FILE...]\n"; for (size_t i = 0; i < params_.num_generators; ++i) { const Generator &g = params_.generators[i]; AppendOption(ss, g.option, 80, 25); @@ -320,16 +325,48 @@ std::string FlatCompiler::GetUsageString(const char *program_name) const { std::string files_description = "FILEs may be schemas (must end in .fbs), binary schemas (must end in " - ".bfbs) or JSON files (conforming to preceding schema). FILEs after the " - "-- must be binary flatbuffer format files. Output files are named using " - "the base file name of the input, and written to the current directory " - "or the path given by -o. example: " + + ".bfbs) or JSON files (conforming to preceding schema). BINARY_FILEs " + "after the -- must be binary flatbuffer format files. Output files are " + "named using the base file name of the input, and written to the current " + "directory or the path given by -o. example: " + std::string(program_name) + " -c -b schema1.fbs schema2.fbs data.json"; AppendTextWrappedString(ss, files_description, 80, 0); ss << "\n"; return ss.str(); } +void FlatCompiler::AnnotateBinaries( + const uint8_t *binary_schema, const uint64_t binary_schema_size, + const std::string &schema_filename, + const std::vector &binary_files) { + for (const std::string &filename : binary_files) { + std::string binary_contents; + if (!flatbuffers::LoadFile(filename.c_str(), true, &binary_contents)) { + Warn("unable to load binary file: " + filename); + continue; + } + + const uint8_t *binary = + reinterpret_cast(binary_contents.c_str()); + const size_t binary_size = binary_contents.size(); + + flatbuffers::BinaryAnnotator binary_annotator(binary_schema, + binary_schema_size, binary); + + auto annotations = binary_annotator.Annotate(); + + // TODO(dbaileychess): Right now we just support a single text-based + // output of the annotated binary schema, which we generate here. We + // could output the raw annotations instead and have third-party tools + // use them to generate their own output. + flatbuffers::AnnotatedBinaryTextGenerator text_generator( + flatbuffers::AnnotatedBinaryTextGenerator::Options{}, annotations, + binary, binary_size); + + text_generator.Generate(filename, schema_filename); + } +} + int FlatCompiler::Compile(int argc, const char **argv) { if (params_.generators == nullptr || params_.num_generators == 0) { return 0; @@ -353,6 +390,7 @@ int FlatCompiler::Compile(int argc, const char **argv) { std::vector generator_enabled(params_.num_generators, false); size_t binary_files_from = std::numeric_limits::max(); std::string conform_to_schema; + std::string annotate_schema; const char *program_name = argv[0]; @@ -554,6 +592,9 @@ int FlatCompiler::Compile(int argc, const char **argv) { opts.json_nested_legacy_flatbuffers = true; } else if (arg == "--ts-flat-files") { opts.ts_flat_file = true; + } else if (arg == "--annotate") { + if (++argi >= argc) Error("missing path following: " + arg, true); + annotate_schema = flatbuffers::PosixPath(argv[argi]); } else { for (size_t i = 0; i < params_.num_generators; ++i) { if (arg == "--" + params_.generators[i].option.long_opt || @@ -582,7 +623,8 @@ int FlatCompiler::Compile(int argc, const char **argv) { if (opts.proto_mode) { if (any_generator) Error("cannot generate code directly from .proto files", true); - } else if (!any_generator && conform_to_schema.empty()) { + } else if (!any_generator && conform_to_schema.empty() && + annotate_schema.empty()) { Error("no options: specify at least one generator.", true); } @@ -611,6 +653,53 @@ int FlatCompiler::Compile(int argc, const char **argv) { } } + if (!annotate_schema.empty()) { + const std::string ext = flatbuffers::GetExtension(annotate_schema); + if (!(ext == reflection::SchemaExtension() || ext == "fbs")) { + Error("Expected a `.bfbs` or `.fbs` schema, got: " + annotate_schema); + } + + const bool is_binary_schema = ext == reflection::SchemaExtension(); + + std::string schema_contents; + if (!flatbuffers::LoadFile(annotate_schema.c_str(), + /*binary=*/is_binary_schema, &schema_contents)) { + Error("unable to load schema: " + annotate_schema); + } + + const uint8_t *binary_schema = nullptr; + uint64_t binary_schema_size = 0; + + IDLOptions binary_opts; + binary_opts.lang_to_generate |= flatbuffers::IDLOptions::kBinary; + flatbuffers::Parser parser(binary_opts); + + if (is_binary_schema) { + binary_schema = + reinterpret_cast(schema_contents.c_str()); + binary_schema_size = schema_contents.size(); + } else { + // If we need to generate the .bfbs file from the provided schema file + // (.fbs) + ParseFile(parser, annotate_schema, schema_contents, include_directories); + parser.Serialize(); + + binary_schema = parser.builder_.GetBufferPointer(); + binary_schema_size = parser.builder_.GetSize(); + } + + if (binary_schema == nullptr || !binary_schema_size) { + Error("could not parse a value binary schema from: " + annotate_schema); + } + + // Annotate the provided files with the binary_schema. + AnnotateBinaries(binary_schema, binary_schema_size, annotate_schema, + filenames); + + // We don't support doing anything else after annotating a binary. + return 0; + } + std::unique_ptr parser(new flatbuffers::Parser(opts)); for (auto file_it = filenames.begin(); file_it != filenames.end(); diff --git a/src/reflection.cpp b/src/reflection.cpp index 6bba28c53..1d0f1d6b5 100644 --- a/src/reflection.cpp +++ b/src/reflection.cpp @@ -120,6 +120,23 @@ std::string GetAnyValueS(reflection::BaseType type, const uint8_t *data, } } +void ForAllFields(const reflection::Object *object, bool reverse, + std::function func) { + std::vector field_to_id_map; + field_to_id_map.resize(object->fields()->size()); + + // Create the mapping of field ID to the index into the vector. + for (uint32_t i = 0; i < object->fields()->size(); ++i) { + auto field = object->fields()->Get(i); + field_to_id_map[field->id()] = i; + } + + for (size_t i = 0; i < field_to_id_map.size(); ++i) { + func(object->fields()->Get( + field_to_id_map[reverse ? field_to_id_map.size() - i + 1 : i])); + } +} + void SetAnyValueI(reflection::BaseType type, uint8_t *data, int64_t val) { // clang-format off #define FLATBUFFERS_SET(T) WriteScalar(data, static_cast(val)) diff --git a/tests/annotated_binary/README.md b/tests/annotated_binary/README.md new file mode 100644 index 000000000..0cb046114 --- /dev/null +++ b/tests/annotated_binary/README.md @@ -0,0 +1,101 @@ +# Annotated Flatbuffer Binary + +This directory demonstrates the ability of flatc to annotate binary flatbuffers +with helpful annotations. The resulting annotated flatbuffer binary (afb) +contains all the binary data with line-by-line annotations. + +## Usage + +Given a `schema` in either plain-text (.fbs) or already compiled to a binary +schema (.bfbs) and `binary` file(s) that was created by the `schema`. + +```sh +flatc --annotate {schema_file} -- {binary_file}... +``` + +### Example + +The following command should produce `annotated_binary.afb` in this directory: + +```sh +cd tests\annotated_binary +..\..\flatc --annotate annotated_binary.fbs -- annotated_binary.bin +``` + +The `annotated_binary.bin` is the flatbufer binary of the data contained within + `annotated_binary.json`, which was made by the following command: + +```sh +..\..\flatc -b annotated_binary.fbs annotated_binary.json +``` + +## Text Format + +Currently there is a built-in text-based format for outputting the annotations. +The `annotated_binary.afb` is an example of the text format of a binary +`annotated_binary.bin` and the `annotated_binary.fbs` (or +`annotated_binary.bfbs`) schema. + +The file is ordered in increasing the offsets from the beginning of the binary. +The offset is the 1st column, expressed in hexadecimal format (e.g. `+0x003c`). + +### Binary Sections + +Binary sections are comprised of contigious [binary regions](#binary-regions) +that are logically grouped together. For example, a binary section may be a +single instance of a flatbuffer `Table` or its `vtable`. The sections may be +labelled with the name of the associated type, as defined in the input schema. + +Example of a `vtable` Binary Section that is associated with the user-defined +`AnnotateBinary.Bar` table. + +``` +vtable (AnnotatedBinary.Bar): + +0x00A0 | 08 00 | uint16_t | 0x0008 (8) | size of this vtable + +0x00A2 | 13 00 | uint16_t | 0x0013 (19) | size of referring table + +0x00A4 | 08 00 | VOffset16 | 0x0008 (8) | offset to field `a` (id: 0) + +0x00A6 | 04 00 | VOffset16 | 0x0004 (4) | offset to field `b` (id: 1) +``` + +### Binary Regions + +Binary regions are contigious bytes regions that are grouped together to form +some sort of value, e.g. a `scalar` or an array of scalars. A binary region may +be split up over multiple text lines, if the size of the region is large. + +Looking at an example binary region: + +``` +vtable (AnnotatedBinary.Bar): + +0x00A0 | 08 00 | uint16_t | 0x0008 (8) | size of this vtable +``` + +The first column (`+0x00A0`) is the offset to this region from the beginning of +the buffer. + +The second column are the raw bytes (hexadecimal) that make up this +region. These are expressed in the little-endian format that flatbuffers uses +for the wire format. + +The third column is the type to interpret the bytes as. Some types are special +to flatbuffer internals (e.g. `SOffet32`, `Offset32`, and `VOffset16`) which are +used by flatbuffers to point to various offsetes. The other types are specified +as C++-like types which are the standard fix-width scalars. For the above +example, the type is `uint16_t` which is a 16-bit unsigned integer type. + +The fourth column shows the raw bytes as a compacted, big-endian value. The raw +bytes are duplicated in this fashion since it is more intutive to read the data +in the big-endian format (e.g., `0x0008`). This value is followed by the decimal +representation of the value (e.g., `(8)`). (For strings, the raw string value +is shown instead). + +The fifth column is a textual comment on what the value is. As much metadata as +known is provided. + +#### Offsets + +If the type in the 3rd column is of an absolute offset (`SOffet32` or +`Offset32`), the fourth column also shows an `Loc: +0x025A` value which shows +where in the binary this region is pointing to. These values are absolute from +the beginning of the file, their calculation from the raw value in the 4th +column depends on the context. diff --git a/tests/annotated_binary/annotated_binary.afb b/tests/annotated_binary/annotated_binary.afb new file mode 100644 index 000000000..804d112ad --- /dev/null +++ b/tests/annotated_binary/annotated_binary.afb @@ -0,0 +1,297 @@ +// Annotated Flatbuffer Binary +// +// Schema file: annotated_binary.fbs +// Binary file: annotated_binary.bin + +header: + +0x0000 | 44 00 00 00 | UOffset32 | 0x00000044 (68) Loc: +0x0044 | offset to root table `AnnotatedBinary.Foo` + +0x0004 | 41 4E 4E 4F | char[4] | ANNO | File Identifier + +padding: + +0x0008 | 00 00 | uint8_t[2] | .. | padding + +vtable (AnnotatedBinary.Foo): + +0x000A | 3A 00 | uint16_t | 0x003A (58) | size of this vtable + +0x000C | 68 00 | uint16_t | 0x0068 (104) | size of referring table + +0x000E | 0C 00 | VOffset16 | 0x000C (12) | offset to field `counter` (id: 0) + +0x0010 | 07 00 | VOffset16 | 0x0007 (7) | offset to field `healthy` (id: 1) + +0x0012 | 00 00 | VOffset16 | 0x0000 (0) | offset to field `level` (id: 2) (Long) + +0x0014 | 08 00 | VOffset16 | 0x0008 (8) | offset to field `meal` (id: 3) + +0x0016 | 10 00 | VOffset16 | 0x0010 (16) | offset to field `bar` (id: 4) + +0x0018 | 14 00 | VOffset16 | 0x0014 (20) | offset to field `home` (id: 5) + +0x001A | 30 00 | VOffset16 | 0x0030 (48) | offset to field `name` (id: 6) + +0x001C | 34 00 | VOffset16 | 0x0034 (52) | offset to field `bars` (id: 7) + +0x001E | 09 00 | VOffset16 | 0x0009 (9) | offset to field `bar_baz_type` (id: 8) + +0x0020 | 38 00 | VOffset16 | 0x0038 (56) | offset to field `bar_baz` (id: 9) + +0x0022 | 3C 00 | VOffset16 | 0x003C (60) | offset to field `accounts` (id: 10) + +0x0024 | 40 00 | VOffset16 | 0x0040 (64) | offset to field `bob` (id: 11) + +0x0026 | 44 00 | VOffset16 | 0x0044 (68) | offset to field `alice` (id: 12) + +0x0028 | 00 00 | VOffset16 | 0x0000 (0) | offset to field `maybe_i32` (id: 13) (Int) + +0x002A | 00 00 | VOffset16 | 0x0000 (0) | offset to field `default_i32` (id: 14) (Int) + +0x002C | 48 00 | VOffset16 | 0x0048 (72) | offset to field `just_i32` (id: 15) + +0x002E | 4C 00 | VOffset16 | 0x004C (76) | offset to field `names` (id: 16) + +0x0030 | 50 00 | VOffset16 | 0x0050 (80) | offset to field `points_of_interest` (id: 17) + +0x0032 | 54 00 | VOffset16 | 0x0054 (84) | offset to field `foobars_type` (id: 18) + +0x0034 | 58 00 | VOffset16 | 0x0058 (88) | offset to field `foobars` (id: 19) + +0x0036 | 0A 00 | VOffset16 | 0x000A (10) | offset to field `measurement_type` (id: 20) + +0x0038 | 5C 00 | VOffset16 | 0x005C (92) | offset to field `measurement` (id: 21) + +0x003A | 0B 00 | VOffset16 | 0x000B (11) | offset to field `anything_type` (id: 22) + +0x003C | 60 00 | VOffset16 | 0x0060 (96) | offset to field `anything` (id: 23) + +0x003E | 00 00 | VOffset16 | 0x0000 (0) | offset to field `temperature` (id: 24) (Float) + +0x0040 | 00 00 | VOffset16 | 0x0000 (0) | offset to field `teetotaler` (id: 25) (Obj) + +0x0042 | 64 00 | VOffset16 | 0x0064 (100) | offset to field `charlie` (id: 26) + +root_table (AnnotatedBinary.Foo): + +0x0044 | 3A 00 00 00 | SOffset32 | 0x0000003A (58) Loc: +0x000A | offset to vtable + +0x0048 | 00 00 00 | uint8_t[3] | ... | padding + +0x004B | 01 | uint8_t | 0x01 (1) | table field `healthy` (Bool) + +0x004C | 02 | uint8_t | 0x02 (2) | table field `meal` (Byte) + +0x004D | 02 | uint8_t | 0x02 (2) | table field `bar_baz_type` (UType) + +0x004E | 01 | uint8_t | 0x01 (1) | table field `measurement_type` (UType) + +0x004F | 01 | uint8_t | 0x01 (1) | table field `anything_type` (UType) + +0x0050 | D2 04 00 00 | uint32_t | 0x000004D2 (1234) | table field `counter` (Int) + +0x0054 | 28 02 00 00 | UOffset32 | 0x00000228 (552) Loc: +0x027C | offset to field `bar` + +0x0058 | 01 00 00 00 | uint32_t | 0x00000001 (1) | struct field `AnnotatedBinary.Building.floors` (Int) + +0x005C | 02 00 00 00 | uint32_t | 0x00000002 (2) | struct field `AnnotatedBinary.Building.doors` (Int) + +0x0060 | 0C 00 00 00 | uint32_t | 0x0000000C (12) | struct field `AnnotatedBinary.Building.windows` (Int) + +0x0064 | 0A 00 00 00 | uint32_t | 0x0000000A (10) | array field `AnnotatedBinary.Dimension.values[0]` (Int) + +0x0068 | 0C 00 00 00 | uint32_t | 0x0000000C (12) | array field `AnnotatedBinary.Dimension.values[1]` (Int) + +0x006C | 14 00 00 00 | uint32_t | 0x00000014 (20) | array field `AnnotatedBinary.Dimension.values[2]` (Int) + +0x0070 | 01 | uint8_t | 0x01 (1) | struct field `AnnotatedBinary.Tolerance.width` (UByte) + +0x0071 | 02 | uint8_t | 0x02 (2) | struct field `AnnotatedBinary.Tolerance.width` (UByte) + +0x0072 | 03 | uint8_t | 0x03 (3) | struct field `AnnotatedBinary.Tolerance.width` (UByte) + +0x0073 | 00 | uint8_t[1] | . | padding + +0x0074 | C8 01 00 00 | UOffset32 | 0x000001C8 (456) Loc: +0x023C | offset to field `name` + +0x0078 | 5C 01 00 00 | UOffset32 | 0x0000015C (348) Loc: +0x01D4 | offset to field `bars` + +0x007C | 50 01 00 00 | UOffset32 | 0x00000150 (336) Loc: +0x01CC | offset to field `bar_baz` (union of type `Baz`) + +0x0080 | 34 01 00 00 | UOffset32 | 0x00000134 (308) Loc: +0x01B4 | offset to field `accounts` + +0x0084 | 24 01 00 00 | UOffset32 | 0x00000124 (292) Loc: +0x01A8 | offset to field `bob` + +0x0088 | 14 01 00 00 | UOffset32 | 0x00000114 (276) Loc: +0x019C | offset to field `alice` + +0x008C | 0D 00 00 00 | uint32_t | 0x0000000D (13) | table field `just_i32` (Int) + +0x0090 | DC 00 00 00 | UOffset32 | 0x000000DC (220) Loc: +0x016C | offset to field `names` + +0x0094 | A0 00 00 00 | UOffset32 | 0x000000A0 (160) Loc: +0x0134 | offset to field `points_of_interest` + +0x0098 | 94 00 00 00 | UOffset32 | 0x00000094 (148) Loc: +0x012C | offset to field `foobars_type` + +0x009C | 38 00 00 00 | UOffset32 | 0x00000038 (56) Loc: +0x00D4 | offset to field `foobars` + +0x00A0 | 33 00 00 00 | UOffset32 | 0x00000033 (51) Loc: +0x00D3 | offset to field `measurement` (union of type `Tolerance`) + +0x00A4 | 1C 00 00 00 | UOffset32 | 0x0000001C (28) Loc: +0x00C0 | offset to field `anything` (union of type `Bar`) + +0x00A8 | 04 00 00 00 | UOffset32 | 0x00000004 (4) Loc: +0x00AC | offset to field `charlie` + +string (AnnotatedBinary.Foo.charlie): + +0x00AC | 05 00 00 00 | uint32_t | 0x00000005 (5) | length of string + +0x00B0 | 61 6C 69 63 65 | char[5] | alice + +0x00B5 | 00 | char | 0x00 (0) | string terminator + +padding: + +0x00B6 | 00 00 | uint8_t[2] | .. | padding + +vtable (AnnotatedBinary.Bar): + +0x00B8 | 08 00 | uint16_t | 0x0008 (8) | size of this vtable + +0x00BA | 13 00 | uint16_t | 0x0013 (19) | size of referring table + +0x00BC | 08 00 | VOffset16 | 0x0008 (8) | offset to field `a` (id: 0) + +0x00BE | 04 00 | VOffset16 | 0x0004 (4) | offset to field `b` (id: 1) + +table (AnnotatedBinary.Bar): + +0x00C0 | 08 00 00 00 | SOffset32 | 0x00000008 (8) Loc: +0x00B8 | offset to vtable + +0x00C4 | 00 80 23 44 | float | 0x44238000 (654.000000) | table field `b` (Float) + +0x00C8 | 00 00 00 00 00 10 74 40 | double | 0x4074100000000000 (321.000000) | table field `a` (Double) + +0x00D0 | 00 00 00 | uint8_t[3] | ... | padding + +union (AnnotatedBinary.Tolerance.measurement): + +0x00D3 | 05 | uint8_t | 0x05 (5) | struct field `AnnotatedBinary.Tolerance.width` (UByte) + +vector (AnnotatedBinary.Foo.foobars): + +0x00D4 | 03 00 00 00 | uint32_t | 0x00000003 (3) | length of vector (# items) + +0x00D8 | 34 00 00 00 | UOffset32 | 0x00000034 (52) Loc: +0x010C | offset to union[0] (`Bar`) + +0x00DC | 2C 00 00 00 | UOffset32 | 0x0000002C (44) Loc: +0x0108 | offset to union[1] (`Baz`) + +0x00E0 | 04 00 00 00 | UOffset32 | 0x00000004 (4) Loc: +0x00E4 | offset to union[2] (`Bar`) + +table (AnnotatedBinary.Bar): + +0x00E4 | D2 FE FF FF | SOffset32 | 0xFFFFFED2 (-302) Loc: +0x0212 | offset to vtable + +0x00E8 | 00 80 23 44 | float | 0x44238000 (654.000000) | table field `b` (Float) + +0x00EC | 10 00 00 00 | UOffset32 | 0x00000010 (16) Loc: +0x00FC | offset to field `c` + +0x00F0 | 00 00 00 00 00 D8 8E 40 | double | 0x408ED80000000000 (987.000000) | table field `a` (Double) + +0x00F8 | 00 00 00 00 | uint8_t[4] | .... | padding + +table (AnnotatedBinary.Baz): + +0x00FC | 6A FE FF FF | SOffset32 | 0xFFFFFE6A (-406) Loc: +0x0292 | offset to vtable + +0x0100 | 00 00 00 | uint8_t[3] | ... | padding + +0x0103 | 03 | uint8_t | 0x03 (3) | table field `meal` (Byte) + +vtable (AnnotatedBinary.Baz): + +0x0104 | 04 00 | uint16_t | 0x0004 (4) | size of this vtable + +0x0106 | 04 00 | uint16_t | 0x0004 (4) | size of referring table + +table (AnnotatedBinary.Baz): + +0x0108 | 04 00 00 00 | SOffset32 | 0x00000004 (4) Loc: +0x0104 | offset to vtable + +table (AnnotatedBinary.Bar): + +0x010C | FA FE FF FF | SOffset32 | 0xFFFFFEFA (-262) Loc: +0x0212 | offset to vtable + +0x0110 | 00 00 E4 43 | float | 0x43E40000 (456.000000) | table field `b` (Float) + +0x0114 | 10 00 00 00 | UOffset32 | 0x00000010 (16) Loc: +0x0124 | offset to field `c` + +0x0118 | 00 00 00 00 00 C0 5E 40 | double | 0x405EC00000000000 (123.000000) | table field `a` (Double) + +0x0120 | 00 00 00 00 | uint8_t[4] | .... | padding + +table (AnnotatedBinary.Baz): + +0x0124 | 92 FE FF FF | SOffset32 | 0xFFFFFE92 (-366) Loc: +0x0292 | offset to vtable + +0x0128 | 00 00 00 | uint8_t[3] | ... | padding + +0x012B | 01 | uint8_t | 0x01 (1) | table field `meal` (Byte) + +vector (AnnotatedBinary.Foo.foobars_type): + +0x012C | 03 00 00 00 | uint32_t | 0x00000003 (3) | length of vector (# items) + +0x0130 | 01 | uint8_t | 0x01 (1) | value[0] + +0x0131 | 02 | uint8_t | 0x02 (2) | value[1] + +0x0132 | 01 | uint8_t | 0x01 (1) | value[2] + +vector (AnnotatedBinary.Foo.points_of_interest): + +0x0134 | 03 00 00 00 | uint32_t | 0x00000003 (3) | length of vector (# items) + +0x0138 | 33 33 33 33 33 A3 45 40 | double | 0x4045A33333333333 (43.275000) | struct field `AnnotatedBinary.Location.latitude` (Double) + +0x0140 | 7E 57 04 FF 5B 87 53 C0 | double | 0xC053875BFF04577E (-78.114990) | struct field `AnnotatedBinary.Location.longitude` (Double) + +0x0148 | 8D F0 F6 20 04 B6 42 40 | double | 0x4042B60420F6F08D (37.422001) | struct field `AnnotatedBinary.Location.latitude` (Double) + +0x0150 | 9F 77 63 41 61 85 5E C0 | double | 0xC05E85614163779F (-122.084061) | struct field `AnnotatedBinary.Location.longitude` (Double) + +0x0158 | 8F 35 23 83 DC 35 4B C0 | double | 0xC04B35DC8323358F (-54.420792) | struct field `AnnotatedBinary.Location.latitude` (Double) + +0x0160 | F6 97 DD 93 87 C5 0A 40 | double | 0x400AC58793DD97F6 (3.346450) | struct field `AnnotatedBinary.Location.longitude` (Double) + +padding: + +0x0168 | 00 00 00 00 | uint8_t[4] | .... | padding + +vector (AnnotatedBinary.Foo.names): + +0x016C | 03 00 00 00 | uint32_t | 0x00000003 (3) | length of vector (# items) + +0x0170 | 20 00 00 00 | UOffset32 | 0x00000020 (32) Loc: +0x0190 | offset to string[0] + +0x0174 | 14 00 00 00 | UOffset32 | 0x00000014 (20) Loc: +0x0188 | offset to string[1] + +0x0178 | 04 00 00 00 | UOffset32 | 0x00000004 (4) Loc: +0x017C | offset to string[2] + +string (AnnotatedBinary.Foo.names): + +0x017C | 07 00 00 00 | uint32_t | 0x00000007 (7) | length of string + +0x0180 | 63 68 61 72 6C 69 65 | char[7] | charlie + +0x0187 | 00 | char | 0x00 (0) | string terminator + +string (AnnotatedBinary.Foo.names): + +0x0188 | 03 00 00 00 | uint32_t | 0x00000003 (3) | length of string + +0x018C | 62 6F 62 | char[3] | bob + +0x018F | 00 | char | 0x00 (0) | string terminator + +string (AnnotatedBinary.Foo.names): + +0x0190 | 05 00 00 00 | uint32_t | 0x00000005 (5) | length of string + +0x0194 | 61 6C 69 63 65 | char[5] | alice + +0x0199 | 00 | char | 0x00 (0) | string terminator + +padding: + +0x019A | 00 00 | uint8_t[2] | .. | padding + +string (AnnotatedBinary.Foo.alice): + +0x019C | 07 00 00 00 | uint32_t | 0x00000007 (7) | length of string + +0x01A0 | 63 68 61 72 6C 69 65 | char[7] | charlie + +0x01A7 | 00 | char | 0x00 (0) | string terminator + +string (AnnotatedBinary.Foo.bob): + +0x01A8 | 07 00 00 00 | uint32_t | 0x00000007 (7) | length of string + +0x01AC | 63 68 61 72 6C 69 65 | char[7] | charlie + +0x01B3 | 00 | char | 0x00 (0) | string terminator + +vector (AnnotatedBinary.Foo.accounts): + +0x01B4 | 09 00 00 00 | uint32_t | 0x00000009 (9) | length of vector (# items) + +0x01B8 | 09 00 | uint16_t | 0x0009 (9) | value[0] + +0x01BA | 08 00 | uint16_t | 0x0008 (8) | value[1] + +0x01BC | 07 00 | uint16_t | 0x0007 (7) | value[2] + +0x01BE | 01 00 | uint16_t | 0x0001 (1) | value[3] + +0x01C0 | 02 00 | uint16_t | 0x0002 (2) | value[4] + +0x01C2 | 03 00 | uint16_t | 0x0003 (3) | value[5] + +0x01C4 | 06 00 | uint16_t | 0x0006 (6) | value[6] + +0x01C6 | 05 00 | uint16_t | 0x0005 (5) | value[7] + +0x01C8 | 04 00 | uint16_t | 0x0004 (4) | value[8] + +padding: + +0x01CA | 00 00 | uint8_t[2] | .. | padding + +table (AnnotatedBinary.Baz): + +0x01CC | 3A FF FF FF | SOffset32 | 0xFFFFFF3A (-198) Loc: +0x0292 | offset to vtable + +0x01D0 | 00 00 00 | uint8_t[3] | ... | padding + +0x01D3 | 03 | uint8_t | 0x03 (3) | table field `meal` (Byte) + +vector (AnnotatedBinary.Foo.bars): + +0x01D4 | 02 00 00 00 | uint32_t | 0x00000002 (2) | length of vector (# items) + +0x01D8 | 44 00 00 00 | UOffset32 | 0x00000044 (68) Loc: +0x021C | offset to table[0] + +0x01DC | 10 00 00 00 | UOffset32 | 0x00000010 (16) Loc: +0x01EC | offset to table[1] + +padding: + +0x01E0 | 00 00 | uint8_t[2] | .. | padding + +vtable (AnnotatedBinary.Bar): + +0x01E2 | 0A 00 | uint16_t | 0x000A (10) | size of this vtable + +0x01E4 | 1A 00 | uint16_t | 0x001A (26) | size of referring table + +0x01E6 | 0C 00 | VOffset16 | 0x000C (12) | offset to field `a` (id: 0) + +0x01E8 | 04 00 | VOffset16 | 0x0004 (4) | offset to field `b` (id: 1) + +0x01EA | 08 00 | VOffset16 | 0x0008 (8) | offset to field `c` (id: 2) + +table (AnnotatedBinary.Bar): + +0x01EC | 0A 00 00 00 | SOffset32 | 0x0000000A (10) Loc: +0x01E2 | offset to vtable + +0x01F0 | 00 80 23 44 | float | 0x44238000 (654.000000) | table field `b` (Float) + +0x01F4 | 18 00 00 00 | UOffset32 | 0x00000018 (24) Loc: +0x020C | offset to field `c` + +0x01F8 | 00 00 00 00 00 D8 8E 40 | double | 0x408ED80000000000 (987.000000) | table field `a` (Double) + +0x0200 | 00 00 00 00 00 00 | uint8_t[6] | ...... | padding + +vtable (AnnotatedBinary.Baz): + +0x0206 | 06 00 | uint16_t | 0x0006 (6) | size of this vtable + +0x0208 | 06 00 | uint16_t | 0x0006 (6) | size of referring table + +0x020A | 05 00 | VOffset16 | 0x0005 (5) | offset to field `meal` (id: 0) + +table (AnnotatedBinary.Baz): + +0x020C | 06 00 00 00 | SOffset32 | 0x00000006 (6) Loc: +0x0206 | offset to vtable + +0x0210 | 00 | uint8_t[1] | . | padding + +0x0211 | 03 | uint8_t | 0x03 (3) | table field `meal` (Byte) + +vtable (AnnotatedBinary.Bar): + +0x0212 | 0A 00 | uint16_t | 0x000A (10) | size of this vtable + +0x0214 | 18 00 | uint16_t | 0x0018 (24) | size of referring table + +0x0216 | 0C 00 | VOffset16 | 0x000C (12) | offset to field `a` (id: 0) + +0x0218 | 04 00 | VOffset16 | 0x0004 (4) | offset to field `b` (id: 1) + +0x021A | 08 00 | VOffset16 | 0x0008 (8) | offset to field `c` (id: 2) + +table (AnnotatedBinary.Bar): + +0x021C | 0A 00 00 00 | SOffset32 | 0x0000000A (10) Loc: +0x0212 | offset to vtable + +0x0220 | 00 00 E4 43 | float | 0x43E40000 (456.000000) | table field `b` (Float) + +0x0224 | 10 00 00 00 | UOffset32 | 0x00000010 (16) Loc: +0x0234 | offset to field `c` + +0x0228 | 00 00 00 00 00 C0 5E 40 | double | 0x405EC00000000000 (123.000000) | table field `a` (Double) + +0x0230 | 00 00 00 00 | uint8_t[4] | .... | padding + +table (AnnotatedBinary.Baz): + +0x0234 | A2 FF FF FF | SOffset32 | 0xFFFFFFA2 (-94) Loc: +0x0292 | offset to vtable + +0x0238 | 00 00 00 | uint8_t[3] | ... | padding + +0x023B | 01 | uint8_t | 0x01 (1) | table field `meal` (Byte) + +string (AnnotatedBinary.Foo.name): + +0x023C | 2F 00 00 00 | uint32_t | 0x0000002F (47) | length of string + +0x0240 | 54 68 69 73 20 69 73 20 | char[47] | This is + +0x0248 | 61 20 6C 6F 6E 67 20 73 | | a long s + +0x0250 | 74 72 69 6E 67 20 74 6F | | tring to + +0x0258 | 20 73 68 6F 77 20 68 6F | | show ho + +0x0260 | 77 20 69 74 20 62 72 65 | | w it bre + +0x0268 | 61 6B 73 20 75 70 2E | | aks up. + +0x026F | 00 | char | 0x00 (0) | string terminator + +padding: + +0x0270 | 00 00 | uint8_t[2] | .. | padding + +vtable (AnnotatedBinary.Bar): + +0x0272 | 0A 00 | uint16_t | 0x000A (10) | size of this vtable + +0x0274 | 16 00 | uint16_t | 0x0016 (22) | size of referring table + +0x0276 | 0C 00 | VOffset16 | 0x000C (12) | offset to field `a` (id: 0) + +0x0278 | 04 00 | VOffset16 | 0x0004 (4) | offset to field `b` (id: 1) + +0x027A | 08 00 | VOffset16 | 0x0008 (8) | offset to field `c` (id: 2) + +table (AnnotatedBinary.Bar): + +0x027C | 0A 00 00 00 | SOffset32 | 0x0000000A (10) Loc: +0x0272 | offset to vtable + +0x0280 | 65 20 71 49 | float | 0x49712065 (987654.312500) | table field `b` (Float) + +0x0284 | 14 00 00 00 | UOffset32 | 0x00000014 (20) Loc: +0x0298 | offset to field `c` + +0x0288 | C9 76 BE 9F 0C 24 FE 40 | double | 0x40FE240C9FBE76C9 (123456.789000) | table field `a` (Double) + +0x0290 | 00 00 | uint8_t[2] | .. | padding + +vtable (AnnotatedBinary.Baz): + +0x0292 | 06 00 | uint16_t | 0x0006 (6) | size of this vtable + +0x0294 | 08 00 | uint16_t | 0x0008 (8) | size of referring table + +0x0296 | 07 00 | VOffset16 | 0x0007 (7) | offset to field `meal` (id: 0) + +table (AnnotatedBinary.Baz): + +0x0298 | 06 00 00 00 | SOffset32 | 0x00000006 (6) Loc: +0x0292 | offset to vtable + +0x029C | 00 00 00 | uint8_t[3] | ... | padding + +0x029F | 01 | uint8_t | 0x01 (1) | table field `meal` (Byte) diff --git a/tests/annotated_binary/annotated_binary.bfbs b/tests/annotated_binary/annotated_binary.bfbs new file mode 100644 index 000000000..90195d7f7 Binary files /dev/null and b/tests/annotated_binary/annotated_binary.bfbs differ diff --git a/tests/annotated_binary/annotated_binary.bin b/tests/annotated_binary/annotated_binary.bin new file mode 100644 index 000000000..2557aaf53 Binary files /dev/null and b/tests/annotated_binary/annotated_binary.bin differ diff --git a/tests/annotated_binary/annotated_binary.fbs b/tests/annotated_binary/annotated_binary.fbs new file mode 100644 index 000000000..60e320ff3 --- /dev/null +++ b/tests/annotated_binary/annotated_binary.fbs @@ -0,0 +1,92 @@ +namespace AnnotatedBinary; + +enum Food : byte { + None = 0, + Apple = 1, + Banana = 2, + Kiwi = 3, +} + +table Baz { + meal:Food = Banana; +} + +table Bar { + a:double = 3.14; + b:float = 1.68; + c:Baz; +} + +union BarBaz { + Bar, Baz +} + +union Measurement { + Tolerance, Dimension +} + +struct Tolerance { + width:uint8; +} + +union Any { + Bar, Tolerance +} + +struct Dimension { + values:[int:3]; + tolerances:[Tolerance:3]; +} + +struct Building { + floors:int; + doors:int; + windows:int; + dimensions:Dimension; +} + +struct Location { + latitude:double; + longitude:double; +} + +table Foo { + counter:int; + healthy:bool; + level:long = 99; + meal:Food = Apple; + bar:Bar; + home:Building; + name:string; + // Vector of tables + bars:[Bar]; + // Union of tables + bar_baz:BarBaz; + // Vector of Scalars + accounts:[uint16]; + bob:string; + alice:string; + // Optional Scalars + maybe_i32: int32 = null; + default_i32: int32 = 42; + just_i32: int32; + // Vector of strings + names:[string]; + // Vector of structs + points_of_interest:[Location]; + // Vector of unions + foobars:[BarBaz]; + // Union of structs + measurement:Measurement; + // Union of struct/table + anything:Any; + // Default floating point + temperature:float=98.6; + // Not present object + teetotaler:Bar; + charlie:string; +} + +file_identifier "ANNO"; + +root_type Foo; \ No newline at end of file diff --git a/tests/annotated_binary/annotated_binary.json b/tests/annotated_binary/annotated_binary.json new file mode 100644 index 000000000..744e27409 --- /dev/null +++ b/tests/annotated_binary/annotated_binary.json @@ -0,0 +1,124 @@ +{ + "counter": 1234, + "healthy": true, + "meal": "Banana", + "bar": { + "a": 123456.789, + "b": 987654.321, + "c": { + "meal": "Apple" + } + }, + "home": { + "floors": 1, + "doors": 2, + "windows": 12, + "dimensions": { + "values": [ + 10, + 12, + 20 + ], + "tolerances": [ + { + "width": 1 + }, + { + "width": 2 + }, + { + "width": 3 + } + ] + } + }, + "name": "This is a long string to show how it breaks up.", + "bars": [ + { + "a": 123, + "b": 456, + "c": { + "meal": "Apple" + } + }, + { + "a": 987, + "b": 654, + "c": { + "meal": "Kiwi" + } + } + ], + "bar_baz_type": "Baz", + "bar_baz": { + "meal": "Kiwi" + }, + "accounts": [ + 9, + 8, + 7, + 1, + 2, + 3, + 6, + 5, + 4, + ], + // Use the same string to show shared string support + "bob": "charlie", + "alice": "charlie", + "just_i32": 13, + "names": [ + "alice", + "bob", + "charlie" + ], + "points_of_interest": [ + { + "latitude": 43.275, + "longitude": -78.114990 + }, + { + "latitude": 37.422001, + "longitude": -122.084061 + }, + { + "latitude": -54.420792, + "longitude": 3.346450 + } + ], + "foobars_type": [ + "Bar", + "Baz", + "Bar" + ], + "foobars" : [ + { + "a": 123, + "b": 456, + "c": { + "meal": "Apple" + } + }, + { + "meal": "Banana" + }, + { + "a": 987, + "b": 654, + "c": { + "meal": "Kiwi" + } + } + ], + "measurement_type": "Tolerance", + "measurement": { + "width": 5 + }, + "anything_type": "Bar", + "anything": { + "a": 321, + "b": 654 + }, + "charlie": "alice" +} \ No newline at end of file diff --git a/tests/annotated_binary/annotated_binary_old.afb b/tests/annotated_binary/annotated_binary_old.afb new file mode 100644 index 000000000..9a0e5b022 --- /dev/null +++ b/tests/annotated_binary/annotated_binary_old.afb @@ -0,0 +1,293 @@ +// Annotated Flatbuffer Binary +// +// Schema file: annotated_binary_old.fbs +// Binary file: annotated_binary.bin + +header: + +0x0000 | 44 00 00 00 | UOffset32 | 0x00000044 (68) Loc: +0x0044 | offset to root table `AnnotatedBinary.Foo` + +0x0004 | 41 4E 4E 4F | char[4] | ANNO | File Identifier + +padding: + +0x0008 | 00 00 | uint8_t[2] | .. | padding + +vtable (AnnotatedBinary.Foo): + +0x000A | 3A 00 | uint16_t | 0x003A (58) | size of this vtable + +0x000C | 68 00 | uint16_t | 0x0068 (104) | size of referring table + +0x000E | 0C 00 | VOffset16 | 0x000C (12) | offset to field `counter` (id: 0) + +0x0010 | 07 00 | VOffset16 | 0x0007 (7) | offset to field `healthy` (id: 1) + +0x0012 | 00 00 | VOffset16 | 0x0000 (0) | offset to field `level` (id: 2) (Long) + +0x0014 | 08 00 | VOffset16 | 0x0008 (8) | offset to field `meal` (id: 3) + +0x0016 | 10 00 | VOffset16 | 0x0010 (16) | offset to field `bar` (id: 4) + +0x0018 | 14 00 | VOffset16 | 0x0014 (20) | offset to field `home` (id: 5) + +0x001A | 30 00 | VOffset16 | 0x0030 (48) | offset to field `name` (id: 6) + +0x001C | 34 00 | VOffset16 | 0x0034 (52) | offset to field `bars` (id: 7) + +0x001E | 09 00 | VOffset16 | 0x0009 (9) | offset to field `bar_baz_type` (id: 8) + +0x0020 | 38 00 | VOffset16 | 0x0038 (56) | offset to field `bar_baz` (id: 9) + +0x0022 | 3C 00 | VOffset16 | 0x003C (60) | offset to field `accounts` (id: 10) + +0x0024 | 40 00 | VOffset16 | 0x0040 (64) | offset to field `bob` (id: 11) + +0x0026 | 44 00 | VOffset16 | 0x0044 (68) | offset to field `alice` (id: 12) + +0x0028 | 00 00 | VOffset16 | 0x0000 (0) | offset to field `maybe_i32` (id: 13) (Int) + +0x002A | 00 00 | VOffset16 | 0x0000 (0) | offset to field `default_i32` (id: 14) (Int) + +0x002C | 48 00 | VOffset16 | 0x0048 (72) | offset to field `just_i32` (id: 15) + +0x002E | 4C 00 | VOffset16 | 0x004C (76) | offset to field `names` (id: 16) + +0x0030 | 50 00 | VOffset16 | 0x0050 (80) | offset to field `points_of_interest` (id: 17) + +0x0032 | 54 00 | VOffset16 | 0x0054 (84) | offset to field `foobars_type` (id: 18) + +0x0034 | 58 00 | VOffset16 | 0x0058 (88) | offset to field `foobars` (id: 19) + +0x0036 | 0A 00 | VOffset16 | 0x000A (10) | offset to field `measurement_type` (id: 20) + +0x0038 | 5C 00 | VOffset16 | 0x005C (92) | offset to field `measurement` (id: 21) + +0x003A | 0B 00 | VOffset16 | 0x000B (11) | offset to field `anything_type` (id: 22) + +0x003C | 60 00 | VOffset16 | 0x0060 (96) | offset to field `anything` (id: 23) + +0x003E | 00 00 | VOffset16 | 0x0000 (0) | offset to field `temperature` (id: 24) (Float) + +0x0040 | 00 00 | VOffset16 | 0x0000 (0) | offset to field `teetotaler` (id: 25) (Obj) + +0x0042 | 64 00 | VOffset16 | 0x0064 (100) | offset to unknown field (id: 26) + +root_table (AnnotatedBinary.Foo): + +0x0044 | 3A 00 00 00 | SOffset32 | 0x0000003A (58) Loc: +0x000A | offset to vtable + +0x0048 | 00 00 00 | uint8_t[3] | ... | padding + +0x004B | 01 | uint8_t | 0x01 (1) | table field `healthy` (Bool) + +0x004C | 02 | uint8_t | 0x02 (2) | table field `meal` (Byte) + +0x004D | 02 | uint8_t | 0x02 (2) | table field `bar_baz_type` (UType) + +0x004E | 01 | uint8_t | 0x01 (1) | table field `measurement_type` (UType) + +0x004F | 01 | uint8_t | 0x01 (1) | table field `anything_type` (UType) + +0x0050 | D2 04 00 00 | uint32_t | 0x000004D2 (1234) | table field `counter` (Int) + +0x0054 | 28 02 00 00 | UOffset32 | 0x00000228 (552) Loc: +0x027C | offset to field `bar` + +0x0058 | 01 00 00 00 | uint32_t | 0x00000001 (1) | struct field `AnnotatedBinary.Building.floors` (Int) + +0x005C | 02 00 00 00 | uint32_t | 0x00000002 (2) | struct field `AnnotatedBinary.Building.doors` (Int) + +0x0060 | 0C 00 00 00 | uint32_t | 0x0000000C (12) | struct field `AnnotatedBinary.Building.windows` (Int) + +0x0064 | 0A 00 00 00 | uint32_t | 0x0000000A (10) | array field `AnnotatedBinary.Dimension.values[0]` (Int) + +0x0068 | 0C 00 00 00 | uint32_t | 0x0000000C (12) | array field `AnnotatedBinary.Dimension.values[1]` (Int) + +0x006C | 14 00 00 00 | uint32_t | 0x00000014 (20) | array field `AnnotatedBinary.Dimension.values[2]` (Int) + +0x0070 | 01 | uint8_t | 0x01 (1) | struct field `AnnotatedBinary.Tolerance.width` (UByte) + +0x0071 | 02 | uint8_t | 0x02 (2) | struct field `AnnotatedBinary.Tolerance.width` (UByte) + +0x0072 | 03 | uint8_t | 0x03 (3) | struct field `AnnotatedBinary.Tolerance.width` (UByte) + +0x0073 | 00 | uint8_t[1] | . | padding + +0x0074 | C8 01 00 00 | UOffset32 | 0x000001C8 (456) Loc: +0x023C | offset to field `name` + +0x0078 | 5C 01 00 00 | UOffset32 | 0x0000015C (348) Loc: +0x01D4 | offset to field `bars` + +0x007C | 50 01 00 00 | UOffset32 | 0x00000150 (336) Loc: +0x01CC | offset to field `bar_baz` (union of type `Baz`) + +0x0080 | 34 01 00 00 | UOffset32 | 0x00000134 (308) Loc: +0x01B4 | offset to field `accounts` + +0x0084 | 24 01 00 00 | UOffset32 | 0x00000124 (292) Loc: +0x01A8 | offset to field `bob` + +0x0088 | 14 01 00 00 | UOffset32 | 0x00000114 (276) Loc: +0x019C | offset to field `alice` + +0x008C | 0D 00 00 00 | uint32_t | 0x0000000D (13) | table field `just_i32` (Int) + +0x0090 | DC 00 00 00 | UOffset32 | 0x000000DC (220) Loc: +0x016C | offset to field `names` + +0x0094 | A0 00 00 00 | UOffset32 | 0x000000A0 (160) Loc: +0x0134 | offset to field `points_of_interest` + +0x0098 | 94 00 00 00 | UOffset32 | 0x00000094 (148) Loc: +0x012C | offset to field `foobars_type` + +0x009C | 38 00 00 00 | UOffset32 | 0x00000038 (56) Loc: +0x00D4 | offset to field `foobars` + +0x00A0 | 33 00 00 00 | UOffset32 | 0x00000033 (51) Loc: +0x00D3 | offset to field `measurement` (union of type `Tolerance`) + +0x00A4 | 1C 00 00 00 | UOffset32 | 0x0000001C (28) Loc: +0x00C0 | offset to field `anything` (union of type `Bar`) + +0x00A8 | 04 00 00 00 | ?uint8_t[4] | .... | Unknown field + +unknown (no known references): + +0x00AC | 05 00 00 00 61 6C 69 63 | ?uint8_t[12] | ....alic | WARN: nothing refers to this. Check if any `Unkown Field`s point to this. + +0x00B4 | 65 00 00 00 | | e... + +vtable (AnnotatedBinary.Bar): + +0x00B8 | 08 00 | uint16_t | 0x0008 (8) | size of this vtable + +0x00BA | 13 00 | uint16_t | 0x0013 (19) | size of referring table + +0x00BC | 08 00 | VOffset16 | 0x0008 (8) | offset to field `a` (id: 0) + +0x00BE | 04 00 | VOffset16 | 0x0004 (4) | offset to field `b` (id: 1) + +table (AnnotatedBinary.Bar): + +0x00C0 | 08 00 00 00 | SOffset32 | 0x00000008 (8) Loc: +0x00B8 | offset to vtable + +0x00C4 | 00 80 23 44 | float | 0x44238000 (654.000000) | table field `b` (Float) + +0x00C8 | 00 00 00 00 00 10 74 40 | double | 0x4074100000000000 (321.000000) | table field `a` (Double) + +0x00D0 | 00 00 00 | uint8_t[3] | ... | padding + +union (AnnotatedBinary.Tolerance.measurement): + +0x00D3 | 05 | uint8_t | 0x05 (5) | struct field `AnnotatedBinary.Tolerance.width` (UByte) + +vector (AnnotatedBinary.Foo.foobars): + +0x00D4 | 03 00 00 00 | uint32_t | 0x00000003 (3) | length of vector (# items) + +0x00D8 | 34 00 00 00 | UOffset32 | 0x00000034 (52) Loc: +0x010C | offset to union[0] (`Bar`) + +0x00DC | 2C 00 00 00 | UOffset32 | 0x0000002C (44) Loc: +0x0108 | offset to union[1] (`Baz`) + +0x00E0 | 04 00 00 00 | UOffset32 | 0x00000004 (4) Loc: +0x00E4 | offset to union[2] (`Bar`) + +table (AnnotatedBinary.Bar): + +0x00E4 | D2 FE FF FF | SOffset32 | 0xFFFFFED2 (-302) Loc: +0x0212 | offset to vtable + +0x00E8 | 00 80 23 44 | float | 0x44238000 (654.000000) | table field `b` (Float) + +0x00EC | 10 00 00 00 | UOffset32 | 0x00000010 (16) Loc: +0x00FC | offset to field `c` + +0x00F0 | 00 00 00 00 00 D8 8E 40 | double | 0x408ED80000000000 (987.000000) | table field `a` (Double) + +0x00F8 | 00 00 00 00 | uint8_t[4] | .... | padding + +table (AnnotatedBinary.Baz): + +0x00FC | 6A FE FF FF | SOffset32 | 0xFFFFFE6A (-406) Loc: +0x0292 | offset to vtable + +0x0100 | 00 00 00 | uint8_t[3] | ... | padding + +0x0103 | 03 | uint8_t | 0x03 (3) | table field `meal` (Byte) + +vtable (AnnotatedBinary.Baz): + +0x0104 | 04 00 | uint16_t | 0x0004 (4) | size of this vtable + +0x0106 | 04 00 | uint16_t | 0x0004 (4) | size of referring table + +table (AnnotatedBinary.Baz): + +0x0108 | 04 00 00 00 | SOffset32 | 0x00000004 (4) Loc: +0x0104 | offset to vtable + +table (AnnotatedBinary.Bar): + +0x010C | FA FE FF FF | SOffset32 | 0xFFFFFEFA (-262) Loc: +0x0212 | offset to vtable + +0x0110 | 00 00 E4 43 | float | 0x43E40000 (456.000000) | table field `b` (Float) + +0x0114 | 10 00 00 00 | UOffset32 | 0x00000010 (16) Loc: +0x0124 | offset to field `c` + +0x0118 | 00 00 00 00 00 C0 5E 40 | double | 0x405EC00000000000 (123.000000) | table field `a` (Double) + +0x0120 | 00 00 00 00 | uint8_t[4] | .... | padding + +table (AnnotatedBinary.Baz): + +0x0124 | 92 FE FF FF | SOffset32 | 0xFFFFFE92 (-366) Loc: +0x0292 | offset to vtable + +0x0128 | 00 00 00 | uint8_t[3] | ... | padding + +0x012B | 01 | uint8_t | 0x01 (1) | table field `meal` (Byte) + +vector (AnnotatedBinary.Foo.foobars_type): + +0x012C | 03 00 00 00 | uint32_t | 0x00000003 (3) | length of vector (# items) + +0x0130 | 01 | uint8_t | 0x01 (1) | value[0] + +0x0131 | 02 | uint8_t | 0x02 (2) | value[1] + +0x0132 | 01 | uint8_t | 0x01 (1) | value[2] + +vector (AnnotatedBinary.Foo.points_of_interest): + +0x0134 | 03 00 00 00 | uint32_t | 0x00000003 (3) | length of vector (# items) + +0x0138 | 33 33 33 33 33 A3 45 40 | double | 0x4045A33333333333 (43.275000) | struct field `AnnotatedBinary.Location.latitude` (Double) + +0x0140 | 7E 57 04 FF 5B 87 53 C0 | double | 0xC053875BFF04577E (-78.114990) | struct field `AnnotatedBinary.Location.longitude` (Double) + +0x0148 | 8D F0 F6 20 04 B6 42 40 | double | 0x4042B60420F6F08D (37.422001) | struct field `AnnotatedBinary.Location.latitude` (Double) + +0x0150 | 9F 77 63 41 61 85 5E C0 | double | 0xC05E85614163779F (-122.084061) | struct field `AnnotatedBinary.Location.longitude` (Double) + +0x0158 | 8F 35 23 83 DC 35 4B C0 | double | 0xC04B35DC8323358F (-54.420792) | struct field `AnnotatedBinary.Location.latitude` (Double) + +0x0160 | F6 97 DD 93 87 C5 0A 40 | double | 0x400AC58793DD97F6 (3.346450) | struct field `AnnotatedBinary.Location.longitude` (Double) + +padding: + +0x0168 | 00 00 00 00 | uint8_t[4] | .... | padding + +vector (AnnotatedBinary.Foo.names): + +0x016C | 03 00 00 00 | uint32_t | 0x00000003 (3) | length of vector (# items) + +0x0170 | 20 00 00 00 | UOffset32 | 0x00000020 (32) Loc: +0x0190 | offset to string[0] + +0x0174 | 14 00 00 00 | UOffset32 | 0x00000014 (20) Loc: +0x0188 | offset to string[1] + +0x0178 | 04 00 00 00 | UOffset32 | 0x00000004 (4) Loc: +0x017C | offset to string[2] + +string (AnnotatedBinary.Foo.names): + +0x017C | 07 00 00 00 | uint32_t | 0x00000007 (7) | length of string + +0x0180 | 63 68 61 72 6C 69 65 | char[7] | charlie + +0x0187 | 00 | char | 0x00 (0) | string terminator + +string (AnnotatedBinary.Foo.names): + +0x0188 | 03 00 00 00 | uint32_t | 0x00000003 (3) | length of string + +0x018C | 62 6F 62 | char[3] | bob + +0x018F | 00 | char | 0x00 (0) | string terminator + +string (AnnotatedBinary.Foo.names): + +0x0190 | 05 00 00 00 | uint32_t | 0x00000005 (5) | length of string + +0x0194 | 61 6C 69 63 65 | char[5] | alice + +0x0199 | 00 | char | 0x00 (0) | string terminator + +padding: + +0x019A | 00 00 | uint8_t[2] | .. | padding + +string (AnnotatedBinary.Foo.alice): + +0x019C | 07 00 00 00 | uint32_t | 0x00000007 (7) | length of string + +0x01A0 | 63 68 61 72 6C 69 65 | char[7] | charlie + +0x01A7 | 00 | char | 0x00 (0) | string terminator + +string (AnnotatedBinary.Foo.bob): + +0x01A8 | 07 00 00 00 | uint32_t | 0x00000007 (7) | length of string + +0x01AC | 63 68 61 72 6C 69 65 | char[7] | charlie + +0x01B3 | 00 | char | 0x00 (0) | string terminator + +vector (AnnotatedBinary.Foo.accounts): + +0x01B4 | 09 00 00 00 | uint32_t | 0x00000009 (9) | length of vector (# items) + +0x01B8 | 09 00 | uint16_t | 0x0009 (9) | value[0] + +0x01BA | 08 00 | uint16_t | 0x0008 (8) | value[1] + +0x01BC | 07 00 | uint16_t | 0x0007 (7) | value[2] + +0x01BE | 01 00 | uint16_t | 0x0001 (1) | value[3] + +0x01C0 | 02 00 | uint16_t | 0x0002 (2) | value[4] + +0x01C2 | 03 00 | uint16_t | 0x0003 (3) | value[5] + +0x01C4 | 06 00 | uint16_t | 0x0006 (6) | value[6] + +0x01C6 | 05 00 | uint16_t | 0x0005 (5) | value[7] + +0x01C8 | 04 00 | uint16_t | 0x0004 (4) | value[8] + +padding: + +0x01CA | 00 00 | uint8_t[2] | .. | padding + +table (AnnotatedBinary.Baz): + +0x01CC | 3A FF FF FF | SOffset32 | 0xFFFFFF3A (-198) Loc: +0x0292 | offset to vtable + +0x01D0 | 00 00 00 | uint8_t[3] | ... | padding + +0x01D3 | 03 | uint8_t | 0x03 (3) | table field `meal` (Byte) + +vector (AnnotatedBinary.Foo.bars): + +0x01D4 | 02 00 00 00 | uint32_t | 0x00000002 (2) | length of vector (# items) + +0x01D8 | 44 00 00 00 | UOffset32 | 0x00000044 (68) Loc: +0x021C | offset to table[0] + +0x01DC | 10 00 00 00 | UOffset32 | 0x00000010 (16) Loc: +0x01EC | offset to table[1] + +padding: + +0x01E0 | 00 00 | uint8_t[2] | .. | padding + +vtable (AnnotatedBinary.Bar): + +0x01E2 | 0A 00 | uint16_t | 0x000A (10) | size of this vtable + +0x01E4 | 1A 00 | uint16_t | 0x001A (26) | size of referring table + +0x01E6 | 0C 00 | VOffset16 | 0x000C (12) | offset to field `a` (id: 0) + +0x01E8 | 04 00 | VOffset16 | 0x0004 (4) | offset to field `b` (id: 1) + +0x01EA | 08 00 | VOffset16 | 0x0008 (8) | offset to field `c` (id: 2) + +table (AnnotatedBinary.Bar): + +0x01EC | 0A 00 00 00 | SOffset32 | 0x0000000A (10) Loc: +0x01E2 | offset to vtable + +0x01F0 | 00 80 23 44 | float | 0x44238000 (654.000000) | table field `b` (Float) + +0x01F4 | 18 00 00 00 | UOffset32 | 0x00000018 (24) Loc: +0x020C | offset to field `c` + +0x01F8 | 00 00 00 00 00 D8 8E 40 | double | 0x408ED80000000000 (987.000000) | table field `a` (Double) + +0x0200 | 00 00 00 00 00 00 | uint8_t[6] | ...... | padding + +vtable (AnnotatedBinary.Baz): + +0x0206 | 06 00 | uint16_t | 0x0006 (6) | size of this vtable + +0x0208 | 06 00 | uint16_t | 0x0006 (6) | size of referring table + +0x020A | 05 00 | VOffset16 | 0x0005 (5) | offset to field `meal` (id: 0) + +table (AnnotatedBinary.Baz): + +0x020C | 06 00 00 00 | SOffset32 | 0x00000006 (6) Loc: +0x0206 | offset to vtable + +0x0210 | 00 | uint8_t[1] | . | padding + +0x0211 | 03 | uint8_t | 0x03 (3) | table field `meal` (Byte) + +vtable (AnnotatedBinary.Bar): + +0x0212 | 0A 00 | uint16_t | 0x000A (10) | size of this vtable + +0x0214 | 18 00 | uint16_t | 0x0018 (24) | size of referring table + +0x0216 | 0C 00 | VOffset16 | 0x000C (12) | offset to field `a` (id: 0) + +0x0218 | 04 00 | VOffset16 | 0x0004 (4) | offset to field `b` (id: 1) + +0x021A | 08 00 | VOffset16 | 0x0008 (8) | offset to field `c` (id: 2) + +table (AnnotatedBinary.Bar): + +0x021C | 0A 00 00 00 | SOffset32 | 0x0000000A (10) Loc: +0x0212 | offset to vtable + +0x0220 | 00 00 E4 43 | float | 0x43E40000 (456.000000) | table field `b` (Float) + +0x0224 | 10 00 00 00 | UOffset32 | 0x00000010 (16) Loc: +0x0234 | offset to field `c` + +0x0228 | 00 00 00 00 00 C0 5E 40 | double | 0x405EC00000000000 (123.000000) | table field `a` (Double) + +0x0230 | 00 00 00 00 | uint8_t[4] | .... | padding + +table (AnnotatedBinary.Baz): + +0x0234 | A2 FF FF FF | SOffset32 | 0xFFFFFFA2 (-94) Loc: +0x0292 | offset to vtable + +0x0238 | 00 00 00 | uint8_t[3] | ... | padding + +0x023B | 01 | uint8_t | 0x01 (1) | table field `meal` (Byte) + +string (AnnotatedBinary.Foo.name): + +0x023C | 2F 00 00 00 | uint32_t | 0x0000002F (47) | length of string + +0x0240 | 54 68 69 73 20 69 73 20 | char[47] | This is + +0x0248 | 61 20 6C 6F 6E 67 20 73 | | a long s + +0x0250 | 74 72 69 6E 67 20 74 6F | | tring to + +0x0258 | 20 73 68 6F 77 20 68 6F | | show ho + +0x0260 | 77 20 69 74 20 62 72 65 | | w it bre + +0x0268 | 61 6B 73 20 75 70 2E | | aks up. + +0x026F | 00 | char | 0x00 (0) | string terminator + +padding: + +0x0270 | 00 00 | uint8_t[2] | .. | padding + +vtable (AnnotatedBinary.Bar): + +0x0272 | 0A 00 | uint16_t | 0x000A (10) | size of this vtable + +0x0274 | 16 00 | uint16_t | 0x0016 (22) | size of referring table + +0x0276 | 0C 00 | VOffset16 | 0x000C (12) | offset to field `a` (id: 0) + +0x0278 | 04 00 | VOffset16 | 0x0004 (4) | offset to field `b` (id: 1) + +0x027A | 08 00 | VOffset16 | 0x0008 (8) | offset to field `c` (id: 2) + +table (AnnotatedBinary.Bar): + +0x027C | 0A 00 00 00 | SOffset32 | 0x0000000A (10) Loc: +0x0272 | offset to vtable + +0x0280 | 65 20 71 49 | float | 0x49712065 (987654.312500) | table field `b` (Float) + +0x0284 | 14 00 00 00 | UOffset32 | 0x00000014 (20) Loc: +0x0298 | offset to field `c` + +0x0288 | C9 76 BE 9F 0C 24 FE 40 | double | 0x40FE240C9FBE76C9 (123456.789000) | table field `a` (Double) + +0x0290 | 00 00 | uint8_t[2] | .. | padding + +vtable (AnnotatedBinary.Baz): + +0x0292 | 06 00 | uint16_t | 0x0006 (6) | size of this vtable + +0x0294 | 08 00 | uint16_t | 0x0008 (8) | size of referring table + +0x0296 | 07 00 | VOffset16 | 0x0007 (7) | offset to field `meal` (id: 0) + +table (AnnotatedBinary.Baz): + +0x0298 | 06 00 00 00 | SOffset32 | 0x00000006 (6) Loc: +0x0292 | offset to vtable + +0x029C | 00 00 00 | uint8_t[3] | ... | padding + +0x029F | 01 | uint8_t | 0x01 (1) | table field `meal` (Byte) diff --git a/tests/annotated_binary/annotated_binary_old.fbs b/tests/annotated_binary/annotated_binary_old.fbs new file mode 100644 index 000000000..c22d6d2f3 --- /dev/null +++ b/tests/annotated_binary/annotated_binary_old.fbs @@ -0,0 +1,94 @@ +namespace AnnotatedBinary; + +enum Food : byte { + None = 0, + Apple = 1, + Banana = 2, + Kiwi = 3, +} + +table Baz { + meal:Food = Banana; +} + +table Bar { + a:double = 3.14; + b:float = 1.68; + c:Baz; +} + +union BarBaz { + Bar, Baz +} + +union Measurement { + Tolerance, Dimension +} + +struct Tolerance { + width:uint8; +} + +union Any { + Bar, Tolerance +} + +struct Dimension { + values:[int:3]; + tolerances:[Tolerance:3]; +} + +struct Building { + floors:int; + doors:int; + windows:int; + dimensions:Dimension; +} + +struct Location { + latitude:double; + longitude:double; +} + +table Foo { + counter:int; + healthy:bool; + level:long = 99; + meal:Food = Apple; + bar:Bar; + home:Building; + name:string; + // Vector of tables + bars:[Bar]; + // Union of tables + bar_baz:BarBaz; + // Vector of Scalars + accounts:[uint16]; + bob:string; + alice:string; + // Optional Scalars + maybe_i32: int32 = null; + default_i32: int32 = 42; + just_i32: int32; + // Vector of strings + names:[string]; + // Vector of structs + points_of_interest:[Location]; + // Vector of unions + foobars:[BarBaz]; + // Union of structs + measurement:Measurement; + // Union of struct/table + anything:Any; + // Default floating point + temperature:float=98.6; + // Not present object + teetotaler:Bar; + + // NOTE THIS IS A PURPOSELY OLD VERSION OF annotated_binary.fbs TO TEST + // PROCESSING OF NEWER BINARIES THAN THE SCHEMA. DO NOT ADD TO THIS. +} + +file_identifier "ANNO"; + +root_type Foo; \ No newline at end of file