BigfootDev/flatbuffers
1
0
Fork 1
mirror of https://github.com/google/flatbuffers.git synced 2026-06-09 22:56:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

FlatBuffers 64 for C++ (#7935)

Browse Source 
* First working hack of adding 64-bit. Don't judge :)

* Made vector_downward work on 64 bit types

* vector_downward uses size_t, added offset64 to reflection

* cleaned up adding offset64 in parser

* Add C++ testing skeleton for 64-bit

* working test for CreateVector64

* working >2 GiB buffers

* support for large strings

* simplified CreateString<> to just provide the offset type

* generalize CreateVector template

* update test_64.afb due to upstream format change

* Added Vector64 type, which is just an alias for vector ATM

* Switch to Offset64 for Vector64

* Update for reflection bfbs output change

* Starting to add support for vector64 type in C++

* made a generic CreateVector that can handle different offsets and vector types

* Support for 32-vector with 64-addressing

* Vector64 basic builder + tests working

* basic support for json vector64 support

* renamed fields in test_64bit.fbs to better reflect their use

* working C++ vector64 builder

* Apply --annotate-sparse-vector to 64-bit tests

* Enable Vector64 for --annotate-sparse-vectors

* Merged from upstream

* Add `near_string` field for testing 32-bit offsets alongside

* keep track of where the 32-bit and 64-bit regions are for flatbufferbuilder

* move template<> outside class body for GCC

* update run.sh to build and run tests

* basic assertion for adding 64-bit offset at the wrong time

* started to separate `FlatBufferBuilder` into two classes, 1 64-bit aware, the other not

* add test for nested flatbuffer vector64, fix bug in alignment of big vectors

* fixed CreateDirect method by iterating by Offset64 first

* internal refactoring of flatbufferbuilder

* block not supported languages in the parser from using 64-bit

* evolution tests for adding a vector64 field

* conformity tests for adding/removing offset64 attributes

* ensure test is for a big buffer

* add parser error tests for `offset64` and `vector64` attributes

* add missing static that GCC only complains about

* remove stdint-uintn.h header that gets automatically added

* move 64-bit CalculateOffset internal

* fixed return size of EndVector

* various fixes on windows

* add SizeT to vector_downward

* minimze range of size changes in vector and builder

* reworked how tracking if 64-offsets are added

* Add ReturnT to EndVector

* small cleanups

* remove need for second Array definition

* combine IndirectHelpers into one definition

* started support for vector of struct

* Support for 32/64-vectors of structs + Offset64

* small cleanups

* add verification for vector64

* add sized prefix for 64-bit buffers

* add fuzzer for 64-bit

* add example of adding many vectors using a wrapper table

* run the new -bfbs-gen-embed logic on the 64-bit tests

* remove run.sh and fix cmakelist issue

* fixed bazel rules

* fixed some PR comments

* add 64-bit tests to cmakelist
This commit is contained in:
Derek Bailey
2023-05-09 09:16:30 -07:00
committed by GitHub
parent 13fc75cb6b
commit 63b7b25289
49 changed files with 3274 additions and 529 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/annotated_binary_text_gen.cpp
View File

@@ -1,6 +1,7 @@
#include "annotated_binary_text_gen.h"
#include <algorithm>
#include <cstdint>
#include <fstream>
#include <ostream>
#include <sstream>
@@ -36,6 +37,7 @@ static std::string ToString(const BinarySectionType type) {
case BinarySectionType::Struct: return "struct";
case BinarySectionType::String: return "string";
case BinarySectionType::Vector: return "vector";
case BinarySectionType::Vector64: return "vector64";
case BinarySectionType::Unknown: return "unknown";
case BinarySectionType::Union: return "union";
case BinarySectionType::Padding: return "padding";
@@ -44,7 +46,9 @@ static std::string ToString(const BinarySectionType type) {
}
static bool IsOffset(const BinaryRegionType type) {
return type == BinaryRegionType::UOffset || type == BinaryRegionType::SOffset;
return type == BinaryRegionType::UOffset ||
type == BinaryRegionType::SOffset ||
type == BinaryRegionType::UOffset64;
}
template<typename T> std::string ToString(T value) {
@@ -119,6 +123,9 @@ static std::string ToValueString(const BinaryRegion &region,
case BinaryRegionType::UType: return ToValueString<uint8_t>(region, binary);
// Handle Offsets separately, incase they add additional details.
case BinaryRegionType::UOffset64:
s += ToValueString<uint64_t>(region, binary);
break;
case BinaryRegionType::UOffset:
s += ToValueString<uint32_t>(region, binary);
break;
@@ -368,7 +375,8 @@ static void GenerateSection(std::ostream &os, const BinarySection &section,
// As a space saving measure, skip generating every vector element, just put
// the first and last elements in the output. Skip the whole thing if there
// are only three or fewer elements, as it doesn't save space.
if (section.type == BinarySectionType::Vector &&
if ((section.type == BinarySectionType::Vector ||
section.type == BinarySectionType::Vector64) &&
!output_config.include_vector_contents && section.regions.size() > 4) {
// Generate the length region which should be first.
GenerateRegion(os, section.regions[0], section, binary, output_config);

161
src/binary_annotator.cpp
View File

@@ -1,10 +1,13 @@
#include "binary_annotator.h"
#include <algorithm>
#include <cstdint>
#include <iostream>
#include <limits>
#include <string>
#include <vector>
#include "flatbuffers/base.h"
#include "flatbuffers/reflection.h"
#include "flatbuffers/util.h"
#include "flatbuffers/verifier.h"
@@ -37,9 +40,9 @@ static BinaryRegion MakeBinaryRegion(
return region;
}
static BinarySection MakeBinarySection(
const std::string &name, const BinarySectionType type,
std::vector<BinaryRegion> regions) {
static BinarySection MakeBinarySection(const std::string &name,
const BinarySectionType type,
std::vector<BinaryRegion> regions) {
BinarySection section;
section.name = name;
section.type = type;
@@ -118,12 +121,15 @@ static BinarySection GenerateMissingSection(const uint64_t offset,
std::map<uint64_t, BinarySection> BinaryAnnotator::Annotate() {
flatbuffers::Verifier verifier(bfbs_, static_cast<size_t>(bfbs_length_));
if (!reflection::VerifySchemaBuffer(verifier)) { return {}; }
if ((is_size_prefixed_ &&
!reflection::VerifySizePrefixedSchemaBuffer(verifier)) ||
!reflection::VerifySchemaBuffer(verifier)) {
return {};
}
// The binary is too short to read as a flatbuffers.
// TODO(dbaileychess): We could spit out the annotated buffer sections, but
// I'm not sure if it is worth it.
if (binary_length_ < 4) { return {}; }
if (binary_length_ < FLATBUFFERS_MIN_BUFFER_SIZE) { return {}; }
// Make sure we start with a clean slate.
vtables_.clear();
@@ -151,7 +157,41 @@ std::map<uint64_t, BinarySection> BinaryAnnotator::Annotate() {
}
uint64_t BinaryAnnotator::BuildHeader(const uint64_t header_offset) {
const auto root_table_offset = ReadScalar<uint32_t>(header_offset);
uint64_t offset = header_offset;
std::vector<BinaryRegion> regions;
// If this binary is a size prefixed one, attempt to parse the size.
if (is_size_prefixed_) {
BinaryRegionComment prefix_length_comment;
prefix_length_comment.type = BinaryRegionCommentType::SizePrefix;
bool has_prefix_value = false;
const auto prefix_length = ReadScalar<uoffset64_t>(offset);
if (*prefix_length <= binary_length_) {
regions.push_back(MakeBinaryRegion(offset, sizeof(uoffset64_t),
BinaryRegionType::Uint64, 0, 0,
prefix_length_comment));
offset += sizeof(uoffset64_t);
has_prefix_value = true;
}
if (!has_prefix_value) {
const auto prefix_length = ReadScalar<uoffset_t>(offset);
if (*prefix_length <= binary_length_) {
regions.push_back(MakeBinaryRegion(offset, sizeof(uoffset_t),
BinaryRegionType::Uint32, 0, 0,
prefix_length_comment));
offset += sizeof(uoffset_t);
has_prefix_value = true;
}
}
if (!has_prefix_value) {
SetError(prefix_length_comment, BinaryRegionStatus::ERROR);
}
}
const auto root_table_offset = ReadScalar<uint32_t>(offset);
if (!root_table_offset.has_value()) {
// This shouldn't occur, since we validate the min size of the buffer
@@ -159,22 +199,20 @@ uint64_t BinaryAnnotator::BuildHeader(const uint64_t header_offset) {
return std::numeric_limits<uint64_t>::max();
}
std::vector<BinaryRegion> regions;
uint64_t offset = header_offset;
// TODO(dbaileychess): sized prefixed value
const auto root_table_loc = offset + *root_table_offset;
BinaryRegionComment root_offset_comment;
root_offset_comment.type = BinaryRegionCommentType::RootTableOffset;
root_offset_comment.name = schema_->root_table()->name()->str();
if (!IsValidOffset(root_table_offset.value())) {
if (!IsValidOffset(root_table_loc)) {
SetError(root_offset_comment,
BinaryRegionStatus::ERROR_OFFSET_OUT_OF_BINARY);
}
regions.push_back(
MakeBinaryRegion(offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0,
root_table_offset.value(), root_offset_comment));
regions.push_back(MakeBinaryRegion(offset, sizeof(uint32_t),
BinaryRegionType::UOffset, 0,
root_table_loc, root_offset_comment));
offset += sizeof(uint32_t);
if (IsValidRead(offset, flatbuffers::kFileIdentifierLength) &&
@@ -193,7 +231,7 @@ uint64_t BinaryAnnotator::BuildHeader(const uint64_t header_offset) {
AddSection(header_offset, MakeBinarySection("", BinarySectionType::Header,
std::move(regions)));
return root_table_offset.value();
return root_table_loc;
}
BinaryAnnotator::VTable *BinaryAnnotator::GetOrBuildVTable(
@@ -656,7 +694,18 @@ void BinaryAnnotator::BuildTable(const uint64_t table_offset,
}
// Read the offset
const auto offset_from_field = ReadScalar<uint32_t>(field_offset);
uint64_t offset = 0;
uint64_t length = sizeof(uint32_t);
BinaryRegionType region_type = BinaryRegionType::UOffset;
if (field->offset64()) {
length = sizeof(uint64_t);
region_type = BinaryRegionType::UOffset64;
offset = ReadScalar<uint64_t>(field_offset).value_or(0);
} else {
offset = ReadScalar<uint32_t>(field_offset).value_or(0);
}
// const auto offset_from_field = ReadScalar<uint32_t>(field_offset);
uint64_t offset_of_next_item = 0;
BinaryRegionComment offset_field_comment;
offset_field_comment.type = BinaryRegionCommentType::TableOffsetField;
@@ -666,7 +715,7 @@ void BinaryAnnotator::BuildTable(const uint64_t table_offset,
// Validate any field that isn't inline (i.e., non-structs).
if (!IsInlineField(field)) {
if (!offset_from_field.has_value()) {
if (offset == 0) {
const uint64_t remaining = RemainingBytes(field_offset);
SetError(offset_field_comment,
@@ -678,14 +727,14 @@ void BinaryAnnotator::BuildTable(const uint64_t table_offset,
continue;
}
offset_of_next_item = field_offset + offset_from_field.value();
offset_of_next_item = field_offset + offset;
if (!IsValidOffset(offset_of_next_item)) {
SetError(offset_field_comment,
BinaryRegionStatus::ERROR_OFFSET_OUT_OF_BINARY);
regions.push_back(MakeBinaryRegion(
field_offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0,
offset_of_next_item, offset_field_comment));
regions.push_back(MakeBinaryRegion(field_offset, length, region_type, 0,
offset_of_next_item,
offset_field_comment));
continue;
}
}
@@ -702,9 +751,9 @@ void BinaryAnnotator::BuildTable(const uint64_t table_offset,
} else {
offset_field_comment.default_value = "(table)";
regions.push_back(MakeBinaryRegion(
field_offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0,
offset_of_next_item, offset_field_comment));
regions.push_back(MakeBinaryRegion(field_offset, length, region_type,
0, offset_of_next_item,
offset_field_comment));
BuildTable(offset_of_next_item, BinarySectionType::Table,
next_object);
@@ -713,17 +762,25 @@ void BinaryAnnotator::BuildTable(const uint64_t table_offset,
case reflection::BaseType::String: {
offset_field_comment.default_value = "(string)";
regions.push_back(MakeBinaryRegion(
field_offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0,
offset_of_next_item, offset_field_comment));
regions.push_back(MakeBinaryRegion(field_offset, length, region_type, 0,
offset_of_next_item,
offset_field_comment));
BuildString(offset_of_next_item, table, field);
} break;
case reflection::BaseType::Vector: {
offset_field_comment.default_value = "(vector)";
regions.push_back(MakeBinaryRegion(
field_offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0,
offset_of_next_item, offset_field_comment));
regions.push_back(MakeBinaryRegion(field_offset, length, region_type, 0,
offset_of_next_item,
offset_field_comment));
BuildVector(offset_of_next_item, table, field, table_offset,
vtable->fields);
} break;
case reflection::BaseType::Vector64: {
offset_field_comment.default_value = "(vector64)";
regions.push_back(MakeBinaryRegion(field_offset, length, region_type, 0,
offset_of_next_item,
offset_field_comment));
BuildVector(offset_of_next_item, table, field, table_offset,
vtable->fields);
} break;
@@ -768,8 +825,7 @@ void BinaryAnnotator::BuildTable(const uint64_t table_offset,
offset_field_comment.default_value =
"(union of type `" + enum_type + "`)";
regions.push_back(MakeBinaryRegion(field_offset, sizeof(uint32_t),
BinaryRegionType::UOffset, 0,
regions.push_back(MakeBinaryRegion(field_offset, length, region_type, 0,
union_offset, offset_field_comment));
} break;
@@ -986,7 +1042,28 @@ void BinaryAnnotator::BuildVector(
BinaryRegionComment vector_length_comment;
vector_length_comment.type = BinaryRegionCommentType::VectorLength;
const auto vector_length = ReadScalar<uint32_t>(vector_offset);
const bool is_64_bit_vector =
field->type()->base_type() == reflection::BaseType::Vector64;
flatbuffers::Optional<uint64_t> vector_length;
uint32_t vector_length_size_type = 0;
BinaryRegionType region_type = BinaryRegionType::Uint32;
BinarySectionType section_type = BinarySectionType::Vector;
if (is_64_bit_vector) {
auto v = ReadScalar<uint64_t>(vector_offset);
if (v.has_value()) { vector_length = v.value(); }
vector_length_size_type = sizeof(uint64_t);
region_type = BinaryRegionType::Uint64;
section_type = BinarySectionType::Vector64;
} else {
auto v = ReadScalar<uint32_t>(vector_offset);
if (v.has_value()) { vector_length = v.value(); }
vector_length_size_type = sizeof(uint32_t);
region_type = BinaryRegionType::Uint32;
section_type = BinarySectionType::Vector;
}
if (!vector_length.has_value()) {
const uint64_t remaining = RemainingBytes(vector_offset);
SetError(vector_length_comment, BinaryRegionStatus::ERROR_INCOMPLETE_BINARY,
@@ -1006,7 +1083,7 @@ void BinaryAnnotator::BuildVector(
// Validate there are enough bytes left in the binary to process all the
// items.
const uint64_t last_item_offset =
vector_offset + sizeof(uint32_t) +
vector_offset + vector_length_size_type +
vector_length.value() * GetElementSize(field);
if (!IsValidOffset(last_item_offset - 1)) {
@@ -1016,20 +1093,18 @@ void BinaryAnnotator::BuildVector(
MakeSingleRegionBinarySection(
std::string(table->name()->c_str()) + "." + field->name()->c_str(),
BinarySectionType::Vector,
MakeBinaryRegion(vector_offset, sizeof(uint32_t),
BinaryRegionType::Uint32, 0, 0,
vector_length_comment)));
MakeBinaryRegion(vector_offset, vector_length_size_type,
region_type, 0, 0, vector_length_comment)));
return;
}
std::vector<BinaryRegion> regions;
regions.push_back(MakeBinaryRegion(vector_offset, sizeof(uint32_t),
BinaryRegionType::Uint32, 0, 0,
vector_length_comment));
regions.push_back(MakeBinaryRegion(vector_offset, vector_length_size_type,
region_type, 0, 0, vector_length_comment));
// Consume the vector length offset.
uint64_t offset = vector_offset + sizeof(uint32_t);
uint64_t offset = vector_offset + vector_length_size_type;
switch (field->type()->element()) {
case reflection::BaseType::Obj: {
@@ -1302,7 +1377,7 @@ void BinaryAnnotator::BuildVector(
AddSection(vector_offset,
MakeBinarySection(std::string(table->name()->c_str()) + "." +
field->name()->c_str(),
BinarySectionType::Vector, std::move(regions)));
section_type, std::move(regions)));
}
std::string BinaryAnnotator::BuildUnion(const uint64_t union_offset,

10
src/binary_annotator.h
View File

@@ -48,6 +48,7 @@ enum class BinaryRegionType {
Float = 15,
Double = 16,
UType = 17,
UOffset64 = 18,
};
template<typename T>
@@ -179,6 +180,7 @@ enum class BinarySectionType {
Vector = 7,
Union = 8,
Padding = 9,
Vector64 = 10,
};
// A section of the binary that is grouped together in some logical manner, and
@@ -216,6 +218,7 @@ inline static BinaryRegionType GetRegionType(reflection::BaseType base_type) {
inline static std::string ToString(const BinaryRegionType type) {
switch (type) {
case BinaryRegionType::UOffset: return "UOffset32";
case BinaryRegionType::UOffset64: return "UOffset64";
case BinaryRegionType::SOffset: return "SOffset32";
case BinaryRegionType::VOffset: return "VOffset16";
case BinaryRegionType::Bool: return "bool";
@@ -242,12 +245,14 @@ class BinaryAnnotator {
explicit BinaryAnnotator(const uint8_t *const bfbs,
const uint64_t bfbs_length,
const uint8_t *const binary,
const uint64_t binary_length)
const uint64_t binary_length,
const bool is_size_prefixed)
: bfbs_(bfbs),
bfbs_length_(bfbs_length),
schema_(reflection::GetSchema(bfbs)),
binary_(binary),
binary_length_(binary_length) {}
binary_length_(binary_length),
is_size_prefixed_(is_size_prefixed) {}
std::map<uint64_t, BinarySection> Annotate();
@@ -387,6 +392,7 @@ class BinaryAnnotator {
// The binary data itself.
const uint8_t *binary_;
const uint64_t binary_length_;
const bool is_size_prefixed_;
// Map of binary offset to vtables, to dedupe vtables.
std::map<uint64_t, std::list<VTable>> vtables_;

11
src/flatc.cpp
View File

@@ -252,10 +252,9 @@ const static FlatCOption flatc_options[] = {
"Currently this is required to generate private types in Rust" },
{ "", "python-no-type-prefix-suffix", "",
"Skip emission of Python functions that are prefixed with typenames" },
{ "", "python-typing", "",
"Generate Python type annotations" },
{ "", "python-typing", "", "Generate Python type annotations" },
{ "", "file-names-only", "",
"Print out generated file names without writing to the files"},
"Print out generated file names without writing to the files" },
};
auto cmp = [](FlatCOption a, FlatCOption b) { return a.long_opt < b.long_opt; };
@@ -394,9 +393,11 @@ void FlatCompiler::AnnotateBinaries(const uint8_t *binary_schema,
const uint8_t *binary =
reinterpret_cast<const uint8_t *>(binary_contents.c_str());
const size_t binary_size = binary_contents.size();
const bool is_size_prefixed = options.opts.size_prefixed;
flatbuffers::BinaryAnnotator binary_annotator(
binary_schema, binary_schema_size, binary, binary_size);
binary_schema, binary_schema_size, binary, binary_size,
is_size_prefixed);
auto annotations = binary_annotator.Annotate();
@@ -663,7 +664,7 @@ FlatCOptions FlatCompiler::ParseFromCommandLineArguments(int argc,
} else if (arg == "--annotate") {
if (++argi >= argc) Error("missing path following: " + arg, true);
options.annotate_schema = flatbuffers::PosixPath(argv[argi]);
} else if(arg == "--file-names-only") {
} else if (arg == "--file-names-only") {
// TODO (khhn): Provide 2 implementation
options.file_names_only = true;
} else {

306
src/idl_gen_cpp.cpp
View File

@@ -77,8 +77,7 @@ static std::string GenIncludeGuard(const std::string &file_name,
static bool IsVectorOfPointers(const FieldDef &field) {
const auto &type = field.value.type;
const auto &vector_type = type.VectorType();
return type.base_type == BASE_TYPE_VECTOR &&
vector_type.base_type == BASE_TYPE_STRUCT &&
return IsVector(type) && vector_type.base_type == BASE_TYPE_STRUCT &&
!vector_type.struct_def->fixed && !field.native_inline;
}
@@ -107,6 +106,21 @@ struct IDLOptionsCpp : public IDLOptions {
: IDLOptions(opts), g_cpp_std(CPP_STD_11), g_only_fixed_enums(true) {}
};
// Iterates over all the fields of the object first by Offset type (Offset64
// before Offset32) and then by definition order.
static void ForAllFieldsOrderedByOffset(
const StructDef &object, std::function<void(const FieldDef *field)> func) {
// Loop over all the fields and call the func on all offset64 fields.
for (const FieldDef *field_def : object.fields.vec) {
if (field_def->offset64) { func(field_def); }
}
// Loop over all the fields a second time and call the func on all offset
// fields.
for (const FieldDef *field_def : object.fields.vec) {
if (!field_def->offset64) { func(field_def); }
}
}
class CppGenerator : public BaseGenerator {
public:
CppGenerator(const Parser &parser, const std::string &path,
@@ -273,6 +287,25 @@ class CppGenerator : public BaseGenerator {
}
}
void MarkIf64BitBuilderIsNeeded() {
if (needs_64_bit_builder_) { return; }
for (auto t : parser_.structs_.vec) {
if (t == nullptr) continue;
for (auto f : t->fields.vec) {
if (f == nullptr) continue;
if (f->offset64) {
needs_64_bit_builder_ = true;
break;
}
}
}
}
std::string GetBuilder() {
return std::string("::flatbuffers::FlatBufferBuilder") +
(needs_64_bit_builder_ ? "64" : "");
}
void GenExtraIncludes() {
for (const std::string &cpp_include : opts_.cpp_includes) {
code_ += "#include \"" + cpp_include + "\"";
@@ -396,6 +429,9 @@ class CppGenerator : public BaseGenerator {
// Iterate through all definitions we haven't generate code for (enums,
// structs, and tables) and output them to a single file.
bool generate() {
// Check if we require a 64-bit flatbuffer builder.
MarkIf64BitBuilderIsNeeded();
code_.Clear();
code_ += "// " + std::string(FlatBuffersGeneratedWarning()) + "\n\n";
@@ -530,6 +566,8 @@ class CppGenerator : public BaseGenerator {
code_.SetValue("STRUCT_NAME", name);
code_.SetValue("CPP_NAME", cpp_name);
code_.SetValue("NULLABLE_EXT", NullableExtension());
code_.SetValue(
"SIZE_T", needs_64_bit_builder_ ? ",::flatbuffers::uoffset64_t" : "");
// The root datatype accessor:
code_ += "inline \\";
@@ -546,7 +584,8 @@ class CppGenerator : public BaseGenerator {
"*{{NULLABLE_EXT}}GetSizePrefixed{{STRUCT_NAME}}(const void "
"*buf) {";
code_ +=
" return ::flatbuffers::GetSizePrefixedRoot<{{CPP_NAME}}>(buf);";
" return "
"::flatbuffers::GetSizePrefixedRoot<{{CPP_NAME}}{{SIZE_T}}>(buf);";
code_ += "}";
code_ += "";
@@ -565,7 +604,8 @@ class CppGenerator : public BaseGenerator {
"*buf) {";
code_ +=
" return "
"::flatbuffers::GetMutableSizePrefixedRoot<{{CPP_NAME}}>(buf);";
"::flatbuffers::GetMutableSizePrefixedRoot<{{CPP_NAME}}{{SIZE_T}}>("
"buf);";
code_ += "}";
code_ += "";
}
@@ -612,7 +652,8 @@ class CppGenerator : public BaseGenerator {
code_ += "inline bool VerifySizePrefixed{{STRUCT_NAME}}Buffer(";
code_ += " ::flatbuffers::Verifier &verifier) {";
code_ +=
" return verifier.VerifySizePrefixedBuffer<{{CPP_NAME}}>({{ID}});";
" return "
"verifier.VerifySizePrefixedBuffer<{{CPP_NAME}}{{SIZE_T}}>({{ID}});";
code_ += "}";
code_ += "";
@@ -626,7 +667,7 @@ class CppGenerator : public BaseGenerator {
// Finish a buffer with a given root object:
code_ += "inline void Finish{{STRUCT_NAME}}Buffer(";
code_ += " ::flatbuffers::FlatBufferBuilder &fbb,";
code_ += " " + GetBuilder() + " &fbb,";
code_ += " ::flatbuffers::Offset<{{CPP_NAME}}> root) {";
if (parser_.file_identifier_.length())
code_ += " fbb.Finish(root, {{STRUCT_NAME}}Identifier());";
@@ -636,7 +677,7 @@ class CppGenerator : public BaseGenerator {
code_ += "";
code_ += "inline void FinishSizePrefixed{{STRUCT_NAME}}Buffer(";
code_ += " ::flatbuffers::FlatBufferBuilder &fbb,";
code_ += " " + GetBuilder() + " &fbb,";
code_ += " ::flatbuffers::Offset<{{CPP_NAME}}> root) {";
if (parser_.file_identifier_.length())
code_ += " fbb.FinishSizePrefixed(root, {{STRUCT_NAME}}Identifier());";
@@ -696,6 +737,7 @@ class CppGenerator : public BaseGenerator {
const IDLOptionsCpp opts_;
const TypedFloatConstantGenerator float_const_gen_;
bool needs_64_bit_builder_ = false;
const Namespace *CurrentNameSpace() const { return cur_name_space_; }
@@ -747,10 +789,14 @@ class CppGenerator : public BaseGenerator {
case BASE_TYPE_STRING: {
return "::flatbuffers::String";
}
case BASE_TYPE_VECTOR64:
case BASE_TYPE_VECTOR: {
const auto type_name = GenTypeWire(
type.VectorType(), "", VectorElementUserFacing(type.VectorType()));
return "::flatbuffers::Vector<" + type_name + ">";
return "::flatbuffers::Vector" +
std::string((type.base_type == BASE_TYPE_VECTOR64) ? "64<"
: "<") +
type_name + ">";
}
case BASE_TYPE_STRUCT: {
return WrapInNameSpace(*type.struct_def);
@@ -766,13 +812,15 @@ class CppGenerator : public BaseGenerator {
// Return a C++ type for any type (scalar/pointer) specifically for
// building a flatbuffer.
std::string GenTypeWire(const Type &type, const char *postfix,
bool user_facing_type) const {
bool user_facing_type,
bool _64_bit_offset = false) const {
if (IsScalar(type.base_type)) {
return GenTypeBasic(type, user_facing_type) + postfix;
} else if (IsStruct(type)) {
return "const " + GenTypePointer(type) + " *";
} else {
return "::flatbuffers::Offset<" + GenTypePointer(type) + ">" + postfix;
return "::flatbuffers::Offset" + std::string(_64_bit_offset ? "64" : "") +
"<" + GenTypePointer(type) + ">" + postfix;
}
}
@@ -858,6 +906,7 @@ class CppGenerator : public BaseGenerator {
case BASE_TYPE_STRING: {
return NativeString(&field);
}
case BASE_TYPE_VECTOR64:
case BASE_TYPE_VECTOR: {
const auto type_name = GenTypeNative(type.VectorType(), true, field);
if (type.struct_def &&
@@ -866,8 +915,9 @@ class CppGenerator : public BaseGenerator {
type.struct_def->attributes.Lookup("native_custom_alloc");
return "std::vector<" + type_name + "," +
native_custom_alloc->constant + "<" + type_name + ">>";
} else
} else {
return "std::vector<" + type_name + ">";
}
}
case BASE_TYPE_STRUCT: {
auto type_name = WrapInNameSpace(*type.struct_def);
@@ -1015,8 +1065,8 @@ class CppGenerator : public BaseGenerator {
std::string UnionPackSignature(const EnumDef &enum_def, bool inclass) {
return "::flatbuffers::Offset<void> " +
(inclass ? "" : Name(enum_def) + "Union::") +
"Pack(::flatbuffers::FlatBufferBuilder &_fbb, " +
(inclass ? "" : Name(enum_def) + "Union::") + "Pack(" +
GetBuilder() + " &_fbb, " +
"const ::flatbuffers::rehasher_function_t *_rehasher" +
(inclass ? " = nullptr" : "") + ") const";
}
@@ -1024,8 +1074,7 @@ class CppGenerator : public BaseGenerator {
std::string TableCreateSignature(const StructDef &struct_def, bool predecl,
const IDLOptions &opts) {
return "::flatbuffers::Offset<" + Name(struct_def) + "> Create" +
Name(struct_def) +
"(::flatbuffers::FlatBufferBuilder &_fbb, const " +
Name(struct_def) + "(" + GetBuilder() + " &_fbb, const " +
NativeName(Name(struct_def), &struct_def, opts) +
" *_o, const ::flatbuffers::rehasher_function_t *_rehasher" +
(predecl ? " = nullptr" : "") + ")";
@@ -1035,7 +1084,7 @@ class CppGenerator : public BaseGenerator {
const IDLOptions &opts) {
return std::string(inclass ? "static " : "") + "::flatbuffers::Offset<" +
Name(struct_def) + "> " + (inclass ? "" : Name(struct_def) + "::") +
"Pack(::flatbuffers::FlatBufferBuilder &_fbb, " + "const " +
"Pack(" + GetBuilder() + " &_fbb, " + "const " +
NativeName(Name(struct_def), &struct_def, opts) + "* _o, " +
"const ::flatbuffers::rehasher_function_t *_rehasher" +
(inclass ? " = nullptr" : "") + ")";
@@ -1791,7 +1840,8 @@ class CppGenerator : public BaseGenerator {
if (IsStruct(vtype)) {
type = WrapInNameSpace(*vtype.struct_def);
} else {
type = GenTypeWire(vtype, "", VectorElementUserFacing(vtype));
type = GenTypeWire(vtype, "", VectorElementUserFacing(vtype),
field.offset64);
}
if (TypeHasKey(vtype)) {
code_.SetValue("PARAM_TYPE", "std::vector<" + type + "> *");
@@ -1805,7 +1855,8 @@ class CppGenerator : public BaseGenerator {
if (field.IsScalarOptional())
code_.SetValue("PARAM_TYPE", GenOptionalDecl(type) + " ");
else
code_.SetValue("PARAM_TYPE", GenTypeWire(type, " ", true));
code_.SetValue("PARAM_TYPE",
GenTypeWire(type, " ", true, field.offset64));
}
code_ += "{{PRE}}{{PARAM_TYPE}}{{PARAM_NAME}} = {{PARAM_VALUE}}\\";
}
@@ -1814,7 +1865,7 @@ class CppGenerator : public BaseGenerator {
void GenMember(const FieldDef &field) {
if (!field.deprecated && // Deprecated fields won't be accessible.
field.value.type.base_type != BASE_TYPE_UTYPE &&
(field.value.type.base_type != BASE_TYPE_VECTOR ||
(!IsVector(field.value.type) ||
field.value.type.element != BASE_TYPE_UTYPE)) {
auto type = GenTypeNative(field.value.type, false, field);
auto cpp_type = field.attributes.Lookup("cpp_type");
@@ -1918,7 +1969,7 @@ class CppGenerator : public BaseGenerator {
Name(field) + "(" + native_default->constant + ")";
}
}
} else if (cpp_type && field.value.type.base_type != BASE_TYPE_VECTOR) {
} else if (cpp_type && !IsVector(field.value.type)) {
if (!initializer_list.empty()) { initializer_list += ",\n "; }
initializer_list += Name(field) + "(0)";
}
@@ -2063,7 +2114,7 @@ class CppGenerator : public BaseGenerator {
const auto rhs_accessor = "rhs." + accessor;
if (!field.deprecated && // Deprecated fields won't be accessible.
field.value.type.base_type != BASE_TYPE_UTYPE &&
(field.value.type.base_type != BASE_TYPE_VECTOR ||
(!IsVector(field.value.type) ||
field.value.type.element != BASE_TYPE_UTYPE)) {
if (!compare_op.empty()) { compare_op += " &&\n "; }
if (struct_def.fixed || field.native_inline ||
@@ -2195,7 +2246,10 @@ class CppGenerator : public BaseGenerator {
"{{PRE}}VerifyField{{REQUIRED}}<{{SIZE}}>(verifier, "
"{{OFFSET}}, {{ALIGN}})\\";
} else {
code_ += "{{PRE}}VerifyOffset{{REQUIRED}}(verifier, {{OFFSET}})\\";
code_.SetValue("OFFSET_SIZE", field.offset64 ? "64" : "");
code_ +=
"{{PRE}}VerifyOffset{{OFFSET_SIZE}}{{REQUIRED}}(verifier, "
"{{OFFSET}})\\";
}
switch (field.value.type.base_type) {
@@ -2217,6 +2271,7 @@ class CppGenerator : public BaseGenerator {
code_ += "{{PRE}}verifier.VerifyString({{NAME}}())\\";
break;
}
case BASE_TYPE_VECTOR64:
case BASE_TYPE_VECTOR: {
code_ += "{{PRE}}verifier.VerifyVector({{NAME}}())\\";
@@ -2468,12 +2523,18 @@ class CppGenerator : public BaseGenerator {
if (!field.IsScalarOptional()) {
const bool is_scalar = IsScalar(type.base_type);
std::string accessor;
if (is_scalar)
std::string offset_size = "";
if (is_scalar) {
accessor = "GetField<";
else if (IsStruct(type))
} else if (IsStruct(type)) {
accessor = "GetStruct<";
else
accessor = "GetPointer<";
} else {
if (field.offset64) {
accessor = "GetPointer64<";
} else {
accessor = "GetPointer<";
}
}
auto offset_type = GenTypeGet(type, "", "const ", " *", false);
auto call = accessor + offset_type + ">(" + offset_str;
// Default value as second arg for non-pointer types.
@@ -2633,7 +2694,7 @@ class CppGenerator : public BaseGenerator {
auto offset_str = GenFieldOffsetName(field);
if (is_scalar) {
const auto wire_type = GenTypeWire(type, "", false);
const auto wire_type = GenTypeWire(type, "", false, field.offset64);
code_.SetValue("SET_FN", "SetField<" + wire_type + ">");
code_.SetValue("OFFSET_NAME", offset_str);
code_.SetValue("FIELD_TYPE", GenTypeBasic(type, true));
@@ -2665,7 +2726,11 @@ class CppGenerator : public BaseGenerator {
} else {
auto postptr = " *" + NullableExtension();
auto wire_type = GenTypeGet(type, " ", "", postptr.c_str(), true);
std::string accessor = IsStruct(type) ? "GetStruct<" : "GetPointer<";
const std::string accessor = [&]() {
if (IsStruct(type)) { return "GetStruct<"; }
if (field.offset64) { return "GetPointer64<"; }
return "GetPointer<";
}();
auto underlying = accessor + wire_type + ">(" + offset_str + ")";
code_.SetValue("FIELD_TYPE", wire_type);
code_.SetValue("FIELD_VALUE", GenUnderlyingCast(field, true, underlying));
@@ -2859,9 +2924,9 @@ class CppGenerator : public BaseGenerator {
// Generate code to do force_align for the vector.
if (align > 1) {
const auto vtype = field.value.type.VectorType();
const std::string &type = IsStruct(vtype)
? WrapInNameSpace(*vtype.struct_def)
: GenTypeWire(vtype, "", false);
const std::string &type =
IsStruct(vtype) ? WrapInNameSpace(*vtype.struct_def)
: GenTypeWire(vtype, "", false, field.offset64);
return "_fbb.ForceVectorAlignment(" + field_size + ", sizeof(" + type +
"), " + std::to_string(static_cast<long long>(align)) + ");";
}
@@ -2874,7 +2939,7 @@ class CppGenerator : public BaseGenerator {
// Generate a builder struct:
code_ += "struct {{STRUCT_NAME}}Builder {";
code_ += " typedef {{STRUCT_NAME}} Table;";
code_ += " ::flatbuffers::FlatBufferBuilder &fbb_;";
code_ += " " + GetBuilder() + " &fbb_;";
code_ += " ::flatbuffers::uoffset_t start_;";
bool has_string_or_vector_fields = false;
@@ -2897,12 +2962,14 @@ class CppGenerator : public BaseGenerator {
// fbb_.AddElement<type>(offset, name, default);
// }
code_.SetValue("FIELD_NAME", Name(field));
code_.SetValue("FIELD_TYPE", GenTypeWire(field.value.type, " ", true));
code_.SetValue("FIELD_TYPE",
GenTypeWire(field.value.type, " ", true, field.offset64));
code_.SetValue("ADD_OFFSET", Name(struct_def) + "::" + offset);
code_.SetValue("ADD_NAME", name);
code_.SetValue("ADD_VALUE", value);
if (is_scalar) {
const auto type = GenTypeWire(field.value.type, "", false);
const auto type =
GenTypeWire(field.value.type, "", false, field.offset64);
code_.SetValue("ADD_FN", "AddElement<" + type + ">");
} else if (IsStruct(field.value.type)) {
code_.SetValue("ADD_FN", "AddStruct");
@@ -2921,9 +2988,9 @@ class CppGenerator : public BaseGenerator {
}
// Builder constructor
code_ +=
" explicit {{STRUCT_NAME}}Builder(::flatbuffers::FlatBufferBuilder "
"&_fbb)";
code_ += " explicit {{STRUCT_NAME}}Builder(" + GetBuilder() +
" "
"&_fbb)";
code_ += " : fbb_(_fbb) {";
code_ += " start_ = fbb_.StartTable();";
code_ += " }";
@@ -2950,7 +3017,7 @@ class CppGenerator : public BaseGenerator {
code_ +=
"inline ::flatbuffers::Offset<{{STRUCT_NAME}}> "
"Create{{STRUCT_NAME}}(";
code_ += " ::flatbuffers::FlatBufferBuilder &_fbb\\";
code_ += " " + GetBuilder() + " &_fbb\\";
for (const auto &field : struct_def.fields.vec) {
if (!field->deprecated) { GenParam(*field, false, ",\n "); }
}
@@ -2988,7 +3055,7 @@ class CppGenerator : public BaseGenerator {
code_ +=
"inline ::flatbuffers::Offset<{{STRUCT_NAME}}> "
"Create{{STRUCT_NAME}}Direct(";
code_ += " ::flatbuffers::FlatBufferBuilder &_fbb\\";
code_ += " " + GetBuilder() + " &_fbb\\";
for (const auto &field : struct_def.fields.vec) {
if (!field->deprecated) { GenParam(*field, true, ",\n "); }
}
@@ -2997,54 +3064,85 @@ class CppGenerator : public BaseGenerator {
struct_def.defined_namespace->GetFullyQualifiedName("Create");
code_.SetValue("CREATE_NAME", TranslateNameSpace(qualified_create_name));
code_ += ") {";
for (const auto &field : struct_def.fields.vec) {
if (!field->deprecated) {
code_.SetValue("FIELD_NAME", Name(*field));
if (IsString(field->value.type)) {
if (!field->shared) {
code_.SetValue("CREATE_STRING", "CreateString");
} else {
code_.SetValue("CREATE_STRING", "CreateSharedString");
}
code_ +=
" auto {{FIELD_NAME}}__ = {{FIELD_NAME}} ? "
"_fbb.{{CREATE_STRING}}({{FIELD_NAME}}) : 0;";
} else if (IsVector(field->value.type)) {
const std::string force_align_code =
GenVectorForceAlign(*field, Name(*field) + "->size()");
if (!force_align_code.empty()) {
code_ += " if ({{FIELD_NAME}}) { " + force_align_code + " }";
}
code_ += " auto {{FIELD_NAME}}__ = {{FIELD_NAME}} ? \\";
const auto vtype = field->value.type.VectorType();
const auto has_key = TypeHasKey(vtype);
if (IsStruct(vtype)) {
const auto type = WrapInNameSpace(*vtype.struct_def);
code_ += (has_key ? "_fbb.CreateVectorOfSortedStructs<"
: "_fbb.CreateVectorOfStructs<") +
type + ">\\";
} else if (has_key) {
const auto type = WrapInNameSpace(*vtype.struct_def);
code_ += "_fbb.CreateVectorOfSortedTables<" + type + ">\\";
} else {
const auto type =
GenTypeWire(vtype, "", VectorElementUserFacing(vtype));
code_ += "_fbb.CreateVector<" + type + ">\\";
}
code_ +=
has_key ? "({{FIELD_NAME}}) : 0;" : "(*{{FIELD_NAME}}) : 0;";
// Offset64 bit fields need to be added to the buffer first, so here we
// loop over the fields in order of their offset size, followed by their
// definition order. Otherwise the emitted code might add a Offset
// followed by an Offset64 which would trigger an assertion.
// TODO(derekbailey): maybe optimize for the case where there is no
// 64offsets in the whole schema?
ForAllFieldsOrderedByOffset(struct_def, [&](const FieldDef *field) {
if (field->deprecated) { return; }
code_.SetValue("FIELD_NAME", Name(*field));
if (IsString(field->value.type)) {
if (!field->shared) {
code_.SetValue(
"CREATE_STRING",
"CreateString" + std::string(field->offset64
? "<::flatbuffers::Offset64>"
: ""));
} else {
code_.SetValue("CREATE_STRING", "CreateSharedString");
}
code_ +=
" auto {{FIELD_NAME}}__ = {{FIELD_NAME}} ? "
"_fbb.{{CREATE_STRING}}({{FIELD_NAME}}) : 0;";
} else if (IsVector(field->value.type)) {
const std::string force_align_code =
GenVectorForceAlign(*field, Name(*field) + "->size()");
if (!force_align_code.empty()) {
code_ += " if ({{FIELD_NAME}}) { " + force_align_code + " }";
}
code_ += " auto {{FIELD_NAME}}__ = {{FIELD_NAME}} ? \\";
const auto vtype = field->value.type.VectorType();
const auto has_key = TypeHasKey(vtype);
if (IsStruct(vtype)) {
const std::string type = WrapInNameSpace(*vtype.struct_def);
if (has_key) {
code_ += "_fbb.CreateVectorOfSortedStructs<" + type + ">\\";
} else {
// If the field uses 64-bit addressing, create a 64-bit vector.
if (field->value.type.base_type == BASE_TYPE_VECTOR64) {
code_ += "_fbb.CreateVectorOfStructs64\\";
} else {
code_ += "_fbb.CreateVectorOfStructs\\";
if (field->offset64) {
// This is normal 32-bit vector, with 64-bit addressing.
code_ += "64<::flatbuffers::Vector>\\";
} else {
code_ += "<" + type + ">\\";
}
}
}
} else if (has_key) {
const auto type = WrapInNameSpace(*vtype.struct_def);
code_ += "_fbb.CreateVectorOfSortedTables<" + type + ">\\";
} else {
const auto type = GenTypeWire(
vtype, "", VectorElementUserFacing(vtype), field->offset64);
if (field->value.type.base_type == BASE_TYPE_VECTOR64) {
code_ += "_fbb.CreateVector64\\";
} else {
// If the field uses 64-bit addressing, create a 64-bit vector.
code_.SetValue("64OFFSET", field->offset64 ? "64" : "");
code_.SetValue("TYPE",
field->offset64 ? "::flatbuffers::Vector" : type);
code_ += "_fbb.CreateVector{{64OFFSET}}<{{TYPE}}>\\";
}
}
code_ += has_key ? "({{FIELD_NAME}}) : 0;" : "(*{{FIELD_NAME}}) : 0;";
}
}
});
code_ += " return {{CREATE_NAME}}{{STRUCT_NAME}}(";
code_ += " _fbb\\";
for (const auto &field : struct_def.fields.vec) {
if (!field->deprecated) {
code_.SetValue("FIELD_NAME", Name(*field));
code_ += ",\n {{FIELD_NAME}}\\";
if (IsString(field->value.type) || IsVector(field->value.type)) {
code_ += "__\\";
}
if (field->deprecated) { continue; }
code_.SetValue("FIELD_NAME", Name(*field));
code_ += ",\n {{FIELD_NAME}}\\";
if (IsString(field->value.type) || IsVector(field->value.type)) {
code_ += "__\\";
}
}
code_ += ");";
@@ -3115,6 +3213,7 @@ class CppGenerator : public BaseGenerator {
const FieldDef *union_field) {
std::string code;
switch (field.value.type.base_type) {
case BASE_TYPE_VECTOR64:
case BASE_TYPE_VECTOR: {
auto name = Name(field);
if (field.value.type.element == BASE_TYPE_UTYPE) {
@@ -3151,8 +3250,11 @@ class CppGenerator : public BaseGenerator {
? ".type"
: (field.value.type.element == BASE_TYPE_UNION ? ".value"
: "");
code += "for (::flatbuffers::uoffset_t _i = 0;";
if (field.value.type.base_type == BASE_TYPE_VECTOR64) {
code += "for (::flatbuffers::uoffset64_t _i = 0;";
} else {
code += "for (::flatbuffers::uoffset_t _i = 0;";
}
code += " _i < _e->size(); _i++) { ";
auto cpp_type = field.attributes.Lookup("cpp_type");
if (cpp_type) {
@@ -3265,8 +3367,7 @@ class CppGenerator : public BaseGenerator {
} else {
value += Name(field);
}
if (field.value.type.base_type != BASE_TYPE_VECTOR &&
field.attributes.Lookup("cpp_type")) {
if (!IsVector(field.value.type) && field.attributes.Lookup("cpp_type")) {
auto type = GenTypeBasic(field.value.type, false);
value =
"_rehasher ? "
@@ -3282,7 +3383,10 @@ class CppGenerator : public BaseGenerator {
// _fbb.CreateSharedString(_o->field)
case BASE_TYPE_STRING: {
if (!field.shared) {
code += "_fbb.CreateString(";
code +=
"_fbb.CreateString" +
std::string(field.offset64 ? "<::flatbuffers::Offset64>" : "") +
"(";
} else {
code += "_fbb.CreateSharedString(";
}
@@ -3309,6 +3413,7 @@ class CppGenerator : public BaseGenerator {
// _fbb.CreateVector<Offset<T>>(_o->field.size() [&](size_t i) {
// return CreateT(_fbb, _o->Get(i), rehasher);
// });
case BASE_TYPE_VECTOR64:
case BASE_TYPE_VECTOR: {
auto vector_type = field.value.type.VectorType();
switch (vector_type.base_type) {
@@ -3347,7 +3452,16 @@ class CppGenerator : public BaseGenerator {
}
code += ")";
} else {
code += "_fbb.CreateVectorOfStructs";
// If the field uses 64-bit addressing, create a 64-bit vector.
if (field.value.type.base_type == BASE_TYPE_VECTOR64) {
code += "_fbb.CreateVectorOfStructs64";
} else {
code += "_fbb.CreateVectorOfStructs";
if (field.offset64) {
// This is normal 32-bit vector, with 64-bit addressing.
code += "64<::flatbuffers::Vector>";
}
}
code += "(" + value + ")";
}
} else {
@@ -3413,7 +3527,17 @@ class CppGenerator : public BaseGenerator {
code += "(__va->_" + value + "[i]" + GenPtrGet(field) + ")) : 0";
code += "; }, &_va )";
} else {
code += "_fbb.CreateVector(" + value + ")";
// If the field uses 64-bit addressing, create a 64-bit vector.
if (field.value.type.base_type == BASE_TYPE_VECTOR64) {
code += "_fbb.CreateVector64(" + value + ")";
} else {
code += "_fbb.CreateVector";
if (field.offset64) {
// This is normal 32-bit vector, with 64-bit addressing.
code += "64<::flatbuffers::Vector>";
}
code += "(" + value + ")";
}
}
break;
}
@@ -3540,7 +3664,9 @@ class CppGenerator : public BaseGenerator {
code_ +=
" struct _VectorArgs "
"{ ::flatbuffers::FlatBufferBuilder *__fbb; "
"{ " +
GetBuilder() +
" *__fbb; "
"const " +
NativeName(Name(struct_def), &struct_def, opts_) +
"* __o; "

24
src/idl_gen_text.cpp
View File

@@ -19,6 +19,7 @@
#include <algorithm>
#include "flatbuffers/base.h"
#include "flatbuffers/code_generator.h"
#include "flatbuffers/flatbuffers.h"
#include "flatbuffers/flexbuffers.h"
@@ -101,13 +102,13 @@ struct JsonPrinter {
// Print a vector or an array of JSON values, comma seperated, wrapped in
// "[]".
template<typename Container>
const char *PrintContainer(PrintScalarTag, const Container &c, size_t size,
template<typename Container, typename SizeT = typename Container::size_type>
const char *PrintContainer(PrintScalarTag, const Container &c, SizeT size,
const Type &type, int indent, const uint8_t *) {
const auto elem_indent = indent + Indent();
text += '[';
AddNewLine();
for (uoffset_t i = 0; i < size; i++) {
for (SizeT i = 0; i < size; i++) {
if (i) {
AddComma();
AddNewLine();
@@ -123,14 +124,14 @@ struct JsonPrinter {
// Print a vector or an array of JSON values, comma seperated, wrapped in
// "[]".
template<typename Container>
const char *PrintContainer(PrintPointerTag, const Container &c, size_t size,
template<typename Container, typename SizeT = typename Container::size_type>
const char *PrintContainer(PrintPointerTag, const Container &c, SizeT size,
const Type &type, int indent, const uint8_t *prev_val) {
const auto is_struct = IsStruct(type);
const auto elem_indent = indent + Indent();
text += '[';
AddNewLine();
for (uoffset_t i = 0; i < size; i++) {
for (SizeT i = 0; i < size; i++) {
if (i) {
AddComma();
AddNewLine();
@@ -149,10 +150,10 @@ struct JsonPrinter {
return nullptr;
}
template<typename T>
template<typename T, typename SizeT = uoffset_t>
const char *PrintVector(const void *val, const Type &type, int indent,
const uint8_t *prev_val) {
typedef Vector<T> Container;
typedef Vector<T, SizeT> Container;
typedef typename PrintTag<typename Container::return_type>::type tag;
auto &vec = *reinterpret_cast<const Container *>(val);
return PrintContainer<Container>(tag(), vec, vec.size(), type, indent,
@@ -161,8 +162,9 @@ struct JsonPrinter {
// Print an array a sequence of JSON values, comma separated, wrapped in "[]".
template<typename T>
const char *PrintArray(const void *val, size_t size, const Type &type,
int indent) {
const char *PrintArray(const void *val, uint16_t size, const Type &type,
int indent) {
typedef Array<T, 0xFFFF> Container;
typedef typename PrintTag<typename Container::return_type>::type tag;
auto &arr = *reinterpret_cast<const Container *>(val);
@@ -240,7 +242,7 @@ struct JsonPrinter {
}
template<typename T> static T GetFieldDefault(const FieldDef &fd) {
T val;
T val{};
auto check = StringToNumber(fd.value.constant.c_str(), &val);
(void)check;
FLATBUFFERS_ASSERT(check);

178
src/idl_parser.cpp
View File

@@ -16,12 +16,15 @@
#include <algorithm>
#include <cmath>
#include <iostream>
#include <list>
#include <string>
#include <utility>
#include "flatbuffers/base.h"
#include "flatbuffers/buffer.h"
#include "flatbuffers/idl.h"
#include "flatbuffers/reflection_generated.h"
#include "flatbuffers/util.h"
namespace flatbuffers {
@@ -42,7 +45,8 @@ static const double kPi = 3.14159265358979323846;
// The enums in the reflection schema should match the ones we use internally.
// Compare the last element to check if these go out of sync.
static_assert(BASE_TYPE_UNION == static_cast<BaseType>(reflection::Union),
static_assert(BASE_TYPE_VECTOR64 ==
static_cast<BaseType>(reflection::MaxBaseType - 1),
"enums don't match");
// Any parsing calls have to be wrapped in this macro, which automates
@@ -124,6 +128,14 @@ CheckedError atot<Offset<void>>(const char *s, Parser &parser,
return NoError();
}
template<>
CheckedError atot<Offset64<void>>(const char *s, Parser &parser,
Offset64<void> *val) {
(void)parser;
*val = Offset64<void>(atoi(s));
return NoError();
}
template<typename T>
static T *LookupTableByName(const SymbolTable<T> &table,
const std::string &name,
@@ -957,11 +969,11 @@ CheckedError Parser::ParseField(StructDef &struct_def) {
ECHECK(AddField(struct_def, name, type, &field));
if (typefield) {
// We preserve the relation between the typefield
// and field, so we can easily map it in the code
// generators.
typefield->sibling_union_field = field;
field->sibling_union_field = typefield;
// We preserve the relation between the typefield
// and field, so we can easily map it in the code
// generators.
typefield->sibling_union_field = field;
field->sibling_union_field = typefield;
}
if (token_ == '=') {
@@ -1036,6 +1048,65 @@ CheckedError Parser::ParseField(StructDef &struct_def) {
}
}
if (field->attributes.Lookup("vector64") != nullptr) {
if (!IsVector(type)) {
return Error("`vector64` attribute can only be applied on vectors.");
}
// Upgrade the type to be a BASE_TYPE_VECTOR64, since the attributes are
// parsed after the type.
const BaseType element_base_type = type.element;
type = Type(BASE_TYPE_VECTOR64, type.struct_def, type.enum_def);
type.element = element_base_type;
// Since the field was already added to the parent object, update the type
// in place.
field->value.type = type;
// 64-bit vectors imply the offset64 attribute.
field->offset64 = true;
}
// Record that this field uses 64-bit offsets.
if (field->attributes.Lookup("offset64") != nullptr) {
// TODO(derekbailey): would be nice to have this be a recommendation or hint
// instead of a warning.
if (type.base_type == BASE_TYPE_VECTOR64) {
Warning("attribute `vector64` implies `offset64` and isn't required.");
}
field->offset64 = true;
}
// Check for common conditions with Offset64 fields.
if (field->offset64) {
// TODO(derekbailey): this is where we can disable string support for
// offset64, as that is not a hard requirement to have.
if (!IsString(type) && !IsVector(type)) {
return Error(
"only string and vectors can have `offset64` attribute applied");
}
// If this is a Vector, only scalar and scalar-like (structs) items are
// allowed.
// TODO(derekbailey): allow vector of strings, just require that the strings
// are Offset64<string>.
if (IsVector(type) &&
!((IsScalar(type.element) && !IsEnum(type.VectorType())) ||
IsStruct(type.VectorType()))) {
return Error("only vectors of scalars are allowed to be 64-bit.");
}
// Lastly, check if it is supported by the specified generated languages. Do
// this last so the above checks can inform the user of schema errors to fix
// first.
if (!Supports64BitOffsets()) {
return Error(
"fields using 64-bit offsets are not yet supported in at least one "
"of the specified programming languages.");
}
}
// For historical convenience reasons, string keys are assumed required.
// Scalars are kDefault unless otherwise specified.
// Nonscalars are kOptional unless required;
@@ -1058,7 +1129,8 @@ CheckedError Parser::ParseField(StructDef &struct_def) {
if (field->key) {
if (struct_def.has_key) return Error("only one field may be set as 'key'");
struct_def.has_key = true;
auto is_valid = IsScalar(type.base_type) || IsString(type) || IsStruct(type);
auto is_valid =
IsScalar(type.base_type) || IsString(type) || IsStruct(type);
if (IsArray(type)) {
is_valid |=
IsScalar(type.VectorType().base_type) || IsStruct(type.VectorType());
@@ -1161,7 +1233,7 @@ CheckedError Parser::ParseField(StructDef &struct_def) {
if (nested->type.base_type != BASE_TYPE_STRING)
return Error(
"nested_flatbuffer attribute must be a string (the root type)");
if (type.base_type != BASE_TYPE_VECTOR || type.element != BASE_TYPE_UCHAR)
if (!IsVector(type.base_type) || type.element != BASE_TYPE_UCHAR)
return Error(
"nested_flatbuffer attribute may only apply to a vector of ubyte");
// This will cause an error if the root type of the nested flatbuffer
@@ -1230,7 +1302,7 @@ CheckedError Parser::ParseComma() {
CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
size_t parent_fieldn,
const StructDef *parent_struct_def,
uoffset_t count, bool inside_vector) {
size_t count, bool inside_vector) {
switch (val.type.base_type) {
case BASE_TYPE_UNION: {
FLATBUFFERS_ASSERT(field);
@@ -1300,7 +1372,7 @@ CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
return Error(
"union types vector smaller than union values vector for: " +
field->name);
enum_idx = vector_of_union_types->Get(count);
enum_idx = vector_of_union_types->Get(static_cast<uoffset_t>(count));
} else {
ECHECK(atot(constant.c_str(), *this, &enum_idx));
}
@@ -1329,9 +1401,10 @@ CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
ECHECK(ParseString(val, field->shared));
break;
}
case BASE_TYPE_VECTOR64:
case BASE_TYPE_VECTOR: {
uoffset_t off;
ECHECK(ParseVector(val.type.VectorType(), &off, field, parent_fieldn));
ECHECK(ParseVector(val.type, &off, field, parent_fieldn));
val.constant = NumToString(off);
break;
}
@@ -1503,6 +1576,9 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
for (size_t size = struct_def.sortbysize ? sizeof(largest_scalar_t) : 1; size;
size /= 2) {
// Go through elements in reverse, since we're building the data backwards.
// TODO(derekbailey): this doesn't work when there are Offset64 fields, as
// those have to be built first. So this needs to be changed to iterate over
// Offset64 then Offset32 fields.
for (auto it = field_stack_.rbegin();
it != field_stack_.rbegin() + fieldn_outer; ++it) {
auto &field_value = it->first;
@@ -1510,7 +1586,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
if (!struct_def.sortbysize ||
size == SizeOf(field_value.type.base_type)) {
switch (field_value.type.base_type) {
// clang-format off
// clang-format off
#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \
case BASE_TYPE_ ## ENUM: \
builder_.Pad(field->padding); \
@@ -1541,9 +1617,16 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
if (IsStruct(field->value.type)) { \
SerializeStruct(*field->value.type.struct_def, field_value); \
} else { \
CTYPE val; \
ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
builder_.AddOffset(field_value.offset, val); \
/* Special case for fields that use 64-bit addressing */ \
if(field->offset64) { \
Offset64<void> offset; \
ECHECK(atot(field_value.constant.c_str(), *this, &offset)); \
builder_.AddOffset(field_value.offset, offset); \
} else { \
CTYPE val; \
ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
builder_.AddOffset(field_value.offset, val); \
} \
} \
break;
FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD)
@@ -1581,7 +1664,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
}
template<typename F>
CheckedError Parser::ParseVectorDelimiters(uoffset_t &count, F body) {
CheckedError Parser::ParseVectorDelimiters(size_t &count, F body) {
EXPECT('[');
for (;;) {
if ((!opts.strict_json || !count) && Is(']')) break;
@@ -1611,10 +1694,11 @@ CheckedError Parser::ParseAlignAttribute(const std::string &align_constant,
NumToString(FLATBUFFERS_MAX_ALIGNMENT));
}
CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue,
CheckedError Parser::ParseVector(const Type &vector_type, uoffset_t *ovalue,
FieldDef *field, size_t fieldn) {
uoffset_t count = 0;
auto err = ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
Type type = vector_type.VectorType();
size_t count = 0;
auto err = ParseVectorDelimiters(count, [&](size_t &) -> CheckedError {
Value val;
val.type = type;
ECHECK(ParseAnyValue(val, field, fieldn, nullptr, count, true));
@@ -1634,12 +1718,18 @@ CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue,
}
// TODO Fix using element alignment as size (`elemsize`)!
builder_.StartVector(len, elemsize, alignment);
for (uoffset_t i = 0; i < count; i++) {
if (vector_type.base_type == BASE_TYPE_VECTOR64) {
// TODO(derekbailey): this requires a 64-bit builder.
// builder_.StartVector<Offset64, uoffset64_t>(len, elemsize, alignment);
builder_.StartVector(len, elemsize, alignment);
} else {
builder_.StartVector(len, elemsize, alignment);
}
for (size_t i = 0; i < count; i++) {
// start at the back, since we're building the data backwards.
auto &val = field_stack_.back().first;
switch (val.type.base_type) {
// clang-format off
// clang-format off
#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE,...) \
case BASE_TYPE_ ## ENUM: \
if (IsStruct(val.type)) SerializeStruct(*val.type.struct_def, val); \
@@ -1657,7 +1747,11 @@ CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue,
}
builder_.ClearOffsets();
*ovalue = builder_.EndVector(count);
if (vector_type.base_type == BASE_TYPE_VECTOR64) {
*ovalue = builder_.EndVector<uoffset64_t>(count);
} else {
*ovalue = builder_.EndVector(count);
}
if (type.base_type == BASE_TYPE_STRUCT && type.struct_def->has_key) {
// We should sort this vector. Find the key first.
@@ -1725,8 +1819,8 @@ CheckedError Parser::ParseArray(Value &array) {
FlatBufferBuilder builder;
const auto &type = array.type.VectorType();
auto length = array.type.fixed_length;
uoffset_t count = 0;
auto err = ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
size_t count = 0;
auto err = ParseVectorDelimiters(count, [&](size_t &) -> CheckedError {
stack.emplace_back(Value());
auto &val = stack.back();
val.type = type;
@@ -1977,8 +2071,7 @@ CheckedError Parser::TryTypedValue(const std::string *name, int dtoken,
e.type.base_type = req;
} else {
return Error(std::string("type mismatch: expecting: ") +
TypeName(e.type.base_type) +
", found: " + TypeName(req) +
TypeName(e.type.base_type) + ", found: " + TypeName(req) +
", name: " + (name ? *name : "") + ", value: " + e.constant);
}
}
@@ -2595,6 +2688,11 @@ bool Parser::SupportsAdvancedArrayFeatures() const {
IDLOptions::kBinary | IDLOptions::kRust | IDLOptions::kTs)) == 0;
}
bool Parser::Supports64BitOffsets() const {
return (opts.lang_to_generate &
~(IDLOptions::kCpp | IDLOptions::kJson | IDLOptions::kBinary)) == 0;
}
Namespace *Parser::UniqueNamespace(Namespace *ns) {
for (auto it = namespaces_.begin(); it != namespaces_.end(); ++it) {
if (ns->components == (*it)->components) {
@@ -3217,10 +3315,9 @@ CheckedError Parser::SkipAnyJsonValue() {
});
}
case '[': {
uoffset_t count = 0;
return ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
return SkipAnyJsonValue();
});
size_t count = 0;
return ParseVectorDelimiters(
count, [&](size_t &) -> CheckedError { return SkipAnyJsonValue(); });
}
case kTokenStringConstant:
case kTokenIntegerConstant:
@@ -3269,8 +3366,8 @@ CheckedError Parser::ParseFlexBufferValue(flexbuffers::Builder *builder) {
}
case '[': {
auto start = builder->StartVector();
uoffset_t count = 0;
ECHECK(ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
size_t count = 0;
ECHECK(ParseVectorDelimiters(count, [&](size_t &) -> CheckedError {
return ParseFlexBufferValue(builder);
}));
builder->EndVector(start, false, false);
@@ -3922,7 +4019,7 @@ Offset<reflection::Field> FieldDef::Serialize(FlatBufferBuilder *builder,
IsInteger(value.type.base_type) ? StringToInt(value.constant.c_str()) : 0,
// result may be platform-dependent if underlying is float (not double)
IsFloat(value.type.base_type) ? d : 0.0, deprecated, IsRequired(), key,
attr__, docs__, IsOptional(), static_cast<uint16_t>(padding));
attr__, docs__, IsOptional(), static_cast<uint16_t>(padding), offset64);
// TODO: value.constant is almost always "0", we could save quite a bit of
// space by sharing it. Same for common values of value.type.
}
@@ -3940,6 +4037,7 @@ bool FieldDef::Deserialize(Parser &parser, const reflection::Field *field) {
presence = FieldDef::MakeFieldPresence(field->optional(), field->required());
padding = field->padding();
key = field->key();
offset64 = field->offset64();
if (!DeserializeAttributes(parser, field->attributes())) return false;
// TODO: this should probably be handled by a separate attribute
if (attributes.Lookup("flexbuffer")) {
@@ -4264,12 +4362,18 @@ std::string Parser::ConformTo(const Parser &base) {
auto field_base = struct_def_base->fields.Lookup(field.name);
const auto qualified_field_name = qualified_name + "." + field.name;
if (field_base) {
if (field.value.offset != field_base->value.offset)
if (field.value.offset != field_base->value.offset) {
return "offsets differ for field: " + qualified_field_name;
if (field.value.constant != field_base->value.constant)
}
if (field.value.constant != field_base->value.constant) {
return "defaults differ for field: " + qualified_field_name;
if (!EqualByName(field.value.type, field_base->value.type))
}
if (!EqualByName(field.value.type, field_base->value.type)) {
return "types differ for field: " + qualified_field_name;
}
if (field.offset64 != field_base->offset64) {
return "offset types differ for field: " + qualified_field_name;
}
} else {
// Doesn't have to exist, deleting fields is fine.
// But we should check if there is a field that has the same offset