FlatBuffers 64 for C++ (#7935)

* First working hack of adding 64-bit. Don't judge :)

* Made vector_downward work on 64 bit types

* vector_downward uses size_t, added offset64 to reflection

* cleaned up adding offset64 in parser

* Add C++ testing skeleton for 64-bit

* working test for CreateVector64

* working >2 GiB buffers

* support for large strings

* simplified CreateString<> to just provide the offset type

* generalize CreateVector template

* update test_64.afb due to upstream format change

* Added Vector64 type, which is just an alias for vector ATM

* Switch to Offset64 for Vector64

* Update for reflection bfbs output change

* Starting to add support for vector64 type in C++

* made a generic CreateVector that can handle different offsets and vector types

* Support for 32-vector with 64-addressing

* Vector64 basic builder + tests working

* basic support for json vector64 support

* renamed fields in test_64bit.fbs to better reflect their use

* working C++ vector64 builder

* Apply --annotate-sparse-vector to 64-bit tests

* Enable Vector64 for --annotate-sparse-vectors

* Merged from upstream

* Add `near_string` field for testing 32-bit offsets alongside

* keep track of where the 32-bit and 64-bit regions are for flatbufferbuilder

* move template<> outside class body for GCC

* update run.sh to build and run tests

* basic assertion for adding 64-bit offset at the wrong time

* started to separate `FlatBufferBuilder` into two classes, 1 64-bit aware, the other not

* add test for nested flatbuffer vector64, fix bug in alignment of big vectors

* fixed CreateDirect method by iterating by Offset64 first

* internal refactoring of flatbufferbuilder

* block not supported languages in the parser from using 64-bit

* evolution tests for adding a vector64 field

* conformity tests for adding/removing offset64 attributes

* ensure test is for a big buffer

* add parser error tests for `offset64` and `vector64` attributes

* add missing static that GCC only complains about

* remove stdint-uintn.h header that gets automatically added

* move 64-bit CalculateOffset internal

* fixed return size of EndVector

* various fixes on windows

* add SizeT to vector_downward

* minimze range of size changes in vector and builder

* reworked how tracking if 64-offsets are added

* Add ReturnT to EndVector

* small cleanups

* remove need for second Array definition

* combine IndirectHelpers into one definition

* started support for vector of struct

* Support for 32/64-vectors of structs + Offset64

* small cleanups

* add verification for vector64

* add sized prefix for 64-bit buffers

* add fuzzer for 64-bit

* add example of adding many vectors using a wrapper table

* run the new -bfbs-gen-embed logic on the 64-bit tests

* remove run.sh and fix cmakelist issue

* fixed bazel rules

* fixed some PR comments

* add 64-bit tests to cmakelist
This commit is contained in:
Derek Bailey
2023-05-09 09:16:30 -07:00
committed by GitHub
parent 13fc75cb6b
commit 63b7b25289
49 changed files with 3274 additions and 529 deletions

View File

@@ -19,6 +19,7 @@
#include <algorithm>
#include "flatbuffers/base.h"
#include "flatbuffers/code_generator.h"
#include "flatbuffers/flatbuffers.h"
#include "flatbuffers/flexbuffers.h"
@@ -101,13 +102,13 @@ struct JsonPrinter {
// Print a vector or an array of JSON values, comma seperated, wrapped in
// "[]".
template<typename Container>
const char *PrintContainer(PrintScalarTag, const Container &c, size_t size,
template<typename Container, typename SizeT = typename Container::size_type>
const char *PrintContainer(PrintScalarTag, const Container &c, SizeT size,
const Type &type, int indent, const uint8_t *) {
const auto elem_indent = indent + Indent();
text += '[';
AddNewLine();
for (uoffset_t i = 0; i < size; i++) {
for (SizeT i = 0; i < size; i++) {
if (i) {
AddComma();
AddNewLine();
@@ -123,14 +124,14 @@ struct JsonPrinter {
// Print a vector or an array of JSON values, comma seperated, wrapped in
// "[]".
template<typename Container>
const char *PrintContainer(PrintPointerTag, const Container &c, size_t size,
template<typename Container, typename SizeT = typename Container::size_type>
const char *PrintContainer(PrintPointerTag, const Container &c, SizeT size,
const Type &type, int indent, const uint8_t *prev_val) {
const auto is_struct = IsStruct(type);
const auto elem_indent = indent + Indent();
text += '[';
AddNewLine();
for (uoffset_t i = 0; i < size; i++) {
for (SizeT i = 0; i < size; i++) {
if (i) {
AddComma();
AddNewLine();
@@ -149,10 +150,10 @@ struct JsonPrinter {
return nullptr;
}
template<typename T>
template<typename T, typename SizeT = uoffset_t>
const char *PrintVector(const void *val, const Type &type, int indent,
const uint8_t *prev_val) {
typedef Vector<T> Container;
typedef Vector<T, SizeT> Container;
typedef typename PrintTag<typename Container::return_type>::type tag;
auto &vec = *reinterpret_cast<const Container *>(val);
return PrintContainer<Container>(tag(), vec, vec.size(), type, indent,
@@ -161,8 +162,9 @@ struct JsonPrinter {
// Print an array a sequence of JSON values, comma separated, wrapped in "[]".
template<typename T>
const char *PrintArray(const void *val, size_t size, const Type &type,
int indent) {
const char *PrintArray(const void *val, uint16_t size, const Type &type,
int indent) {
typedef Array<T, 0xFFFF> Container;
typedef typename PrintTag<typename Container::return_type>::type tag;
auto &arr = *reinterpret_cast<const Container *>(val);
@@ -240,7 +242,7 @@ struct JsonPrinter {
}
template<typename T> static T GetFieldDefault(const FieldDef &fd) {
T val;
T val{};
auto check = StringToNumber(fd.value.constant.c_str(), &val);
(void)check;
FLATBUFFERS_ASSERT(check);