BigfootDev/flatbuffers-bigfoot
1
0
Fork 0
forked from BigfootDev/flatbuffers
Code Pull Requests Activity

FlatBuffers 64 for C++ (#7935)

Browse Source 
* First working hack of adding 64-bit. Don't judge :)

* Made vector_downward work on 64 bit types

* vector_downward uses size_t, added offset64 to reflection

* cleaned up adding offset64 in parser

* Add C++ testing skeleton for 64-bit

* working test for CreateVector64

* working >2 GiB buffers

* support for large strings

* simplified CreateString<> to just provide the offset type

* generalize CreateVector template

* update test_64.afb due to upstream format change

* Added Vector64 type, which is just an alias for vector ATM

* Switch to Offset64 for Vector64

* Update for reflection bfbs output change

* Starting to add support for vector64 type in C++

* made a generic CreateVector that can handle different offsets and vector types

* Support for 32-vector with 64-addressing

* Vector64 basic builder + tests working

* basic support for json vector64 support

* renamed fields in test_64bit.fbs to better reflect their use

* working C++ vector64 builder

* Apply --annotate-sparse-vector to 64-bit tests

* Enable Vector64 for --annotate-sparse-vectors

* Merged from upstream

* Add `near_string` field for testing 32-bit offsets alongside

* keep track of where the 32-bit and 64-bit regions are for flatbufferbuilder

* move template<> outside class body for GCC

* update run.sh to build and run tests

* basic assertion for adding 64-bit offset at the wrong time

* started to separate `FlatBufferBuilder` into two classes, 1 64-bit aware, the other not

* add test for nested flatbuffer vector64, fix bug in alignment of big vectors

* fixed CreateDirect method by iterating by Offset64 first

* internal refactoring of flatbufferbuilder

* block not supported languages in the parser from using 64-bit

* evolution tests for adding a vector64 field

* conformity tests for adding/removing offset64 attributes

* ensure test is for a big buffer

* add parser error tests for `offset64` and `vector64` attributes

* add missing static that GCC only complains about

* remove stdint-uintn.h header that gets automatically added

* move 64-bit CalculateOffset internal

* fixed return size of EndVector

* various fixes on windows

* add SizeT to vector_downward

* minimze range of size changes in vector and builder

* reworked how tracking if 64-offsets are added

* Add ReturnT to EndVector

* small cleanups

* remove need for second Array definition

* combine IndirectHelpers into one definition

* started support for vector of struct

* Support for 32/64-vectors of structs + Offset64

* small cleanups

* add verification for vector64

* add sized prefix for 64-bit buffers

* add fuzzer for 64-bit

* add example of adding many vectors using a wrapper table

* run the new -bfbs-gen-embed logic on the 64-bit tests

* remove run.sh and fix cmakelist issue

* fixed bazel rules

* fixed some PR comments

* add 64-bit tests to cmakelist
This commit is contained in:
Derek Bailey
2023-05-09 09:16:30 -07:00
committed by GitHub
parent 13fc75cb6b
commit 63b7b25289
49 changed files with 3274 additions and 529 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
include/flatbuffers/idl.h
View File

@@ -45,26 +45,27 @@ namespace flatbuffers {
// of type tokens.
// clang-format off
#define FLATBUFFERS_GEN_TYPES_SCALAR(TD) \
TD(NONE, "", uint8_t, byte, byte, byte, uint8, u8, UByte, UInt8, 0) \
TD(UTYPE, "", uint8_t, byte, byte, byte, uint8, u8, UByte, UInt8, 1) /* begin scalar/int */ \
TD(BOOL, "bool", uint8_t, boolean,bool, bool, bool, bool, Boolean, Bool, 2) \
TD(CHAR, "byte", int8_t, byte, int8, sbyte, int8, i8, Byte, Int8, 3) \
TD(UCHAR, "ubyte", uint8_t, byte, byte, byte, uint8, u8, UByte, UInt8, 4) \
TD(SHORT, "short", int16_t, short, int16, short, int16, i16, Short, Int16, 5) \
TD(USHORT, "ushort", uint16_t, short, uint16, ushort, uint16, u16, UShort, UInt16, 6) \
TD(INT, "int", int32_t, int, int32, int, int32, i32, Int, Int32, 7) \
TD(UINT, "uint", uint32_t, int, uint32, uint, uint32, u32, UInt, UInt32, 8) \
TD(LONG, "long", int64_t, long, int64, long, int64, i64, Long, Int64, 9) \
TD(ULONG, "ulong", uint64_t, long, uint64, ulong, uint64, u64, ULong, UInt64, 10) /* end int */ \
TD(FLOAT, "float", float, float, float32, float, float32, f32, Float, Float32, 11) /* begin float */ \
TD(DOUBLE, "double", double, double, float64, double, float64, f64, Double, Double, 12) /* end float/scalar */
TD(NONE, "", uint8_t, byte, byte, byte, uint8, u8, UByte, UInt8, 0) \
TD(UTYPE, "", uint8_t, byte, byte, byte, uint8, u8, UByte, UInt8, 1) /* begin scalar/int */ \
TD(BOOL, "bool", uint8_t, boolean,bool, bool, bool, bool, Boolean, Bool, 2) \
TD(CHAR, "byte", int8_t, byte, int8, sbyte, int8, i8, Byte, Int8, 3) \
TD(UCHAR, "ubyte", uint8_t, byte, byte, byte, uint8, u8, UByte, UInt8, 4) \
TD(SHORT, "short", int16_t, short, int16, short, int16, i16, Short, Int16, 5) \
TD(USHORT, "ushort", uint16_t, short, uint16, ushort, uint16, u16, UShort, UInt16, 6) \
TD(INT, "int", int32_t, int, int32, int, int32, i32, Int, Int32, 7) \
TD(UINT, "uint", uint32_t, int, uint32, uint, uint32, u32, UInt, UInt32, 8) \
TD(LONG, "long", int64_t, long, int64, long, int64, i64, Long, Int64, 9) \
TD(ULONG, "ulong", uint64_t, long, uint64, ulong, uint64, u64, ULong, UInt64, 10) /* end int */ \
TD(FLOAT, "float", float, float, float32, float, float32, f32, Float, Float32, 11) /* begin float */ \
TD(DOUBLE, "double", double, double, float64, double, float64, f64, Double, Double, 12) /* end float/scalar */
#define FLATBUFFERS_GEN_TYPES_POINTER(TD) \
TD(STRING, "string", Offset<void>, int, int, StringOffset, int, unused, Int, Offset<String>, 13) \
TD(VECTOR, "", Offset<void>, int, int, VectorOffset, int, unused, Int, Offset<UOffset>, 14) \
TD(STRUCT, "", Offset<void>, int, int, int, int, unused, Int, Offset<UOffset>, 15) \
TD(UNION, "", Offset<void>, int, int, int, int, unused, Int, Offset<UOffset>, 16)
TD(STRING, "string", Offset<void>, int, int, StringOffset, int, unused, Int, Offset<String>, 13) \
TD(VECTOR, "", Offset<void>, int, int, VectorOffset, int, unused, Int, Offset<UOffset>, 14) \
TD(VECTOR64, "", Offset64<void>, int, int, VectorOffset, int, unused, Int, Offset<UOffset>, 18) \
TD(STRUCT, "", Offset<void>, int, int, int, int, unused, Int, Offset<UOffset>, 15) \
TD(UNION, "", Offset<void>, int, int, int, int, unused, Int, Offset<UOffset>, 16)
#define FLATBUFFERS_GEN_TYPE_ARRAY(TD) \
TD(ARRAY, "", int, int, int, int, int, unused, Int, Offset<UOffset>, 17)
TD(ARRAY, "", int, int, int, int, int, unused, Int, Offset<UOffset>, 17)
// The fields are:
// - enum
// - FlatBuffers schema type.
@@ -139,6 +140,8 @@ inline bool IsLong (BaseType t) { return t == BASE_TYPE_LONG ||
inline bool IsBool (BaseType t) { return t == BASE_TYPE_BOOL; }
inline bool IsOneByte(BaseType t) { return t >= BASE_TYPE_UTYPE &&
t <= BASE_TYPE_UCHAR; }
inline bool IsVector (BaseType t) { return t == BASE_TYPE_VECTOR ||
t == BASE_TYPE_VECTOR64; }
inline bool IsUnsigned(BaseType t) {
return (t == BASE_TYPE_UTYPE) || (t == BASE_TYPE_UCHAR) ||
@@ -210,7 +213,8 @@ struct Type {
bool Deserialize(const Parser &parser, const reflection::Type *type);
BaseType base_type;
BaseType element; // only set if t == BASE_TYPE_VECTOR
BaseType element; // only set if t == BASE_TYPE_VECTOR or
// BASE_TYPE_VECTOR64
StructDef *struct_def; // only set if t or element == BASE_TYPE_STRUCT
EnumDef *enum_def; // set if t == BASE_TYPE_UNION / BASE_TYPE_UTYPE,
// or for an integral type derived from an enum.
@@ -326,6 +330,7 @@ struct FieldDef : public Definition {
shared(false),
native_inline(false),
flexbuffer(false),
offset64(false),
presence(kDefault),
nested_flatbuffer(nullptr),
padding(0),
@@ -352,6 +357,7 @@ struct FieldDef : public Definition {
bool native_inline; // Field will be defined inline (instead of as a pointer)
// for native tables if field is a struct.
bool flexbuffer; // This field contains FlexBuffer data.
bool offset64; // If the field uses 64-bit offsets.
enum Presence {
// Field must always be present.
@@ -528,9 +534,7 @@ inline bool IsUnionType(const Type &type) {
return IsUnion(type) && IsInteger(type.base_type);
}
inline bool IsVector(const Type &type) {
return type.base_type == BASE_TYPE_VECTOR;
}
inline bool IsVector(const Type &type) { return IsVector(type.base_type); }
inline bool IsVectorOfStruct(const Type &type) {
return IsVector(type) && IsStruct(type.VectorType());
@@ -952,6 +956,13 @@ class Parser : public ParserState {
known_attributes_["native_default"] = true;
known_attributes_["flexbuffer"] = true;
known_attributes_["private"] = true;
// An attribute added to a field to indicate that is uses 64-bit addressing.
known_attributes_["offset64"] = true;
// An attribute added to a vector field to indicate that it uses 64-bit
// addressing and it has a 64-bit length.
known_attributes_["vector64"] = true;
}
// Copying is not allowed
@@ -1062,7 +1073,7 @@ class Parser : public ParserState {
FLATBUFFERS_CHECKED_ERROR ParseAnyValue(Value &val, FieldDef *field,
size_t parent_fieldn,
const StructDef *parent_struct_def,
uoffset_t count,
size_t count,
bool inside_vector = false);
template<typename F>
FLATBUFFERS_CHECKED_ERROR ParseTableDelimiters(size_t &fieldn,
@@ -1074,7 +1085,7 @@ class Parser : public ParserState {
void SerializeStruct(FlatBufferBuilder &builder, const StructDef &struct_def,
const Value &val);
template<typename F>
FLATBUFFERS_CHECKED_ERROR ParseVectorDelimiters(uoffset_t &count, F body);
FLATBUFFERS_CHECKED_ERROR ParseVectorDelimiters(size_t &count, F body);
FLATBUFFERS_CHECKED_ERROR ParseVector(const Type &type, uoffset_t *ovalue,
FieldDef *field, size_t fieldn);
FLATBUFFERS_CHECKED_ERROR ParseArray(Value &array);
@@ -1139,6 +1150,7 @@ class Parser : public ParserState {
bool SupportsAdvancedArrayFeatures() const;
bool SupportsOptionalScalars() const;
bool SupportsDefaultVectorsAndStrings() const;
bool Supports64BitOffsets() const;
Namespace *UniqueNamespace(Namespace *ns);
FLATBUFFERS_CHECKED_ERROR RecurseError();
@@ -1288,8 +1300,7 @@ extern bool GenerateSwift(const Parser &parser, const std::string &path,
// Generate a schema file from the internal representation, useful after
// parsing a .proto schema.
extern std::string GenerateFBS(const Parser &parser,
const std::string &file_name,
bool no_log);
const std::string &file_name, bool no_log);
extern bool GenerateFBS(const Parser &parser, const std::string &path,
const std::string &file_name, bool no_log);