FlatBuffers 64 for C++ (#7935)

* First working hack of adding 64-bit. Don't judge :)

* Made vector_downward work on 64 bit types

* vector_downward uses size_t, added offset64 to reflection

* cleaned up adding offset64 in parser

* Add C++ testing skeleton for 64-bit

* working test for CreateVector64

* working >2 GiB buffers

* support for large strings

* simplified CreateString<> to just provide the offset type

* generalize CreateVector template

* update test_64.afb due to upstream format change

* Added Vector64 type, which is just an alias for vector ATM

* Switch to Offset64 for Vector64

* Update for reflection bfbs output change

* Starting to add support for vector64 type in C++

* made a generic CreateVector that can handle different offsets and vector types

* Support for 32-vector with 64-addressing

* Vector64 basic builder + tests working

* basic support for json vector64 support

* renamed fields in test_64bit.fbs to better reflect their use

* working C++ vector64 builder

* Apply --annotate-sparse-vector to 64-bit tests

* Enable Vector64 for --annotate-sparse-vectors

* Merged from upstream

* Add `near_string` field for testing 32-bit offsets alongside

* keep track of where the 32-bit and 64-bit regions are for flatbufferbuilder

* move template<> outside class body for GCC

* update run.sh to build and run tests

* basic assertion for adding 64-bit offset at the wrong time

* started to separate `FlatBufferBuilder` into two classes, 1 64-bit aware, the other not

* add test for nested flatbuffer vector64, fix bug in alignment of big vectors

* fixed CreateDirect method by iterating by Offset64 first

* internal refactoring of flatbufferbuilder

* block not supported languages in the parser from using 64-bit

* evolution tests for adding a vector64 field

* conformity tests for adding/removing offset64 attributes

* ensure test is for a big buffer

* add parser error tests for `offset64` and `vector64` attributes

* add missing static that GCC only complains about

* remove stdint-uintn.h header that gets automatically added

* move 64-bit CalculateOffset internal

* fixed return size of EndVector

* various fixes on windows

* add SizeT to vector_downward

* minimze range of size changes in vector and builder

* reworked how tracking if 64-offsets are added

* Add ReturnT to EndVector

* small cleanups

* remove need for second Array definition

* combine IndirectHelpers into one definition

* started support for vector of struct

* Support for 32/64-vectors of structs + Offset64

* small cleanups

* add verification for vector64

* add sized prefix for 64-bit buffers

* add fuzzer for 64-bit

* add example of adding many vectors using a wrapper table

* run the new -bfbs-gen-embed logic on the 64-bit tests

* remove run.sh and fix cmakelist issue

* fixed bazel rules

* fixed some PR comments

* add 64-bit tests to cmakelist

include/flatbuffers/reflection_generated.h

@@ -64,10 +64,11 @@ enum BaseType {
   Obj = 15,
   Union = 16,
   Array = 17,
-  MaxBaseType = 18
+  Vector64 = 18,
+  MaxBaseType = 19
 };
 
-inline const BaseType (&EnumValuesBaseType())[19] {
+inline const BaseType (&EnumValuesBaseType())[20] {
   static const BaseType values[] = {
     None,
     UType,
@@ -87,13 +88,14 @@ inline const BaseType (&EnumValuesBaseType())[19] {
     Obj,
     Union,
     Array,
+    Vector64,
     MaxBaseType
   };
   return values;
 }
 
 inline const char * const *EnumNamesBaseType() {
-  static const char * const names[20] = {
+  static const char * const names[21] = {
     "None",
     "UType",
     "Bool",
@@ -112,6 +114,7 @@ inline const char * const *EnumNamesBaseType() {
     "Obj",
     "Union",
     "Array",
+    "Vector64",
     "MaxBaseType",
     nullptr
   };
@@ -601,7 +604,8 @@ struct Field FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table {
     VT_ATTRIBUTES = 22,
     VT_DOCUMENTATION = 24,
     VT_OPTIONAL = 26,
-    VT_PADDING = 28
+    VT_PADDING = 28,
+    VT_OFFSET64 = 30
   };
   const ::flatbuffers::String *name() const {
     return GetPointer<const ::flatbuffers::String *>(VT_NAME);
@@ -649,6 +653,10 @@ struct Field FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table {
   uint16_t padding() const {
     return GetField<uint16_t>(VT_PADDING, 0);
   }
+  /// If the field uses 64-bit offsets.
+  bool offset64() const {
+    return GetField<uint8_t>(VT_OFFSET64, 0) != 0;
+  }
   bool Verify(::flatbuffers::Verifier &verifier) const {
     return VerifyTableStart(verifier) &&
            VerifyOffsetRequired(verifier, VT_NAME) &&
@@ -670,6 +678,7 @@ struct Field FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table {
            verifier.VerifyVectorOfStrings(documentation()) &&
            VerifyField<uint8_t>(verifier, VT_OPTIONAL, 1) &&
            VerifyField<uint16_t>(verifier, VT_PADDING, 2) &&
+           VerifyField<uint8_t>(verifier, VT_OFFSET64, 1) &&
            verifier.EndTable();
   }
 };
@@ -717,6 +726,9 @@ struct FieldBuilder {
   void add_padding(uint16_t padding) {
     fbb_.AddElement<uint16_t>(Field::VT_PADDING, padding, 0);
   }
+  void add_offset64(bool offset64) {
+    fbb_.AddElement<uint8_t>(Field::VT_OFFSET64, static_cast<uint8_t>(offset64), 0);
+  }
   explicit FieldBuilder(::flatbuffers::FlatBufferBuilder &_fbb)
         : fbb_(_fbb) {
     start_ = fbb_.StartTable();
@@ -744,7 +756,8 @@ inline ::flatbuffers::Offset<Field> CreateField(
     ::flatbuffers::Offset<::flatbuffers::Vector<::flatbuffers::Offset<reflection::KeyValue>>> attributes = 0,
     ::flatbuffers::Offset<::flatbuffers::Vector<::flatbuffers::Offset<::flatbuffers::String>>> documentation = 0,
     bool optional = false,
-    uint16_t padding = 0) {
+    uint16_t padding = 0,
+    bool offset64 = false) {
   FieldBuilder builder_(_fbb);
   builder_.add_default_real(default_real);
   builder_.add_default_integer(default_integer);
@@ -755,6 +768,7 @@ inline ::flatbuffers::Offset<Field> CreateField(
   builder_.add_padding(padding);
   builder_.add_offset(offset);
   builder_.add_id(id);
+  builder_.add_offset64(offset64);
   builder_.add_optional(optional);
   builder_.add_key(key);
   builder_.add_required(required);
@@ -776,7 +790,8 @@ inline ::flatbuffers::Offset<Field> CreateFieldDirect(
     std::vector<::flatbuffers::Offset<reflection::KeyValue>> *attributes = nullptr,
     const std::vector<::flatbuffers::Offset<::flatbuffers::String>> *documentation = nullptr,
     bool optional = false,
-    uint16_t padding = 0) {
+    uint16_t padding = 0,
+    bool offset64 = false) {
   auto name__ = name ? _fbb.CreateString(name) : 0;
   auto attributes__ = attributes ? _fbb.CreateVectorOfSortedTables<reflection::KeyValue>(attributes) : 0;
   auto documentation__ = documentation ? _fbb.CreateVector<::flatbuffers::Offset<::flatbuffers::String>>(*documentation) : 0;
@@ -794,7 +809,8 @@ inline ::flatbuffers::Offset<Field> CreateFieldDirect(
       attributes__,
       documentation__,
       optional,
-      padding);
+      padding,
+      offset64);
 }
 
 struct Object FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table {