FlatBuffers 64 for C++ (#7935)

* First working hack of adding 64-bit. Don't judge :)

* Made vector_downward work on 64 bit types

* vector_downward uses size_t, added offset64 to reflection

* cleaned up adding offset64 in parser

* Add C++ testing skeleton for 64-bit

* working test for CreateVector64

* working >2 GiB buffers

* support for large strings

* simplified CreateString<> to just provide the offset type

* generalize CreateVector template

* update test_64.afb due to upstream format change

* Added Vector64 type, which is just an alias for vector ATM

* Switch to Offset64 for Vector64

* Update for reflection bfbs output change

* Starting to add support for vector64 type in C++

* made a generic CreateVector that can handle different offsets and vector types

* Support for 32-vector with 64-addressing

* Vector64 basic builder + tests working

* basic support for json vector64 support

* renamed fields in test_64bit.fbs to better reflect their use

* working C++ vector64 builder

* Apply --annotate-sparse-vector to 64-bit tests

* Enable Vector64 for --annotate-sparse-vectors

* Merged from upstream

* Add `near_string` field for testing 32-bit offsets alongside

* keep track of where the 32-bit and 64-bit regions are for flatbufferbuilder

* move template<> outside class body for GCC

* update run.sh to build and run tests

* basic assertion for adding 64-bit offset at the wrong time

* started to separate `FlatBufferBuilder` into two classes, 1 64-bit aware, the other not

* add test for nested flatbuffer vector64, fix bug in alignment of big vectors

* fixed CreateDirect method by iterating by Offset64 first

* internal refactoring of flatbufferbuilder

* block not supported languages in the parser from using 64-bit

* evolution tests for adding a vector64 field

* conformity tests for adding/removing offset64 attributes

* ensure test is for a big buffer

* add parser error tests for `offset64` and `vector64` attributes

* add missing static that GCC only complains about

* remove stdint-uintn.h header that gets automatically added

* move 64-bit CalculateOffset internal

* fixed return size of EndVector

* various fixes on windows

* add SizeT to vector_downward

* minimze range of size changes in vector and builder

* reworked how tracking if 64-offsets are added

* Add ReturnT to EndVector

* small cleanups

* remove need for second Array definition

* combine IndirectHelpers into one definition

* started support for vector of struct

* Support for 32/64-vectors of structs + Offset64

* small cleanups

* add verification for vector64

* add sized prefix for 64-bit buffers

* add fuzzer for 64-bit

* add example of adding many vectors using a wrapper table

* run the new -bfbs-gen-embed logic on the 64-bit tests

* remove run.sh and fix cmakelist issue

* fixed bazel rules

* fixed some PR comments

* add 64-bit tests to cmakelist

include/flatbuffers/vector.h

@@ -27,7 +27,8 @@ struct String;
 
 // An STL compatible iterator implementation for Vector below, effectively
 // calling Get() for every element.
-template<typename T, typename IT, typename Data = uint8_t *>
+template<typename T, typename IT, typename Data = uint8_t *,
+         typename SizeT = uoffset_t>
 struct VectorIterator {
   typedef std::random_access_iterator_tag iterator_category;
   typedef IT value_type;
@@ -35,8 +36,9 @@ struct VectorIterator {
   typedef IT *pointer;
   typedef IT &reference;
 
-  VectorIterator(Data data, uoffset_t i)
-      : data_(data + IndirectHelper<T>::element_stride * i) {}
+  static const SizeT element_stride = IndirectHelper<T>::element_stride;
+
+  VectorIterator(Data data, SizeT i) : data_(data + element_stride * i) {}
   VectorIterator(const VectorIterator &other) : data_(other.data_) {}
   VectorIterator() : data_(nullptr) {}
 
@@ -63,7 +65,7 @@ struct VectorIterator {
   }
 
   difference_type operator-(const VectorIterator &other) const {
-    return (data_ - other.data_) / IndirectHelper<T>::element_stride;
+    return (data_ - other.data_) / element_stride;
   }
 
   // Note: return type is incompatible with the standard
@@ -75,44 +77,42 @@ struct VectorIterator {
   IT operator->() const { return IndirectHelper<T>::Read(data_, 0); }
 
   VectorIterator &operator++() {
-    data_ += IndirectHelper<T>::element_stride;
+    data_ += element_stride;
     return *this;
   }
 
   VectorIterator operator++(int) {
     VectorIterator temp(data_, 0);
-    data_ += IndirectHelper<T>::element_stride;
+    data_ += element_stride;
     return temp;
   }
 
-  VectorIterator operator+(const uoffset_t &offset) const {
-    return VectorIterator(data_ + offset * IndirectHelper<T>::element_stride,
-                          0);
+  VectorIterator operator+(const SizeT &offset) const {
+    return VectorIterator(data_ + offset * element_stride, 0);
   }
 
-  VectorIterator &operator+=(const uoffset_t &offset) {
-    data_ += offset * IndirectHelper<T>::element_stride;
+  VectorIterator &operator+=(const SizeT &offset) {
+    data_ += offset * element_stride;
     return *this;
   }
 
   VectorIterator &operator--() {
-    data_ -= IndirectHelper<T>::element_stride;
+    data_ -= element_stride;
     return *this;
   }
 
   VectorIterator operator--(int) {
     VectorIterator temp(data_, 0);
-    data_ -= IndirectHelper<T>::element_stride;
+    data_ -= element_stride;
     return temp;
   }
 
-  VectorIterator operator-(const uoffset_t &offset) const {
-    return VectorIterator(data_ - offset * IndirectHelper<T>::element_stride,
-                          0);
+  VectorIterator operator-(const SizeT &offset) const {
+    return VectorIterator(data_ - offset * element_stride, 0);
   }
 
-  VectorIterator &operator-=(const uoffset_t &offset) {
-    data_ -= offset * IndirectHelper<T>::element_stride;
+  VectorIterator &operator-=(const SizeT &offset) {
+    data_ -= offset * element_stride;
     return *this;
   }
 
@@ -120,8 +120,8 @@ struct VectorIterator {
   Data data_;
 };
 
-template<typename T, typename IT>
-using VectorConstIterator = VectorIterator<T, IT, const uint8_t *>;
+template<typename T, typename IT, typename SizeT = uoffset_t>
+using VectorConstIterator = VectorIterator<T, IT, const uint8_t *, SizeT>;
 
 template<typename Iterator>
 struct VectorReverseIterator : public std::reverse_iterator<Iterator> {
@@ -145,11 +145,14 @@ struct VectorReverseIterator : public std::reverse_iterator<Iterator> {
 
 // This is used as a helper type for accessing vectors.
 // Vector::data() assumes the vector elements start after the length field.
-template<typename T> class Vector {
+template<typename T, typename SizeT = uoffset_t> class Vector {
  public:
-  typedef VectorIterator<T, typename IndirectHelper<T>::mutable_return_type>
+  typedef VectorIterator<T,
+                         typename IndirectHelper<T>::mutable_return_type,
+                         uint8_t *, SizeT>
       iterator;
-  typedef VectorConstIterator<T, typename IndirectHelper<T>::return_type>
+  typedef VectorConstIterator<T, typename IndirectHelper<T>::return_type,
+                              SizeT>
       const_iterator;
   typedef VectorReverseIterator<iterator> reverse_iterator;
   typedef VectorReverseIterator<const_iterator> const_reverse_iterator;
@@ -160,39 +163,41 @@ template<typename T> class Vector {
   static FLATBUFFERS_CONSTEXPR bool is_span_observable =
       scalar_tag::value && (FLATBUFFERS_LITTLEENDIAN || sizeof(T) == 1);
 
-  uoffset_t size() const { return EndianScalar(length_); }
+  SizeT size() const { return EndianScalar(length_); }
 
   // Deprecated: use size(). Here for backwards compatibility.
   FLATBUFFERS_ATTRIBUTE([[deprecated("use size() instead")]])
-  uoffset_t Length() const { return size(); }
+  SizeT Length() const { return size(); }
 
+  typedef SizeT size_type;
   typedef typename IndirectHelper<T>::return_type return_type;
-  typedef typename IndirectHelper<T>::mutable_return_type mutable_return_type;
+  typedef typename IndirectHelper<T>::mutable_return_type
+      mutable_return_type;
   typedef return_type value_type;
 
-  return_type Get(uoffset_t i) const {
+  return_type Get(SizeT i) const {
     FLATBUFFERS_ASSERT(i < size());
     return IndirectHelper<T>::Read(Data(), i);
   }
 
-  return_type operator[](uoffset_t i) const { return Get(i); }
+  return_type operator[](SizeT i) const { return Get(i); }
 
   // If this is a Vector of enums, T will be its storage type, not the enum
   // type. This function makes it convenient to retrieve value with enum
   // type E.
-  template<typename E> E GetEnum(uoffset_t i) const {
+  template<typename E> E GetEnum(SizeT i) const {
     return static_cast<E>(Get(i));
   }
 
   // If this a vector of unions, this does the cast for you. There's no check
   // to make sure this is the right type!
-  template<typename U> const U *GetAs(uoffset_t i) const {
+  template<typename U> const U *GetAs(SizeT i) const {
     return reinterpret_cast<const U *>(Get(i));
   }
 
   // If this a vector of unions, this does the cast for you. There's no check
   // to make sure this is actually a string!
-  const String *GetAsString(uoffset_t i) const {
+  const String *GetAsString(SizeT i) const {
     return reinterpret_cast<const String *>(Get(i));
   }
 
@@ -226,7 +231,7 @@ template<typename T> class Vector {
 
   // Change elements if you have a non-const pointer to this object.
   // Scalars only. See reflection.h, and the documentation.
-  void Mutate(uoffset_t i, const T &val) {
+  void Mutate(SizeT i, const T &val) {
     FLATBUFFERS_ASSERT(i < size());
     WriteScalar(data() + i, val);
   }
@@ -234,15 +239,15 @@ template<typename T> class Vector {
   // Change an element of a vector of tables (or strings).
   // "val" points to the new table/string, as you can obtain from
   // e.g. reflection::AddFlatBuffer().
-  void MutateOffset(uoffset_t i, const uint8_t *val) {
+  void MutateOffset(SizeT i, const uint8_t *val) {
     FLATBUFFERS_ASSERT(i < size());
-    static_assert(sizeof(T) == sizeof(uoffset_t), "Unrelated types");
+    static_assert(sizeof(T) == sizeof(SizeT), "Unrelated types");
     WriteScalar(data() + i,
-                static_cast<uoffset_t>(val - (Data() + i * sizeof(uoffset_t))));
+                static_cast<SizeT>(val - (Data() + i * sizeof(SizeT))));
   }
 
   // Get a mutable pointer to tables/strings inside this vector.
-  mutable_return_type GetMutableObject(uoffset_t i) const {
+  mutable_return_type GetMutableObject(SizeT i) const {
     FLATBUFFERS_ASSERT(i < size());
     return const_cast<mutable_return_type>(IndirectHelper<T>::Read(Data(), i));
   }
@@ -280,7 +285,7 @@ template<typename T> class Vector {
   // try to construct these manually.
   Vector();
 
-  uoffset_t length_;
+  SizeT length_;
 
  private:
   // This class is a pointer. Copying will therefore create an invalid object.
@@ -299,6 +304,8 @@ template<typename T> class Vector {
   }
 };
 
+template<typename T> using Vector64 = Vector<T, uoffset64_t>;
+
 template<class U>
 FLATBUFFERS_CONSTEXPR_CPP11 flatbuffers::span<U> make_span(Vector<U> &vec)
     FLATBUFFERS_NOEXCEPT {