FlatBuffers 64 for C++ (#7935)

* First working hack of adding 64-bit. Don't judge :)

* Made vector_downward work on 64 bit types

* vector_downward uses size_t, added offset64 to reflection

* cleaned up adding offset64 in parser

* Add C++ testing skeleton for 64-bit

* working test for CreateVector64

* working >2 GiB buffers

* support for large strings

* simplified CreateString<> to just provide the offset type

* generalize CreateVector template

* update test_64.afb due to upstream format change

* Added Vector64 type, which is just an alias for vector ATM

* Switch to Offset64 for Vector64

* Update for reflection bfbs output change

* Starting to add support for vector64 type in C++

* made a generic CreateVector that can handle different offsets and vector types

* Support for 32-vector with 64-addressing

* Vector64 basic builder + tests working

* basic support for json vector64 support

* renamed fields in test_64bit.fbs to better reflect their use

* working C++ vector64 builder

* Apply --annotate-sparse-vector to 64-bit tests

* Enable Vector64 for --annotate-sparse-vectors

* Merged from upstream

* Add `near_string` field for testing 32-bit offsets alongside

* keep track of where the 32-bit and 64-bit regions are for flatbufferbuilder

* move template<> outside class body for GCC

* update run.sh to build and run tests

* basic assertion for adding 64-bit offset at the wrong time

* started to separate `FlatBufferBuilder` into two classes, 1 64-bit aware, the other not

* add test for nested flatbuffer vector64, fix bug in alignment of big vectors

* fixed CreateDirect method by iterating by Offset64 first

* internal refactoring of flatbufferbuilder

* block not supported languages in the parser from using 64-bit

* evolution tests for adding a vector64 field

* conformity tests for adding/removing offset64 attributes

* ensure test is for a big buffer

* add parser error tests for `offset64` and `vector64` attributes

* add missing static that GCC only complains about

* remove stdint-uintn.h header that gets automatically added

* move 64-bit CalculateOffset internal

* fixed return size of EndVector

* various fixes on windows

* add SizeT to vector_downward

* minimze range of size changes in vector and builder

* reworked how tracking if 64-offsets are added

* Add ReturnT to EndVector

* small cleanups

* remove need for second Array definition

* combine IndirectHelpers into one definition

* started support for vector of struct

* Support for 32/64-vectors of structs + Offset64

* small cleanups

* add verification for vector64

* add sized prefix for 64-bit buffers

* add fuzzer for 64-bit

* add example of adding many vectors using a wrapper table

* run the new -bfbs-gen-embed logic on the 64-bit tests

* remove run.sh and fix cmakelist issue

* fixed bazel rules

* fixed some PR comments

* add 64-bit tests to cmakelist

tests/fuzzer/.gitignore

@@ -15,4 +15,6 @@ fuzz-*.log
 annotated_binary.bfbs
 annotated_binary.bin
 
+test_64bit.bin
+
 monster_test.bfbs

tests/fuzzer/CMakeLists.txt

@@ -117,6 +117,7 @@ set(FlatBuffers_Library_SRCS
     ${FLATBUFFERS_DIR}/src/binary_annotator.cpp
     ${FLATBUFFERS_DIR}/src/util.cpp
     ${FLATBUFFERS_DIR}/tests/test_assert.cpp
+    ${FLATBUFFERS_DIR}/tests/64bit/test_64bit_bfbs_generated.h
 )
 
 include_directories(${FLATBUFFERS_DIR}/include)
@@ -175,6 +176,16 @@ add_custom_command(
   ${CMAKE_CURRENT_BINARY_DIR}/seed_annotator/annotated_binary.bin
 )
 
+add_executable(64bit_fuzzer flatbuffers_64bit_fuzzer.cc)
+target_link_libraries(64bit_fuzzer PRIVATE flatbuffers_fuzzed)
+add_custom_command(
+  TARGET 64bit_fuzzer PRE_BUILD
+
+  COMMAND ${CMAKE_COMMAND} -E copy
+  ${CMAKE_SOURCE_DIR}/../64bit/test_64bit.bin
+  ${CMAKE_CURRENT_BINARY_DIR}/seed_64bit/test_64bit.bin
+)
+
 # Build debugger for weird cases found with fuzzer.
 if(BUILD_DEBUGGER)
   add_library(flatbuffers_nonfuzz STATIC ${FlatBuffers_Library_SRCS})

tests/fuzzer/flatbuffers_64bit_fuzzer.cc

@@ -0,0 +1,121 @@
+#include <cstdint>
+#include <filesystem>
+#include <type_traits>
+
+#include "64bit/test_64bit_bfbs_generated.h"
+#include "64bit/test_64bit_generated.h"
+#include "flatbuffers/base.h"
+#include "flatbuffers/flatbuffer_builder.h"
+#include "flatbuffers/flatbuffers.h"
+#include "flatbuffers/reflection.h"
+#include "flatbuffers/verifier.h"
+#include "test_assert.h"
+#include "test_init.h"
+
+OneTimeTestInit OneTimeTestInit::one_time_init_;
+
+static RootTableBinarySchema schema;
+
+static constexpr uint8_t flags_sized_prefixed = 0b00000001;
+
+static const uint64_t kFnvPrime = 0x00000100000001b3ULL;
+static const uint64_t kOffsetBasis = 0xcbf29ce484222645ULL;
+
+namespace flatbuffers {
+
+template<typename T, typename = std::enable_if_t<std::is_integral_v<T>>>
+uint64_t Hash(T value, uint64_t hash) {
+  return (hash * kFnvPrime) ^ value;
+}
+
+uint64_t Hash(double value, uint64_t hash) {
+  static_assert(sizeof(double) == sizeof(uint64_t));
+  return (hash * kFnvPrime) ^ static_cast<uint64_t>(value);
+}
+
+uint64_t Hash(const flatbuffers::String *value, uint64_t hash) {
+  if (value == nullptr) { return hash * kFnvPrime; }
+  for (auto &c : value->str()) { hash = Hash(static_cast<uint8_t>(c), hash); }
+  return hash;
+}
+
+uint64_t Hash(const LeafStruct *value, uint64_t hash) {
+  if (value == nullptr) { return hash * kFnvPrime; }
+  hash = Hash(value->a(), hash);
+  hash = Hash(value->b(), hash);
+  return hash;
+}
+
+template<typename T> uint64_t Hash(const Vector<T> *value, uint64_t hash) {
+  if (value == nullptr) { return hash * kFnvPrime; }
+  for (const T c : *value) { hash = Hash(c, hash); }
+  return hash;
+}
+
+template<typename T> uint64_t Hash(const Vector64<T> *value, uint64_t hash) {
+  if (value == nullptr) { return hash * kFnvPrime; }
+  for (const T c : *value) { hash = Hash(c, hash); }
+  return hash;
+}
+
+uint64_t Hash(const RootTable *value, uint64_t hash) {
+  if (value == nullptr) { return hash * kFnvPrime; }
+  // Hash all the fields so we can exercise all parts of the code.
+  hash = Hash(value->far_vector(), hash);
+  hash = Hash(value->a(), hash);
+  hash = Hash(value->far_string(), hash);
+  hash = Hash(value->big_vector(), hash);
+  hash = Hash(value->near_string(), hash);
+  hash = Hash(value->nested_root(), hash);
+  hash = Hash(value->far_struct_vector(), hash);
+  hash = Hash(value->big_struct_vector(), hash);
+  return hash;
+}
+
+static int AccessBuffer(const uint8_t *data, size_t size,
+                        bool is_size_prefixed) {
+  const RootTable *root_table =
+      is_size_prefixed ? GetSizePrefixedRootTable(data) : GetRootTable(data);
+  TEST_NOTNULL(root_table);
+
+  uint64_t hash = kOffsetBasis;
+  hash = Hash(root_table, hash);
+  hash = Hash(root_table->nested_root_nested_root(), hash);
+
+  return 0;
+}
+
+extern "C" int LLVMFuzzerInitialize(int *, char ***argv) {
+  Verifier verifier(schema.begin(), schema.size());
+  TEST_EQ(true, reflection::VerifySchemaBuffer(verifier));
+
+  return 0;
+}
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  if (size < FLATBUFFERS_MIN_BUFFER_SIZE) { return 0; }
+
+  // Take the first bit of data as a flag to control things.
+  const uint8_t flags = data[0];
+  data++;
+  size--;
+
+  Verifier::Options options;
+  options.assert = true;
+  options.check_alignment = true;
+  options.check_nested_flatbuffers = true;
+
+  Verifier verifier(data, size, options);
+
+  const bool is_size_prefixed = flags & flags_sized_prefixed;
+
+  // Filter out data that isn't valid.
+  if ((is_size_prefixed && !VerifySizePrefixedRootTableBuffer(verifier)) ||
+      !VerifyRootTableBuffer(verifier)) {
+    return 0;
+  }
+
+  return AccessBuffer(data, size, is_size_prefixed);
+}
+
+}  // namespace flatbuffers

tests/fuzzer/flatbuffers_annotator_fuzzer.cc

@@ -46,7 +46,7 @@ extern "C" int LLVMFuzzerInitialize(int *, char ***argv) {
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   flatbuffers::BinaryAnnotator annotator(schema_bfbs_, schema_bfbs_length_,
-                                         data, size);
+                                         data, size, false);
 
   annotator.Annotate();
   return 0;