BigfootDev/flatbuffers
1
0
Fork 1
mirror of https://github.com/google/flatbuffers.git synced 2026-06-16 09:12:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

FlatBuffers 64 for C++ (#7935)

Browse Source 
* First working hack of adding 64-bit. Don't judge :)

* Made vector_downward work on 64 bit types

* vector_downward uses size_t, added offset64 to reflection

* cleaned up adding offset64 in parser

* Add C++ testing skeleton for 64-bit

* working test for CreateVector64

* working >2 GiB buffers

* support for large strings

* simplified CreateString<> to just provide the offset type

* generalize CreateVector template

* update test_64.afb due to upstream format change

* Added Vector64 type, which is just an alias for vector ATM

* Switch to Offset64 for Vector64

* Update for reflection bfbs output change

* Starting to add support for vector64 type in C++

* made a generic CreateVector that can handle different offsets and vector types

* Support for 32-vector with 64-addressing

* Vector64 basic builder + tests working

* basic support for json vector64 support

* renamed fields in test_64bit.fbs to better reflect their use

* working C++ vector64 builder

* Apply --annotate-sparse-vector to 64-bit tests

* Enable Vector64 for --annotate-sparse-vectors

* Merged from upstream

* Add `near_string` field for testing 32-bit offsets alongside

* keep track of where the 32-bit and 64-bit regions are for flatbufferbuilder

* move template<> outside class body for GCC

* update run.sh to build and run tests

* basic assertion for adding 64-bit offset at the wrong time

* started to separate `FlatBufferBuilder` into two classes, 1 64-bit aware, the other not

* add test for nested flatbuffer vector64, fix bug in alignment of big vectors

* fixed CreateDirect method by iterating by Offset64 first

* internal refactoring of flatbufferbuilder

* block not supported languages in the parser from using 64-bit

* evolution tests for adding a vector64 field

* conformity tests for adding/removing offset64 attributes

* ensure test is for a big buffer

* add parser error tests for `offset64` and `vector64` attributes

* add missing static that GCC only complains about

* remove stdint-uintn.h header that gets automatically added

* move 64-bit CalculateOffset internal

* fixed return size of EndVector

* various fixes on windows

* add SizeT to vector_downward

* minimze range of size changes in vector and builder

* reworked how tracking if 64-offsets are added

* Add ReturnT to EndVector

* small cleanups

* remove need for second Array definition

* combine IndirectHelpers into one definition

* started support for vector of struct

* Support for 32/64-vectors of structs + Offset64

* small cleanups

* add verification for vector64

* add sized prefix for 64-bit buffers

* add fuzzer for 64-bit

* add example of adding many vectors using a wrapper table

* run the new -bfbs-gen-embed logic on the 64-bit tests

* remove run.sh and fix cmakelist issue

* fixed bazel rules

* fixed some PR comments

* add 64-bit tests to cmakelist
This commit is contained in:
Derek Bailey
2023-05-09 09:16:30 -07:00
committed by GitHub
parent 13fc75cb6b
commit 63b7b25289
49 changed files with 3274 additions and 529 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
include/flatbuffers/verifier.h
View File

@@ -34,12 +34,16 @@ class Verifier FLATBUFFERS_FINAL_CLASS {
bool check_alignment = true;
// If true, run verifier on nested flatbuffers
bool check_nested_flatbuffers = true;
// The maximum size of a buffer.
size_t max_size = FLATBUFFERS_MAX_BUFFER_SIZE;
// Use assertions to check for errors.
bool assert = false;
};
explicit Verifier(const uint8_t *const buf, const size_t buf_len,
const Options &opts)
: buf_(buf), size_(buf_len), opts_(opts) {
FLATBUFFERS_ASSERT(size_ < FLATBUFFERS_MAX_BUFFER_SIZE);
FLATBUFFERS_ASSERT(size_ < opts.max_size);
}
// Deprecated API, please construct with Verifier::Options.
@@ -58,7 +62,7 @@ class Verifier FLATBUFFERS_FINAL_CLASS {
bool Check(const bool ok) const {
// clang-format off
#ifdef FLATBUFFERS_DEBUG_VERIFICATION_FAILURE
FLATBUFFERS_ASSERT(ok);
if (opts_.assert) { FLATBUFFERS_ASSERT(ok); }
#endif
#ifdef FLATBUFFERS_TRACK_VERIFIER_BUFFER_SIZE
if (!ok)
@@ -113,41 +117,43 @@ class Verifier FLATBUFFERS_FINAL_CLASS {
}
// Verify a pointer (may be NULL) of any vector type.
template<typename T> bool VerifyVector(const Vector<T> *const vec) const {
return !vec || VerifyVectorOrString(reinterpret_cast<const uint8_t *>(vec),
sizeof(T));
template<int &..., typename T, typename LenT>
bool VerifyVector(const Vector<T, LenT> *const vec) const {
return !vec || VerifyVectorOrString<LenT>(
reinterpret_cast<const uint8_t *>(vec), sizeof(T));
}
// Verify a pointer (may be NULL) of a vector to struct.
template<typename T>
bool VerifyVector(const Vector<const T *> *const vec) const {
return VerifyVector(reinterpret_cast<const Vector<T> *>(vec));
template<int &..., typename T, typename LenT>
bool VerifyVector(const Vector<const T *, LenT> *const vec) const {
return VerifyVector(reinterpret_cast<const Vector<T, LenT> *>(vec));
}
// Verify a pointer (may be NULL) to string.
bool VerifyString(const String *const str) const {
size_t end;
return !str || (VerifyVectorOrString(reinterpret_cast<const uint8_t *>(str),
1, &end) &&
return !str || (VerifyVectorOrString<uoffset_t>(
reinterpret_cast<const uint8_t *>(str), 1, &end) &&
Verify(end, 1) && // Must have terminator
Check(buf_[end] == '\0')); // Terminating byte must be 0.
}
// Common code between vectors and strings.
template<typename LenT = uoffset_t>
bool VerifyVectorOrString(const uint8_t *const vec, const size_t elem_size,
size_t *const end = nullptr) const {
const auto veco = static_cast<size_t>(vec - buf_);
const auto vec_offset = static_cast<size_t>(vec - buf_);
// Check we can read the size field.
if (!Verify<uoffset_t>(veco)) return false;
if (!Verify<LenT>(vec_offset)) return false;
// Check the whole array. If this is a string, the byte past the array must
// be 0.
const auto size = ReadScalar<uoffset_t>(vec);
const auto max_elems = FLATBUFFERS_MAX_BUFFER_SIZE / elem_size;
const LenT size = ReadScalar<LenT>(vec);
const auto max_elems = opts_.max_size / elem_size;
if (!Check(size < max_elems))
return false; // Protect against byte_size overflowing.
const auto byte_size = sizeof(size) + elem_size * size;
if (end) *end = veco + byte_size;
return Verify(veco, byte_size);
const auto byte_size = sizeof(LenT) + elem_size * size;
if (end) *end = vec_offset + byte_size;
return Verify(vec_offset, byte_size);
}
// Special case for string contents, after the above has been called.
@@ -203,7 +209,7 @@ class Verifier FLATBUFFERS_FINAL_CLASS {
}
// Call T::Verify, which must be in the generated code for this type.
const auto o = VerifyOffset(start);
const auto o = VerifyOffset<uoffset_t>(start);
return Check(o != 0) &&
reinterpret_cast<const T *>(buf_ + start + o)->Verify(*this)
// clang-format off
@@ -214,8 +220,8 @@ class Verifier FLATBUFFERS_FINAL_CLASS {
// clang-format on
}
template<typename T>
bool VerifyNestedFlatBuffer(const Vector<uint8_t> *const buf,
template<typename T, int &..., typename SizeT>
bool VerifyNestedFlatBuffer(const Vector<uint8_t, SizeT> *const buf,
const char *const identifier) {
// Caller opted out of this.
if (!opts_.check_nested_flatbuffers) return true;
@@ -226,7 +232,7 @@ class Verifier FLATBUFFERS_FINAL_CLASS {
// If there is a nested buffer, it must be greater than the min size.
if (!Check(buf->size() >= FLATBUFFERS_MIN_BUFFER_SIZE)) return false;
Verifier nested_verifier(buf->data(), buf->size());
Verifier nested_verifier(buf->data(), buf->size(), opts_);
return nested_verifier.VerifyBuffer<T>(identifier);
}
@@ -237,29 +243,30 @@ class Verifier FLATBUFFERS_FINAL_CLASS {
return VerifyBufferFromStart<T>(identifier, 0);
}
template<typename T>
template<typename T, typename SizeT = uoffset_t>
bool VerifySizePrefixedBuffer(const char *const identifier) {
return Verify<uoffset_t>(0U) &&
Check(ReadScalar<uoffset_t>(buf_) == size_ - sizeof(uoffset_t)) &&
VerifyBufferFromStart<T>(identifier, sizeof(uoffset_t));
return Verify<SizeT>(0U) &&
Check(ReadScalar<SizeT>(buf_) == size_ - sizeof(SizeT)) &&
VerifyBufferFromStart<T>(identifier, sizeof(SizeT));
}
uoffset_t VerifyOffset(const size_t start) const {
if (!Verify<uoffset_t>(start)) return 0;
const auto o = ReadScalar<uoffset_t>(buf_ + start);
template<typename OffsetT = uoffset_t, typename SOffsetT = soffset_t>
size_t VerifyOffset(const size_t start) const {
if (!Verify<OffsetT>(start)) return 0;
const auto o = ReadScalar<OffsetT>(buf_ + start);
// May not point to itself.
if (!Check(o != 0)) return 0;
// Can't wrap around / buffers are max 2GB.
if (!Check(static_cast<soffset_t>(o) >= 0)) return 0;
// Can't wrap around larger than the max size.
if (!Check(static_cast<SOffsetT>(o) >= 0)) return 0;
// Must be inside the buffer to create a pointer from it (pointer outside
// buffer is UB).
if (!Verify(start + o, 1)) return 0;
return o;
}
uoffset_t VerifyOffset(const uint8_t *const base,
const voffset_t start) const {
return VerifyOffset(static_cast<size_t>(base - buf_) + start);
template<typename OffsetT = uoffset_t>
size_t VerifyOffset(const uint8_t *const base, const voffset_t start) const {
return VerifyOffset<OffsetT>(static_cast<size_t>(base - buf_) + start);
}
// Called at the start of a table to increase counters measuring data
@@ -312,6 +319,12 @@ class Verifier FLATBUFFERS_FINAL_CLASS {
std::vector<uint8_t> *flex_reuse_tracker_ = nullptr;
};
// Specialization for 64-bit offsets.
template<>
inline size_t Verifier::VerifyOffset<uoffset64_t>(const size_t start) const {
return VerifyOffset<uoffset64_t, soffset64_t>(start);
}
} // namespace flatbuffers
#endif // FLATBUFFERS_VERIFIER_H_