From 486c048a0d5963f83f3a0d6957e4dde41602e2e7 Mon Sep 17 00:00:00 2001 From: Wouter van Oortmerssen Date: Mon, 10 Oct 2016 15:55:26 -0700 Subject: [PATCH] Added support for size prefixed buffers. These are useful for streaming FlatBuffers. The functionality ensures proper alignment of the whole buffer. Tested: on OS X. Bug: 27123865 Change-Id: Ic7d75a618c1bb470ea44c4dcf202ff71f2b3f4f1 Signed-off-by: Wouter van Oortmerssen --- include/flatbuffers/flatbuffers.h | 84 ++++++++++++++++++++++++------- tests/test.cpp | 21 ++++++++ 2 files changed, 87 insertions(+), 18 deletions(-) diff --git a/include/flatbuffers/flatbuffers.h b/include/flatbuffers/flatbuffers.h index 8c1c7f0ee..39942e271 100644 --- a/include/flatbuffers/flatbuffers.h +++ b/include/flatbuffers/flatbuffers.h @@ -658,6 +658,16 @@ FLATBUFFERS_FINAL_CLASS } #endif + /// @brief get the minimum alignment this buffer needs to be accessed + /// properly. This is only known once all elements have been written (after + /// you call Finish()). You can use this information if you need to embed + /// a FlatBuffer in some other buffer, such that you can later read it + /// without first having to copy it into its own buffer. + size_t GetBufferMinAlignment() { + Finished(); + return minalign_; + } + /// @cond FLATBUFFERS_INTERNAL void Finished() const { // If you get this assert, you're attempting to get access a buffer @@ -1153,17 +1163,20 @@ FLATBUFFERS_FINAL_CLASS /// will be prefixed with a standard FlatBuffers file header. template void Finish(Offset root, const char *file_identifier = nullptr) { - NotNested(); - // This will cause the whole buffer to be aligned. - PreAlign(sizeof(uoffset_t) + (file_identifier ? kFileIdentifierLength : 0), - minalign_); - if (file_identifier) { - assert(strlen(file_identifier) == kFileIdentifierLength); - buf_.push(reinterpret_cast(file_identifier), - kFileIdentifierLength); - } - PushElement(ReferTo(root.o)); // Location of root. - finished = true; + + Finish(root.o, file_identifier, false); + } + + /// @brief Finish a buffer with a 32 bit size field pre-fixed (size of the + /// buffer following the size field). These buffers are NOT compatible + /// with standard buffers created by Finish, i.e. you can't call GetRoot + /// on them, you have to use GetSizePrefixedRoot instead. + /// All >32 bit quantities in this buffer will be aligned when the whole + /// size pre-fixed buffer is aligned. + /// These kinds of buffers are useful for creating a stream of FlatBuffers. + template void FinishSizePrefixed(Offset root, + const char *file_identifier = nullptr) { + Finish(root.o, file_identifier, true); } private: @@ -1171,6 +1184,25 @@ FLATBUFFERS_FINAL_CLASS FlatBufferBuilder(const FlatBufferBuilder &); FlatBufferBuilder &operator=(const FlatBufferBuilder &); + void Finish(uoffset_t root, const char *file_identifier, bool size_prefix) { + NotNested(); + // This will cause the whole buffer to be aligned. + PreAlign((size_prefix ? sizeof(uoffset_t) : 0) + + sizeof(uoffset_t) + + (file_identifier ? kFileIdentifierLength : 0), + minalign_); + if (file_identifier) { + assert(strlen(file_identifier) == kFileIdentifierLength); + buf_.push(reinterpret_cast(file_identifier), + kFileIdentifierLength); + } + PushElement(ReferTo(root)); // Location of root. + if (size_prefix) { + PushElement(GetSize()); + } + finished = true; + } + struct FieldLoc { uoffset_t off; voffset_t id; @@ -1224,7 +1256,11 @@ template const T *GetRoot(const void *buf) { return GetMutableRoot(const_cast(buf)); } -/// Helpers to get a typed pointer to objects that are currently beeing built. +template const T *GetSizePrefixedRoot(const void *buf) { + return GetRoot(reinterpret_cast(buf) + sizeof(uoffset_t)); +} + +/// Helpers to get a typed pointer to objects that are currently being built. /// @warning Creating new objects will lead to reallocations and invalidates /// the pointer! template T *GetMutableTemporaryPointer(FlatBufferBuilder &fbb, @@ -1348,16 +1384,17 @@ class Verifier FLATBUFFERS_FINAL_CLASS { return true; } - // Verify this whole buffer, starting with root type T. - template bool VerifyBuffer(const char *identifier) { - if (identifier && (size_t(end_ - buf_) < 2 * sizeof(flatbuffers::uoffset_t) || - !BufferHasIdentifier(buf_, identifier))) { + template bool VerifyBufferFromStart(const char *identifier, + const uint8_t *start) { + if (identifier && + (size_t(end_ - start) < 2 * sizeof(flatbuffers::uoffset_t) || + !BufferHasIdentifier(start, identifier))) { return false; } // Call T::Verify, which must be in the generated code for this type. - return Verify(buf_) && - reinterpret_cast(buf_ + ReadScalar(buf_))-> + return Verify(start) && + reinterpret_cast(start + ReadScalar(start))-> Verify(*this) #ifdef FLATBUFFERS_TRACK_VERIFIER_BUFFER_SIZE && GetComputedSize() @@ -1365,6 +1402,17 @@ class Verifier FLATBUFFERS_FINAL_CLASS { ; } + // Verify this whole buffer, starting with root type T. + template bool VerifyBuffer(const char *identifier) { + return VerifyBufferFromStart(identifier, buf_); + } + + template bool VerifySizePrefixedBuffer(const char *identifier) { + return Verify(buf_) && + ReadScalar(buf_) == end_ - buf_ - sizeof(uoffset_t) && + VerifyBufferFromStart(identifier, buf_ + sizeof(uoffset_t)); + } + // Called at the start of a table to increase counters measuring data // structure depth and amount, and possibly bails out with false if // limits set by the constructor have been hit. Needs to be balanced diff --git a/tests/test.cpp b/tests/test.cpp index 6567b9139..d704fb625 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -396,6 +396,25 @@ void ObjectFlatBuffersTest(uint8_t *flatbuf) { TEST_EQ(tests[1].b(), 40); } +// Prefix a FlatBuffer with a size field. +void SizePrefixedTest() { + // Create size prefixed buffer. + flatbuffers::FlatBufferBuilder fbb; + fbb.FinishSizePrefixed(CreateMonster(fbb, 0, 200, 300, + fbb.CreateString("bob"))); + + // Verify it. + flatbuffers::Verifier verifier(fbb.GetBufferPointer(), fbb.GetSize()); + TEST_EQ(verifier.VerifySizePrefixedBuffer(nullptr), true); + + // Access it. + auto m = flatbuffers::GetSizePrefixedRoot( + fbb.GetBufferPointer()); + TEST_EQ(m->mana(), 200); + TEST_EQ(m->hp(), 300); + TEST_EQ_STR(m->name()->c_str(), "bob"); +} + // example of parsing text straight into a buffer, and generating // text back from it: void ParseAndGenerateTextTest() { @@ -1242,6 +1261,8 @@ int main(int /*argc*/, const char * /*argv*/[]) { ObjectFlatBuffersTest(flatbuf.get()); + SizePrefixedTest(); + #ifndef FLATBUFFERS_NO_FILE_TESTS ParseAndGenerateTextTest(); ReflectionTest(flatbuf.get(), rawbuf.length());