mirror of
https://github.com/google/flatbuffers.git
synced 2026-06-02 20:15:34 +00:00
4e582b0c1d
A blob is an array of bytes and has no intrinsic alignment (i.e. the alignment is 1). The alignment of the existing flexbuffers blob is solely affected by the width of the integer needed to store the blob's size: that integer's width becomes the alignment of the blob. The proposed AlignedBlob function here piggybacks on this effect and simply uses a user-defined alignment for the width of the integer that stores the blob's size; this automatically imparts that same alignment on the blob itself. (The width is bounded below by the actual width needed to store the blob's size.) The ability to control the alignment of a blob is important for use cases in which the blob itself stores structured data that we want to access without further copies (e.g. other flatbuffer messages).
319 lines
13 KiB
C++
319 lines
13 KiB
C++
#include "flexbuffers_test.h"
|
|
|
|
#include <memory>
|
|
#include <limits>
|
|
|
|
#include "flatbuffers/flexbuffers.h"
|
|
#include "flatbuffers/idl.h"
|
|
#include "is_quiet_nan.h"
|
|
#include "test_assert.h"
|
|
|
|
namespace flatbuffers {
|
|
namespace tests {
|
|
|
|
// Shortcuts for the infinity.
|
|
static const auto infinity_d = std::numeric_limits<double>::infinity();
|
|
|
|
static bool IsAligned(const void* ptr, std::size_t alignment) {
|
|
void* p = const_cast<void*>(ptr);
|
|
std::size_t space = 2 * alignment;
|
|
void* q = std::align(alignment, alignment, p, space);
|
|
return q != nullptr && p == ptr && space == 2 * alignment;
|
|
}
|
|
|
|
void FlexBuffersTest() {
|
|
flexbuffers::Builder slb(512,
|
|
flexbuffers::BUILDER_FLAG_SHARE_KEYS_AND_STRINGS);
|
|
|
|
// Write the equivalent of:
|
|
// { vec: [ -100, "Fred", 4.0, false ], bar: [ 1, 2, 3 ], bar3: [ 1, 2, 3 ],
|
|
// foo: 100, bool: true, mymap: { foo: "Fred" } }
|
|
|
|
// It's possible to do this without std::function support as well.
|
|
slb.Map([&]() {
|
|
slb.Vector("vec", [&]() {
|
|
slb += -100; // Equivalent to slb.Add(-100) or slb.Int(-100);
|
|
slb += "Fred";
|
|
slb.IndirectFloat(4.0f);
|
|
auto i_f = slb.LastValue();
|
|
uint8_t blob[] = {77};
|
|
uint32_t aligned_blob[] = {88, 99};
|
|
slb.Blob(blob, sizeof blob);
|
|
slb.AlignedBlob(aligned_blob, sizeof aligned_blob,
|
|
flexbuffers::BIT_WIDTH_32);
|
|
slb += false;
|
|
slb.ReuseValue(i_f);
|
|
});
|
|
int ints[] = {1, 2, 3};
|
|
slb.Vector("bar", ints, 3);
|
|
slb.FixedTypedVector("bar3", ints, 3);
|
|
bool bools[] = {true, false, true, false};
|
|
slb.Vector("bools", bools, 4);
|
|
slb.Bool("bool", true);
|
|
slb.Double("foo", 100);
|
|
slb.Map("mymap", [&]() {
|
|
slb.String("foo", "Fred"); // Testing key and string reuse.
|
|
});
|
|
});
|
|
slb.Finish();
|
|
|
|
// clang-format off
|
|
#ifdef FLATBUFFERS_TEST_VERBOSE
|
|
for (size_t i = 0; i < slb.GetBuffer().size(); i++)
|
|
printf("%d ", slb.GetBuffer().data()[i]);
|
|
printf("\n");
|
|
#endif
|
|
// clang-format on
|
|
|
|
std::vector<uint8_t> reuse_tracker;
|
|
TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
|
|
slb.GetBuffer().size(), &reuse_tracker),
|
|
true);
|
|
|
|
auto map = flexbuffers::GetRoot(slb.GetBuffer()).AsMap();
|
|
TEST_EQ(map.size(), 7);
|
|
auto vec = map["vec"].AsVector();
|
|
TEST_EQ(vec.size(), 7);
|
|
TEST_EQ(vec[0].AsInt64(), -100);
|
|
TEST_EQ_STR(vec[1].AsString().c_str(), "Fred");
|
|
TEST_EQ(vec[1].AsInt64(), 0); // Number parsing failed.
|
|
TEST_EQ(vec[2].AsDouble(), 4.0);
|
|
TEST_EQ(vec[2].AsString().IsTheEmptyString(), true); // Wrong Type.
|
|
TEST_EQ_STR(vec[2].AsString().c_str(), ""); // This still works though.
|
|
TEST_EQ_STR(vec[2].ToString().c_str(), "4.0"); // Or have it converted.
|
|
// Few tests for templated version of As.
|
|
TEST_EQ(vec[0].As<int64_t>(), -100);
|
|
TEST_EQ_STR(vec[1].As<std::string>().c_str(), "Fred");
|
|
TEST_EQ(vec[1].As<int64_t>(), 0); // Number parsing failed.
|
|
TEST_EQ(vec[2].As<double>(), 4.0);
|
|
// Test that the blob can be accessed.
|
|
TEST_EQ(vec[3].IsBlob(), true);
|
|
auto blob = vec[3].AsBlob();
|
|
TEST_EQ(blob.size(), 1);
|
|
TEST_EQ(blob.data()[0], 77);
|
|
TEST_EQ(vec[4].IsBlob(), true);
|
|
auto aligned_blob = vec[4].AsBlob();
|
|
TEST_EQ(aligned_blob.size(), 8);
|
|
TEST_EQ(reinterpret_cast<const uint32_t*>(aligned_blob.data())[0], 88);
|
|
TEST_EQ(reinterpret_cast<const uint32_t*>(aligned_blob.data())[1], 99);
|
|
TEST_EQ(IsAligned(aligned_blob.data(), 4), true);
|
|
TEST_EQ(vec[5].IsBool(), true); // Check if type is a bool
|
|
TEST_EQ(vec[5].AsBool(), false); // Check if value is false
|
|
TEST_EQ(vec[6].AsDouble(), 4.0); // This is shared with vec[2] !
|
|
auto tvec = map["bar"].AsTypedVector();
|
|
TEST_EQ(tvec.size(), 3);
|
|
TEST_EQ(tvec[2].AsInt8(), 3);
|
|
auto tvec3 = map["bar3"].AsFixedTypedVector();
|
|
TEST_EQ(tvec3.size(), 3);
|
|
TEST_EQ(tvec3[2].AsInt8(), 3);
|
|
TEST_EQ(map["bool"].AsBool(), true);
|
|
auto tvecb = map["bools"].AsTypedVector();
|
|
TEST_EQ(tvecb.ElementType(), flexbuffers::FBT_BOOL);
|
|
TEST_EQ(map["foo"].AsUInt8(), 100);
|
|
TEST_EQ(map["unknown"].IsNull(), true);
|
|
auto mymap = map["mymap"].AsMap();
|
|
// These should be equal by pointer equality, since key and value are shared.
|
|
TEST_EQ(mymap.Keys()[0].AsKey(), map.Keys()[4].AsKey());
|
|
TEST_EQ(mymap.Values()[0].AsString().c_str(), vec[1].AsString().c_str());
|
|
// We can mutate values in the buffer.
|
|
TEST_EQ(vec[0].MutateInt(-99), true);
|
|
TEST_EQ(vec[0].AsInt64(), -99);
|
|
TEST_EQ(vec[1].MutateString("John"), true); // Size must match.
|
|
TEST_EQ_STR(vec[1].AsString().c_str(), "John");
|
|
TEST_EQ(vec[1].MutateString("Alfred"), false); // Too long.
|
|
TEST_EQ(vec[2].MutateFloat(2.0f), true);
|
|
TEST_EQ(vec[2].AsFloat(), 2.0f);
|
|
TEST_EQ(vec[2].MutateFloat(3.14159), false); // Double does not fit in float.
|
|
TEST_EQ(vec[5].AsBool(), false); // Is false before change
|
|
TEST_EQ(vec[5].MutateBool(true), true); // Can change a bool
|
|
TEST_EQ(vec[5].AsBool(), true); // Changed bool is now true
|
|
|
|
// Parse from JSON:
|
|
flatbuffers::Parser parser;
|
|
slb.Clear();
|
|
auto jsontest = "{ a: [ 123, 456.0 ], b: \"hello\", c: true, d: false }";
|
|
TEST_EQ(parser.ParseFlexBuffer(jsontest, nullptr, &slb), true);
|
|
TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
|
|
slb.GetBuffer().size(), &reuse_tracker),
|
|
true);
|
|
auto jroot = flexbuffers::GetRoot(slb.GetBuffer());
|
|
auto jmap = jroot.AsMap();
|
|
auto jvec = jmap["a"].AsVector();
|
|
TEST_EQ(jvec[0].AsInt64(), 123);
|
|
TEST_EQ(jvec[1].AsDouble(), 456.0);
|
|
TEST_EQ_STR(jmap["b"].AsString().c_str(), "hello");
|
|
TEST_EQ(jmap["c"].IsBool(), true); // Parsed correctly to a bool
|
|
TEST_EQ(jmap["c"].AsBool(), true); // Parsed correctly to true
|
|
TEST_EQ(jmap["d"].IsBool(), true); // Parsed correctly to a bool
|
|
TEST_EQ(jmap["d"].AsBool(), false); // Parsed correctly to false
|
|
// And from FlexBuffer back to JSON:
|
|
auto jsonback = jroot.ToString();
|
|
TEST_EQ_STR(jsontest, jsonback.c_str());
|
|
// With indentation:
|
|
std::string jsonback_indented;
|
|
jroot.ToString(true, false, jsonback_indented, true, 0, " ");
|
|
auto jsontest_indented =
|
|
"{\n a: [\n 123,\n 456.0\n ],\n b: \"hello\",\n c: true,\n d: "
|
|
"false\n}";
|
|
TEST_EQ_STR(jsontest_indented, jsonback_indented.c_str());
|
|
|
|
slb.Clear();
|
|
slb.Vector([&]() {
|
|
for (int i = 0; i < 130; ++i) slb.Add(static_cast<uint8_t>(255));
|
|
slb.Vector([&]() {
|
|
for (int i = 0; i < 130; ++i) slb.Add(static_cast<uint8_t>(255));
|
|
slb.Vector([] {});
|
|
});
|
|
});
|
|
slb.Finish();
|
|
TEST_EQ(slb.GetSize(), 664);
|
|
}
|
|
|
|
void FlexBuffersReuseBugTest() {
|
|
flexbuffers::Builder slb;
|
|
slb.Map([&]() {
|
|
slb.Vector("vec", [&]() {});
|
|
slb.Bool("bool", true);
|
|
});
|
|
slb.Finish();
|
|
std::vector<uint8_t> reuse_tracker;
|
|
// This would fail before, since the reuse_tracker would use the address of
|
|
// the vector reference to check for reuse, but in this case we have an empty
|
|
// vector, and since the size field is before the pointer, its address is the
|
|
// same as thing after it, the key "bool".
|
|
// We fix this by using the address of the size field for tracking reuse.
|
|
TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
|
|
slb.GetBuffer().size(), &reuse_tracker),
|
|
true);
|
|
}
|
|
|
|
void FlexBuffersFloatingPointTest() {
|
|
#if defined(FLATBUFFERS_HAS_NEW_STRTOD) && (FLATBUFFERS_HAS_NEW_STRTOD > 0)
|
|
flexbuffers::Builder slb(512,
|
|
flexbuffers::BUILDER_FLAG_SHARE_KEYS_AND_STRINGS);
|
|
// Parse floating-point values from JSON:
|
|
flatbuffers::Parser parser;
|
|
slb.Clear();
|
|
auto jsontest =
|
|
"{ a: [1.0, nan, inf, infinity, -inf, +inf, -infinity, 8.0] }";
|
|
TEST_EQ(parser.ParseFlexBuffer(jsontest, nullptr, &slb), true);
|
|
auto jroot = flexbuffers::GetRoot(slb.GetBuffer());
|
|
TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
|
|
slb.GetBuffer().size(), nullptr),
|
|
true);
|
|
auto jmap = jroot.AsMap();
|
|
auto jvec = jmap["a"].AsVector();
|
|
TEST_EQ(8, jvec.size());
|
|
TEST_EQ(1.0, jvec[0].AsDouble());
|
|
TEST_ASSERT(is_quiet_nan(jvec[1].AsDouble()));
|
|
TEST_EQ(infinity_d, jvec[2].AsDouble());
|
|
TEST_EQ(infinity_d, jvec[3].AsDouble());
|
|
TEST_EQ(-infinity_d, jvec[4].AsDouble());
|
|
TEST_EQ(+infinity_d, jvec[5].AsDouble());
|
|
TEST_EQ(-infinity_d, jvec[6].AsDouble());
|
|
TEST_EQ(8.0, jvec[7].AsDouble());
|
|
#endif
|
|
}
|
|
|
|
void FlexBuffersDeprecatedTest() {
|
|
// FlexBuffers as originally designed had a flaw involving the
|
|
// FBT_VECTOR_STRING datatype, and this test documents/tests the fix for it.
|
|
// Discussion: https://github.com/google/flatbuffers/issues/5627
|
|
flexbuffers::Builder slb;
|
|
// FBT_VECTOR_* are "typed vectors" where all elements are of the same type.
|
|
// Problem is, when storing FBT_STRING elements, it relies on that type to
|
|
// get the bit-width for the size field of the string, which in this case
|
|
// isn't present, and instead defaults to 8-bit. This means that any strings
|
|
// stored inside such a vector, when accessed thru the old API that returns
|
|
// a String reference, will appear to be truncated if the string stored is
|
|
// actually >=256 bytes.
|
|
std::string test_data(300, 'A');
|
|
auto start = slb.StartVector();
|
|
// This one will have a 16-bit size field.
|
|
slb.String(test_data);
|
|
// This one will have an 8-bit size field.
|
|
slb.String("hello");
|
|
// We're asking this to be serialized as a typed vector (true), but not
|
|
// fixed size (false). The type will be FBT_VECTOR_STRING with a bit-width
|
|
// of whatever the offsets in the vector need, the bit-widths of the strings
|
|
// are not stored(!) <- the actual design flaw.
|
|
// Note that even in the fixed code, we continue to serialize the elements of
|
|
// FBT_VECTOR_STRING as FBT_STRING, since there may be old code out there
|
|
// reading new data that we want to continue to function.
|
|
// Thus, FBT_VECTOR_STRING, while deprecated, will always be represented the
|
|
// same way, the fix lies on the reading side.
|
|
slb.EndVector(start, true, false);
|
|
slb.Finish();
|
|
// Verify because why not.
|
|
TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
|
|
slb.GetBuffer().size(), nullptr),
|
|
true);
|
|
// So now lets read this data back.
|
|
// For existing data, since we have no way of knowing what the actual
|
|
// bit-width of the size field of the string is, we are going to ignore this
|
|
// field, and instead treat these strings as FBT_KEY (null-terminated), so we
|
|
// can deal with strings of arbitrary length. This of course truncates strings
|
|
// with embedded nulls, but we think that that is preferrable over truncating
|
|
// strings >= 256 bytes.
|
|
auto vec = flexbuffers::GetRoot(slb.GetBuffer()).AsTypedVector();
|
|
// Even though this was serialized as FBT_VECTOR_STRING, it is read as
|
|
// FBT_VECTOR_KEY:
|
|
TEST_EQ(vec.ElementType(), flexbuffers::FBT_KEY);
|
|
// Access the long string. Previously, this would return a string of size 1,
|
|
// since it would read the high-byte of the 16-bit length.
|
|
// This should now correctly test the full 300 bytes, using AsKey():
|
|
TEST_EQ_STR(vec[0].AsKey(), test_data.c_str());
|
|
// Old code that called AsString will continue to work, as the String
|
|
// accessor objects now use a cached size that can come from a key as well.
|
|
TEST_EQ_STR(vec[0].AsString().c_str(), test_data.c_str());
|
|
// Short strings work as before:
|
|
TEST_EQ_STR(vec[1].AsKey(), "hello");
|
|
TEST_EQ_STR(vec[1].AsString().c_str(), "hello");
|
|
// So, while existing code and data mostly "just work" with the fixes applied
|
|
// to AsTypedVector and AsString, what do you do going forward?
|
|
// Code accessing existing data doesn't necessarily need to change, though
|
|
// you could consider using AsKey instead of AsString for a) documenting
|
|
// that you are accessing keys, or b) a speedup if you don't actually use
|
|
// the string size.
|
|
// For new data, or data that doesn't need to be backwards compatible,
|
|
// instead serialize as FBT_VECTOR (call EndVector with typed = false, then
|
|
// read elements with AsString), or, for maximum compactness, use
|
|
// FBT_VECTOR_KEY (call slb.Key above instead, read with AsKey or AsString).
|
|
}
|
|
|
|
void ParseFlexbuffersFromJsonWithNullTest() {
|
|
// Test nulls are handled appropriately through flexbuffers to exercise other
|
|
// code paths of ParseSingleValue in the optional scalars change.
|
|
// TODO(cneo): Json -> Flatbuffers test once some language can generate code
|
|
// with optional scalars.
|
|
{
|
|
char json[] = "{\"opt_field\": 123 }";
|
|
flatbuffers::Parser parser;
|
|
flexbuffers::Builder flexbuild;
|
|
parser.ParseFlexBuffer(json, nullptr, &flexbuild);
|
|
auto root = flexbuffers::GetRoot(flexbuild.GetBuffer());
|
|
TEST_EQ(root.AsMap()["opt_field"].AsInt64(), 123);
|
|
}
|
|
{
|
|
char json[] = "{\"opt_field\": 123.4 }";
|
|
flatbuffers::Parser parser;
|
|
flexbuffers::Builder flexbuild;
|
|
parser.ParseFlexBuffer(json, nullptr, &flexbuild);
|
|
auto root = flexbuffers::GetRoot(flexbuild.GetBuffer());
|
|
TEST_EQ(root.AsMap()["opt_field"].AsDouble(), 123.4);
|
|
}
|
|
{
|
|
char json[] = "{\"opt_field\": null }";
|
|
flatbuffers::Parser parser;
|
|
flexbuffers::Builder flexbuild;
|
|
parser.ParseFlexBuffer(json, nullptr, &flexbuild);
|
|
auto root = flexbuffers::GetRoot(flexbuild.GetBuffer());
|
|
TEST_ASSERT(!root.AsMap().IsTheEmptyMap());
|
|
TEST_ASSERT(root.AsMap()["opt_field"].IsNull());
|
|
TEST_EQ(root.ToString(), std::string("{ opt_field: null }"));
|
|
}
|
|
}
|
|
|
|
} // namespace tests
|
|
} // namespace flatbuffers
|