BigfootDev/flatbuffers
1
0
Fork 1
mirror of https://github.com/google/flatbuffers.git synced 2026-06-12 16:00:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

[rust] Genericize flexbuffer reader (#6450)

Browse Source 
* feature/rust-tokio-bytes added feature name for tokio-bytes

* Added flexbuffer implementation, TODO: typecast to avoid recurse

* Converted codebase to utilize FlexBuffer implementation, need to resolve deserialization issues

* Added todo for lifetime issue, may use &'de [u8] for deserializer instead of current method

* Added proper &[u8] implementation

* Removed unused struct

* Added experimental fix to get_slice

* Added experimental fix to get_slice

* Avoided lifetime issues via ref structs, need to check if this hurts peformance

* Updated deserializer implementation to allow for borrowed data from Reader struct

* Fixed bug with str

* Removed unnecessary generic parameter

* Added unsafe to avoid lifetime complaints, current tests pass, need to review alternatives to unsafe

* Opinionated: Removed bytes crate as this implementation could be done in a separate crate

* Cleaned up flatbuffer

* Fixed sample / example

* Resolved PR feedback, need to resolve issues with tests

* Cleaned up FlexBuffer trait to be an auto impl

* Removed TODO

* Reverted Deserializer to only support &'de [u8]

* Cleaned up / renamed function for clarification

* Renamed FlexBuffer -> InternalBuffer for clarification on it's purpose

* Fixed issue with key bytes

* resolved issues with broken tests, confirming this is a breaking change

* Removed FIXME that's solved by splitting String and Key variants

* Implemented associated types approach

* Fixed backward slice logic

* Fixed MapReader compile error

* Added from_buffer for deserialization, removed  function since it's only needed for deserialization

* Removed dead code

* Cleaned up buffer, removed AsRef in favor of Deref

* Renamed Buffer::as_str -> Buffer::buffer_str

* Minor cleanup

* Updated documentation, need to fix tests

* Removed unnecessary &

* Removed unused lifetime

* removed unnecessary as_ref

* Minor optimization wrap-up

* resolved issue with Clone

* Added test to verify no deep-copy

* Added  for optimization

* Updated to use empty fn instead of default

* Updated comments / test name - plus the 0.3.0 version bump

* comment
This commit is contained in:
Colin
2021-02-16 08:04:48 -05:00
committed by GitHub
parent a20f606c29
commit 4174c10e7a
14 changed files with 385 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/rust_usage_test/benches/flexbuffers_benchmarks.rs
View File

@@ -224,7 +224,7 @@ fn serialize_monsters(b: &mut Bencher) {
b.iter(go);
b.bytes = n as u64;
}
fn validate_monster(r: MapReader) {
fn validate_monster(r: MapReader<&[u8]>) {
assert_eq!(r.idx("type").as_str(), "great orc");
assert_eq!(r.idx("age").as_u8(), 100);
assert_eq!(r.idx("name").as_str(), "Mr. Orc");

2
tests/rust_usage_test/tests/flexbuffers_tests/binary_format.rs
View File

@@ -311,7 +311,7 @@ fn utf8_snowman() {
1, // Root bytes
]
);
let r = Reader::get_root(&buf).unwrap();
let r = Reader::get_root(buf.as_ref()).unwrap();
assert_eq!(r.get_str(), Ok("snowman ☃︎"));
}
#[test]

4
tests/rust_usage_test/tests/flexbuffers_tests/interop.rs
View File

@@ -18,14 +18,14 @@ use flexbuffers::*;
fn read_golden_flexbuffer() {
let s =
std::fs::read("../gold_flexbuffer_example.bin").expect("Unable to read golden flexbuffer.");
let r = Reader::get_root(&s).unwrap();
let r = Reader::get_root(s.as_ref()).unwrap();
let m = r.as_map();
let vec = m.idx("vec").as_vector();
assert_eq!(vec.idx(0).as_i8(), -100);
assert_eq!(vec.idx(1).as_str(), "Fred");
assert_eq!(vec.idx(2).as_f32(), 4.0);
assert_eq!(vec.idx(3).as_blob(), Blob(&[77]));
assert_eq!(vec.idx(3).as_blob(), Blob([77].as_ref()));
assert_eq!(vec.idx(4).flexbuffer_type(), FlexBufferType::Bool);
assert_eq!(vec.idx(4).as_bool(), false);
assert_eq!(vec.idx(5).as_f64(), 4.0);

25
tests/rust_usage_test/tests/flexbuffers_tests/other_api.rs
View File

@@ -20,7 +20,7 @@ use quickcheck::QuickCheck;
#[cfg(not(miri))] // slow.
fn qc_reader_no_crash() {
fn no_crash(xs: Vec<u8>) -> bool {
let r = Reader::get_root(&xs);
let r = Reader::get_root(xs.as_ref());
r.is_err() || r.is_ok()
}
QuickCheck::new()
@@ -133,7 +133,7 @@ fn string_as_num() {
}
#[test]
fn null_reader() {
let n = Reader::default();
let n = Reader::<&[u8]>::default();
assert_eq!(n.as_i8(), 0);
assert_eq!(n.as_i16(), 0);
assert_eq!(n.as_i32(), 0);
@@ -159,7 +159,7 @@ fn get_root_deref_oob() {
(FlexBufferType::Vector as u8) << 2 | BitWidth::W8 as u8,
1,
];
assert!(Reader::get_root(s).is_err());
assert!(Reader::get_root(s.as_ref()).is_err());
}
#[test]
fn get_root_deref_u64() {
@@ -170,7 +170,24 @@ fn get_root_deref_u64() {
1,
];
// The risk of crashing is reading 8 bytes from index 0.
assert_eq!(Reader::get_root(s).unwrap().as_u64(), 0);
assert_eq!(Reader::get_root(s.as_ref()).unwrap().as_u64(), 0);
}
/// Verifies that the clone operation is shallow / zero copy.
#[test]
fn clone_is_shallow() {
let mut fxb = Builder::default();
let mut m = fxb.start_map();
m.push("a", &[-1i8, -2, -3, -4]);
m.push("b", 250i64);
m.push("c", 5000u16);
m.end_map();
let r = Reader::get_root(fxb.view()).unwrap();
let r2 = r.clone();
assert_eq!(r.buffer().as_ptr(), r2.buffer().as_ptr());
}
#[test]

50
tests/rust_usage_test/tests/flexbuffers_tests/rwyw.rs
View File

@@ -43,7 +43,7 @@ quickcheck! {
v.push(x);
}
v.end_vector();
let r = Reader::get_root(&builder.view()).unwrap().as_vector();
let r = Reader::get_root(builder.view()).unwrap().as_vector();
xs.iter().enumerate().all(|(i, &x)| r.index(i).unwrap().get_bool().unwrap() == x)
}
fn qc_vec_uint(xs: Vec<u64>) -> bool {
@@ -53,7 +53,7 @@ quickcheck! {
v.push(x);
}
v.end_vector();
let r = Reader::get_root(&builder.view()).unwrap().as_vector();
let r = Reader::get_root(builder.view()).unwrap().as_vector();
xs.iter().enumerate().all(|(i, &x)| r.idx(i).as_u64() == x)
}
fn qc_vec_int(xs: Vec<i64>) -> bool {
@@ -63,7 +63,7 @@ quickcheck! {
v.push(x);
}
v.end_vector();
let r = Reader::get_root(&builder.view()).unwrap().as_vector();
let r = Reader::get_root(builder.view()).unwrap().as_vector();
xs.iter().enumerate().all(|(i, &x)| r.idx(i).as_i64() == x)
}
fn qc_vec_float(xs: Vec<f64>) -> bool {
@@ -73,7 +73,7 @@ quickcheck! {
v.push(x);
}
v.end_vector();
let r = Reader::get_root(&builder.view()).unwrap().as_vector();
let r = Reader::get_root(builder.view()).unwrap().as_vector();
xs.iter().enumerate().all(|(i, &x)| (r.idx(i).as_f64() - x).abs() < std::f64::EPSILON)
}
fn qc_vec_string(xs: Vec<String>) -> bool {
@@ -83,7 +83,7 @@ quickcheck! {
v.push(x as &str);
}
v.end_vector();
let r = Reader::get_root(&builder.view()).unwrap().as_vector();
let r = Reader::get_root(builder.view()).unwrap().as_vector();
xs.iter().enumerate().all(|(i, x)| (r.idx(i).as_str() == x))
}
fn qc_map_int(xs: std::collections::BTreeMap<NonNullString, i64>) -> bool {
@@ -93,7 +93,7 @@ quickcheck! {
m.push(&k.0, v);
}
m.end_map();
let r = Reader::get_root(&builder.view()).unwrap().as_map();
let r = Reader::get_root(builder.view()).unwrap().as_map();
xs.iter().enumerate().all(|(i, (k, &v))| {
r.idx(i).as_i64() == v && r.idx(k.0.as_str()).as_i64() == v
})
@@ -105,7 +105,7 @@ quickcheck! {
m.push(&k.0, v as &str);
}
m.end_map();
let r = Reader::get_root(&builder.view()).unwrap().as_map();
let r = Reader::get_root(builder.view()).unwrap().as_map();
xs.iter().enumerate().all(|(i, (k, v))| {
r.idx(i).as_str() == v && r.idx(k.0.as_str()).as_str() == v
})
@@ -114,10 +114,10 @@ quickcheck! {
let mut builder = Builder::default();
let mut v = builder.start_vector();
for x in &xs {
v.push(Blob(&x));
v.push(Blob(x.as_ref()));
}
v.end_vector();
let r = Reader::get_root(&builder.view()).unwrap().as_vector();
let r = Reader::get_root(builder.view()).unwrap().as_vector();
xs.iter().enumerate().all(
|(i, x)| r.idx(i).get_blob().unwrap().0.iter().eq(x.iter())
)
@@ -209,7 +209,7 @@ fn string() {
v.push("barrr");
v.push("bazzzzzz");
v.end_vector();
let r = Reader::get_root(&builder.view()).unwrap().as_vector();
let r = Reader::get_root(builder.view()).unwrap().as_vector();
assert_eq!(r.idx(0).as_str(), "foo");
assert_eq!(r.idx(1).as_str(), "barrr");
assert_eq!(r.idx(2).as_str(), "bazzzzzz");
@@ -218,7 +218,7 @@ fn string() {
#[test]
fn store_13() {
let finished = singleton::<i32>(13);
let r = Reader::get_root(&finished).unwrap();
let r = Reader::get_root(finished.as_ref()).unwrap();
assert_eq!(r.as_i32(), 13);
}
#[test]
@@ -233,7 +233,7 @@ fn singleton_vector_uint_4_16bit() {
let buf2 = singleton(&[2u8, 3, 5]);
assert_eq!(buf1, buf2.as_slice());
let r = Reader::get_root(&buf1).unwrap().as_vector();
let r = Reader::get_root(buf1).unwrap().as_vector();
assert_eq!(r.idx(0).get_u64(), Ok(2));
assert_eq!(r.idx(1).get_u64(), Ok(3));
assert_eq!(r.idx(2).get_u64(), Ok(5));
@@ -248,7 +248,7 @@ fn vector_uint4() {
v.push(5u8);
v.push(7u8);
v.end_vector();
let r = Reader::get_root(&fxb.view()).unwrap();
let r = Reader::get_root(fxb.view()).unwrap();
let v = r.as_vector();
assert_eq!(v.idx(0).get_u64(), Ok(2));
assert_eq!(v.idx(1).get_u64(), Ok(3));
@@ -264,13 +264,13 @@ fn vector_uint4() {
fn store_and_read_blob() {
let mut fxb = Builder::default();
let mut v = fxb.start_vector();
v.push(Blob(&[1, 2, 3, 4]));
v.push(Blob(&[5, 6, 7]));
v.push(Blob([1, 2, 3, 4].as_ref()));
v.push(Blob([5, 6, 7].as_ref()));
v.end_vector();
let r = Reader::get_root(&fxb.view()).unwrap().as_vector();
assert_eq!(r.idx(0).get_blob(), Ok(Blob(&[1, 2, 3, 4])));
assert_eq!(r.idx(1).get_blob(), Ok(Blob(&[5, 6, 7])));
let r = Reader::get_root(fxb.view()).unwrap().as_vector();
assert_eq!(r.idx(0).get_blob(), Ok(Blob([1, 2, 3, 4].as_ref())));
assert_eq!(r.idx(1).get_blob(), Ok(Blob([5, 6, 7].as_ref())));
}
#[test]
fn map_64bit() {
@@ -280,7 +280,7 @@ fn map_64bit() {
m.push("b", u64::max_value() - 3);
m.end_map();
let r = Reader::get_root(&fxb.view()).unwrap().as_map();
let r = Reader::get_root(fxb.view()).unwrap().as_map();
assert_eq!(r.idx("a").as_u16(), 257);
assert_eq!(r.idx("b").as_u64(), u64::max_value() - 3);
}
@@ -341,7 +341,7 @@ fn map_strings() {
#[test]
fn store_u64() {
let finished = singleton(u64::max_value() - 10);
let r = Reader::get_root(&finished).unwrap();
let r = Reader::get_root(finished.as_ref()).unwrap();
assert_eq!(r.get_u64(), Ok(u64::max_value() - 10));
}
#[test]
@@ -444,7 +444,7 @@ fn serialize_serde_with_bytes_as_blob() {
Foo(vec![5, 6, 7, 8]).serialize(&mut s).unwrap();
let reader = Reader::get_root(s.view()).unwrap();
assert_eq!(reader.flexbuffer_type(), FlexBufferType::Blob);
assert_eq!(reader.as_blob(), Blob(&[5, 6, 7, 8]));
assert_eq!(reader.as_blob(), Blob([5, 6, 7, 8].as_ref()));
}
#[test]
fn iter() {
@@ -466,7 +466,7 @@ fn deserialize_newtype_i8() {
#[derive(Deserialize)]
struct Foo(u8);
let data = [13, 4, 1];
let r = Reader::get_root(&data).unwrap();
let r = Reader::get_root(data.as_ref()).unwrap();
let foo = Foo::deserialize(r).unwrap();
assert_eq!(foo.0, 13);
}
@@ -475,7 +475,7 @@ fn deserialize_newtype_str() {
#[derive(Deserialize)]
struct Foo<'a>(&'a str);
let data = [5, b'h', b'e', b'l', b'l', b'o', b'\0', 6, 5 << 2, 1];
let r = Reader::get_root(&data).unwrap();
let r = Reader::get_root(data.as_ref()).unwrap();
let foo = Foo::deserialize(r).unwrap();
assert_eq!(foo.0, "hello");
}
@@ -490,7 +490,7 @@ fn deserialize_tuple_struct_to_vec_uint4() {
23 << 2 | 1, // (VectorUInt4, W16 - referring to data).
1, // Root width W8 - referring to vector.
];
let r = Reader::get_root(&data).unwrap();
let r = Reader::get_root(data.as_ref()).unwrap();
let foo = Foo::deserialize(r).unwrap();
assert_eq!(foo.0, 4);
assert_eq!(foo.1, 16);
@@ -503,7 +503,7 @@ fn deserialize_tuple_struct_to_vec_uint4() {
23 << 2, // Root type: VectorUInt4, W8.
1, // Root width: W8.
];
let r = Reader::get_root(&data).unwrap();
let r = Reader::get_root(data.as_ref()).unwrap();
let foo = Foo::deserialize(r).unwrap();
assert_eq!(foo.0, 1);
assert_eq!(foo.1, 2);