From 8fd10606c14434ffafd827658f674060f704811b Mon Sep 17 00:00:00 2001 From: Casper Date: Mon, 10 May 2021 23:15:46 -0400 Subject: [PATCH] Implement Serialize for flexbuffer::Reader (#6635) * Implement Serialize for flexbuffer::Reader * bump version * Remove use of experimantal or-patterns * Remove more use of experimantal or-patterns Co-authored-by: Casper Neo --- rust/flexbuffers/src/buffer.rs | 9 +-- rust/flexbuffers/src/builder/map.rs | 3 +- rust/flexbuffers/src/builder/mod.rs | 21 +++-- rust/flexbuffers/src/builder/ser.rs | 4 + rust/flexbuffers/src/lib.rs | 6 +- rust/flexbuffers/src/reader/mod.rs | 29 ++++--- rust/flexbuffers/src/reader/serialize.rs | 76 +++++++++++++++++++ rust/flexbuffers/src/reader/vector.rs | 2 +- .../tests/flexbuffers_tests/qc_serious.rs | 15 ++-- 9 files changed, 125 insertions(+), 40 deletions(-) create mode 100644 rust/flexbuffers/src/reader/serialize.rs diff --git a/rust/flexbuffers/src/buffer.rs b/rust/flexbuffers/src/buffer.rs index b56224939..4643cb8d4 100644 --- a/rust/flexbuffers/src/buffer.rs +++ b/rust/flexbuffers/src/buffer.rs @@ -1,6 +1,6 @@ use std::ops::{Deref, Range}; -/// The underlying buffer that is used by a flexbuffer Reader. +/// The underlying buffer that is used by a flexbuffer Reader. /// /// This allows for custom buffer implementations as long as they can be viewed as a &[u8]. pub trait Buffer: Deref + Sized { @@ -14,12 +14,12 @@ pub trait Buffer: Deref + Sized { /// A BufferString which will live at least as long as the Buffer itself. /// /// Deref's to UTF-8 `str`, and only generated from the `buffer_str` function Result. - type BufferString: Deref + Sized; + type BufferString: Deref + Sized + serde::ser::Serialize; /// This method returns an instance of type Self. This allows for lifetimes to be tracked /// in cases of deserialization. /// - /// It also lets custom buffers manage reference counts. + /// It also lets custom buffers manage reference counts. /// /// Returns None if: /// - range start is greater than end @@ -48,7 +48,7 @@ pub trait Buffer: Deref + Sized { Self::empty().buffer_str().unwrap() } - /// Attempts to convert the given buffer to a custom string type. + /// Attempts to convert the given buffer to a custom string type. /// /// This should fail if the type does not have valid UTF-8 bytes, and must be zero copy. fn buffer_str(&self) -> Result; @@ -78,4 +78,3 @@ impl<'de> Buffer for &'de [u8] { std::str::from_utf8(self) } } - diff --git a/rust/flexbuffers/src/builder/map.rs b/rust/flexbuffers/src/builder/map.rs index 1635f6488..18d768794 100644 --- a/rust/flexbuffers/src/builder/map.rs +++ b/rust/flexbuffers/src/builder/map.rs @@ -97,8 +97,7 @@ pub(super) fn sort_map_by_keys(values: &mut [Value], buffer: &[u8]) { // preferred over custom sorting or adding another dependency. By construction, this part // of the values stack must be alternating (key, value) pairs. The public API must not be // able to trigger the above debug_assets that protect this unsafe usage. - let pairs: &mut [[Value; 2]] = - unsafe { std::slice::from_raw_parts_mut(raw_pairs, pairs_len) }; + let pairs: &mut [[Value; 2]] = unsafe { std::slice::from_raw_parts_mut(raw_pairs, pairs_len) }; #[rustfmt::skip] pairs.sort_unstable_by(|[key1, _], [key2, _]| { if let Value::Key(a1) = *key1 { diff --git a/rust/flexbuffers/src/builder/mod.rs b/rust/flexbuffers/src/builder/mod.rs index 46ee79e3d..97838c21c 100644 --- a/rust/flexbuffers/src/builder/mod.rs +++ b/rust/flexbuffers/src/builder/mod.rs @@ -32,10 +32,11 @@ macro_rules! push_slice { fn $push_name(&mut self, xs: S) where T: Into<$scalar> + Copy, - S: AsRef<[T]> + S: AsRef<[T]>, { let mut value = Value::$new_vec(xs.as_ref().len()); - let mut width = xs.as_ref() + let mut width = xs + .as_ref() .iter() .map(|x| BitWidth::from((*x).into())) .max() @@ -56,7 +57,7 @@ macro_rules! push_slice { value.set_child_width_or_panic(width); self.values.push(value); } - } + }; } macro_rules! push_indirect { ($push_name: ident, $scalar: ty, $Direct: ident, $Indirect: ident) => { @@ -65,15 +66,13 @@ macro_rules! push_indirect { let child_width = x.width_or_child_width(); let address = self.buffer.len(); store_value(&mut self.buffer, x, child_width); - self.values.push( - Value::Reference { - address, - child_width, - fxb_type: FlexBufferType::$Indirect, - } - ); + self.values.push(Value::Reference { + address, + child_width, + fxb_type: FlexBufferType::$Indirect, + }); } - } + }; } bitflags! { diff --git a/rust/flexbuffers/src/builder/ser.rs b/rust/flexbuffers/src/builder/ser.rs index 8e483baf8..7ef77577d 100644 --- a/rust/flexbuffers/src/builder/ser.rs +++ b/rust/flexbuffers/src/builder/ser.rs @@ -36,6 +36,10 @@ impl FlexbufferSerializer { pub fn take_buffer(&mut self) -> Vec { self.builder.take_buffer() } + pub fn reset(&mut self) { + self.builder.reset(); + self.nesting.clear(); + } fn finish_if_not_nested(&mut self) -> Result<(), Error> { if self.nesting.is_empty() { assert_eq!(self.builder.values.len(), 1); diff --git a/rust/flexbuffers/src/lib.rs b/rust/flexbuffers/src/lib.rs index 614b2b368..293519050 100644 --- a/rust/flexbuffers/src/lib.rs +++ b/rust/flexbuffers/src/lib.rs @@ -41,18 +41,18 @@ extern crate num_enum; extern crate serde; mod bitwidth; +mod buffer; mod builder; mod flexbuffer_type; mod reader; -mod buffer; pub use bitwidth::BitWidth; +pub use buffer::Buffer; pub use builder::Error as SerializationError; pub use builder::{ singleton, Builder, BuilderOptions, FlexbufferSerializer, MapBuilder, Pushable, VectorBuilder, }; pub use flexbuffer_type::FlexBufferType; -pub use buffer::Buffer; pub use reader::Error as ReaderError; pub use reader::{DeserializationError, MapReader, Reader, ReaderIterator, VectorReader}; use serde::{Deserialize, Serialize}; @@ -76,7 +76,7 @@ pub fn from_slice<'de, T: Deserialize<'de>>(buf: &'de [u8]) -> Result, B: Buffer>( - buf: &'de B + buf: &'de B, ) -> Result { let r = Reader::get_root(buf as &'de [u8])?; T::deserialize(r) diff --git a/rust/flexbuffers/src/reader/mod.rs b/rust/flexbuffers/src/reader/mod.rs index 311f91d37..c6f51d066 100644 --- a/rust/flexbuffers/src/reader/mod.rs +++ b/rust/flexbuffers/src/reader/mod.rs @@ -14,7 +14,7 @@ use crate::bitwidth::BitWidth; use crate::flexbuffer_type::FlexBufferType; -use crate::{Buffer, Blob}; +use crate::{Blob, Buffer}; use std::convert::{TryFrom, TryInto}; use std::fmt; use std::ops::Rem; @@ -22,6 +22,7 @@ use std::str::FromStr; mod de; mod iter; mod map; +mod serialize; mod vector; pub use de::DeserializationError; pub use iter::ReaderIterator; @@ -184,13 +185,12 @@ impl std::fmt::Debug for Reader { } } - macro_rules! try_cast_fn { ($name: ident, $full_width: ident, $Ty: ident) => { pub fn $name(&self) -> $Ty { self.$full_width().try_into().unwrap_or_default() } - } + }; } fn safe_sub(a: usize, b: usize) -> Result { @@ -242,7 +242,7 @@ impl Reader { } /// Convenience function to get the underlying buffer. By using `shallow_copy`, this preserves - /// the lifetime that the underlying buffer has. + /// the lifetime that the underlying buffer has. pub fn buffer(&self) -> B { self.buffer.shallow_copy() } @@ -263,7 +263,11 @@ impl Reader { if let Some(len) = self.fxb_type.fixed_length_vector_length() { len } else if self.fxb_type.has_length_slot() && self.address >= self.width.n_bytes() { - read_usize(&self.buffer, self.address - self.width.n_bytes(), self.width) + read_usize( + &self.buffer, + self.address - self.width.n_bytes(), + self.width, + ) } else { 0 } @@ -359,7 +363,8 @@ impl Reader { /// Retrieves the string value up until the first `\0` character. pub fn get_key(&self) -> Result { - let bytes = self.buffer + let bytes = self + .buffer .slice(self.address..self.address + self.get_key_len()?) .ok_or(Error::IndexOutOfBounds)?; Ok(bytes.buffer_str()?) @@ -368,9 +373,9 @@ impl Reader { pub fn get_blob(&self) -> Result, Error> { self.expect_type(FlexBufferType::Blob)?; Ok(Blob( - self.buffer - .slice(self.address..self.address + self.length()) - .ok_or(Error::IndexOutOfBounds)? + self.buffer + .slice(self.address..self.address + self.length()) + .ok_or(Error::IndexOutOfBounds)?, )) } @@ -382,7 +387,9 @@ impl Reader { /// is out of bounds. pub fn get_str(&self) -> Result { self.expect_type(FlexBufferType::String)?; - let bytes = self.buffer.slice(self.address..self.address + self.length()); + let bytes = self + .buffer + .slice(self.address..self.address + self.length()); Ok(bytes.ok_or(Error::ReadUsizeOverflowed)?.buffer_str()?) } @@ -666,4 +673,4 @@ fn unpack_type(ty: u8) -> Result<(FlexBufferType, BitWidth), Error> { let w = BitWidth::try_from(ty & 3u8).map_err(|_| Error::InvalidPackedType)?; let t = FlexBufferType::try_from(ty >> 2).map_err(|_| Error::InvalidPackedType)?; Ok((t, w)) -} \ No newline at end of file +} diff --git a/rust/flexbuffers/src/reader/serialize.rs b/rust/flexbuffers/src/reader/serialize.rs new file mode 100644 index 000000000..b129da126 --- /dev/null +++ b/rust/flexbuffers/src/reader/serialize.rs @@ -0,0 +1,76 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::reader::Reader; +use crate::Buffer; +use crate::{BitWidth::*, FlexBufferType::*}; +use serde::ser; +use serde::ser::{SerializeMap, SerializeSeq}; + +impl ser::Serialize for &Reader { + fn serialize(&self, serializer: S) -> Result + where + S: ser::Serializer, + { + #[allow(deprecated)] + match (self.flexbuffer_type(), self.bitwidth()) { + (Null, _) => serializer.serialize_unit(), + (Int, W8) | (IndirectInt, W8) => serializer.serialize_i8(self.as_i8()), + (Int, W16) | (IndirectInt, W16) => serializer.serialize_i16(self.as_i16()), + (Int, W32) | (IndirectInt, W32) => serializer.serialize_i32(self.as_i32()), + (Int, W64) | (IndirectInt, W64) => serializer.serialize_i64(self.as_i64()), + (UInt, W8) | (IndirectUInt, W8) => serializer.serialize_u8(self.as_u8()), + (UInt, W16) | (IndirectUInt, W16) => serializer.serialize_u16(self.as_u16()), + (UInt, W32) | (IndirectUInt, W32) => serializer.serialize_u32(self.as_u32()), + (UInt, W64) | (IndirectUInt, W64) => serializer.serialize_u64(self.as_u64()), + (Float, W32) | (IndirectFloat, W32) => serializer.serialize_f32(self.as_f32()), + (Float, _) | (IndirectFloat, _) => serializer.serialize_f64(self.as_f64()), + (Bool, _) => serializer.serialize_bool(self.as_bool()), + (Key, _) | (String, _) => serializer.serialize_str(&self.as_str()), + (Map, _) => { + let m = self.as_map(); + let mut map_serializer = serializer.serialize_map(Some(m.len()))?; + for (k, v) in m.iter_keys().zip(m.iter_values()) { + map_serializer.serialize_key(&&k)?; + map_serializer.serialize_value(&&v)?; + } + map_serializer.end() + } + (Vector, _) + | (VectorInt, _) + | (VectorUInt, _) + | (VectorFloat, _) + | (VectorKey, _) + | (VectorString, _) + | (VectorBool, _) + | (VectorInt2, _) + | (VectorUInt2, _) + | (VectorFloat2, _) + | (VectorInt3, _) + | (VectorUInt3, _) + | (VectorFloat3, _) + | (VectorInt4, _) + | (VectorUInt4, _) + | (VectorFloat4, _) => { + let v = self.as_vector(); + let mut seq_serializer = serializer.serialize_seq(Some(v.len()))?; + for x in v.iter() { + seq_serializer.serialize_element(&&x)?; + } + seq_serializer.end() + } + (Blob, _) => serializer.serialize_bytes(&self.as_blob().0), + } + } +} diff --git a/rust/flexbuffers/src/reader/vector.rs b/rust/flexbuffers/src/reader/vector.rs index b23cbb491..6b97e1057 100644 --- a/rust/flexbuffers/src/reader/vector.rs +++ b/rust/flexbuffers/src/reader/vector.rs @@ -40,7 +40,7 @@ impl Default for VectorReader { fn default() -> Self { VectorReader { reader: Reader::default(), - length: usize::default() + length: usize::default(), } } } diff --git a/tests/rust_usage_test/tests/flexbuffers_tests/qc_serious.rs b/tests/rust_usage_test/tests/flexbuffers_tests/qc_serious.rs index abd1ced7e..1d2e1225f 100644 --- a/tests/rust_usage_test/tests/flexbuffers_tests/qc_serious.rs +++ b/tests/rust_usage_test/tests/flexbuffers_tests/qc_serious.rs @@ -134,12 +134,13 @@ impl Arbitrary for Array4 { } quickcheck! { - fn qc_serious(x: Struct) -> bool { - let mut s = FlexbufferSerializer::new(); - x.serialize(&mut s).unwrap(); - let r = Reader::get_root(s.view()).unwrap(); - println!("{}", r); - let x2 = Struct::deserialize(r).unwrap(); - x == x2 + fn qc_serious(original: Struct) -> bool { + let struct_buf = flexbuffers::to_vec(&original).unwrap(); + let root = Reader::get_root(&*struct_buf).unwrap(); + let reader_buf = flexbuffers::to_vec(&root).unwrap(); + let deserialized: Struct = flexbuffers::from_slice(&struct_buf).unwrap(); + let reserialized: Struct = flexbuffers::from_slice(&reader_buf).unwrap(); + + original == deserialized && original == reserialized } }