diff --git a/rust/flexbuffers/.gitignore b/rust/flexbuffers/.gitignore new file mode 100644 index 000000000..693699042 --- /dev/null +++ b/rust/flexbuffers/.gitignore @@ -0,0 +1,3 @@ +/target +**/*.rs.bk +Cargo.lock diff --git a/rust/flexbuffers/Cargo.toml b/rust/flexbuffers/Cargo.toml new file mode 100644 index 000000000..7dec3c948 --- /dev/null +++ b/rust/flexbuffers/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "flexbuffers" +version = "0.1.0" +authors = ["Casper Neo "] +edition = "2018" +license = "Apache-2.0" + +[dependencies] +serde = "1.0.103" +serde_derive = "*" +byteorder = "1.3.2" +num_enum = "0.4.1" +debug_stub_derive = "0.3.0" +bitflags = "1.2.1" + +[dev-dependencies] +bencher = "*" +quickcheck = "*" +quickcheck_derive = "*" +rand = "*" diff --git a/rust/flexbuffers/README.md b/rust/flexbuffers/README.md new file mode 100644 index 000000000..c076c0e42 --- /dev/null +++ b/rust/flexbuffers/README.md @@ -0,0 +1,14 @@ +# Flexbuffers + +[Flexbuffers](https://google.github.io/flatbuffers/flexbuffers.html) is a +schema-less binary format developed at Google. FlexBuffers can be accessed +without parsing, copying, or allocation. This is a huge win for efficiency, +memory friendly-ness, and allows for unique use cases such as mmap-ing large +amounts of free-form data. + +FlexBuffers' design and implementation allows for a very compact encoding, +with automatic sizing of containers to their smallest possible representation +(8/16/32/64 bits). Many values and offsets can be encoded in just 8 bits. + +Flexbuffers is the schema-less cousin of +[Flatbuffers](https://google.github.io/flatbuffers/). diff --git a/rust/flexbuffers/src/bitwidth.rs b/rust/flexbuffers/src/bitwidth.rs new file mode 100644 index 000000000..8e0bfedbe --- /dev/null +++ b/rust/flexbuffers/src/bitwidth.rs @@ -0,0 +1,113 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::bitwidth::BitWidth::*; +use std::slice::Iter; + +/// Represents the size of Flexbuffers data. +/// +/// Flexbuffers automatically compresses numbers to the smallest possible width +/// (`250u64` is stored as `250u8`). +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Serialize, + Deserialize, + Ord, + num_enum::TryFromPrimitive, +)] +#[repr(u8)] +pub enum BitWidth { + W8 = 0, + W16 = 1, + W32 = 2, + W64 = 3, +} +impl BitWidth { + pub(crate) fn iter() -> Iter<'static, Self> { + [W8, W16, W32, W64].iter() + } + pub fn n_bytes(self) -> usize { + 1 << self as usize + } + pub fn from_nbytes(n: impl std::convert::Into) -> Option { + match n.into() { + 1 => Some(W8), + 2 => Some(W16), + 4 => Some(W32), + 8 => Some(W64), + _ => None, + } + } +} + +impl Default for BitWidth { + fn default() -> Self { + W8 + } +} + +// TODO(cneo): Overloading with `from` is probably not the most readable idea in hindsight. +macro_rules! impl_bitwidth_from { + ($from: ident, $w64: ident, $w32: ident, $w16: ident, $w8: ident) => { + impl From<$from> for BitWidth { + fn from(x: $from) -> BitWidth { + let x = x as $w64; + if x >= $w8::min_value() as $w64 && x <= $w8::max_value() as $w64 { + return W8; + } + if x >= $w16::min_value() as $w64 && x <= $w16::max_value() as $w64 { + return W16; + } + if x >= $w32::min_value() as $w64 && x <= $w32::max_value() as $w64 { + return W32; + } + W64 + } + } + }; +} +impl_bitwidth_from!(u64, u64, u32, u16, u8); +impl_bitwidth_from!(usize, u64, u32, u16, u8); +impl_bitwidth_from!(i64, i64, i32, i16, i8); + +#[allow(clippy::float_cmp)] +impl From for BitWidth { + fn from(x: f64) -> BitWidth { + if x != x as f32 as f64 { + W64 + } else { + W32 + } + } +} +impl From for BitWidth { + fn from(_: f32) -> BitWidth { + W32 + } +} + +/// Zero pad `v` until `T` will be byte aligned when pushed. +pub fn align(buffer: &mut Vec, width: BitWidth) { + let bytes = 1 << width as u8; + let alignment = (bytes - buffer.len() % bytes) % bytes; + // Profiling reveals the loop is faster than Vec::resize. + for _ in 0..alignment as usize { + buffer.push(0); + } +} diff --git a/rust/flexbuffers/src/builder/map.rs b/rust/flexbuffers/src/builder/map.rs new file mode 100644 index 000000000..1635f6488 --- /dev/null +++ b/rust/flexbuffers/src/builder/map.rs @@ -0,0 +1,118 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{Builder, Pushable, Value, VectorBuilder}; + +/// Builds a Flexbuffer map, returned by a [Builder](struct.Builder.html). +/// +/// ## Side effect when dropped: +/// When this is dropped, or `end_map` is called, the map is +/// commited to the buffer. If this map is the root of the flexbuffer, then the +/// root is written and the flexbuffer is complete. +/// ## Panics: +/// - Duplicate keys will result in a panic in both debug and release mode. +/// - Keys with internal nulls results in a panic in debug mode and result in silent truncaction +/// in release mode. +pub struct MapBuilder<'a> { + pub(super) builder: &'a mut Builder, + // If the root is this map then start == None. Otherwise start is the + // number of values in the 'values stack' before adding this map. + pub(super) start: Option, +} +impl<'a> MapBuilder<'a> { + /// Push `p` onto this map with key `key`. + /// This will panic (in debug mode) if `key` contains internal nulls. + #[inline] + pub fn push(&mut self, key: &str, p: P) { + self.builder.push_key(key); + self.builder.push(p); + } + /// Starts a nested vector that will be pushed onto this map + /// with key `key` when it is dropped. + /// + /// This will panic (in debug mode) if `key` contains internal nulls. + #[inline] + pub fn start_vector(&mut self, key: &str) -> VectorBuilder { + // Push the key that refers to this nested vector. + self.builder.push_key(key); + // Nested vector. + let start = Some(self.builder.values.len()); + VectorBuilder { + builder: &mut self.builder, + start, + } + } + /// Starts a nested map which that will be pushed onto this map + /// with key `key` when it is dropped. + /// + /// This will panic (in debug mode) if `key` contains internal nulls. + #[inline] + pub fn start_map(&mut self, key: &str) -> MapBuilder { + // Push the key that refers to this nested vector. + self.builder.push_key(key); + // Nested map. + let start = Some(self.builder.values.len()); + MapBuilder { + builder: &mut self.builder, + start, + } + } + /// `end_map` sorts the map by key and writes it to the buffer. This happens anyway + /// when the map builder is dropped. + #[inline] + pub fn end_map(self) {} +} +impl<'a> Drop for MapBuilder<'a> { + #[inline] + fn drop(&mut self) { + self.builder.end_map_or_vector(true, self.start); + } +} + +// Read known keys / strings as iterators over bytes -- skipping utf8 validation and strlen. +pub(super) fn get_key(buffer: &[u8], address: usize) -> impl Iterator { + buffer[address..].iter().take_while(|&&b| b != b'\0') +} + +// `values` is assumed to be of the format [key1, value1, ..., keyN, valueN]. +// The keys refer to cstrings in `buffer`. When this function returns, +// `values` is sorted in place by key. +pub(super) fn sort_map_by_keys(values: &mut [Value], buffer: &[u8]) { + debug_assert_eq!(values.len() % 2, 0); + debug_assert!(values.iter().step_by(2).all(Value::is_key)); + let raw_pairs = values.as_mut_ptr() as *mut [Value; 2]; + let pairs_len = values.len() / 2; + // Unsafe code needed to treat the slice as key-value pairs when sorting in place. This is + // preferred over custom sorting or adding another dependency. By construction, this part + // of the values stack must be alternating (key, value) pairs. The public API must not be + // able to trigger the above debug_assets that protect this unsafe usage. + let pairs: &mut [[Value; 2]] = + unsafe { std::slice::from_raw_parts_mut(raw_pairs, pairs_len) }; + #[rustfmt::skip] + pairs.sort_unstable_by(|[key1, _], [key2, _]| { + if let Value::Key(a1) = *key1 { + if let Value::Key(a2) = *key2 { + let s1 = get_key(buffer, a1); + let s2 = get_key(buffer, a2); + let ord = s1.cmp(s2); + if ord == std::cmp::Ordering::Equal { + let dup: String = get_key(buffer, a1).map(|&b| b as char).collect(); + panic!("Duplicated key in map {:?}", dup); + } + return ord; + } + } + unreachable!(); + }); +} diff --git a/rust/flexbuffers/src/builder/mod.rs b/rust/flexbuffers/src/builder/mod.rs new file mode 100644 index 000000000..e71acd0db --- /dev/null +++ b/rust/flexbuffers/src/builder/mod.rs @@ -0,0 +1,404 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::bitwidth::{align, BitWidth}; +mod value; +use crate::FlexBufferType; +use std::cmp::max; +use value::{find_vector_type, store_value, Value}; +mod map; +mod push; +mod ser; +mod vector; +use map::sort_map_by_keys; +pub use map::MapBuilder; +pub use push::Pushable; +pub use ser::{Error, FlexbufferSerializer}; +pub use vector::VectorBuilder; + +macro_rules! push_slice { + ($push_name: ident, $scalar: ty, $Val: ident, $new_vec: ident) => { + fn $push_name(&mut self, xs: S) + where + T: Into<$scalar> + Copy, + S: AsRef<[T]> + { + let mut value = Value::$new_vec(xs.as_ref().len()); + let mut width = xs.as_ref() + .iter() + .map(|x| BitWidth::from((*x).into())) + .max() + .unwrap_or_default(); + if !value.is_fixed_length_vector() { + let length = Value::UInt(xs.as_ref().len() as u64); + width = std::cmp::max(width, length.width_or_child_width()); + align(&mut self.buffer, width); + store_value(&mut self.buffer, length, width); + } else { + align(&mut self.buffer, width); + } + let address = self.buffer.len(); + for &x in xs.as_ref().iter() { + store_value(&mut self.buffer, Value::$Val(x.into()), width); + } + value.set_address_or_panic(address); + value.set_child_width_or_panic(width); + self.values.push(value); + } + } +} +macro_rules! push_indirect { + ($push_name: ident, $scalar: ty, $Direct: ident, $Indirect: ident) => { + fn $push_name>(&mut self, x: T) { + let x = Value::$Direct(x.into()); + let child_width = x.width_or_child_width(); + let address = self.buffer.len(); + store_value(&mut self.buffer, x, child_width); + self.values.push( + Value::Reference { + address, + child_width, + fxb_type: FlexBufferType::$Indirect, + } + ); + } + } +} + +bitflags! { + /// Options for sharing data within a flexbuffer. + /// + /// These increase serialization time but decrease the size of the resulting buffer. By + /// default, `SHARE_KEYS`. You may wish to turn on `SHARE_STRINGS` if you know your data has + /// many duplicate strings or `SHARE_KEY_VECTORS` if your data has many maps with identical + /// keys. + /// + /// ## Not Yet Implemented + /// - `SHARE_STRINGS` + /// - `SHARE_KEY_VECTORS` + pub struct BuilderOptions: u8 { + const SHARE_NONE = 0; + const SHARE_KEYS = 1; + const SHARE_STRINGS = 2; + const SHARE_KEYS_AND_STRINGS = 3; + const SHARE_KEY_VECTORS = 4; + const SHARE_ALL = 7; + } +} +impl Default for BuilderOptions { + fn default() -> Self { + Self::SHARE_KEYS + } +} + +#[derive(Debug, Clone, Copy)] +// Address of a Key inside of the buffer. +struct CachedKey(usize); + +/// **Use this struct to build a Flexbuffer.** +/// +/// Flexbuffers may only have a single root value, which may be constructed +/// with one of the following functions. +/// * `build_singleton` will push 1 value to the buffer and serialize it as the root. +/// * `start_vector` returns a `VectorBuilder`, into which many (potentially +/// heterogenous) values can be pushed. The vector itself is the root and is serialized +/// when the `VectorBuilder` is dropped (or `end` is called). +/// * `start_map` returns a `MapBuilder`, which is similar to a `VectorBuilder` except +/// every value must be pushed with an associated key. The map is serialized when the +/// `MapBuilder` is dropped (or `end` is called). +/// +/// These functions reset and overwrite the Builder which means, while there are no +/// active `MapBuilder` or `VectorBuilder`, the internal buffer is empty or contains a +/// finished Flexbuffer. The internal buffer is accessed with `view`. +#[derive(Debug, Clone)] +pub struct Builder { + buffer: Vec, + values: Vec, + key_pool: Option>, +} +impl Default for Builder { + fn default() -> Self { + let opts = Default::default(); + Builder::new(opts) + } +} + +impl<'a> Builder { + pub fn new(opts: BuilderOptions) -> Self { + let key_pool = if opts.contains(BuilderOptions::SHARE_KEYS) { + Some(vec![]) + } else { + None + }; + Builder { + key_pool, + values: Vec::new(), + buffer: Vec::new(), + } + } + /// Shows the internal flexbuffer. It will either be empty or populated with the most + /// recently built flexbuffer. + pub fn view(&self) -> &[u8] { + &self.buffer + } + /// Returns the internal buffer, replacing it with a new vector. The returned buffer will + /// either be empty or populated with the most recently built flexbuffer. + pub fn take_buffer(&mut self) -> Vec { + let mut b = Vec::new(); + std::mem::swap(&mut self.buffer, &mut b); + b + } + /// Resets the internal state. Automatically called before building a new flexbuffer. + pub fn reset(&mut self) { + self.buffer.clear(); + self.values.clear(); + if let Some(pool) = self.key_pool.as_mut() { + pool.clear(); + } + } + fn push_key(&mut self, key: &str) { + debug_assert!( + key.bytes().all(|b| b != b'\0'), + "Keys must not have internal nulls." + ); + // Search key pool if there is one. + let found = self.key_pool.as_ref().map(|pool| { + pool.binary_search_by(|&CachedKey(addr)| { + let old_key = map::get_key(&self.buffer, addr); + old_key.cloned().cmp(key.bytes()) + }) + }); + let address = if let Some(Ok(idx)) = found { + // Found key in key pool. + self.key_pool.as_ref().unwrap()[idx].0 + } else { + // Key not in pool (or no pool). + let address = self.buffer.len(); + self.buffer.extend_from_slice(key.as_bytes()); + self.buffer.push(b'\0'); + address + }; + if let Some(Err(idx)) = found { + // Insert into key pool. + let pool = self.key_pool.as_mut().unwrap(); + pool.insert(idx, CachedKey(address)); + } + self.values.push(Value::Key(address)); + } + fn push_uint>(&mut self, x: T) { + self.values.push(Value::UInt(x.into())); + } + fn push_int>(&mut self, x: T) { + self.values.push(Value::Int(x.into())); + } + fn push_float>(&mut self, x: T) { + self.values.push(Value::Float(x.into())); + } + fn push_null(&mut self) { + self.values.push(Value::Null); + } + fn push_bool(&mut self, x: bool) { + self.values.push(Value::Bool(x)); + } + fn store_blob(&mut self, xs: &[u8]) -> Value { + let length = Value::UInt(xs.len() as u64); + let width = length.width_or_child_width(); + align(&mut self.buffer, width); + store_value(&mut self.buffer, length, width); + let address = self.buffer.len(); + self.buffer.extend_from_slice(xs); + Value::Reference { + fxb_type: FlexBufferType::Blob, + address, + child_width: width, + } + } + fn push_str(&mut self, x: &str) { + let mut string = self.store_blob(x.as_bytes()); + self.buffer.push(b'\0'); + string.set_fxb_type_or_panic(FlexBufferType::String); + self.values.push(string); + } + fn push_blob(&mut self, x: &[u8]) { + let blob = self.store_blob(x); + self.values.push(blob); + } + fn push_bools(&mut self, xs: &[bool]) { + let length = Value::UInt(xs.len() as u64); + let width = length.width_or_child_width(); + align(&mut self.buffer, width); + store_value(&mut self.buffer, length, width); + let address = self.buffer.len(); + for &b in xs.iter() { + self.buffer.push(b as u8); + for _ in 0..width as u8 { + self.buffer.push(0); // Well this seems wasteful. + } + } + self.values.push(Value::Reference { + fxb_type: FlexBufferType::VectorBool, + address, + child_width: width, + }); + } + + push_slice!(push_uints, u64, UInt, new_uint_vector); + push_slice!(push_ints, i64, Int, new_int_vector); + push_slice!(push_floats, f64, Float, new_float_vector); + push_indirect!(push_indirect_int, i64, Int, IndirectInt); + push_indirect!(push_indirect_uint, u64, UInt, IndirectUInt); + push_indirect!(push_indirect_float, f64, Float, IndirectFloat); + + /// Resets the builder and starts a new flexbuffer with a vector at the root. + /// The exact Flexbuffer vector type is dynamically inferred. + pub fn start_vector(&'a mut self) -> VectorBuilder<'a> { + self.reset(); + VectorBuilder { + builder: self, + start: None, + } + } + /// Resets the builder and builds a new flexbuffer with a map at the root. + pub fn start_map(&'a mut self) -> MapBuilder<'a> { + self.reset(); + MapBuilder { + builder: self, + start: None, + } + } + /// Resets the builder and builds a new flexbuffer with the pushed value at the root. + pub fn build_singleton(&mut self, p: P) { + self.reset(); + p.push_to_builder(self); + let root = self.values.pop().unwrap(); + store_root(&mut self.buffer, root); + } + fn push(&mut self, p: P) { + p.push_to_builder(self); + } + /// Stores the values past `previous_end` as a map or vector depending on `is_map`. + /// If `previous_end` is None then this was a root map / vector and the last value + /// is stored as the root. + fn end_map_or_vector(&mut self, is_map: bool, previous_end: Option) { + let split = previous_end.unwrap_or(0); + let value = if is_map { + let key_vals = &mut self.values[split..]; + sort_map_by_keys(key_vals, &self.buffer); + let key_vector = store_vector(&mut self.buffer, key_vals, StoreOption::MapKeys); + store_vector(&mut self.buffer, key_vals, StoreOption::Map(key_vector)) + } else { + store_vector(&mut self.buffer, &self.values[split..], StoreOption::Vector) + }; + self.values.truncate(split); + if previous_end.is_some() { + self.values.push(value); + } else { + store_root(&mut self.buffer, value); + } + } +} + +/// Builds a Flexbuffer with the single pushed value as the root. +pub fn singleton(p: P) -> Vec { + let mut b = Builder::default(); + b.build_singleton(p); + let Builder { buffer, .. } = b; + buffer +} + +/// Stores the root value, root type and root width. +/// This should be called to finish the Flexbuffer. +fn store_root(buffer: &mut Vec, root: Value) { + let root_width = root.width_in_vector(buffer.len(), 0); + align(buffer, root_width); + store_value(buffer, root, root_width); + buffer.push(root.packed_type(root_width)); + buffer.push(root_width.n_bytes() as u8); +} + +pub enum StoreOption { + Vector, + Map(Value), + MapKeys, +} +/// Writes a Flexbuffer Vector or Map. +/// StoreOption::Map(Keys) must be a Value::Key or this will panic. +// #[inline(always)] +pub fn store_vector(buffer: &mut Vec, values: &[Value], opt: StoreOption) -> Value { + let (skip, stride) = match opt { + StoreOption::Vector => (0, 1), + StoreOption::MapKeys => (0, 2), + StoreOption::Map(_) => (1, 2), + }; + let iter_values = || values.iter().skip(skip).step_by(stride); + + // Figure out vector type and how long is the prefix. + let mut result = if let StoreOption::Map(_) = opt { + Value::new_map() + } else { + find_vector_type(iter_values()) + }; + let length_slot = if !result.is_fixed_length_vector() { + let length = iter_values().count(); + Some(Value::UInt(length as u64)) + } else { + None + }; + // Measure required width and align to it. + let mut width = BitWidth::W8; + if let StoreOption::Map(keys) = opt { + width = max(width, keys.width_in_vector(buffer.len(), 0)) + } + if let Some(l) = length_slot { + width = max(width, l.width_or_child_width()); + } + let prefix_length = result.prefix_length(); + for (i, &val) in iter_values().enumerate() { + width = max(width, val.width_in_vector(buffer.len(), i + prefix_length)); + } + align(buffer, width); + #[allow(deprecated)] + { + debug_assert_ne!( + result.fxb_type(), + FlexBufferType::VectorString, + "VectorString is deprecated and cannot be written.\ + (https://github.com/google/flatbuffers/issues/5627)" + ); + } + // Write Prefix. + if let StoreOption::Map(keys) = opt { + let key_width = Value::UInt(keys.width_or_child_width().n_bytes() as u64); + store_value(buffer, keys, width); + store_value(buffer, key_width, width); + } + if let Some(len) = length_slot { + store_value(buffer, len, width); + } + // Write data. + let address = buffer.len(); + for &v in iter_values() { + store_value(buffer, v, width); + } + // Write types + if result.is_typed_vector_or_map() { + for v in iter_values() { + buffer.push(v.packed_type(width)); + } + } + // Return Value representing this Vector. + result.set_address_or_panic(address); + result.set_child_width_or_panic(width); + result +} diff --git a/rust/flexbuffers/src/builder/push.rs b/rust/flexbuffers/src/builder/push.rs new file mode 100644 index 000000000..d22b47f10 --- /dev/null +++ b/rust/flexbuffers/src/builder/push.rs @@ -0,0 +1,167 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::builder::Builder; +use crate::private::Sealed; +use crate::{Blob, IndirectFloat, IndirectInt, IndirectUInt}; + +impl<'a> Sealed for Blob<'a> {} +impl Sealed for () {} + +// TODO: String interning +// TODO: Pushable for Map types? + +/// Types that implement the Pushable trait can be written into a Flexbuffer. +/// +/// All Rust's standard numbers, `u8, u16, u32, u64, i8, i16, i32, i64, f32, f64`, +/// can all be pushed. They are `FlexBufferType::{UInt, Int, Float}`. +/// Flexbuffers chooses the smallest width that can represent the given number. +/// Strings can pe pushed, they become `FlexBufferType::String` and are stored +/// with both a length and null terminator. +/// +/// * For convenience and speed push typed vectors using rust arrays and slices. +/// Doing so will immediately serialize the data, skipping the `Builder`'s +/// internal cache. +/// +/// * Pushable cannot not be implemented by any downstream crates. +pub trait Pushable: Sealed + Sized { + fn push_to_builder(self, _: &mut Builder) {} +} + +impl Pushable for () { + fn push_to_builder(self, builder: &mut Builder) { + builder.push_null(); + } +} +impl<'a> Pushable for Blob<'a> { + fn push_to_builder(self, builder: &mut Builder) { + builder.push_blob(self.0); + } +} + +macro_rules! forward_to_builder { + ($T: ty, $method: ident) => { + impl Sealed for $T {} + impl Pushable for $T { + fn push_to_builder(self, builder: &mut Builder) { + builder.$method(self); + } + } + }; + ($T: ty, $method: ident, $asT: ty) => { + impl Sealed for $T {} + impl Pushable for $T { + fn push_to_builder(self, builder: &mut Builder) { + builder.$method(self as $asT); + } + } + }; +} +forward_to_builder!(&str, push_str); +forward_to_builder!(bool, push_bool); +forward_to_builder!(u8, push_uint); +forward_to_builder!(u16, push_uint); +forward_to_builder!(u32, push_uint); +forward_to_builder!(u64, push_uint); +forward_to_builder!(i8, push_int); +forward_to_builder!(i16, push_int); +forward_to_builder!(i32, push_int); +forward_to_builder!(i64, push_int); +forward_to_builder!(f32, push_float); +forward_to_builder!(f64, push_float); +forward_to_builder!(&[u8], push_uints); +forward_to_builder!(&[u16], push_uints); +forward_to_builder!(&[u32], push_uints); +forward_to_builder!(&[u64], push_uints); +forward_to_builder!(&[i8], push_ints); +forward_to_builder!(&[i16], push_ints); +forward_to_builder!(&[i32], push_ints); +forward_to_builder!(&[i64], push_ints); +forward_to_builder!(&[f32], push_floats); +forward_to_builder!(&[f64], push_floats); +forward_to_builder!(&[bool], push_bools); +forward_to_builder!(&Vec, push_uints); +forward_to_builder!(&Vec, push_uints); +forward_to_builder!(&Vec, push_uints); +forward_to_builder!(&Vec, push_uints); +forward_to_builder!(&Vec, push_ints); +forward_to_builder!(&Vec, push_ints); +forward_to_builder!(&Vec, push_ints); +forward_to_builder!(&Vec, push_ints); +forward_to_builder!(&Vec, push_floats); +forward_to_builder!(&Vec, push_floats); +forward_to_builder!(&Vec, push_bools); + +macro_rules! impl_indirects { + ($Indirect: ident, $method: ident) => { + impl Sealed for $Indirect {} + impl Pushable for $Indirect { + fn push_to_builder(self, builder: &mut Builder) { + builder.$method(self.0); + } + } + }; +} +impl_indirects!(IndirectInt, push_indirect_int); +impl_indirects!(IndirectUInt, push_indirect_uint); +impl_indirects!(IndirectFloat, push_indirect_float); + +macro_rules! impl_arrays { + ($num: expr) => { + forward_to_builder!(&[u8; $num], push_uints, &[u8]); + forward_to_builder!(&[u16; $num], push_uints, &[u16]); + forward_to_builder!(&[u32; $num], push_uints, &[u32]); + forward_to_builder!(&[u64; $num], push_uints, &[u64]); + forward_to_builder!(&[i8; $num], push_ints, &[i8]); + forward_to_builder!(&[i16; $num], push_ints, &[i16]); + forward_to_builder!(&[i32; $num], push_ints, &[i32]); + forward_to_builder!(&[i64; $num], push_ints, &[i64]); + forward_to_builder!(&[f32; $num], push_floats, &[f32]); + forward_to_builder!(&[f64; $num], push_floats, &[f64]); + forward_to_builder!(&[bool; $num], push_bools, &[bool]); + }; +} +impl_arrays!(0); +impl_arrays!(1); +impl_arrays!(2); +impl_arrays!(3); +impl_arrays!(4); +impl_arrays!(5); +impl_arrays!(6); +// impl_arrays!(7); +// impl_arrays!(8); +// impl_arrays!(9); +// impl_arrays!(10); +// impl_arrays!(11); +// impl_arrays!(12); +// impl_arrays!(13); +// impl_arrays!(14); +// impl_arrays!(15); +// impl_arrays!(16); +// impl_arrays!(17); +// impl_arrays!(18); +// impl_arrays!(19); +// impl_arrays!(20); +// impl_arrays!(21); +// impl_arrays!(22); +// impl_arrays!(23); +// impl_arrays!(24); +// impl_arrays!(25); +// impl_arrays!(26); +// impl_arrays!(27); +// impl_arrays!(28); +// impl_arrays!(29); +// impl_arrays!(30); +// impl_arrays!(31); +// impl_arrays!(32); diff --git a/rust/flexbuffers/src/builder/ser.rs b/rust/flexbuffers/src/builder/ser.rs new file mode 100644 index 000000000..a0180b678 --- /dev/null +++ b/rust/flexbuffers/src/builder/ser.rs @@ -0,0 +1,530 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::Builder; +use serde::ser; +use serde::ser::*; +use std::fmt::Display; + +// This struct internally tracks the nested vectors representing +// nested structs and such. +// TODO: Add an option field names in a map. +/// Flexbuffer Serializer. This should be used to serialize structs. +#[derive(Debug, Default)] +pub struct FlexbufferSerializer { + builder: Builder, + nesting: Vec>, +} +impl FlexbufferSerializer { + pub fn new() -> Self { + Self::default() + } + pub fn view(&self) -> &[u8] { + self.builder.view() + } + pub fn take_buffer(&mut self) -> Vec { + self.builder.take_buffer() + } + fn finish_if_not_nested(&mut self) -> Result<(), Error> { + if self.nesting.is_empty() { + assert_eq!(self.builder.values.len(), 1); + let root = self.builder.values.pop().unwrap(); + super::store_root(&mut self.builder.buffer, root); + } + Ok(()) + } + fn start_vector(&mut self) { + let previous_end = if self.nesting.is_empty() { + None + } else { + Some(self.builder.values.len()) + }; + self.nesting.push(previous_end); + } + fn start_map(&mut self) { + let previous_end = if self.nesting.is_empty() { + None + } else { + Some(self.builder.values.len()) + }; + self.nesting.push(previous_end); + } + fn end_vector(&mut self) -> Result<(), Error> { + let previous_end = self.nesting.pop().unwrap(); + self.builder.end_map_or_vector(false, previous_end); + Ok(()) + } + fn end_map(&mut self) -> Result<(), Error> { + let previous_end = self.nesting.pop().unwrap(); + self.builder.end_map_or_vector(true, previous_end); + Ok(()) + } +} + +#[derive(Debug)] +/// Errors that may happen with Serde. +pub enum Error { + /// Only `str` and `String` can be serialized as keys in serde maps. + KeyMustBeString, + Serde(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { + write!(f, "{:?}", self) + } +} +impl std::error::Error for Error {} +impl ser::Error for Error { + fn custom(msg: T) -> Self + where + T: Display, + { + Self::Serde(format!("{}", msg)) + } +} +impl<'a> ser::SerializeSeq for &mut FlexbufferSerializer { + type Ok = (); + type Error = Error; + fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> + where + T: Serialize, + { + value.serialize(&mut **self) + } + fn end(self) -> Result { + self.end_vector() + } +} +// This is unlike a flexbuffers map which requires CString like keys. +// Its implemented as alternating keys and values (hopefully). +impl<'a> ser::SerializeMap for &'a mut FlexbufferSerializer { + type Ok = (); + type Error = Error; + fn serialize_key(&mut self, key: &T) -> Result<(), Self::Error> + where + T: Serialize, + { + key.serialize(MapKeySerializer(&mut **self)) + } + fn serialize_value(&mut self, value: &T) -> Result<(), Self::Error> + where + T: Serialize, + { + value.serialize(&mut **self) + } + fn end(self) -> Result { + self.end_map() + } +} +impl<'a> ser::SerializeTuple for &mut FlexbufferSerializer { + type Ok = (); + type Error = Error; + fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> + where + T: Serialize, + { + value.serialize(&mut **self) + } + fn end(self) -> Result { + self.end_vector() + } +} +impl<'a> ser::SerializeTupleStruct for &mut FlexbufferSerializer { + type Ok = (); + type Error = Error; + fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> + where + T: Serialize, + { + value.serialize(&mut **self) + } + fn end(self) -> Result { + self.end_vector() + } +} +impl<'a> ser::SerializeStruct for &mut FlexbufferSerializer { + type Ok = (); + type Error = Error; + fn serialize_field( + &mut self, + key: &'static str, + value: &T, + ) -> Result<(), Self::Error> + where + T: Serialize, + { + self.builder.push_key(key); + value.serialize(&mut **self) + } + fn end(self) -> Result { + self.end_map() + } +} +impl<'a> ser::SerializeTupleVariant for &mut FlexbufferSerializer { + type Ok = (); + type Error = Error; + fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> + where + T: Serialize, + { + value.serialize(&mut **self) + } + fn end(self) -> Result { + self.end_vector()?; + self.end_map() + } +} +impl<'a> ser::SerializeStructVariant for &mut FlexbufferSerializer { + type Ok = (); + type Error = Error; + fn serialize_field( + &mut self, + key: &'static str, + value: &T, + ) -> Result<(), Self::Error> + where + T: Serialize, + { + self.builder.push_key(key); + value.serialize(&mut **self) + } + fn end(self) -> Result { + self.end_map()?; + self.end_map() + } + // TODO: skip field? +} + +impl<'a> ser::Serializer for &'a mut FlexbufferSerializer { + type SerializeSeq = &'a mut FlexbufferSerializer; + type SerializeTuple = &'a mut FlexbufferSerializer; + type SerializeTupleStruct = &'a mut FlexbufferSerializer; + type SerializeTupleVariant = &'a mut FlexbufferSerializer; + type SerializeMap = &'a mut FlexbufferSerializer; + type SerializeStruct = &'a mut FlexbufferSerializer; + type SerializeStructVariant = &'a mut FlexbufferSerializer; + type Ok = (); + type Error = Error; + fn serialize_bool(self, v: bool) -> Result { + self.builder.push(v); + self.finish_if_not_nested() + } + fn serialize_i8(self, v: i8) -> Result { + self.builder.push(v); + self.finish_if_not_nested() + } + fn serialize_i16(self, v: i16) -> Result { + self.builder.push(v); + self.finish_if_not_nested() + } + fn serialize_i32(self, v: i32) -> Result { + self.builder.push(v); + self.finish_if_not_nested() + } + fn serialize_i64(self, v: i64) -> Result { + self.builder.push(v); + self.finish_if_not_nested() + } + fn serialize_u8(self, v: u8) -> Result { + self.builder.push(v); + self.finish_if_not_nested() + } + fn serialize_u16(self, v: u16) -> Result { + self.builder.push(v); + self.finish_if_not_nested() + } + fn serialize_u32(self, v: u32) -> Result { + self.builder.push(v); + self.finish_if_not_nested() + } + fn serialize_u64(self, v: u64) -> Result { + self.builder.push(v); + self.finish_if_not_nested() + } + fn serialize_f32(self, v: f32) -> Result { + self.builder.push(v); + self.finish_if_not_nested() + } + fn serialize_f64(self, v: f64) -> Result { + self.builder.push(v); + self.finish_if_not_nested() + } + fn serialize_char(self, v: char) -> Result { + self.builder.push(v as u8); + self.finish_if_not_nested() + } + fn serialize_str(self, v: &str) -> Result { + self.builder.push(v); + self.finish_if_not_nested() + } + fn serialize_bytes(self, v: &[u8]) -> Result { + self.builder.push(v); + self.finish_if_not_nested() + } + fn serialize_none(self) -> Result { + self.builder.push(()); + self.finish_if_not_nested() + } + fn serialize_some(self, t: &T) -> Result + where + T: Serialize, + { + t.serialize(self) + } + fn serialize_unit(self) -> Result { + self.builder.push(()); + self.finish_if_not_nested() + } + fn serialize_unit_struct(self, _name: &'static str) -> Result { + self.builder.push(()); + self.finish_if_not_nested() + } + fn serialize_unit_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + ) -> Result { + self.builder.push(variant); + self.finish_if_not_nested() + } + fn serialize_newtype_struct( + self, + _name: &'static str, + value: &T, + ) -> Result + where + T: Serialize, + { + value.serialize(self) + } + fn serialize_newtype_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + value: &T, + ) -> Result + where + T: Serialize, + { + self.start_map(); + self.builder.push_key(variant); + value.serialize(&mut *self)?; + self.end_map() + } + fn serialize_seq(self, _len: Option) -> Result { + self.start_vector(); + Ok(self) + } + fn serialize_tuple(self, _len: usize) -> Result { + self.start_vector(); + Ok(self) + } + fn serialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + ) -> Result { + self.start_map(); + Ok(self) + } + fn serialize_tuple_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + _len: usize, + ) -> Result { + self.start_map(); + self.builder.push_key(variant); + self.start_vector(); + Ok(self) + } + fn serialize_map(self, _len: Option) -> Result { + self.start_map(); + Ok(self) + } + fn serialize_struct( + self, + _name: &'static str, + _len: usize, + ) -> Result { + self.start_map(); + Ok(self) + } + fn serialize_struct_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + _len: usize, + ) -> Result { + self.start_map(); + self.builder.push_key(variant); + self.start_map(); + Ok(self) + } +} + +fn key_must_be_a_string() -> Result { + Err(Error::KeyMustBeString) +} +struct MapKeySerializer<'a>(&'a mut FlexbufferSerializer); +impl<'a> Serializer for MapKeySerializer<'a> { + type Ok = (); + type Error = Error; + #[inline] + fn serialize_str(self, value: &str) -> Result<(), Error> { + self.0.builder.push_key(value); + Ok(()) + } + #[inline] + fn serialize_unit_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + ) -> Result<(), Error> { + self.0.builder.push_key(variant); + Ok(()) + } + #[inline] + fn serialize_newtype_struct( + self, + _name: &'static str, + value: &T, + ) -> Result<(), Error> + where + T: Serialize, + { + value.serialize(self) + } + type SerializeSeq = Impossible<(), Error>; + type SerializeTuple = Impossible<(), Error>; + type SerializeTupleStruct = Impossible<(), Error>; + type SerializeTupleVariant = Impossible<(), Error>; + type SerializeMap = Impossible<(), Error>; + type SerializeStruct = Impossible<(), Error>; + type SerializeStructVariant = Impossible<(), Error>; + + fn serialize_bool(self, _value: bool) -> Result<(), Error> { + key_must_be_a_string() + } + fn serialize_i8(self, _value: i8) -> Result<(), Error> { + key_must_be_a_string() + } + fn serialize_i16(self, _value: i16) -> Result<(), Error> { + key_must_be_a_string() + } + fn serialize_i32(self, _value: i32) -> Result<(), Error> { + key_must_be_a_string() + } + fn serialize_i64(self, _value: i64) -> Result<(), Error> { + key_must_be_a_string() + } + fn serialize_u8(self, _value: u8) -> Result<(), Error> { + key_must_be_a_string() + } + fn serialize_u16(self, _value: u16) -> Result<(), Error> { + key_must_be_a_string() + } + fn serialize_u32(self, _value: u32) -> Result<(), Error> { + key_must_be_a_string() + } + fn serialize_u64(self, _value: u64) -> Result<(), Error> { + key_must_be_a_string() + } + fn serialize_f32(self, _value: f32) -> Result<(), Error> { + key_must_be_a_string() + } + fn serialize_f64(self, _value: f64) -> Result<(), Error> { + key_must_be_a_string() + } + fn serialize_char(self, _value: char) -> Result<(), Error> { + key_must_be_a_string() + } + fn serialize_bytes(self, _value: &[u8]) -> Result<(), Error> { + key_must_be_a_string() + } + fn serialize_unit(self) -> Result<(), Error> { + key_must_be_a_string() + } + fn serialize_unit_struct(self, _name: &'static str) -> Result<(), Error> { + key_must_be_a_string() + } + fn serialize_newtype_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _value: &T, + ) -> Result<(), Error> + where + T: Serialize, + { + key_must_be_a_string() + } + fn serialize_none(self) -> Result<(), Error> { + key_must_be_a_string() + } + fn serialize_some(self, _value: &T) -> Result<(), Error> + where + T: Serialize, + { + key_must_be_a_string() + } + fn serialize_seq(self, _len: Option) -> Result { + key_must_be_a_string() + } + fn serialize_tuple(self, _len: usize) -> Result { + key_must_be_a_string() + } + fn serialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + ) -> Result { + key_must_be_a_string() + } + fn serialize_tuple_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + key_must_be_a_string() + } + fn serialize_map(self, _len: Option) -> Result { + key_must_be_a_string() + } + fn serialize_struct( + self, + _name: &'static str, + _len: usize, + ) -> Result { + key_must_be_a_string() + } + fn serialize_struct_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + key_must_be_a_string() + } +} diff --git a/rust/flexbuffers/src/builder/value.rs b/rust/flexbuffers/src/builder/value.rs new file mode 100644 index 000000000..f230c34f1 --- /dev/null +++ b/rust/flexbuffers/src/builder/value.rs @@ -0,0 +1,306 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use byteorder::{LittleEndian, WriteBytesExt}; + +use crate::bitwidth::BitWidth; +use crate::bitwidth::BitWidth::*; +use crate::flexbuffer_type::FlexBufferType; +use crate::flexbuffer_type::FlexBufferType::*; + +/// Internal representation of FlexBuffer Types and Data before writing. +/// These get placed on the builder's stack and are eventually commited. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Value { + // Inline types + Null, + Int(i64), + UInt(u64), + Float(f64), + Bool(bool), + /// Null termintated, c_string. Only used with `Map`s. + Key(usize), + /// The other ~20 or so types. + Reference { + address: usize, + child_width: BitWidth, + fxb_type: FlexBufferType, + }, +} + +macro_rules! new_typed_vector { + ($name: ident, $v2: ident, $v3: ident, $v4: ident, $vn: ident) => { + /// Returns a typed vector, fixed length if possible. + /// Address and child width are zero initialized and must be set. + pub fn $name(n: usize) -> Value { + let address = 0; + let child_width = W8; + match n { + 2 => Value::Reference { + address, + child_width, + fxb_type: $v2, + }, + 3 => Value::Reference { + address, + child_width, + fxb_type: $v3, + }, + 4 => Value::Reference { + address, + child_width, + fxb_type: $v4, + }, + _ => Value::Reference { + address, + child_width, + fxb_type: $vn, + }, + } + } + }; +} + +impl Value { + pub fn new_vector() -> Self { + Value::Reference { + address: 0, + child_width: W8, + fxb_type: Vector, + } + } + pub fn new_map() -> Self { + Value::Reference { + address: 0, + child_width: W8, + fxb_type: Map, + } + } + new_typed_vector!( + new_int_vector, + VectorInt2, + VectorInt3, + VectorInt4, + VectorInt + ); + new_typed_vector!( + new_uint_vector, + VectorUInt2, + VectorUInt3, + VectorUInt4, + VectorUInt + ); + new_typed_vector!( + new_float_vector, + VectorFloat2, + VectorFloat3, + VectorFloat4, + VectorFloat + ); + pub fn fxb_type(&self) -> FlexBufferType { + match *self { + Value::Null => Null, + Value::Int(_) => Int, + Value::UInt(_) => UInt, + Value::Float(_) => Float, + Value::Bool(_) => Bool, + Value::Key(_) => Key, + Value::Reference { fxb_type, .. } => fxb_type, + } + } + pub fn is_fixed_length_vector(&self) -> bool { + self.fxb_type().is_fixed_length_vector() + } + pub fn is_inline(&self) -> bool { + self.fxb_type().is_inline() + } + pub fn is_reference(&self) -> bool { + !self.is_inline() + } + pub fn is_key(&self) -> bool { + match self { + Value::Key(_) => true, + _ => false, + } + } + pub fn is_typed_vector_or_map(&self) -> bool { + if let Value::Reference { fxb_type, .. } = self { + fxb_type.is_heterogenous() + } else { + false + } + } + pub fn prefix_length(&self) -> usize { + if self.is_fixed_length_vector() || self.is_inline() { + return 0; + } + if let Value::Reference { fxb_type, .. } = self { + if *fxb_type == Map { + return 3; + } + } + 1 + } + pub fn set_fxb_type_or_panic(&mut self, new_type: FlexBufferType) { + if let Value::Reference { fxb_type, .. } = self { + *fxb_type = new_type; + } else { + panic!("`set_fxb_type_or_panic` called on {:?}", self) + } + } + pub fn set_child_width_or_panic(&mut self, new_width: BitWidth) { + if let Value::Reference { child_width, .. } = self { + *child_width = new_width; + } else { + panic!("`set_child_width_or_panic` called on {:?}", self); + } + } + pub fn get_address(&self) -> Option { + if let Value::Reference { address, .. } | Value::Key(address) = self { + Some(*address) + } else { + None + } + } + pub fn set_address_or_panic(&mut self, new_address: usize) { + if let Value::Reference { address, .. } | Value::Key(address) = self { + *address = new_address; + } else { + panic!("`set_address_or_panic` called on {:?}", self); + } + } + /// For inline types - the width of the value to be stored. + /// For reference types, the width of the referred. + /// Note Key types always refer to 8 bit data. + pub fn width_or_child_width(&self) -> BitWidth { + match *self { + Value::Int(x) => x.into(), + Value::UInt(x) => x.into(), + Value::Float(x) => x.into(), + Value::Key(_) | Value::Bool(_) | Value::Null => W8, + Value::Reference { child_width, .. } => child_width, + } + } + pub fn relative_address(self, written_at: usize) -> Option { + self.get_address().map(|address| { + let offset = written_at + .checked_sub(address) + .expect("Error: References may only refer backwards in buffer."); + Value::UInt(offset as u64) + }) + } + /// Computes the minimum required width of `value` when stored in a vector + /// starting at `vector_start` at index `idx` (this index includes the prefix). + /// `Value::Reference{..}` variants require location information because + /// offsets are relative. + pub fn width_in_vector(self, vector_start: usize, idx: usize) -> BitWidth { + match self { + Value::Bool(_) => W8, + Value::Null => W8, + Value::Int(x) => x.into(), + Value::UInt(x) => x.into(), + Value::Float(x) => x.into(), + _ => { + debug_assert!(self.is_reference()); + for &width in BitWidth::iter() { + let bytes = width as usize + 1; + let alignment = (bytes - vector_start % bytes) % bytes; + let written_at = vector_start + alignment + idx * bytes; + // This match must always succeed. + if let Some(Value::UInt(offset)) = self.relative_address(written_at) { + if BitWidth::from(offset) == width { + return width; + } + } + } + unreachable!() + } + } + } + pub fn packed_type(self, parent_width: BitWidth) -> u8 { + let width = if self.is_inline() { + std::cmp::max(parent_width, self.width_or_child_width()) + } else { + self.width_or_child_width() + }; + (self.fxb_type() as u8) << 2 | width as u8 + } +} + +pub fn find_vector_type<'a, T>(mut values: T) -> Value +where + T: std::iter::Iterator, +{ + let first = values.next(); + if first.is_none() { + return Value::new_vector(); + } + let mut len = 1; + let init = first.unwrap().fxb_type(); + for v in values { + if v.fxb_type() != init { + return Value::new_vector(); + } + len += 1; + } + let vector_type = match init { + Bool => VectorBool, + UInt => return Value::new_uint_vector(len), + Int => return Value::new_int_vector(len), + Float => return Value::new_float_vector(len), + Key => VectorKey, + // Note that VectorString is deprecated for writing + _ => return Value::new_vector(), + }; + Value::Reference { + address: 0, + child_width: W8, + fxb_type: vector_type, + } +} + +#[inline] +pub fn store_value(buffer: &mut Vec, mut value: Value, width: BitWidth) { + // Remap to number types. + use Value::*; + if let Some(offset) = value.relative_address(buffer.len()) { + value = offset; + } else { + value = match value { + Bool(x) => UInt(x.into()), + Null => UInt(0), // Should this be 0 bytes? + _ => value, + } + } + let write_result = match (value, width) { + (UInt(x), W8) => buffer.write_u8(x as u8), + (UInt(x), W16) => buffer.write_u16::(x as u16), + (UInt(x), W32) => buffer.write_u32::(x as u32), + (UInt(x), W64) => buffer.write_u64::(x), + (Int(x), W8) => buffer.write_i8(x as i8), + (Int(x), W16) => buffer.write_i16::(x as i16), + (Int(x), W32) => buffer.write_i32::(x as i32), + (Int(x), W64) => buffer.write_i64::(x), + (Float(x), W32) => buffer.write_f32::(x as f32), + (Float(x), W64) => buffer.write_f64::(x), + (Float(_), _) => unreachable!("Error: Flatbuffers does not support 8 and 16 bit floats."), + _ => unreachable!("Variant not considered: {:?}", value), + }; + write_result.unwrap_or_else(|err| { + panic!( + "Error writing value {:?} with width {:?}: {:?}", + value, width, err + ) + }); +} diff --git a/rust/flexbuffers/src/builder/vector.rs b/rust/flexbuffers/src/builder/vector.rs new file mode 100644 index 000000000..4d73da3c3 --- /dev/null +++ b/rust/flexbuffers/src/builder/vector.rs @@ -0,0 +1,65 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{Builder, MapBuilder, Pushable}; + +/// Builds a Flexbuffer vector, returned by a [Builder](struct.Builder.html). +/// +/// ## Side effect when dropped: +/// When this is dropped, or `end_vector` is called, the vector is +/// commited to the buffer. If this vector is the root of the flexbuffer, then the +/// root is written and the flexbuffer is complete. The FlexBufferType of this vector +/// is determined by the pushed values when this is dropped. The most compact vector type is +/// automatically chosen. +pub struct VectorBuilder<'a> { + pub(crate) builder: &'a mut Builder, + // If the root is this vector then start == None. Otherwise start is the + // number of values in the 'values stack' before adding this vector. + pub(crate) start: Option, +} +impl<'a> VectorBuilder<'a> { + /// Pushes `p` onto the vector. + #[inline] + pub fn push(&mut self, p: P) { + self.builder.push(p); + } + /// Starts a nested vector that will be pushed onto this vector when it is dropped. + #[inline] + pub fn start_vector(&mut self) -> VectorBuilder { + let start = Some(self.builder.values.len()); + VectorBuilder { + builder: &mut self.builder, + start, + } + } + /// Starts a nested map that will be pushed onto this vector when it is dropped. + #[inline] + pub fn start_map(&mut self) -> MapBuilder { + let start = Some(self.builder.values.len()); + MapBuilder { + builder: &mut self.builder, + start, + } + } + /// `end_vector` determines the type of the vector and writes it to the buffer. + /// This will happen automatically if the VectorBuilder is dropped. + #[inline] + pub fn end_vector(self) {} +} +impl<'a> Drop for VectorBuilder<'a> { + #[inline] + fn drop(&mut self) { + self.builder.end_map_or_vector(false, self.start); + } +} diff --git a/rust/flexbuffers/src/flexbuffer_type.rs b/rust/flexbuffers/src/flexbuffer_type.rs new file mode 100644 index 000000000..1a95ac47e --- /dev/null +++ b/rust/flexbuffers/src/flexbuffer_type.rs @@ -0,0 +1,240 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#![allow(deprecated)] +/// Represents all the valid types in a flexbuffer. +/// +/// Flexbuffers supports +/// heterogenous maps, heterogenous vectors, typed vectors, and fixed length +/// typed vectors for some lengths and types. Rust types are converted into +/// Flexbuffers via the [Pushable](trait.Pushable.html) trait. +/// +/// For exact details see the [internals document]( +/// https://google.github.io/flatbuffers/flatbuffers_internals.html) +/// +/// ### Notes: +/// * In the binary format, Each element of a `Map` or (heterogenous) `Vector` +/// is stored with a byte describing its FlexBufferType and BitWidth. +/// +/// * Typed vectors do not store this extra type information and fixed length +/// typed vectors do not store length. Whether a vector is stored as a typed +/// vector or fixed length typed vector is determined dymaically from the +/// given data. +/// +/// * Indirect numbers are stored as an offset instead of inline. Using +/// indirect numbers instead of their inline counterparts in maps and typed +/// vectors can reduce the minimum element width and therefore bytes used. + +#[repr(u8)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, num_enum::TryFromPrimitive)] +pub enum FlexBufferType { + /// Nulls are represented with `()` in Rust. + Null = 0, + /// Variable width signed integer: `i8, i16, i32, i64` + Int = 1, + /// Variable width unsigned integer: `u8, u16, u32, u64` + UInt = 2, + /// Variable width floating point: `f32, f64` + Float = 3, + Bool = 26, + /// Null termintated, utf8 string. Typically used with `Map`s. + Key = 4, + /// Stored with a unsigned integer length, then UTF-8 bytes, and an extra null terminator that + /// is not counted with the length. + String = 5, + /// An Int, stored by offset rather than inline. Indirect types can keep the bitwidth of a + /// vector or map small when the inline value would have increased the bitwidth. + IndirectInt = 6, + /// A UInt, stored by offset rather than inline. Indirect types can keep the bitwidth of a + /// vector or map small when the inline value would have increased the bitwidth. + IndirectUInt = 7, + /// A Float, stored by offset rather than inline. Indirect types can keep the bitwidth of a + /// vector or map small when the inline value would have increased the bitwidth. + IndirectFloat = 8, + /// Maps are like Vectors except elements are associated with, and sorted by, keys. + Map = 9, + /// Heterogenous Vector (stored with a type table). + Vector = 10, + /// Homogenous Vector of Ints. + VectorInt = 11, + /// Homogenous Vector of UInts. + VectorUInt = 12, + /// Homogenous Vector of Floats. + VectorFloat = 13, + /// Homogenous Vector of Keys. + VectorKey = 14, + /// Homogenous Vector of Strings. + #[deprecated( + note = "Please use Vector or VectorKey instead. See https://github.com/google/flatbuffers/issues/5627" + )] + VectorString = 15, + /// Since the elements of a vector use the same `BitWidth` as the length, + /// Blob is more efficient for >255 element boolean vectors. + VectorBool = 36, + /// Homogenous vector of two Ints + VectorInt2 = 16, + /// Homogenous vector of two UInts + VectorUInt2 = 17, + /// Homogenous vector of two Floats + VectorFloat2 = 18, + /// Homogenous vector of three Ints + VectorInt3 = 19, + /// Homogenous vector of three UInts + VectorUInt3 = 20, + /// Homogenous vector of three Floats + VectorFloat3 = 21, + /// Homogenous vector of four Ints + VectorInt4 = 22, + /// Homogenous vector of four UInts + VectorUInt4 = 23, + /// Homogenous vector of four Floats + VectorFloat4 = 24, + /// An array of bytes. Stored with a variable width length. + Blob = 25, +} +use FlexBufferType::*; + +impl Default for FlexBufferType { + fn default() -> Self { + Null + } +} + +macro_rules! is_ty { + ($is_T: ident, $FTy: ident) => { + #[inline(always)] + pub fn $is_T(self) -> bool { + self == $FTy + } + }; +} + +impl FlexBufferType { + /// Returns true for flexbuffer types that are stored inline. + pub fn is_inline(self) -> bool { + match self { + Null | Int | UInt | Float | Bool => true, + _ => false, + } + } + /// Returns true for flexbuffer types that are stored by offset. + pub fn is_reference(self) -> bool { + !self.is_inline() + } + /// Returns true if called on a map, vector, typed vector, or fixed length typed vector. + pub fn is_vector(self) -> bool { + let d = self as u8; + 9 <= d && d < 25 || self == VectorBool + } + /// True iff the binary format stores the length. + /// This applies to Blob, String, Maps, and Vectors of variable length. + pub fn has_length_slot(self) -> bool { + !self.is_fixed_length_vector() && self.is_vector() || self == String || self == Blob + } + /// Returns true if called on a fixed length typed vector. + pub fn is_fixed_length_vector(self) -> bool { + self.fixed_length_vector_length().is_some() + } + /// If called on a fixed type vector, returns the type of the elements. + pub fn typed_vector_type(self) -> Option { + match self { + VectorInt | VectorInt2 | VectorInt3 | VectorInt4 => Some(Int), + VectorUInt | VectorUInt2 | VectorUInt3 | VectorUInt4 => Some(UInt), + VectorFloat | VectorFloat2 | VectorFloat3 | VectorFloat4 => Some(Float), + VectorKey => Some(Key), + // Treat them as keys because we do not know width of length slot. + // see deprecation link. + VectorString => Some(Key), + VectorBool => Some(Bool), + _ => None, + } + } + /// Return the length of the fixed length vector or None. + pub fn fixed_length_vector_length(self) -> Option { + match self { + VectorInt2 | VectorUInt2 | VectorFloat2 => Some(2), + VectorInt3 | VectorUInt3 | VectorFloat3 => Some(3), + VectorInt4 | VectorUInt4 | VectorFloat4 => Some(4), + _ => None, + } + } + /// Returns true if self is a Map or Vector. Typed vectors are not heterogenous. + pub fn is_heterogenous(self) -> bool { + self == Map || self == Vector + } + /// If `self` is an indirect scalar, remap it to the scalar. Otherwise do nothing. + pub fn to_direct(self) -> Option { + match self { + IndirectInt => Some(Int), + IndirectUInt => Some(UInt), + IndirectFloat => Some(Float), + _ => None, + } + } + /// returns true if and only if the flexbuffer type is `Null`. + is_ty!(is_null, Null); + /// returns true if and only if the flexbuffer type is `Int`. + is_ty!(is_int, Int); + /// returns true if and only if the flexbuffer type is `UInt`. + is_ty!(is_uint, UInt); + /// returns true if and only if the flexbuffer type is `Float`. + is_ty!(is_float, Float); + /// returns true if and only if the flexbuffer type is `Bool`. + is_ty!(is_bool, Bool); + /// returns true if and only if the flexbuffer type is `Key`. + is_ty!(is_key, Key); + /// returns true if and only if the flexbuffer type is `String`. + is_ty!(is_string, String); + /// returns true if and only if the flexbuffer type is `IndirectInt`. + is_ty!(is_indirect_int, IndirectInt); + /// returns true if and only if the flexbuffer type is `IndirectUInt`. + is_ty!(is_indirect_uint, IndirectUInt); + /// returns true if and only if the flexbuffer type is `IndirectFloat`. + is_ty!(is_indirect_float, IndirectFloat); + /// returns true if and only if the flexbuffer type is `Map`. + is_ty!(is_map, Map); + /// returns true if and only if the flexbuffer type is `Vector`. + is_ty!(is_heterogenous_vector, Vector); + /// returns true if and only if the flexbuffer type is `VectorInt`. + is_ty!(is_vector_int, VectorInt); + /// returns true if and only if the flexbuffer type is `VectorUInt`. + is_ty!(is_vector_uint, VectorUInt); + /// returns true if and only if the flexbuffer type is `VectorFloat`. + is_ty!(is_vector_float, VectorFloat); + /// returns true if and only if the flexbuffer type is `VectorKey`. + is_ty!(is_vector_key, VectorKey); + /// returns true if and only if the flexbuffer type is `VectorString`. + is_ty!(is_vector_string, VectorString); + /// returns true if and only if the flexbuffer type is `VectorBool`. + is_ty!(is_vector_bool, VectorBool); + /// returns true if and only if the flexbuffer type is `VectorInt2`. + is_ty!(is_vector_int2, VectorInt2); + /// returns true if and only if the flexbuffer type is `VectorUInt2`. + is_ty!(is_vector_uint2, VectorUInt2); + /// returns true if and only if the flexbuffer type is `VectorFloat2`. + is_ty!(is_vector_float2, VectorFloat2); + /// returns true if and only if the flexbuffer type is `VectorInt3`. + is_ty!(is_vector_int3, VectorInt3); + /// returns true if and only if the flexbuffer type is `VectorUInt3`. + is_ty!(is_vector_uint3, VectorUInt3); + /// returns true if and only if the flexbuffer type is `VectorFloat3`. + is_ty!(is_vector_float3, VectorFloat3); + /// returns true if and only if the flexbuffer type is `VectorInt4`. + is_ty!(is_vector_int4, VectorInt4); + /// returns true if and only if the flexbuffer type is `VectorUInt4`. + is_ty!(is_vector_uint4, VectorUInt4); + /// returns true if and only if the flexbuffer type is `VectorFloat4`. + is_ty!(is_vector_float4, VectorFloat4); + /// returns true if and only if the flexbuffer type is `Blob`. + is_ty!(is_blob, Blob); +} diff --git a/rust/flexbuffers/src/lib.rs b/rust/flexbuffers/src/lib.rs new file mode 100644 index 000000000..8076b3401 --- /dev/null +++ b/rust/flexbuffers/src/lib.rs @@ -0,0 +1,103 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg_attr(test, feature(test))] +//! Flexbuffers is a high performance schemaless binary data format designed at Google. +//! It is complementary to the schema-ed format [Flatbuffers](http://docs.rs/flatbuffers/). +//! See [Flexbuffer Internals](https://google.github.io/flatbuffers/flatbuffers_internals.html) +//! for details on the binary format. +//! +//! * [See the examples for usage.](https://github.com/CasperN/flexbuffers/tree/master/examples) +//! +//! This rust implementation is in progress and, until the 1.0 release, breaking API changes may +/// happen between minor versions. +// TODO(cneo): serde stuff are behind a default-on feature flag +// Reader to Json is behind a default-off feature flag +// Serializable structs are Pushable +// Serde with maps - field names and type names. + +#[macro_use] +extern crate bitflags; +extern crate byteorder; +#[macro_use] +extern crate serde_derive; +#[macro_use] +extern crate debug_stub_derive; +extern crate num_enum; +#[cfg(test)] +extern crate quickcheck; +#[cfg(test)] +extern crate quickcheck_derive; +#[cfg(test)] +extern crate rand; +extern crate serde; +#[cfg(test)] +extern crate test; + +mod bitwidth; +mod builder; +mod flexbuffer_type; +mod reader; +pub use bitwidth::BitWidth; +pub use builder::Error as SerializationError; +pub use builder::{ + singleton, Builder, BuilderOptions, FlexbufferSerializer, MapBuilder, Pushable, VectorBuilder, +}; +pub use flexbuffer_type::FlexBufferType; +pub use reader::Error as ReaderError; +pub use reader::{DeserializationError, MapReader, Reader, ReaderIterator, VectorReader}; +use serde::{Deserialize, Serialize}; + +mod private { + pub trait Sealed {} +} + +/// Serialize as a flexbuffer into a vector. +pub fn to_vec(x: T) -> Result, SerializationError> { + let mut s = FlexbufferSerializer::new(); + x.serialize(&mut s)?; + Ok(s.take_buffer()) +} +/// Deserialize a type from a flexbuffer. +pub fn from_slice<'de, T: Deserialize<'de>>(buf: &'de [u8]) -> Result { + let r = Reader::get_root(buf)?; + T::deserialize(r) +} + +/// This struct, when pushed will be serialized as a `FlexBufferType::Blob`. +/// +/// A `Blob` is a variable width `length` followed by that many bytes of data. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct Blob<'a>(pub &'a [u8]); + +/// This struct, when pushed, will be serialized as a `FlexBufferType::IndirectUInt`. +/// +/// It is an unsigned integer stored by reference in the flexbuffer. This can reduce the +/// size of vectors and maps containing the `IndirectUInt`. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct IndirectUInt(pub u64); + +/// This struct, when pushed, will be serialized as a `FlexBufferType::IndirectInt`. +/// +/// It is a signed integer stored by reference in the flexbuffer. This can reduce the +/// size of vectors and maps containing the `IndirectInt`. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct IndirectInt(pub i64); + +/// This struct, when pushed, will be serialized as a `FlexBufferType::IndirectFloat`. +/// +/// It is a floating point stored by reference in the flexbuffer. This can reduce the +/// size of vectors and maps containing the `IndirectFloat`. +#[derive(Debug, Copy, Clone, PartialEq)] +pub struct IndirectFloat(pub f64); diff --git a/rust/flexbuffers/src/reader/de.rs b/rust/flexbuffers/src/reader/de.rs new file mode 100644 index 000000000..ad35f25c2 --- /dev/null +++ b/rust/flexbuffers/src/reader/de.rs @@ -0,0 +1,250 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::Error; +use crate::{FlexBufferType, Reader, ReaderIterator}; +use serde::de::{ + DeserializeSeed, Deserializer, EnumAccess, IntoDeserializer, MapAccess, SeqAccess, + VariantAccess, Visitor, +}; + +/// Errors that may happen when deserializing a flexbuffer with serde. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DeserializationError { + Reader(Error), + Serde(String), +} + +impl std::error::Error for DeserializationError {} +impl std::fmt::Display for DeserializationError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { + match self { + Self::Reader(r) => write!(f, "Flexbuffer Read Error: {:?}", r), + Self::Serde(s) => write!(f, "Serde Error: {}", s), + } + } +} +impl serde::de::Error for DeserializationError { + fn custom(msg: T) -> Self + where + T: std::fmt::Display, + { + Self::Serde(format!("{}", msg)) + } +} +impl std::convert::From for DeserializationError { + fn from(e: super::Error) -> Self { + Self::Reader(e) + } +} + +impl<'de> SeqAccess<'de> for ReaderIterator<'de> { + type Error = DeserializationError; + fn next_element_seed( + &mut self, + seed: T, + ) -> Result>::Value>, Self::Error> + where + T: DeserializeSeed<'de>, + { + if let Some(elem) = self.next() { + seed.deserialize(elem).map(Some) + } else { + Ok(None) + } + } + fn size_hint(&self) -> Option { + Some(self.len()) + } +} + +struct EnumReader<'de> { + variant: &'de str, + value: Option>, +} + +impl<'de> EnumAccess<'de> for EnumReader<'de> { + type Error = DeserializationError; + type Variant = Reader<'de>; + fn variant_seed(self, seed: V) -> Result<(V::Value, Self::Variant), Self::Error> + where + V: DeserializeSeed<'de>, + { + seed.deserialize(self.variant.into_deserializer()) + .map(|v| (v, self.value.unwrap_or_default())) + } +} + +struct MapAccessor<'de> { + keys: ReaderIterator<'de>, + vals: ReaderIterator<'de>, +} +impl<'de> MapAccess<'de> for MapAccessor<'de> { + type Error = DeserializationError; + + fn next_key_seed(&mut self, seed: K) -> Result, Self::Error> + where + K: DeserializeSeed<'de>, + { + if let Some(k) = self.keys.next() { + seed.deserialize(k).map(Some) + } else { + Ok(None) + } + } + fn next_value_seed(&mut self, seed: V) -> Result + where + V: DeserializeSeed<'de>, + { + let val = self.vals.next().ok_or(Error::IndexOutOfBounds)?; + seed.deserialize(val) + } +} + +impl<'de> VariantAccess<'de> for Reader<'de> { + type Error = DeserializationError; + fn unit_variant(self) -> Result<(), Self::Error> { + Ok(()) + } + fn newtype_variant_seed(self, seed: T) -> Result + where + T: DeserializeSeed<'de>, + { + seed.deserialize(self) + } + // Tuple variants have an internally tagged representation. They are vectors where Index 0 is + // the discriminant and index N is field N-1. + fn tuple_variant(self, _len: usize, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_seq(self.as_vector().iter()) + } + // Struct variants have an internally tagged representation. They are vectors where Index 0 is + // the discriminant and index N is field N-1. + fn struct_variant( + self, + _fields: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + let m = self.get_map()?; + visitor.visit_map(MapAccessor { + keys: m.keys_vector().iter(), + vals: m.iter_values(), + }) + } +} + +impl<'de> Deserializer<'de> for crate::Reader<'de> { + type Error = DeserializationError; + fn deserialize_any(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + use crate::BitWidth::*; + use crate::FlexBufferType::*; + match (self.flexbuffer_type(), self.bitwidth()) { + (Bool, _) => visitor.visit_bool(self.as_bool()), + (UInt, W8) => visitor.visit_u8(self.as_u8()), + (UInt, W16) => visitor.visit_u16(self.as_u16()), + (UInt, W32) => visitor.visit_u32(self.as_u32()), + (UInt, W64) => visitor.visit_u64(self.as_u64()), + (Int, W8) => visitor.visit_i8(self.as_i8()), + (Int, W16) => visitor.visit_i16(self.as_i16()), + (Int, W32) => visitor.visit_i32(self.as_i32()), + (Int, W64) => visitor.visit_i64(self.as_i64()), + (Float, W32) => visitor.visit_f32(self.as_f32()), + (Float, W64) => visitor.visit_f64(self.as_f64()), + (Float, _) => Err(Error::InvalidPackedType.into()), // f8 and f16 are not supported. + (Null, _) => visitor.visit_unit(), + (String, _) | (Key, _) => visitor.visit_borrowed_str(self.as_str()), + (Blob, _) => visitor.visit_borrowed_bytes(self.get_blob()?.0), + (Map, _) => { + let m = self.get_map()?; + visitor.visit_map(MapAccessor { + keys: m.keys_vector().iter(), + vals: m.iter_values(), + }) + } + (ty, _) if ty.is_vector() => visitor.visit_seq(self.as_vector().iter()), + (ty, bw) => unreachable!("TODO deserialize_any {:?} {:?}.", ty, bw), + } + } + serde::forward_to_deserialize_any! { + bool i8 i16 i32 i64 u8 u16 u32 u64 f32 f64 str unit unit_struct bytes + ignored_any map identifier struct tuple tuple_struct seq string + } + fn deserialize_char(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_char(self.as_u8() as char) + } + fn deserialize_byte_buf(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_byte_buf(self.get_blob()?.0.to_vec()) + } + fn deserialize_option(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + if self.flexbuffer_type() == FlexBufferType::Null { + visitor.visit_none() + } else { + visitor.visit_some(self) + } + } + fn deserialize_newtype_struct( + self, + _name: &'static str, + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + visitor.visit_newtype_struct(self) + } + fn deserialize_enum( + self, + _name: &'static str, + _variants: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + let (variant, value) = match self.fxb_type { + FlexBufferType::String => (self.as_str(), None), + FlexBufferType::Map => { + let m = self.get_map()?; + let variant = m.keys_vector().idx(0).get_key()?; + let value = Some(m.idx(0)); + (variant, value) + } + _ => { + return Err(Error::UnexpectedFlexbufferType { + expected: FlexBufferType::Map, + actual: self.fxb_type, + } + .into()); + } + }; + visitor.visit_enum(EnumReader { variant, value }) + } +} diff --git a/rust/flexbuffers/src/reader/iter.rs b/rust/flexbuffers/src/reader/iter.rs new file mode 100644 index 000000000..8e06171cc --- /dev/null +++ b/rust/flexbuffers/src/reader/iter.rs @@ -0,0 +1,63 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{Reader, VectorReader}; +use std::iter::{DoubleEndedIterator, ExactSizeIterator, FusedIterator, Iterator}; + +/// Iterates over a flexbuffer vector, typed vector, or map. Yields [Readers](struct.Reader.html). +/// +/// If any error occurs, the Reader is defaulted to a Null flexbuffer Reader. +pub struct ReaderIterator<'de> { + pub(super) reader: VectorReader<'de>, + pub(super) front: usize, + end: usize, +} +impl<'de> ReaderIterator<'de> { + pub(super) fn new(reader: VectorReader<'de>) -> Self { + let end = reader.len(); + ReaderIterator { + reader, + front: 0, + end, + } + } +} +impl<'de> Iterator for ReaderIterator<'de> { + type Item = Reader<'de>; + fn next(&mut self) -> Option { + if self.front < self.end { + let r = self.reader.idx(self.front); + self.front += 1; + Some(r) + } else { + None + } + } + fn size_hint(&self) -> (usize, Option) { + let remaining = self.end - self.front; + (remaining, Some(remaining)) + } +} +impl<'de> DoubleEndedIterator for ReaderIterator<'de> { + fn next_back(&mut self) -> Option { + if self.front < self.end { + self.end -= 1; + Some(self.reader.idx(self.end)) + } else { + None + } + } +} +impl<'de> ExactSizeIterator for ReaderIterator<'de> {} +impl<'de> FusedIterator for ReaderIterator<'de> {} diff --git a/rust/flexbuffers/src/reader/map.rs b/rust/flexbuffers/src/reader/map.rs new file mode 100644 index 000000000..6769ab3e7 --- /dev/null +++ b/rust/flexbuffers/src/reader/map.rs @@ -0,0 +1,144 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{deref_offset, unpack_type, Error, Reader, ReaderIterator, VectorReader}; +use crate::BitWidth; +use std::cmp::Ordering; +use std::iter::{DoubleEndedIterator, ExactSizeIterator, FusedIterator, Iterator}; + +/// Allows indexing on a flexbuffer map. +/// +/// MapReaders may be indexed with strings or usizes. `index` returns a result type, +/// which may indicate failure due to a missing key or bad data, `idx` returns an Null Reader in +/// cases of error. +#[derive(DebugStub, Default, Clone)] +pub struct MapReader<'de> { + #[debug_stub = "&[..]"] + pub(super) buffer: &'de [u8], + pub(super) values_address: usize, + pub(super) keys_address: usize, + pub(super) values_width: BitWidth, + pub(super) keys_width: BitWidth, + pub(super) length: usize, +} + +impl<'de> MapReader<'de> { + /// Returns the number of key/value pairs are in the map. + pub fn len(&self) -> usize { + self.length + } + /// Returns true if the map has zero key/value pairs. + pub fn is_empty(&self) -> bool { + self.length == 0 + } + // Using &CStr will eagerly compute the length of the key. &str needs length info AND utf8 + // validation. This version is faster than both. + fn lazy_strcmp(&self, key_addr: usize, key: &str) -> Ordering { + // TODO: Can we know this won't OOB and panic? + let k = self.buffer[key_addr..].iter().take_while(|&&b| b != b'\0'); + k.cmp(key.as_bytes().iter()) + } + /// Returns the index of a given key in the map. + pub fn index_key(&self, key: &str) -> Option { + let (mut low, mut high) = (0, self.length); + while low < high { + let i = (low + high) / 2; + let key_offset_address = self.keys_address + i * self.keys_width.n_bytes(); + let key_address = + deref_offset(self.buffer, key_offset_address, self.keys_width).ok()?; + match self.lazy_strcmp(key_address, key) { + Ordering::Equal => return Some(i), + Ordering::Less => low = if i == low { i + 1 } else { i }, + Ordering::Greater => high = i, + } + } + None + } + /// Index into a map with a key or usize. + pub fn index(&self, i: I) -> Result, Error> { + i.index_map_reader(self) + } + /// Index into a map with a key or usize. If any errors occur a Null reader is returned. + pub fn idx(&self, i: I) -> Reader<'de> { + i.index_map_reader(self).unwrap_or_default() + } + fn usize_index(&self, i: usize) -> Result, Error> { + if i >= self.length { + return Err(Error::IndexOutOfBounds); + } + let data_address = self.values_address + self.values_width.n_bytes() * i; + let type_address = self.values_address + self.values_width.n_bytes() * self.length + i; + let (fxb_type, width) = self + .buffer + .get(type_address) + .ok_or(Error::FlexbufferOutOfBounds) + .and_then(|&b| unpack_type(b))?; + Reader::new( + &self.buffer, + data_address, + fxb_type, + width, + self.values_width, + ) + } + fn key_index(&self, k: &str) -> Result, Error> { + let i = self.index_key(k).ok_or(Error::KeyNotFound)?; + self.usize_index(i) + } + /// Iterate over the values of the map. + pub fn iter_values(&self) -> ReaderIterator<'de> { + ReaderIterator::new(VectorReader { + reader: Reader { + buffer: self.buffer, + fxb_type: crate::FlexBufferType::Map, + width: self.values_width, + address: self.values_address, + }, + length: self.length, + }) + } + /// Iterate over the keys of the map. + pub fn iter_keys( + &self, + ) -> impl Iterator + DoubleEndedIterator + ExactSizeIterator + FusedIterator + { + self.keys_vector().iter().map(|k| k.as_str()) + } + pub fn keys_vector(&self) -> VectorReader<'de> { + VectorReader { + reader: Reader { + buffer: self.buffer, + fxb_type: crate::FlexBufferType::VectorKey, + width: self.keys_width, + address: self.keys_address, + }, + length: self.length, + } + } +} +pub trait MapReaderIndexer { + fn index_map_reader<'de>(self, r: &MapReader<'de>) -> Result, Error>; +} +impl MapReaderIndexer for usize { + #[inline] + fn index_map_reader<'de>(self, r: &MapReader<'de>) -> Result, Error> { + r.usize_index(self) + } +} +impl MapReaderIndexer for &str { + #[inline] + fn index_map_reader<'de>(self, r: &MapReader<'de>) -> Result, Error> { + r.key_index(self) + } +} diff --git a/rust/flexbuffers/src/reader/mod.rs b/rust/flexbuffers/src/reader/mod.rs new file mode 100644 index 000000000..6db5b4950 --- /dev/null +++ b/rust/flexbuffers/src/reader/mod.rs @@ -0,0 +1,592 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::bitwidth::BitWidth; +use crate::flexbuffer_type::FlexBufferType; +use crate::Blob; +use std::convert::{TryFrom, TryInto}; +use std::fmt; +use std::ops::Rem; +use std::str::FromStr; +mod de; +mod iter; +mod map; +mod vector; +pub use de::DeserializationError; +pub use iter::ReaderIterator; +pub use map::{MapReader, MapReaderIndexer}; +pub use vector::VectorReader; + +/// All the possible errors when reading a flexbuffer. +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +pub enum Error { + /// One of the following data errors occured: + /// + /// * The read flexbuffer had an offset that pointed outside the flexbuffer. + /// * The 'negative indicies' where length and map keys are stored were out of bounds + /// * The buffer was too small to contain a flexbuffer root. + FlexbufferOutOfBounds, + /// Failed to parse a valid FlexbufferType and Bitwidth from a type byte. + InvalidPackedType, + /// Flexbuffer type of the read data does not match function used. + UnexpectedFlexbufferType { + expected: FlexBufferType, + actual: FlexBufferType, + }, + /// BitWidth type of the read data does not match function used. + UnexpectedBitWidth { + expected: BitWidth, + actual: BitWidth, + }, + /// Read a flexbuffer offset or length that overflowed usize. + ReadUsizeOverflowed, + /// Tried to index a type that's not one of the Flexbuffer vector types. + CannotIndexAsVector, + /// Tried to index a Flexbuffer vector or map out of bounds. + IndexOutOfBounds, + /// A Map was indexed with a key that it did not contain. + KeyNotFound, + /// Failed to parse a Utf8 string. + /// The Option will be `None` if and only if this Error was deserialized. + // NOTE: std::str::Utf8Error does not implement Serialize, Deserialize, nor Default. We tell + // serde to skip the field and default to None. We prefer to have the boxed error so it can be + // used with std::error::Error::source, though another (worse) option could be to drop that + // information. + Utf8Error(#[serde(skip)] Option>), + /// get_slice failed because the given data buffer is misaligned. + AlignmentError, + InvalidRootWidth, + InvalidMapKeysVectorWidth, +} +impl std::convert::From for Error { + fn from(e: std::str::Utf8Error) -> Self { + Self::Utf8Error(Some(Box::new(e))) + } +} +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::UnexpectedBitWidth { expected, actual } => write!( + f, + "Error reading flexbuffer: Expected bitwidth: {:?}, found bitwidth: {:?}", + expected, actual + ), + Self::UnexpectedFlexbufferType { expected, actual } => write!( + f, + "Error reading flexbuffer: Expected type: {:?}, found type: {:?}", + expected, actual + ), + _ => write!(f, "Error reading flexbuffer: {:?}", self), + } + } +} +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + if let Self::Utf8Error(Some(e)) = self { + Some(e) + } else { + None + } + } +} + +pub trait ReadLE: crate::private::Sealed + std::marker::Sized { + const VECTOR_TYPE: FlexBufferType; + const WIDTH: BitWidth; +} +macro_rules! rle { + ($T: ty, $VECTOR_TYPE: ident, $WIDTH: ident) => { + impl ReadLE for $T { + const VECTOR_TYPE: FlexBufferType = FlexBufferType::$VECTOR_TYPE; + const WIDTH: BitWidth = BitWidth::$WIDTH; + } + }; +} +rle!(u8, VectorUInt, W8); +rle!(u16, VectorUInt, W16); +rle!(u32, VectorUInt, W32); +rle!(u64, VectorUInt, W64); +rle!(i8, VectorInt, W8); +rle!(i16, VectorInt, W16); +rle!(i32, VectorInt, W32); +rle!(i64, VectorInt, W64); +rle!(f32, VectorFloat, W32); +rle!(f64, VectorFloat, W64); + +macro_rules! as_default { + ($as: ident, $get: ident, $T: ty) => { + pub fn $as(&self) -> $T { + self.$get().unwrap_or_default() + } + }; +} + +/// `Reader`s allow access to data stored in a Flexbuffer. +/// +/// Each reader represents a single address in the buffer so data is read lazily. Start a reader +/// by calling `get_root` on your flexbuffer `&[u8]`. +/// +/// - The `get_T` methods return a `Result`. They return an OK value if and only if the +/// flexbuffer type matches `T`. This is analogous to the behavior of Rust's json library, though +/// with Result instead of Option. +/// - The `as_T` methods will try their best to return to a value of type `T` +/// (by casting or even parsing a string if necessary) but ultimately returns `T::default` if it +/// fails. This behavior is analogous to that of flexbuffers C++. +#[derive(DebugStub, Default, Clone)] +pub struct Reader<'de> { + fxb_type: FlexBufferType, + width: BitWidth, + address: usize, + #[debug_stub = "&[..]"] + buffer: &'de [u8], +} + +macro_rules! try_cast_fn { + ($name: ident, $full_width: ident, $Ty: ident) => { + pub fn $name(&self) -> $Ty { + self.$full_width().try_into().unwrap_or_default() + } + } +} + +fn safe_sub(a: usize, b: usize) -> Result { + a.checked_sub(b).ok_or(Error::FlexbufferOutOfBounds) +} + +fn deref_offset(buffer: &[u8], address: usize, width: BitWidth) -> Result { + let off = read_usize(buffer, address, width); + safe_sub(address, off) +} + +impl<'de> Reader<'de> { + fn new( + buffer: &'de [u8], + mut address: usize, + mut fxb_type: FlexBufferType, + width: BitWidth, + parent_width: BitWidth, + ) -> Result { + if fxb_type.is_reference() { + address = deref_offset(buffer, address, parent_width)?; + // Indirects were dereferenced. + if let Some(t) = fxb_type.to_direct() { + fxb_type = t; + } + } + Ok(Reader { + address, + fxb_type, + width, + buffer, + }) + } + /// Parses the flexbuffer from the given buffer. Assumes the flexbuffer root is the last byte + /// of the buffer. + pub fn get_root(buffer: &'de [u8]) -> Result { + let end = buffer.len(); + if end < 3 { + return Err(Error::FlexbufferOutOfBounds); + } + // Last byte is the root width. + let root_width = BitWidth::from_nbytes(buffer[end - 1]).ok_or(Error::InvalidRootWidth)?; + // Second last byte is root type. + let (fxb_type, width) = unpack_type(buffer[end - 2])?; + // Location of root data. (BitWidth bits before root type) + let address = safe_sub(end - 2, root_width.n_bytes())?; + Self::new(buffer, address, fxb_type, width, root_width) + } + /// Returns the FlexBufferType of this Reader. + pub fn flexbuffer_type(&self) -> FlexBufferType { + self.fxb_type + } + /// Returns the bitwidth of this Reader. + pub fn bitwidth(&self) -> BitWidth { + self.width + } + /// Returns the length of the Flexbuffer. If the type has no length, or if an error occurs, + /// 0 is returned. + pub fn length(&self) -> usize { + if let Some(len) = self.fxb_type.fixed_length_vector_length() { + len + } else if self.fxb_type.has_length_slot() && self.address >= self.width.n_bytes() { + read_usize(self.buffer, self.address - self.width.n_bytes(), self.width) + } else { + 0 + } + } + /// Returns true if the flexbuffer is aligned to 8 bytes. This guarantees, for valid + /// flexbuffers, that the data is correctly aligned in memory and slices can be read directly + /// e.g. with `get_f64s` or `get_i16s`. + pub fn is_aligned(&self) -> bool { + (self.buffer.as_ptr() as usize).rem(8) == 0 + } + as_default!(as_vector, get_vector, VectorReader<'de>); + as_default!(as_map, get_map, MapReader<'de>); + + fn expect_type(&self, ty: FlexBufferType) -> Result<(), Error> { + if self.fxb_type == ty { + Ok(()) + } else { + Err(Error::UnexpectedFlexbufferType { + expected: ty, + actual: self.fxb_type, + }) + } + } + fn expect_bw(&self, bw: BitWidth) -> Result<(), Error> { + if self.width == bw { + Ok(()) + } else { + Err(Error::UnexpectedBitWidth { + expected: bw, + actual: self.width, + }) + } + } + /// Directly reads a slice of type `T`where `T` is one of `u8,u16,u32,u64,i8,i16,i32,i64,f32,f64`. + /// Returns Err if the type, bitwidth, or memory alignment does not match. Since the bitwidth is + /// dynamic, its better to use a VectorReader unless you know your data and performance is critical. + #[cfg(target_endian = "little")] + pub fn get_slice(&self) -> Result<&'de [T], Error> { + if self.flexbuffer_type().typed_vector_type() != T::VECTOR_TYPE.typed_vector_type() { + self.expect_type(T::VECTOR_TYPE)?; + } + if self.bitwidth().n_bytes() != std::mem::size_of::() { + self.expect_bw(T::WIDTH)?; + } + let end = self.address + self.length() * std::mem::size_of::(); + let slice = &self + .buffer + .get(self.address..end) + .ok_or(Error::FlexbufferOutOfBounds)?; + // `align_to` is required because the point of this function is to directly hand back a + // slice of scalars. This can fail because Rust's default allocator is not 16byte aligned + // (though in practice this only happens for small buffers). + let (pre, mid, suf) = unsafe { slice.align_to::() }; + if pre.is_empty() && suf.is_empty() { + Ok(mid) + } else { + Err(Error::AlignmentError) + } + } + + pub fn get_bool(&self) -> Result { + self.expect_type(FlexBufferType::Bool)?; + Ok( + self.buffer[self.address..self.address + self.width.n_bytes()] + .iter() + .any(|&b| b != 0), + ) + } + pub fn get_key(&self) -> Result<&'de str, Error> { + self.expect_type(FlexBufferType::Key)?; + let (length, _) = self.buffer[self.address..] + .iter() + .enumerate() + .find(|(_, &b)| b == b'\0') + .unwrap_or((0, &0)); + let bytes = &self.buffer[self.address..self.address + length]; + Ok(std::str::from_utf8(bytes)?) + } + pub fn get_blob(&self) -> Result, Error> { + self.expect_type(FlexBufferType::Blob)?; + Ok(Blob( + &self.buffer[self.address..self.address + self.length()], + )) + } + pub fn as_blob(&self) -> Blob<'de> { + self.get_blob().unwrap_or(Blob(&[])) + } + pub fn get_str(&self) -> Result<&'de str, Error> { + self.expect_type(FlexBufferType::String)?; + let bytes = &self.buffer[self.address..self.address + self.length()]; + Ok(std::str::from_utf8(bytes)?) + } + fn get_map_info(&self) -> Result<(usize, BitWidth), Error> { + self.expect_type(FlexBufferType::Map)?; + if 3 * self.width.n_bytes() >= self.address { + return Err(Error::FlexbufferOutOfBounds); + } + let keys_offset_address = self.address - 3 * self.width.n_bytes(); + let keys_width = { + let kw_addr = self.address - 2 * self.width.n_bytes(); + let kw = read_usize(self.buffer, kw_addr, self.width); + BitWidth::from_nbytes(kw).ok_or(Error::InvalidMapKeysVectorWidth) + }?; + Ok((keys_offset_address, keys_width)) + } + pub fn get_map(&self) -> Result, Error> { + let (keys_offset_address, keys_width) = self.get_map_info()?; + let keys_address = deref_offset(self.buffer, keys_offset_address, self.width)?; + // TODO(cneo): Check that vectors length equals keys length. + Ok(MapReader { + buffer: self.buffer, + values_address: self.address, + values_width: self.width, + keys_address, + keys_width, + length: self.length(), + }) + } + /// Tries to read a FlexBufferType::UInt. Returns Err if the type is not a UInt or if the + /// address is out of bounds. + pub fn get_u64(&self) -> Result { + self.expect_type(FlexBufferType::UInt)?; + let cursor = self + .buffer + .get(self.address..self.address + self.width.n_bytes()); + match self.width { + BitWidth::W8 => cursor.map(|s| s[0] as u8).map(Into::into), + BitWidth::W16 => cursor + .and_then(|s| s.try_into().ok()) + .map(::from_le_bytes) + .map(Into::into), + BitWidth::W32 => cursor + .and_then(|s| s.try_into().ok()) + .map(::from_le_bytes) + .map(Into::into), + BitWidth::W64 => cursor + .and_then(|s| s.try_into().ok()) + .map(::from_le_bytes), + } + .ok_or(Error::FlexbufferOutOfBounds) + } + /// Tries to read a FlexBufferType::Int. Returns Err if the type is not a UInt or if the + /// address is out of bounds. + pub fn get_i64(&self) -> Result { + self.expect_type(FlexBufferType::Int)?; + let cursor = self + .buffer + .get(self.address..self.address + self.width.n_bytes()); + match self.width { + BitWidth::W8 => cursor.map(|s| s[0] as i8).map(Into::into), + BitWidth::W16 => cursor + .and_then(|s| s.try_into().ok()) + .map(::from_le_bytes) + .map(Into::into), + BitWidth::W32 => cursor + .and_then(|s| s.try_into().ok()) + .map(::from_le_bytes) + .map(Into::into), + BitWidth::W64 => cursor + .and_then(|s| s.try_into().ok()) + .map(::from_le_bytes), + } + .ok_or(Error::FlexbufferOutOfBounds) + } + /// Tries to read a FlexBufferType::Float. Returns Err if the type is not a UInt, if the + /// address is out of bounds, or if its a f16 or f8 (not currently supported). + pub fn get_f64(&self) -> Result { + self.expect_type(FlexBufferType::Float)?; + let cursor = self + .buffer + .get(self.address..self.address + self.width.n_bytes()); + match self.width { + BitWidth::W8 | BitWidth::W16 => return Err(Error::InvalidPackedType), + BitWidth::W32 => cursor + .and_then(|s| s.try_into().ok()) + .map(f32_from_le_bytes) + .map(Into::into), + BitWidth::W64 => cursor + .and_then(|s| s.try_into().ok()) + .map(f64_from_le_bytes), + } + .ok_or(Error::FlexbufferOutOfBounds) + } + pub fn as_bool(&self) -> bool { + use FlexBufferType::*; + match self.fxb_type { + Bool => self.get_bool().unwrap_or_default(), + UInt => self.as_u64() != 0, + Int => self.as_i64() != 0, + Float => self.as_f64().abs() > std::f64::EPSILON, + String | Key => !self.as_str().is_empty(), + Null => false, + Blob => self.length() != 0, + ty if ty.is_vector() => self.length() != 0, + _ => unreachable!(), + } + } + /// Returns a u64, casting if necessary. For Maps and Vectors, their length is + /// returned. If anything fails, 0 is returned. + pub fn as_u64(&self) -> u64 { + match self.fxb_type { + FlexBufferType::UInt => self.get_u64().unwrap_or_default(), + FlexBufferType::Int => self + .get_i64() + .unwrap_or_default() + .try_into() + .unwrap_or_default(), + FlexBufferType::Float => self.get_f64().unwrap_or_default() as u64, + FlexBufferType::String => { + if let Ok(s) = self.get_str() { + if let Ok(f) = u64::from_str(s) { + return f; + } + } + 0 + } + _ if self.fxb_type.is_vector() => self.length() as u64, + _ => 0, + } + } + try_cast_fn!(as_u32, as_u64, u32); + try_cast_fn!(as_u16, as_u64, u16); + try_cast_fn!(as_u8, as_u64, u8); + + /// Returns an i64, casting if necessary. For Maps and Vectors, their length is + /// returned. If anything fails, 0 is returned. + pub fn as_i64(&self) -> i64 { + match self.fxb_type { + FlexBufferType::Int => self.get_i64().unwrap_or_default(), + FlexBufferType::UInt => self + .get_u64() + .unwrap_or_default() + .try_into() + .unwrap_or_default(), + FlexBufferType::Float => self.get_f64().unwrap_or_default() as i64, + FlexBufferType::String => { + if let Ok(s) = self.get_str() { + if let Ok(f) = i64::from_str(s) { + return f; + } + } + 0 + } + _ if self.fxb_type.is_vector() => self.length() as i64, + _ => 0, + } + } + try_cast_fn!(as_i32, as_i64, i32); + try_cast_fn!(as_i16, as_i64, i16); + try_cast_fn!(as_i8, as_i64, i8); + + /// Returns an f64, casting if necessary. For Maps and Vectors, their length is + /// returned. If anything fails, 0 is returned. + pub fn as_f64(&self) -> f64 { + match self.fxb_type { + FlexBufferType::Int => self.get_i64().unwrap_or_default() as f64, + FlexBufferType::UInt => self.get_u64().unwrap_or_default() as f64, + FlexBufferType::Float => self.get_f64().unwrap_or_default(), + FlexBufferType::String => { + if let Ok(s) = self.get_str() { + if let Ok(f) = f64::from_str(s) { + return f; + } + } + 0.0 + } + _ if self.fxb_type.is_vector() => self.length() as f64, + _ => 0.0, + } + } + pub fn as_f32(&self) -> f32 { + self.as_f64() as f32 + } + + /// Returns empty string if you're not trying to read a string. + pub fn as_str(&self) -> &'de str { + match self.fxb_type { + FlexBufferType::String => self.get_str().unwrap_or_default(), + FlexBufferType::Key => self.get_key().unwrap_or_default(), + _ => "", + } + } + pub fn get_vector(&self) -> Result, Error> { + if !self.fxb_type.is_vector() { + self.expect_type(FlexBufferType::Vector)?; + }; + Ok(VectorReader { + reader: self.clone(), + length: self.length(), + }) + } +} + +impl<'de> fmt::Display for Reader<'de> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use FlexBufferType::*; + match self.flexbuffer_type() { + Null => write!(f, "null"), + UInt => write!(f, "{}", self.as_u64()), + Int => write!(f, "{}", self.as_i64()), + Float => write!(f, "{}", self.as_f64()), + Key | String => write!(f, "{:?}", self.as_str()), + Bool => write!(f, "{}", self.as_bool()), + Blob => write!(f, "blob"), + Map => { + write!(f, "{{")?; + let m = self.as_map(); + let mut pairs = m.iter_keys().zip(m.iter_values()); + if let Some((k, v)) = pairs.next() { + write!(f, "{:?}: {}", k, v)?; + for (k, v) in pairs { + write!(f, ", {:?}: {}", k, v)?; + } + } + write!(f, "}}") + } + t if t.is_vector() => { + write!(f, "[")?; + let mut elems = self.as_vector().iter(); + if let Some(first) = elems.next() { + write!(f, "{}", first)?; + for e in elems { + write!(f, ", {}", e)?; + } + } + write!(f, "]") + } + _ => unreachable!("Display not implemented for {:?}", self), + } + } +} + +// TODO(cneo): Use ::from_le_bytes when we move past rustc 1.39. +fn f32_from_le_bytes(bytes: [u8; 4]) -> f32 { + let bits = ::from_le_bytes(bytes); + ::from_bits(bits) +} +fn f64_from_le_bytes(bytes: [u8; 8]) -> f64 { + let bits = ::from_le_bytes(bytes); + ::from_bits(bits) +} + +fn read_usize(buffer: &[u8], address: usize, width: BitWidth) -> usize { + let cursor = &buffer[address..]; + match width { + BitWidth::W8 => cursor[0] as usize, + BitWidth::W16 => cursor + .get(0..2) + .and_then(|s| s.try_into().ok()) + .map(::from_le_bytes) + .unwrap_or_default() as usize, + BitWidth::W32 => cursor + .get(0..4) + .and_then(|s| s.try_into().ok()) + .map(::from_le_bytes) + .unwrap_or_default() as usize, + BitWidth::W64 => cursor + .get(0..8) + .and_then(|s| s.try_into().ok()) + .map(::from_le_bytes) + .unwrap_or_default() as usize, + } +} + +fn unpack_type(ty: u8) -> Result<(FlexBufferType, BitWidth), Error> { + let w = BitWidth::try_from(ty & 3u8).map_err(|_| Error::InvalidPackedType)?; + let t = FlexBufferType::try_from(ty >> 2).map_err(|_| Error::InvalidPackedType)?; + Ok((t, w)) +} diff --git a/rust/flexbuffers/src/reader/vector.rs b/rust/flexbuffers/src/reader/vector.rs new file mode 100644 index 000000000..8ba8fe505 --- /dev/null +++ b/rust/flexbuffers/src/reader/vector.rs @@ -0,0 +1,74 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{unpack_type, Error, Reader, ReaderIterator}; +use crate::{BitWidth, FlexBufferType}; + +#[derive(Default, Clone)] +/// Allows indexing on any flexbuffer vector type, (heterogenous vector, typed vector, or fixed +/// length typed vector). +/// +/// VectorReaders may be indexed with usize, `index` returns a result type +/// which may indicate failure due to indexing out of bounds or bad data. `idx` returns a +/// Null Reader in the event of any failure. +pub struct VectorReader<'de> { + pub(super) reader: Reader<'de>, + // Cache the length because read_usize can be slow. + pub(super) length: usize, +} + +impl<'de> VectorReader<'de> { + /// Returns the number of elements in the vector. + pub fn len(&self) -> usize { + self.length + } + /// Returns true if there are 0 elements in the vector. + pub fn is_empty(&self) -> bool { + self.length == 0 + } + fn get_elem_type(&self, i: usize) -> Result<(FlexBufferType, BitWidth), Error> { + if let Some(ty) = self.reader.fxb_type.typed_vector_type() { + Ok((ty, self.reader.width)) + } else { + let types_addr = self.reader.address + self.length * self.reader.width.n_bytes(); + self.reader + .buffer + .get(types_addr + i) + .ok_or(Error::FlexbufferOutOfBounds) + .and_then(|&t| unpack_type(t)) + } + } + /// Index into a flexbuffer vector. Any errors are defaulted to Null Readers. + pub fn idx(&self, i: usize) -> Reader<'de> { + self.index(i).unwrap_or_default() + } + /// Index into a flexbuffer. + pub fn index(&self, i: usize) -> Result, Error> { + if i >= self.length { + return Err(Error::IndexOutOfBounds); + } + let (fxb_type, bw) = self.get_elem_type(i)?; + let data_address = self.reader.address + self.reader.width.n_bytes() * i; + Reader::new( + self.reader.buffer, + data_address, + fxb_type, + bw, + self.reader.width, + ) + } + pub fn iter(&self) -> ReaderIterator<'de> { + ReaderIterator::new(self.clone()) + } +} diff --git a/samples/sample_flexbuffers.rs b/samples/sample_flexbuffers.rs new file mode 100644 index 000000000..425279dfa --- /dev/null +++ b/samples/sample_flexbuffers.rs @@ -0,0 +1,169 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +extern crate flexbuffers; + +use flexbuffers::{BitWidth, Builder, Reader, ReaderError}; + +fn main() { + // Create a new Flexbuffer builder. + let mut builder = Builder::default(); + + // The root of the builder can be a singleton, map or vector. + // Our monster will be represented with a map. + let mut monster = builder.start_map(); + + // Use `push` to add elements to a vector or map. Note that it up to the programmer to ensure + // duplicate keys are avoided and the key has no null bytes. + monster.push("hp", 80); + monster.push("mana", 200); + monster.push("enraged", true); + + // Let's give our monster some weapons. Use `start_vector` to store a vector. + let mut weapons = monster.start_vector("weapons"); + + // The first weapon is a fist which has no damage so we'll store it as a string. + // Strings in Flexbuffers are utf8 encoded and are distinct from map Keys which are c strings. + weapons.push("fist"); + + // The monster also has an axe. We'll store it as a map to make it more interesting. + let mut axe = weapons.start_map(); + axe.push("name", "great axe"); + axe.push("damage", 15); + // We're done adding to the axe. + axe.end_map(); + + // The monster also has a hammer. + { + let mut hammer = weapons.start_map(); + hammer.push("name", "hammer"); + hammer.push("damage", 5); + // Instead of calling `hammer.end_map()`, we can just drop the `hammer` for the same effect. + // Vectors and maps are completed and serialized when their builders are dropped. + } + + // We're done adding weapons. + weapons.end_vector(); + + // Give the monster some money. Flexbuffers has typed vectors which are smaller than + // heterogenous vectors. Elements of typed vectors can be pushed one at a time, as above, or + // they can be passed as a slice. This will be stored as a `FlexBufferType::VectorInt`. + monster.push("coins", &[5, 10, 25, 25, 25, 100]); + + // Flexbuffer has special types for fixed-length-typed-vectors (if the length is 3 or 4 and the + // type is int, uint, or float). They're even more compact than typed vectors. + // The monster's position and Velocity will be stored as `FlexbufferType::VectorFloat3`. + monster.push("position", &[0.0; 3]); + monster.push("velocity", &[1.0, 0.0, 0.0]); + + // Give the monster bright red skin. In rust, numbers are assumed integers until proven + // otherwise. We annotate u8 to tell flexbuffers to store it as a FlexbufferType::VectorUInt4. + monster.push("color", &[255, 0, 0, 255u8]); + + // End the map at the root of the builder. This finishes the Flexbuffer. + monster.end_map(); + + // Now the buffer is free to be reused. Let's see the final buffer. + let data = builder.view(); + println!("The monster was serialized in {:?} bytes.", data.len()); + + // Let's read and verify the data. + let root = Reader::get_root(data).unwrap(); + println!("The monster: {}", root); + + let read_monster = root.as_map(); + + // What attributes does this monster have? + let attrs: Vec<_> = read_monster.iter_keys().collect(); + assert_eq!( + attrs, + vec!["coins", "color", "enraged", "hp", "mana", "position", "velocity", "weapons"] + ); + + // index into a vector or map with the `idx` method. + let read_hp = read_monster.idx("hp"); + let read_mana = read_monster.idx("mana"); + // If `idx` fails it will return a Null flexbuffer Reader + + // Use `as_T` to cast the data to your desired type. + assert_eq!(read_hp.as_u8(), 80); + assert_eq!(read_hp.as_f32(), 80.0); + // If it fails it will return T::default(). + assert_eq!(read_hp.as_str(), ""); // Its not a string. + assert_eq!(read_mana.as_i8(), 0); // 200 is not representable in i8. + assert!(read_mana.as_vector().is_empty()); // Its not a vector. + assert_eq!(read_monster.idx("foo").as_i32(), 0); // `foo` is not a monster attribute. + + // To examine how your data is stored, check the flexbuffer type and bitwidth. + assert!(read_hp.flexbuffer_type().is_int()); + assert!(read_mana.flexbuffer_type().is_int()); + // Note that mana=200 is bigger than the maximum i8 so everything in the top layer of the + // monster map is stored in 16 bits. + assert_eq!(read_hp.bitwidth(), BitWidth::W16); + assert_eq!(read_monster.idx("mana").bitwidth(), BitWidth::W16); + + // Use get_T functions if you want to ensure the flexbuffer type matches what you expect. + assert_eq!(read_hp.get_i64(), Ok(80)); + assert!(read_hp.get_u64().is_err()); + assert!(read_hp.get_vector().is_err()); + + // Analogously, the `index` method is the safe version of `idx`. + assert!(read_monster.index("hp").is_ok()); + assert_eq!( + read_monster.index("foo").unwrap_err(), + ReaderError::KeyNotFound + ); + + // Maps can also be indexed by usize. They're stored by key so `coins` are the first element. + let monster_coins = read_monster.idx(0); + // Maps and Vectors can be iterated over. + assert!(monster_coins + .as_vector() + .iter() + .map(|r| r.as_u8()) + .eq(vec![5, 10, 25, 25, 25, 100].into_iter())); + // For very speed sensitive applications, you can directly read the slice if all of the + // following are true: + // + // * The provided data buffer contains a valid flexbuffer. + // * You correctly specify the flexbuffer type and width. + // * The host machine is little endian. + // * The provided data buffer itself is aligned in memory to 8 bytes. + // + // Vec has alignment 1 so special care is needed to get your buffer's alignment to 8. + #[cfg(target_endian = "little")] + { + if monster_coins.is_aligned() { + assert_eq!( + monster_coins.get_slice::().unwrap(), + &[5, 10, 25, 25, 25, 100] + ); + } + } + + // Build the answer to life the universe and everything. Reusing a builder resets it. The + // reused internals won't need to reallocate leading to a potential 2x speedup. + builder.build_singleton(42); + + // The monster is now no more. + assert_eq!(builder.view().len(), 3); // Bytes. + + let the_answer = Reader::get_root(builder.view()).unwrap(); + assert_eq!(the_answer.as_i32(), 42); +} + +#[test] +fn test_main() { + main() +} diff --git a/samples/sample_flexbuffers_serde.rs b/samples/sample_flexbuffers_serde.rs new file mode 100644 index 000000000..359ba4583 --- /dev/null +++ b/samples/sample_flexbuffers_serde.rs @@ -0,0 +1,81 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +extern crate flexbuffers; +extern crate serde; +#[macro_use] +extern crate serde_derive; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +enum Weapon { + Fist, + Equipment { name: String, damage: i32 }, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +struct Color(u8, u8, u8, u8); + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +struct Monster { + hp: u32, + mana: i32, + enraged: bool, + weapons: Vec, + color: Color, + position: [f64; 3], + velocity: [f64; 3], + coins: Vec, +} + +fn main() { + let monster = Monster { + hp: 80, + mana: 200, + enraged: true, + color: Color(255, 255, 255, 255), + position: [0.0; 3], + velocity: [1.0, 0.0, 0.0], + weapons: vec![ + Weapon::Fist, + Weapon::Equipment { + name: "great axe".to_string(), + damage: 15, + }, + Weapon::Equipment { + name: "hammer".to_string(), + damage: 5, + }, + ], + coins: vec![5, 10, 25, 25, 25, 100], + }; + let mut s = flexbuffers::FlexbufferSerializer::new(); + monster.serialize(&mut s).unwrap(); + + let r = flexbuffers::Reader::get_root(s.view()).unwrap(); + + // Serialization is similar to JSON. Field names are stored in the buffer but are reused + // between all maps and structs. + println!("Monster stored in {:?} bytes.", s.view().len()); + println!("{}", r); + + let monster2 = Monster::deserialize(r).unwrap(); + + assert_eq!(monster, monster2); +} + +#[test] +fn test_main() { + main() +} diff --git a/tests/RustTest.bat b/tests/RustTest.bat index ba9cfd298..1473ffbed 100644 --- a/tests/RustTest.bat +++ b/tests/RustTest.bat @@ -19,5 +19,7 @@ rem TODO(rw): how do we make this script abort the calling script in appveyor? cd rust_usage_test cargo test -- --quiet || exit /b 1 -cargo run --bin=alloc_check || exit /b 1 +cargo run --bin=flatbuffers_alloc_check || exit /b 1 +cargo run --bin=flexbuffers_alloc_check || exit /b 1 +cargo run --bin=monster_example || exit /b 1 cd .. diff --git a/tests/RustTest.sh b/tests/RustTest.sh index 0a3974b91..7a7606b4f 100755 --- a/tests/RustTest.sh +++ b/tests/RustTest.sh @@ -31,7 +31,16 @@ else exit 1 fi -cargo run $TARGET_FLAG --bin=alloc_check +cargo run $TARGET_FLAG --bin=flatbuffers_alloc_check +TEST_RESULT=$? +if [[ $TEST_RESULT == 0 ]]; then + echo "OK: Rust heap alloc test passed." +else + echo "KO: Rust heap alloc test failed." + exit 1 +fi + +cargo run $TARGET_FLAG --bin=flexbuffers_alloc_check TEST_RESULT=$? if [[ $TEST_RESULT == 0 ]]; then echo "OK: Rust heap alloc test passed." diff --git a/tests/docker/languages/Dockerfile.testing.rust.1_37_0 b/tests/docker/languages/Dockerfile.testing.rust.1_40_0 similarity index 75% rename from tests/docker/languages/Dockerfile.testing.rust.1_37_0 rename to tests/docker/languages/Dockerfile.testing.rust.1_40_0 index 522937b88..849ad76ad 100644 --- a/tests/docker/languages/Dockerfile.testing.rust.1_37_0 +++ b/tests/docker/languages/Dockerfile.testing.rust.1_40_0 @@ -1,4 +1,4 @@ -FROM rust:1.37.0-slim-stretch as base +FROM rust:1.40.0-slim-stretch as base WORKDIR /code ADD . . RUN cp flatc_debian_stretch flatc diff --git a/tests/docker/languages/Dockerfile.testing.rust.big_endian.1_37_0 b/tests/docker/languages/Dockerfile.testing.rust.big_endian.1_40_0 similarity index 89% rename from tests/docker/languages/Dockerfile.testing.rust.big_endian.1_37_0 rename to tests/docker/languages/Dockerfile.testing.rust.big_endian.1_40_0 index eae0fe140..3abf8df19 100644 --- a/tests/docker/languages/Dockerfile.testing.rust.big_endian.1_37_0 +++ b/tests/docker/languages/Dockerfile.testing.rust.big_endian.1_40_0 @@ -1,4 +1,4 @@ -FROM rust:1.37.0-slim-stretch as base +FROM rust:1.40.0-slim-stretch as base RUN apt -qq update -y && apt -qq install -y \ gcc-mips-linux-gnu \ libexpat1 \ diff --git a/tests/gold_flexbuffer_example.bin b/tests/gold_flexbuffer_example.bin new file mode 100644 index 000000000..f9d24b1e3 Binary files /dev/null and b/tests/gold_flexbuffer_example.bin differ diff --git a/tests/rust_usage_test/Cargo.toml b/tests/rust_usage_test/Cargo.toml index 6178849d3..e02b0b743 100644 --- a/tests/rust_usage_test/Cargo.toml +++ b/tests/rust_usage_test/Cargo.toml @@ -5,23 +5,31 @@ authors = ["Robert Winslow ", "FlatBuffers Maintainers"] [dependencies] flatbuffers = { path = "../../rust/flatbuffers" } +flexbuffers = { path = "../../rust/flexbuffers" } [[bin]] name = "monster_example" path = "bin/monster_example.rs" [[bin]] -name = "alloc_check" -path = "bin/alloc_check.rs" +name = "flatbuffers_alloc_check" +path = "bin/flatbuffers_alloc_check.rs" +[[bin]] +name = "flexbuffers_alloc_check" +path = "bin/flexbuffers_alloc_check.rs" [dev-dependencies] quickcheck = "0.6" # TODO(rw): look into moving to criterion.rs bencher = "0.1.5" static_assertions = "1.0.0" +serde_derive = "*" +serde = "*" +rand = "*" +quickcheck_derive = "*" [[bench]] # setup for bencher -name = "flatbuffers_benchmarks" +name = "benchmarks" harness = false diff --git a/tests/rust_usage_test/benches/benchmarks.rs b/tests/rust_usage_test/benches/benchmarks.rs new file mode 100644 index 000000000..bfe63b6d1 --- /dev/null +++ b/tests/rust_usage_test/benches/benchmarks.rs @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#[macro_use] +extern crate bencher; +extern crate flatbuffers; +extern crate flexbuffers; + +mod flatbuffers_benchmarks; +mod flexbuffers_benchmarks; + +#[allow(dead_code, unused_imports)] +#[path = "../../include_test/include_test1_generated.rs"] +pub mod include_test1_generated; + +#[allow(dead_code, unused_imports)] +#[path = "../../include_test/sub/include_test2_generated.rs"] +pub mod include_test2_generated; + +#[allow(dead_code, unused_imports)] +#[path = "../../monster_test_generated.rs"] +mod monster_test_generated; +pub use monster_test_generated::my_game; + +benchmark_main!( + flatbuffers_benchmarks::benches, + flexbuffers_benchmarks::benches +); diff --git a/tests/rust_usage_test/benches/flatbuffers_benchmarks.rs b/tests/rust_usage_test/benches/flatbuffers_benchmarks.rs index d38b8b4b8..ee6d81db7 100644 --- a/tests/rust_usage_test/benches/flatbuffers_benchmarks.rs +++ b/tests/rust_usage_test/benches/flatbuffers_benchmarks.rs @@ -14,12 +14,8 @@ * limitations under the License. */ -#[macro_use] -extern crate bencher; use bencher::Bencher; -extern crate flatbuffers; - #[allow(dead_code, unused_imports)] #[path = "../../include_test/include_test1_generated.rs"] pub mod include_test1_generated; @@ -63,42 +59,69 @@ fn create_canonical_buffer_then_reset(bench: &mut Bencher) { } #[inline(always)] -fn create_serialized_example_with_generated_code(builder: &mut flatbuffers::FlatBufferBuilder, finish: bool) -> usize{ +fn create_serialized_example_with_generated_code( + builder: &mut flatbuffers::FlatBufferBuilder, + finish: bool, +) -> usize { let s0 = builder.create_string("test1"); let s1 = builder.create_string("test2"); let t0_name = builder.create_string("Barney"); let t1_name = builder.create_string("Fred"); let t2_name = builder.create_string("Wilma"); - let t0 = my_game::example::Monster::create(builder, &my_game::example::MonsterArgs{ - hp: 1000, - name: Some(t0_name), - ..Default::default() - }); - let t1 = my_game::example::Monster::create(builder, &my_game::example::MonsterArgs{ - name: Some(t1_name), - ..Default::default() - }); - let t2 = my_game::example::Monster::create(builder, &my_game::example::MonsterArgs{ - name: Some(t2_name), - ..Default::default() - }); + let t0 = my_game::example::Monster::create( + builder, + &my_game::example::MonsterArgs { + hp: 1000, + name: Some(t0_name), + ..Default::default() + }, + ); + let t1 = my_game::example::Monster::create( + builder, + &my_game::example::MonsterArgs { + name: Some(t1_name), + ..Default::default() + }, + ); + let t2 = my_game::example::Monster::create( + builder, + &my_game::example::MonsterArgs { + name: Some(t2_name), + ..Default::default() + }, + ); let mon = { let name = builder.create_string("MyMonster"); let fred_name = builder.create_string("Fred"); let inventory = builder.create_vector_direct(&[0u8, 1, 2, 3, 4]); - let test4 = builder.create_vector_direct(&[my_game::example::Test::new(10, 20), - my_game::example::Test::new(30, 40)]); - let pos = my_game::example::Vec3::new(1.0, 2.0, 3.0, 3.0, my_game::example::Color::Green, &my_game::example::Test::new(5i16, 6i8)); - let args = my_game::example::MonsterArgs{ + let test4 = builder.create_vector_direct(&[ + my_game::example::Test::new(10, 20), + my_game::example::Test::new(30, 40), + ]); + let pos = my_game::example::Vec3::new( + 1.0, + 2.0, + 3.0, + 3.0, + my_game::example::Color::Green, + &my_game::example::Test::new(5i16, 6i8), + ); + let args = my_game::example::MonsterArgs { hp: 80, mana: 150, name: Some(name), pos: Some(&pos), test_type: my_game::example::Any::Monster, - test: Some(my_game::example::Monster::create(builder, &my_game::example::MonsterArgs{ - name: Some(fred_name), - ..Default::default() - }).as_union_value()), + test: Some( + my_game::example::Monster::create( + builder, + &my_game::example::MonsterArgs { + name: Some(fred_name), + ..Default::default() + }, + ) + .as_union_value(), + ), inventory: Some(inventory), test4: Some(test4), testarrayofstring: Some(builder.create_vector(&[s0, s1])), @@ -155,7 +178,7 @@ fn traverse_serialized_example_with_generated_code(bytes: &[u8]) { } fn create_string_10(bench: &mut Bencher) { - let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1<<20); + let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1 << 20); let mut i = 0; bench.iter(|| { builder.create_string("foobarbaz"); // zero-terminated -> 10 bytes @@ -170,7 +193,7 @@ fn create_string_10(bench: &mut Bencher) { } fn create_string_100(bench: &mut Bencher) { - let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1<<20); + let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1 << 20); let s_owned = (0..99).map(|_| "x").collect::(); let s: &str = &s_owned; @@ -188,7 +211,7 @@ fn create_string_100(bench: &mut Bencher) { } fn create_byte_vector_100_naive(bench: &mut Bencher) { - let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1<<20); + let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1 << 20); let v_owned = (0u8..100).map(|i| i).collect::>(); let v: &[u8] = &v_owned; @@ -206,7 +229,7 @@ fn create_byte_vector_100_naive(bench: &mut Bencher) { } fn create_byte_vector_100_optimal(bench: &mut Bencher) { - let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1<<20); + let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1 << 20); let v_owned = (0u8..100).map(|i| i).collect::>(); let v: &[u8] = &v_owned; @@ -223,5 +246,12 @@ fn create_byte_vector_100_optimal(bench: &mut Bencher) { bench.bytes = v.len() as u64; } -benchmark_group!(benches, create_byte_vector_100_naive, create_byte_vector_100_optimal, traverse_canonical_buffer, create_canonical_buffer_then_reset, create_string_10, create_string_100); -benchmark_main!(benches); +benchmark_group!( + benches, + create_byte_vector_100_naive, + create_byte_vector_100_optimal, + traverse_canonical_buffer, + create_canonical_buffer_then_reset, + create_string_10, + create_string_100 +); diff --git a/tests/rust_usage_test/benches/flexbuffers_benchmarks.rs b/tests/rust_usage_test/benches/flexbuffers_benchmarks.rs new file mode 100644 index 000000000..1e9c516e4 --- /dev/null +++ b/tests/rust_usage_test/benches/flexbuffers_benchmarks.rs @@ -0,0 +1,295 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use bencher::Bencher; +use flexbuffers::*; + +fn push_vec_u64_to_map(b: &mut Bencher) { + let va = vec![u64::max_value() - 10; 512]; + let vb = vec![u64::max_value() - 20; 512]; + let vc = vec![u64::max_value() - 30; 512]; + let mut n = 0; + + b.iter(|| { + let mut fxb = Builder::default(); + let mut m = fxb.start_map(); + let mut ma = m.start_vector("a"); + for &a in va.iter() { + ma.push(a); + } + ma.end_vector(); + let mut mb = m.start_vector("b"); + for &b in vb.iter() { + mb.push(b); + } + mb.end_vector(); + let mut mc = m.start_vector("c"); + for &c in vc.iter() { + mc.push(c); + } + mc.end_vector(); + m.end_map(); + n = fxb.view().len(); + }); + b.bytes = n as u64; +} +fn push_vec_u64_to_map_reused(b: &mut Bencher) { + let va = vec![u64::max_value() - 10; 512]; + let vb = vec![u64::max_value() - 20; 512]; + let vc = vec![u64::max_value() - 30; 512]; + let mut fxb = Builder::default(); + let mut n = 0; + let mut go = || { + let mut m = fxb.start_map(); + let mut ma = m.start_vector("a"); + for &a in va.iter() { + ma.push(a); + } + ma.end_vector(); + let mut mb = m.start_vector("b"); + for &b in vb.iter() { + mb.push(b); + } + mb.end_vector(); + let mut mc = m.start_vector("c"); + for &c in vc.iter() { + mc.push(c); + } + mc.end_vector(); + m.end_map(); + n = fxb.view().len(); + }; + go(); // warm up allocations. + b.iter(go); + b.bytes = n as u64; +} +fn push_vec_u64_to_map_direct(b: &mut Bencher) { + let va = vec![u64::max_value() - 10; 512]; + let vb = vec![u64::max_value() - 20; 512]; + let vc = vec![u64::max_value() - 30; 512]; + let mut n = 0; + + b.iter(|| { + let mut fxb = Builder::default(); + let mut m = fxb.start_map(); + m.push("a", &va); + m.push("b", &vb); + m.push("c", &vc); + m.end_map(); + n = fxb.view().len(); + }); + b.bytes = n as u64; +} +fn push_vec_u64_to_map_direct_reused(b: &mut Bencher) { + let va = vec![u64::max_value() - 10; 512]; + let vb = vec![u64::max_value() - 20; 512]; + let vc = vec![u64::max_value() - 30; 512]; + let mut n = 0; + let mut fxb = Builder::default(); + let mut go = || { + let mut m = fxb.start_map(); + m.push("a", &va); + m.push("b", &vb); + m.push("c", &vc); + m.end_map(); + n = fxb.view().len(); + }; + go(); // warm up allocations. + b.iter(go); + b.bytes = n as u64; +} + +fn push_vec_without_indirect(b: &mut Bencher) { + let mut builder = Builder::default(); + let mut n = 0; + let mut go = || { + let mut b = builder.start_vector(); + for i in 0..1024u16 { + b.push(i); + } + b.push(i64::max_value()); + b.end_vector(); + n = builder.view().len(); + }; + go(); // warm up allocations. + b.iter(go); + b.bytes = n as u64; +} +// This isn't actually faster than the alternative but it is a lot smaller. +// Based on the above benchmarks a lot of time is stuck in the `values` stack. +fn push_vec_with_indirect(b: &mut Bencher) { + let mut builder = Builder::default(); + let mut n = 0; + let mut go = || { + let mut b = builder.start_vector(); + for i in 0..1024u16 { + b.push(i); + } + b.push(IndirectInt(i64::max_value())); + b.end_vector(); + n = builder.view().len(); + }; + go(); // warm up allocations. + b.iter(go); + b.bytes = n as u64; +} + +fn example_map<'a>(m: &mut MapBuilder<'a>) { + m.push("some_ints", &[256; 5]); + m.push("some_uints", &[256u16; 5]); + m.push("some_floats", &[256f32; 5]); + m.push("some_strings", "muahahahahaha"); +} +fn hundred_maps(b: &mut Bencher) { + let mut builder = Builder::default(); + let mut n = 0; + let mut go = || { + let mut v = builder.start_vector(); + for _ in 0..100 { + example_map(&mut v.start_map()); + } + v.end_vector(); + n = builder.view().len(); + }; + go(); // Warm up allocations. + b.iter(go); + b.bytes = n as u64; +} +fn hundred_maps_pooled(b: &mut Bencher) { + let mut builder = Builder::default(); + let mut n = 0; + let mut go = || { + let mut v = builder.start_vector(); + for _ in 0..100 { + example_map(&mut v.start_map()); + } + v.end_vector(); + n = builder.view().len(); + }; + go(); // Warm up allocations. + b.iter(go); + b.bytes = n as u64; +} +fn make_monster(mut monster: MapBuilder) { + monster.push("type", "great orc"); + monster.push("age", 100u8); + monster.push("name", "Mr. Orc"); + monster.push("coins", &[1, 25, 50, 100, 250]); + monster.push("color", &[255u8, 0, 0, 0]); + { + let mut weapons = monster.start_vector("weapons"); + { + let mut hammer = weapons.start_map(); + hammer.push("name", "hammer"); + hammer.push("damage type", "crush"); + hammer.push("damage", 20); + } + { + let mut axe = weapons.start_map(); + axe.push("name", "Great Axe"); + axe.push("damage type", "slash"); + axe.push("damage", 30); + } + } + { + let mut sounds = monster.start_vector("sounds"); + sounds.push("grr"); + sounds.push("rawr"); + sounds.push("muahaha"); + } +} +fn serialize_monsters(b: &mut Bencher) { + let mut builder = Builder::default(); + let mut n = 0; + let mut go = || { + let mut monsters = builder.start_vector(); + for _ in 0..100 { + make_monster(monsters.start_map()) + } + monsters.end_vector(); + n = builder.view().len(); + }; + go(); // Warm up allocations. + b.iter(go); + b.bytes = n as u64; +} +fn validate_monster(r: MapReader) { + assert_eq!(r.idx("type").as_str(), "great orc"); + assert_eq!(r.idx("age").as_u8(), 100); + assert_eq!(r.idx("name").as_str(), "Mr. Orc"); + assert!(r + .idx("coins") + .as_vector() + .iter() + .map(|c| c.as_i16()) + .eq([1, 25, 50, 100, 250].iter().cloned())); + assert!(r + .idx("color") + .as_vector() + .iter() + .map(|c| c.as_u8()) + .eq([255, 0, 0, 0].iter().cloned())); + + let weapons = r.idx("weapons").as_vector(); + assert_eq!(weapons.len(), 2); + + let hammer = weapons.idx(0).as_map(); + assert_eq!(hammer.idx("name").as_str(), "hammer"); + assert_eq!(hammer.idx("damage type").as_str(), "crush"); + assert_eq!(hammer.idx("damage").as_u64(), 20); + + let axe = weapons.idx(1).as_map(); + assert_eq!(axe.idx("name").as_str(), "Great Axe"); + assert_eq!(axe.idx("damage type").as_str(), "slash"); + assert_eq!(axe.idx("damage").as_u64(), 30); + + assert!(r + .idx("sounds") + .as_vector() + .iter() + .map(|s| s.as_str()) + .eq(["grr", "rawr", "muahaha"].iter().cloned())); +} +fn read_monsters(b: &mut Bencher) { + let mut builder = Builder::default(); + let mut monsters = builder.start_vector(); + for _ in 0..100 { + make_monster(monsters.start_map()); + } + monsters.end_vector(); + b.bytes = builder.view().len() as u64; + let go = || { + let r = Reader::get_root(builder.view()).unwrap().as_vector(); + assert_eq!(r.len(), 100); + for i in 0..100 { + validate_monster(r.idx(i).as_map()); + } + }; + b.iter(go); +} + +benchmark_group!( + benches, + push_vec_u64_to_map, + push_vec_u64_to_map_reused, + push_vec_u64_to_map_direct, + push_vec_u64_to_map_direct_reused, + push_vec_without_indirect, + push_vec_with_indirect, + hundred_maps, + hundred_maps_pooled, + serialize_monsters, + read_monsters, +); +benchmark_main!(benches); diff --git a/tests/rust_usage_test/bin/alloc_check.rs b/tests/rust_usage_test/bin/flatbuffers_alloc_check.rs similarity index 79% rename from tests/rust_usage_test/bin/alloc_check.rs rename to tests/rust_usage_test/bin/flatbuffers_alloc_check.rs index 26f38e3aa..c47e86e19 100644 --- a/tests/rust_usage_test/bin/alloc_check.rs +++ b/tests/rust_usage_test/bin/flatbuffers_alloc_check.rs @@ -44,7 +44,15 @@ pub use monster_test_generated::my_game; // verbatim from the test suite: fn create_serialized_example_with_generated_code(builder: &mut flatbuffers::FlatBufferBuilder) { let mon = { - let _ = builder.create_vector_of_strings(&["these", "unused", "strings", "check", "the", "create_vector_of_strings", "function"]); + let _ = builder.create_vector_of_strings(&[ + "these", + "unused", + "strings", + "check", + "the", + "create_vector_of_strings", + "function", + ]); let s0 = builder.create_string("test1"); let s1 = builder.create_string("test2"); @@ -52,21 +60,36 @@ fn create_serialized_example_with_generated_code(builder: &mut flatbuffers::Flat // can't inline creation of this Vec3 because we refer to it by reference, so it must live // long enough to be used by MonsterArgs. - let pos = my_game::example::Vec3::new(1.0, 2.0, 3.0, 3.0, my_game::example::Color::Green, &my_game::example::Test::new(5i16, 6i8)); + let pos = my_game::example::Vec3::new( + 1.0, + 2.0, + 3.0, + 3.0, + my_game::example::Color::Green, + &my_game::example::Test::new(5i16, 6i8), + ); - let args = my_game::example::MonsterArgs{ + let args = my_game::example::MonsterArgs { hp: 80, mana: 150, name: Some(builder.create_string("MyMonster")), pos: Some(&pos), test_type: my_game::example::Any::Monster, - test: Some(my_game::example::Monster::create(builder, &my_game::example::MonsterArgs{ - name: Some(fred_name), - ..Default::default() - }).as_union_value()), + test: Some( + my_game::example::Monster::create( + builder, + &my_game::example::MonsterArgs { + name: Some(fred_name), + ..Default::default() + }, + ) + .as_union_value(), + ), inventory: Some(builder.create_vector_direct(&[0u8, 1, 2, 3, 4][..])), - test4: Some(builder.create_vector_direct(&[my_game::example::Test::new(10, 20), - my_game::example::Test::new(30, 40)])), + test4: Some(builder.create_vector_direct(&[ + my_game::example::Test::new(10, 20), + my_game::example::Test::new(30, 40), + ])), testarrayofstring: Some(builder.create_vector(&[s0, s1])), ..Default::default() }; @@ -137,8 +160,13 @@ fn main() { let test4 = m.test4().unwrap(); assert_eq!(test4.len(), 2); - assert_eq!(i32::from(test4[0].a()) + i32::from(test4[1].a()) + - i32::from(test4[0].b()) + i32::from(test4[1].b()), 100); + assert_eq!( + i32::from(test4[0].a()) + + i32::from(test4[1].a()) + + i32::from(test4[0].b()) + + i32::from(test4[1].b()), + 100 + ); let testarrayofstring = m.testarrayofstring().unwrap(); assert_eq!(testarrayofstring.len(), 2); diff --git a/tests/rust_usage_test/bin/flexbuffers_alloc_check.rs b/tests/rust_usage_test/bin/flexbuffers_alloc_check.rs new file mode 100644 index 000000000..310d1a95f --- /dev/null +++ b/tests/rust_usage_test/bin/flexbuffers_alloc_check.rs @@ -0,0 +1,138 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +extern crate flexbuffers; + +use flexbuffers::*; +use std::alloc::{GlobalAlloc, Layout, System}; + +/// We take over the Rust allocator to count allocations. This is super not thread safe. +static mut NUM_ALLOCS: usize = 0; +fn current_allocs() -> usize { + unsafe { NUM_ALLOCS } +} +struct TrackingAllocator; +unsafe impl GlobalAlloc for TrackingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + NUM_ALLOCS += 1; + System.alloc(layout) + } + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + System.dealloc(ptr, layout) + } +} +#[global_allocator] +static T: TrackingAllocator = TrackingAllocator; + +/// Make some example data +fn make_monster(mut monster: MapBuilder) { + monster.push("type", "great orc"); + monster.push("age", 100u8); + monster.push("name", "Mr. Orc"); + monster.push("coins", &[1, 25, 50, 100, 250]); + monster.push("color", &[255u8, 0, 0, 0]); + { + let mut weapons = monster.start_vector("weapons"); + { + let mut hammer = weapons.start_map(); + hammer.push("name", "hammer"); + hammer.push("damage type", "crush"); + hammer.push("damage", 20); + } + { + let mut axe = weapons.start_map(); + axe.push("name", "Great Axe"); + axe.push("damage type", "slash"); + axe.push("damage", 30); + } + } + { + let mut sounds = monster.start_vector("sounds"); + sounds.push("grr"); + sounds.push("rawr"); + sounds.push("muahaha"); + } + // TODO(cneo): Directly pushing string slices has alloc. +} + +// Read back the data from make_monster. +fn validate_monster(flexbuffer: &[u8]) { + let r = Reader::get_root(flexbuffer).unwrap().as_map(); + + assert_eq!(r.idx("type").as_str(), "great orc"); + assert_eq!(r.idx("age").as_u8(), 100); + assert_eq!(r.idx("name").as_str(), "Mr. Orc"); + + let coins = r.idx("coins").as_vector(); + for (i, &c) in [1, 25, 50, 100, 250].iter().enumerate() { + assert_eq!(coins.idx(i).as_u16(), c); + } + let color = r.idx("color").as_vector(); + for (i, &c) in [255, 0, 0, 0].iter().enumerate() { + assert_eq!(color.idx(i).as_i32(), c); + } + let weapons = r.idx("weapons").as_vector(); + assert_eq!(weapons.len(), 2); + + let hammer = weapons.idx(0).as_map(); + assert_eq!(hammer.idx("name").as_str(), "hammer"); + assert_eq!(hammer.idx("damage type").as_str(), "crush"); + assert_eq!(hammer.idx("damage").as_u64(), 20); + + let axe = weapons.idx(1).as_map(); + assert_eq!(axe.idx("name").as_str(), "Great Axe"); + assert_eq!(axe.idx("damage type").as_str(), "slash"); + assert_eq!(axe.idx("damage").as_u64(), 30); + + let sounds = r.idx("sounds").as_vector(); + for (i, &s) in ["grr", "rawr", "muahaha"].iter().enumerate() { + assert_eq!(sounds.idx(i).as_str(), s); + } +} + +// This is in a separate binary than tests because taking over the global allocator is not +// hermetic and not thread safe. +fn main() { + let start_up = current_allocs(); + + // Let's build a flexbuffer from a new (cold) flexbuffer builder. + let mut builder = Builder::default(); + make_monster(builder.start_map()); + let after_warmup = current_allocs(); + + // The builder makes some allocations while warming up. + assert!(after_warmup > start_up); + assert!(after_warmup < start_up + 20); + + // A warm builder should make no allocations. + make_monster(builder.start_map()); + assert_eq!(after_warmup, current_allocs()); + + // Nor should a reader. + validate_monster(builder.view()); + assert_eq!(after_warmup, current_allocs()); + + // Do it again just for kicks. + make_monster(builder.start_map()); + validate_monster(builder.view()); + assert_eq!(after_warmup, current_allocs()); + + let final_allocs = current_allocs(); // dbg! does allocate. + dbg!(start_up, after_warmup, final_allocs); +} + +#[test] +fn no_extra_allocations() { + main() +} diff --git a/tests/rust_usage_test/bin/monster_example.rs b/tests/rust_usage_test/bin/monster_example.rs index fe172db1f..d0b75d72d 100644 --- a/tests/rust_usage_test/bin/monster_example.rs +++ b/tests/rust_usage_test/bin/monster_example.rs @@ -13,7 +13,6 @@ pub mod include_test2_generated; mod monster_test_generated; pub use monster_test_generated::my_game; - use std::io::Read; fn main() { @@ -22,7 +21,7 @@ fn main() { f.read_to_end(&mut buf).expect("file reading failed"); let monster = my_game::example::get_root_as_monster(&buf[..]); - println!("{}", monster.hp()); // `80` - println!("{}", monster.mana()); // default value of `150` + println!("{}", monster.hp()); // `80` + println!("{}", monster.mana()); // default value of `150` println!("{:?}", monster.name()); // Some("MyMonster") } diff --git a/tests/rust_usage_test/tests/flexbuffers_tests/binary_format.rs b/tests/rust_usage_test/tests/flexbuffers_tests/binary_format.rs new file mode 100644 index 000000000..ce6951192 --- /dev/null +++ b/tests/rust_usage_test/tests/flexbuffers_tests/binary_format.rs @@ -0,0 +1,536 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use flexbuffers::*; +use serde::Serialize; + +#[test] +fn store_13() { + let buf = singleton(13i32); + assert_eq!(&buf, &[13, 4, 1]); +} +#[test] +fn store_2pow20() { + let buf = singleton(1_048_576i32); + assert_eq!( + &buf, + &[ + 0, + 0, + 16, + 0, // 2^20 in LE bytes. + 1 << 2 | 2, // Int 32bit + 4 // Root width 32 bit + ] + ); +} + +#[test] +fn heterogenous_vector_of_string_because_width() { + // Each string is 32 characters. They are 256 bytes altogether. + // This forces the vector to be W16 because of the large offsets. + let test_data = [ + "0aaabbbbccccddddeeeeffffgggghhh", + "1aaabbbbccccddddeeeeffffgggghhh", + "2aaabbbbccccddddeeeeffffgggghhh", + "3aaabbbbccccddddeeeeffffgggghhh", + "4aaabbbbccccddddeeeeffffgggghhh", + "5aaabbbbccccddddeeeeffffgggghhh", + "6aaabbbbccccddddeeeeffffgggghhh", + "7aaabbbbccccddddeeeeffffgggghhh", + ]; + let mut fxb = Builder::default(); + let mut v = fxb.start_vector(); + for &s in test_data.iter() { + v.push(s); + } + v.end_vector(); + let mut expected = vec![]; + for &s in test_data.iter() { + expected.push(s.len() as u8); + expected.extend(s.bytes()); + expected.push(b'\0'); + } + expected.extend(8u16.to_le_bytes().iter()); // Length. + for i in 0..test_data.len() as u16 { + let offset = 32 * (8 - i) + 9 + i; + expected.extend(offset.to_le_bytes().iter()); + } + for _ in 0..test_data.len() { + expected.push(5 << 2 | 0); // String, W8. + } + expected.push(24); // Offset to Vector. + expected.push(10 << 2 | 1); // Vector, W16. + expected.push(1); // Root width W8. + assert_eq!(fxb.view(), expected.as_slice()); +} + +#[test] +fn store_vec_uint_16() { + let mut fxb = Builder::default(); + let mut v = fxb.start_vector(); + v.push(256u16); + v.push(257u16); + v.push(258u16); + v.push(259u16); + v.push(0u8); // This still becomes u16. + v.end_vector(); + assert_eq!( + fxb.view(), + &[ + 5, + 0, + 0, + 1, + 1, + 1, + 2, + 1, + 3, + 1, + 0, + 0, // Data + 10, // Vector offset. + 12 << 2 | 1, // (VectorUInt, W16 - referring to data). + 1, // Root width W8 - referring to vector. + ] + ); +} + +quickcheck! { + fn qc_f32(x: f32) -> bool { + let fxb = singleton(x); + let mut expected = x.to_le_bytes().to_vec(); + expected.push(3 << 2 | 2); // Float W32. + expected.push(4); // Root width W32. + println!("{:?}: {:?} vs {:?} cmp {:?}", x, &fxb, &expected, fxb==expected); + fxb == expected + } +} + +#[test] +fn singleton_vector_uint_4_16bit() { + let buf = singleton(&[4u16, 16, 64, 256]); + assert_eq!( + &buf, + &[ + 4, + 0, + 16, + 0, + 64, + 0, + 0, + 1, // Data + 8, // Vector offset. + 23 << 2 | 1, // (VectorUInt, W16 - referring to data). + 1, // Root width W8 - referring to vector. + ] + ); +} +#[test] +fn store_u64() { + let buf = singleton(u64::max_value() - 10); + assert_eq!( + &buf, + &[ + 245, + 255, + 255, + 255, + 255, + 255, + 255, + 255, // max value - 10. + 2 << 2 | 3, // (UInt, W64) + 8, // Root width W64. + ] + ); +} +#[test] +fn vector_uint4() { + let mut fxb = Builder::default(); + let mut v = fxb.start_vector(); + v.push(2u8); + v.push(3u8); + v.push(5u8); + v.push(7u8); + v.end_vector(); + assert_eq!( + &fxb.view(), + &[ + 2, + 3, + 5, + 7, // data + 4, // Root (offset) + 23 << 2 | 0, // Root type VectorUInt4, BitWidth::W8 + 1, // Root bitwidth W8 + ] + ); +} +#[test] +fn nested_vector() { + let mut fxb = Builder::default(); + let mut v = fxb.start_vector(); + v.push(0u8); + { + let mut nested = v.start_vector(); + nested.push(1u8); + nested.push(2u8); + nested.push(3u8); + } + v.push(-42i8); + v.end_vector(); + assert_eq!( + fxb.view(), + &[ + 1, + 2, + 3, // Nested vector + 3, + 0, + 5, + 214, // Root Vector: size, v[0], v[1] (offset), v[2] as u8 + 2 << 2 | 0, // v[0]: (UInt, W8) + 20 << 2 | 0, // v[1]: (VectorUInt3, W8) + 1 << 2 | 0, // v[2]: (Int, W8) + 6, // Root points to Root vector + 10 << 2 | 0, // Root type and width (Vector, W8) + 1, // Root bytes + ] + ) +} + +#[test] +fn nested_vector_push_direct() { + let mut fxb = Builder::default(); + let mut v = fxb.start_vector(); + v.push(0u8); + v.push(&[1u8, 2, 3]); + v.push(-42i8); + v.end_vector(); + assert_eq!( + fxb.view(), + &[ + 1, + 2, + 3, // Nested VectorUInt3 + 3, + 0, + 5, + 214, // Root Vector: size, v[0], v[1] (offset), v[2] as u8 + 2 << 2 | 0, // v[0]: (UInt, W8) + 20 << 2 | 0, // v[1]: (VectorUInt3, W8) + 1 << 2 | 0, // v[2]: (Int, W8) + 6, // Root points to Root vector + 10 << 2 | 0, // Root type and width (Vector, W8) + 1, // Root bytes + ] + ) +} +#[test] +fn store_map_index_into_it() { + let mut fxb = Builder::default(); + { + let mut m = fxb.start_map(); + m.push("foo", 17u8); + m.push("bar", 33u16); + m.push("baz", 41u32); + } + assert_eq!( + fxb.view(), + &[ + b'f', + b'o', + b'o', + b'\0', + b'b', + b'a', + b'r', + b'\0', + b'b', + b'a', + b'z', + b'\0', + 3, + 9, + 6, + 15, // Keys vector (note "bar" < "baz" < "foo"). + 3, + 1, + 3, // map prefix + 33, + 41, + 17, // values + 8, + 8, + 8, // types (UInt, W8) ~ (2 << 2 | 0) + 6, // Offset to map (root) + 9 << 2 | 0, // Root type (map) + 1, // Root bytes + ] + ); +} +#[test] +fn utf8_snowman() { + let buf = singleton("snowman ☃︎"); + assert_eq!( + &buf, + &[ + 14, // Byte length (besides extra null terminator). + b's', + b'n', + b'o', + b'w', + b'm', + b'a', + b'n', + b' ', + 226, + 152, + 131, // snowman bytes + 239, + 184, + 142, // UTF Variation selector 15 + 0, // extra null terminator. + 15, // Offset to string start. + 5 << 2, // String, W8 + 1, // Root bytes + ] + ); + let r = Reader::get_root(&buf).unwrap(); + assert_eq!(r.get_str(), Ok("snowman ☃︎")); +} +#[test] +fn indirect_numbers() { + let mut fxb = Builder::default(); + let mut v = fxb.start_vector(); + v.push(IndirectUInt(u64::max_value())); + v.push(IndirectInt(i64::min_value())); + // TODO(cneo): Something about Float EPSILON and casting leads to a different binary format. + v.push(IndirectFloat(std::f64::consts::PI)); + v.push(0u32); // This is stored in 8 bits instead of 64 because of indirection. + v.end_vector(); + assert_eq!( + fxb.view(), + vec![ + 255, + 255, + 255, + 255, + 255, + 255, + 255, + 255, // u64 max + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 128, // i64 min value + 24, + 45, + 68, + 84, + 251, + 33, + 9, + 64, // f64 PI. + 4, // Vector length + 25, + 18, + 11, + 0, // offsets to the indirect numbers and zero. + 7 << 2 | 3, // IndirectUInt 64 bit + 6 << 2 | 3, // IndirectInt 64 bit + 8 << 2 | 3, // IndirectFloat 64 bit + 2 << 2 | 0, // (inline) UInt 8 bit + 8, // Offset to Root. + 10 << 2 | 0, // Vector 8 bit + 1, // 1 byte root + ] + .as_slice() + ) +} +#[test] +fn indirect_2p5x_smaller() { + let mut builder = Builder::default(); + let mut v = builder.start_vector(); + for i in 0..512 { + v.push(i); + } + v.push(i64::max_value()); + v.end_vector(); + let len_without_indirect = builder.view().len() as f32; + + let mut v = builder.start_vector(); + for i in 0..512 { + v.push(i); + } + v.push(IndirectInt(i64::max_value())); + v.end_vector(); + let len_with_indirect = builder.view().len() as f32; + dbg!(len_with_indirect, len_without_indirect); + assert!(len_with_indirect * 2.5 < len_without_indirect); +} +#[test] +fn key_pool() { + let mut builder = Builder::default(); + let mut vector = builder.start_vector(); + for _ in 0..2 { + let mut m = vector.start_map(); + m.push("a", 42u8); + m.push("b", 42u8); + m.push("c", 42u8); + } + vector.end_vector(); + + assert_eq!( + builder.view(), + vec![ + b'a', + b'\0', + b'b', + b'\0', + b'c', + b'\0', + 3, + 7, + 6, + 5, // Key vector 0 + 3, + 1, + 3, + 42, + 42, + 42, + 2 << 2, + 2 << 2, + 2 << 2, // Map 0. + 3, + 20, + 19, + 18, // Key vector 1 (shares keys with key vector 0). + 3, + 1, + 3, + 42, + 42, + 42, + 2 << 2, + 2 << 2, + 2 << 2, // Map 1. + 2, + 20, + 8, + 9 << 2, + 9 << 2, // Vector containing the maps. + 4, + 10 << 2, + 1, // Root. + ] + .as_slice() + ); +} + +#[test] +fn serialize_unit() { + #[derive(Serialize)] + struct Foo; + let mut s = FlexbufferSerializer::new(); + Foo.serialize(&mut s).unwrap(); + assert_eq!(s.view(), &[0, 0, 1]); +} + +#[test] +fn serialize_i8() { + let mut s = FlexbufferSerializer::new(); + 13i8.serialize(&mut s).unwrap(); + assert_eq!(s.view(), &[13, 4, 1]); +} +#[test] +fn serialize_tuple_struct_i8() { + #[derive(Serialize)] + struct Foo(i32); + let mut s = FlexbufferSerializer::new(); + Foo(13).serialize(&mut s).unwrap(); + assert_eq!(s.view(), &[13, 4, 1]); +} +#[test] +fn serialize_tuple_tuple_struct_i8_is_inlined() { + #[derive(Serialize)] + struct Foo(i32); + #[derive(Serialize)] + struct Bar(Foo); + let mut s = FlexbufferSerializer::new(); + Bar(Foo(13)).serialize(&mut s).unwrap(); + assert_eq!(s.view(), &[13, 4, 1]); +} +#[test] +fn align_8byte() { + let mut b = Builder::default(); + let mut v = b.start_vector(); + v.push(IndirectUInt(42)); + v.push(&[u64::max_value(); 2]); + v.end_vector(); + assert_eq!( + b.view()[..16], + [ + 42, 0, 0, 0, 0, 0, 0, 0, // padding + 255, 255, 255, 255, 255, 255, 255, 255, // the first u64 max value. + ] + ); +} +#[test] +fn align_4byte() { + let mut b = Builder::default(); + let mut v = b.start_vector(); + v.push(IndirectUInt(42)); + v.push(&[u32::max_value(); 2]); + v.end_vector(); + assert_eq!( + b.view()[..8], + [ + 42, 0, 0, 0, // padding + 255, 255, 255, 255, // the first u32 max value. + ] + ); +} +#[test] +fn align_2byte() { + let mut b = Builder::default(); + let mut v = b.start_vector(); + v.push(IndirectUInt(42)); + v.push(&[u16::max_value(); 2]); + v.end_vector(); + assert_eq!( + b.view()[..4], + [ + 42, 0, // padding + 255, 255, // the first u16 max value. + ] + ); +} +#[test] +fn align_1byte() { + let mut b = Builder::default(); + let mut v = b.start_vector(); + v.push(IndirectUInt(42)); + v.push(&[u8::max_value(); 2]); + v.end_vector(); + assert_eq!(b.view()[..2], [42, 255]); // No padding. +} diff --git a/tests/rust_usage_test/tests/flexbuffers_tests/interop.rs b/tests/rust_usage_test/tests/flexbuffers_tests/interop.rs new file mode 100644 index 000000000..54ae1fded --- /dev/null +++ b/tests/rust_usage_test/tests/flexbuffers_tests/interop.rs @@ -0,0 +1,50 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use flexbuffers::*; + +#[test] +fn read_golden_flexbuffer() { + let s = + std::fs::read("../gold_flexbuffer_example.bin").expect("Unable to read golden flexbuffer."); + let r = Reader::get_root(&s).unwrap(); + let m = r.as_map(); + + let vec = m.idx("vec").as_vector(); + assert_eq!(vec.idx(0).as_i8(), -100); + assert_eq!(vec.idx(1).as_str(), "Fred"); + assert_eq!(vec.idx(2).as_f32(), 4.0); + assert_eq!(vec.idx(3).as_blob(), Blob(&[77])); + assert_eq!(vec.idx(4).flexbuffer_type(), FlexBufferType::Bool); + assert_eq!(vec.idx(4).as_bool(), false); + assert_eq!(vec.idx(5).as_f64(), 4.0); + + let bar = m.idx("bar").as_vector(); + for (i, &x) in [1, 2, 3].iter().enumerate() { + assert_eq!(bar.idx(i).as_i8(), x); + } + let bar3 = m.idx("bar3").as_vector(); + for (i, &x) in [1, 2, 3].iter().enumerate() { + assert_eq!(bar3.idx(i).as_i8(), x); + } + let bools = m.idx("bools").as_vector(); + for (i, &b) in [true, false, true, false].iter().enumerate() { + assert_eq!(bools.idx(i).as_bool(), b) + } + + assert_eq!(m.idx("bool").as_bool(), true); + assert_eq!(m.idx("foo").as_f64(), 100.0); + let mymap = m.idx("mymap").as_map(); + assert_eq!(mymap.idx("foo").as_str(), "Fred"); +} diff --git a/tests/rust_usage_test/tests/flexbuffers_tests/mod.rs b/tests/rust_usage_test/tests/flexbuffers_tests/mod.rs new file mode 100644 index 000000000..2fccdb312 --- /dev/null +++ b/tests/rust_usage_test/tests/flexbuffers_tests/mod.rs @@ -0,0 +1,19 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod binary_format; +mod interop; +mod other_api; +mod qc_serious; +mod rwyw; diff --git a/tests/rust_usage_test/tests/flexbuffers_tests/other_api.rs b/tests/rust_usage_test/tests/flexbuffers_tests/other_api.rs new file mode 100644 index 000000000..430cae5e3 --- /dev/null +++ b/tests/rust_usage_test/tests/flexbuffers_tests/other_api.rs @@ -0,0 +1,190 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use flexbuffers::*; +use quickcheck::QuickCheck; + +#[test] +fn qc_reader_no_crash() { + fn no_crash(xs: Vec) -> bool { + let r = Reader::get_root(&xs); + r.is_err() || r.is_ok() + } + QuickCheck::new() + .min_tests_passed(10_000_000) + .quicktest(no_crash as fn(Vec) -> bool) + .unwrap(); + + no_crash(vec![0, 10 << 2 | 2, 0]); +} +#[test] +fn as_num() { + let mut fxb = Builder::default(); + let mut m = fxb.start_map(); + m.push("a", &[-1i8, -2, -3, -4]); + m.push("b", 250i64); + m.push("c", 5000u16); + m.end_map(); + + let r = Reader::get_root(fxb.view()).unwrap(); + assert_eq!(r.as_i8(), 3); // length. + assert_eq!(r.as_i16(), 3); + assert_eq!(r.as_i32(), 3); + assert_eq!(r.as_i64(), 3); + assert_eq!(r.as_u8(), 3); + assert_eq!(r.as_u16(), 3); + assert_eq!(r.as_u32(), 3); + assert_eq!(r.as_u64(), 3); + assert_eq!(r.as_f32(), 3.0); + assert_eq!(r.as_f64(), 3.0); + + let m = r.as_map(); + let a = m.index("a").unwrap(); + assert_eq!(a.as_f32(), 4.0); // length. + assert_eq!(a.as_f64(), 4.0); // length. + assert_eq!(a.as_vector().idx(0).as_i8(), -1); + assert_eq!(a.as_vector().idx(1).as_i16(), -2); + assert_eq!(a.as_vector().idx(2).as_i32(), -3); + assert_eq!(a.as_vector().idx(3).as_i64(), -4); + + let b = m.index("b").unwrap(); + assert_eq!(b.as_u8(), 250); + assert_eq!(b.as_u16(), 250); + assert_eq!(b.as_u32(), 250); + assert_eq!(b.as_u64(), 250); + assert_eq!(b.as_i8(), 0); // overflow + assert_eq!(b.as_i16(), 250); + assert_eq!(b.as_i32(), 250); + assert_eq!(b.as_i64(), 250); + + let c = m.index("c").unwrap(); + assert_eq!(c.as_i64(), 5000); + assert_eq!(c.as_u64(), 5000); + assert_eq!(c.as_f32(), 5000.0); + assert_eq!(c.as_u8(), 0); // overflow + assert_eq!(c.as_u16(), 5000); + assert_eq!(c.as_u32(), 5000); + assert_eq!(c.as_u64(), 5000); + assert_eq!(c.as_i8(), 0); // overflow + assert_eq!(c.as_i16(), 5000); + assert_eq!(c.as_i32(), 5000); + assert_eq!(c.as_i64(), 5000); +} +#[test] +fn string_as_num() { + let mut fxb = Builder::default(); + let mut v = fxb.start_vector(); + v.push("3.1415"); + v.push("9.001e3"); + v.push("42"); + v.end_vector(); + let r = Reader::get_root(fxb.view()).unwrap(); + + let v0 = r.as_vector().idx(0); + assert_eq!(v0.as_f64(), 3.1415); + assert_eq!(v0.as_f32(), 3.1415); + assert_eq!(v0.as_u8(), 0); + assert_eq!(v0.as_u16(), 0); + assert_eq!(v0.as_u32(), 0); + assert_eq!(v0.as_u64(), 0); + assert_eq!(v0.as_i8(), 0); + assert_eq!(v0.as_i16(), 0); + assert_eq!(v0.as_i32(), 0); + assert_eq!(v0.as_i64(), 0); + + let v1 = r.as_vector().idx(1); + assert_eq!(v1.as_f64(), 9001.0); + assert_eq!(v1.as_f32(), 9001.0); + assert_eq!(v1.as_u8(), 0); + assert_eq!(v1.as_u16(), 0); + assert_eq!(v1.as_u32(), 0); + assert_eq!(v1.as_u64(), 0); + assert_eq!(v1.as_i8(), 0); + assert_eq!(v1.as_i16(), 0); + assert_eq!(v1.as_i32(), 0); + assert_eq!(v1.as_i64(), 0); + assert_eq!(v1.as_i32(), 0); + + let v2 = r.as_vector().idx(2); + assert_eq!(v2.as_f64(), 42.0); + assert_eq!(v2.as_f32(), 42.0); + assert_eq!(v2.as_u8(), 42); + assert_eq!(v2.as_u16(), 42); + assert_eq!(v2.as_u32(), 42); + assert_eq!(v2.as_u64(), 42); + assert_eq!(v2.as_i8(), 42); + assert_eq!(v2.as_i16(), 42); + assert_eq!(v2.as_i32(), 42); + assert_eq!(v2.as_i64(), 42); + assert_eq!(v2.as_i32(), 42); +} +#[test] +fn null_reader() { + let n = Reader::default(); + assert_eq!(n.as_i8(), 0); + assert_eq!(n.as_i16(), 0); + assert_eq!(n.as_i32(), 0); + assert_eq!(n.as_i64(), 0); + assert_eq!(n.as_u8(), 0); + assert_eq!(n.as_u16(), 0); + assert_eq!(n.as_u32(), 0); + assert_eq!(n.as_u64(), 0); + assert_eq!(n.as_f32(), 0.0); + assert_eq!(n.as_f64(), 0.0); + assert!(n.get_i64().is_err()); + assert!(n.get_u64().is_err()); + assert!(n.get_f64().is_err()); + assert!(n.as_vector().is_empty()); + assert!(n.as_map().is_empty()); + assert_eq!(n.as_vector().idx(1).flexbuffer_type(), FlexBufferType::Null); + assert_eq!(n.as_map().idx("1").flexbuffer_type(), FlexBufferType::Null); +} +#[test] +fn get_root_deref_oob() { + let s = &[ + 4, // Deref out of bounds + (FlexBufferType::Vector as u8) << 2 | BitWidth::W8 as u8, + 1, + ]; + assert!(Reader::get_root(s).is_err()); +} +#[test] +fn get_root_deref_u64() { + let s = &[ + 0, + 0, + (FlexBufferType::IndirectUInt as u8) << 2 | BitWidth::W64 as u8, + 1, + ]; + // The risk of crashing is reading 8 bytes from index 0. + assert_eq!(Reader::get_root(s).unwrap().as_u64(), 0); +} + +#[test] +#[should_panic] +fn build_map_panic_on_repeated_key() { + let mut b = Builder::default(); + let mut m = b.start_map(); + m.push("foo", 5u8); + m.push("foo", 6u8); + m.end_map(); +} +#[test] +#[should_panic] +fn build_map_panic_on_internal_null() { + let mut b = Builder::default(); + let mut m = b.start_map(); + m.push("foo\0", 5u8); + m.end_map(); +} diff --git a/tests/rust_usage_test/tests/flexbuffers_tests/qc_serious.rs b/tests/rust_usage_test/tests/flexbuffers_tests/qc_serious.rs new file mode 100644 index 000000000..3533a4a80 --- /dev/null +++ b/tests/rust_usage_test/tests/flexbuffers_tests/qc_serious.rs @@ -0,0 +1,146 @@ +#![allow(unused_imports)] + +use super::rwyw::NonNullString; +use flexbuffers::*; +use quickcheck::{Arbitrary, Gen}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +enum Enum { + Unit, + U8(u8), + U16(u16), + U32(u32), + U64(u64), + Us(u8, u16, u32, u64), + I8(i8), + I16(i16), + I32(i32), + I64(i64), + Is(i8, i16, i32, i64), + F32(f32), + F64(f64), + Fs(f32, f64), + String(String), + Strings(String, String), + Everything(u8, u16, u32, u64, i8, i16, i32, i64, f32, f64, String), + Arrays { + a: Array3, + b: Array4, + c: Array2, + }, +} + +// There is some upstream bug in deriving Arbitrary for Enum so we manually implement it here. +impl Arbitrary for Enum { + fn arbitrary(g: &mut G) -> Self { + match g.gen_range(0, 18) { + 0 => Enum::Unit, + 1 => Enum::U8(::arbitrary(g)), + 2 => Enum::U16(::arbitrary(g)), + 3 => Enum::U32(::arbitrary(g)), + 4 => Enum::U64(::arbitrary(g)), + 5 => { + let (a, b, c, d) = <(u8, u16, u32, u64)>::arbitrary(g); + Enum::Us(a, b, c, d) + } + 6 => Enum::I8(::arbitrary(g)), + 7 => Enum::I16(::arbitrary(g)), + 8 => Enum::I32(::arbitrary(g)), + 9 => Enum::I64(::arbitrary(g)), + 10 => { + let (a, b, c, d) = <(i8, i16, i32, i64)>::arbitrary(g); + Enum::Is(a, b, c, d) + } + 11 => Enum::F32(::arbitrary(g)), + 12 => Enum::F64(::arbitrary(g)), + 13 => { + let (a, b) = <(f32, f64)>::arbitrary(g); + Enum::Fs(a, b) + } + 14 => Enum::String(String::arbitrary(g)), + 15 => { + let (a, b) = <(String, String)>::arbitrary(g); + Enum::Strings(a, b) + } + 16 => Enum::Everything( + ::arbitrary(g), + ::arbitrary(g), + ::arbitrary(g), + ::arbitrary(g), + ::arbitrary(g), + ::arbitrary(g), + ::arbitrary(g), + ::arbitrary(g), + ::arbitrary(g), + ::arbitrary(g), + ::arbitrary(g), + ), + 17 => { + let a = Array3::arbitrary(g); + let b = Array4::arbitrary(g); + let c = Array2::arbitrary(g); + Enum::Arrays { a, b, c } + } + _ => unreachable!(), + } + } +} + +#[derive(Debug, Clone, Arbitrary, PartialEq, Serialize, Deserialize)] +struct Unit; + +#[derive(Debug, Clone, Arbitrary, PartialEq, Serialize, Deserialize)] +struct NewType(bool); + +#[derive(Debug, Clone, Arbitrary, PartialEq, Serialize, Deserialize)] +struct Tuple(bool, u8, i16, f32, String); + +#[derive(Debug, Clone, Arbitrary, PartialEq, Serialize, Deserialize)] +struct Struct { + a: Vec, + b: BTreeMap, + c: Tuple, + d: (Unit, Unit), + e: Array4, +} + +#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize)] +struct Array2([A; 2]); +#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize)] +struct Array3([A; 3]); +#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize)] +struct Array4([A; 4]); + +impl Arbitrary for Array2 { + fn arbitrary(g: &mut G) -> Self { + Array2([A::arbitrary(g), A::arbitrary(g)]) + } +} +impl Arbitrary for Array3 { + fn arbitrary(g: &mut G) -> Self { + Array3([A::arbitrary(g), A::arbitrary(g), A::arbitrary(g)]) + } +} +impl Arbitrary for Array4 { + fn arbitrary(g: &mut G) -> Self { + Array4([ + A::arbitrary(g), + A::arbitrary(g), + A::arbitrary(g), + A::arbitrary(g), + ]) + } +} + +quickcheck! { + fn qc_serious(x: Struct) -> bool { + let mut s = FlexbufferSerializer::new(); + x.serialize(&mut s).unwrap(); + let r = Reader::get_root(s.view()).unwrap(); + println!("{}", r); + let x2 = Struct::deserialize(r).unwrap(); + x == x2 + } +} diff --git a/tests/rust_usage_test/tests/flexbuffers_tests/rwyw.rs b/tests/rust_usage_test/tests/flexbuffers_tests/rwyw.rs new file mode 100644 index 000000000..a83223a28 --- /dev/null +++ b/tests/rust_usage_test/tests/flexbuffers_tests/rwyw.rs @@ -0,0 +1,499 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Read what you wrote. +use flexbuffers::*; +use quickcheck; +use serde::{Deserialize, Serialize}; + +// TODO(cneo): Upstream this to the quickcheck crate. Also, write a macro to derive Arbitrary. +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Serialize, Deserialize)] +pub struct NonNullString(String); +impl quickcheck::Arbitrary for NonNullString { + fn arbitrary(g: &mut G) -> Self { + let size = std::cmp::min(1, usize::arbitrary(g)); + NonNullString( + (0..) + .map(|_| ::arbitrary(g)) + .filter(|&b| b != '\0') + .take(size) + .collect(), + ) + } +} + +quickcheck! { + fn qc_vec_bool(xs: Vec) -> bool { + let mut builder = Builder::default(); + let mut v = builder.start_vector(); + for &x in &xs { + v.push(x); + } + v.end_vector(); + let r = Reader::get_root(&builder.view()).unwrap().as_vector(); + xs.iter().enumerate().all(|(i, &x)| r.index(i).unwrap().get_bool().unwrap() == x) + } + fn qc_vec_uint(xs: Vec) -> bool { + let mut builder = Builder::default(); + let mut v = builder.start_vector(); + for &x in &xs { + v.push(x); + } + v.end_vector(); + let r = Reader::get_root(&builder.view()).unwrap().as_vector(); + xs.iter().enumerate().all(|(i, &x)| r.idx(i).as_u64() == x) + } + fn qc_vec_int(xs: Vec) -> bool { + let mut builder = Builder::default(); + let mut v = builder.start_vector(); + for &x in &xs { + v.push(x); + } + v.end_vector(); + let r = Reader::get_root(&builder.view()).unwrap().as_vector(); + xs.iter().enumerate().all(|(i, &x)| r.idx(i).as_i64() == x) + } + fn qc_vec_float(xs: Vec) -> bool { + let mut builder = Builder::default(); + let mut v = builder.start_vector(); + for &x in &xs { + v.push(x); + } + v.end_vector(); + let r = Reader::get_root(&builder.view()).unwrap().as_vector(); + xs.iter().enumerate().all(|(i, &x)| (r.idx(i).as_f64() - x).abs() < std::f64::EPSILON) + } + fn qc_vec_string(xs: Vec) -> bool { + let mut builder = Builder::default(); + let mut v = builder.start_vector(); + for x in &xs { + v.push(x as &str); + } + v.end_vector(); + let r = Reader::get_root(&builder.view()).unwrap().as_vector(); + xs.iter().enumerate().all(|(i, x)| (r.idx(i).as_str() == x)) + } + fn qc_map_int(xs: std::collections::BTreeMap) -> bool { + let mut builder = Builder::default(); + let mut m = builder.start_map(); + for (k, &v) in &xs { + m.push(&k.0, v); + } + m.end_map(); + let r = Reader::get_root(&builder.view()).unwrap().as_map(); + xs.iter().enumerate().all(|(i, (k, &v))| { + r.idx(i).as_i64() == v && r.idx(k.0.as_str()).as_i64() == v + }) + } + fn qc_map_string(xs: std::collections::BTreeMap) -> bool { + let mut builder = Builder::default(); + let mut m = builder.start_map(); + for (k, v) in &xs { + m.push(&k.0, v as &str); + } + m.end_map(); + let r = Reader::get_root(&builder.view()).unwrap().as_map(); + xs.iter().enumerate().all(|(i, (k, v))| { + r.idx(i).as_str() == v && r.idx(k.0.as_str()).as_str() == v + }) + } + fn qc_blob(xs: Vec>) -> bool { + let mut builder = Builder::default(); + let mut v = builder.start_vector(); + for x in &xs { + v.push(Blob(&x)); + } + v.end_vector(); + let r = Reader::get_root(&builder.view()).unwrap().as_vector(); + xs.iter().enumerate().all( + |(i, x)| r.idx(i).get_blob().unwrap().0.iter().eq(x.iter()) + ) + } + fn qc_serde_ints( + u8s: Vec, + u16s: Vec, + u32s: Vec, + u64s: Vec, + i8s: Vec, + i16s: Vec, + i32s: Vec, + i64s: Vec + ) -> bool { + #[derive(Serialize, Deserialize, PartialEq)] + struct Foo { + u8s: Vec, + u16s: Vec, + u32s: Vec, + u64s: Vec, + i8s: Vec, + i16s: Vec, + i32s: Vec, + i64s: Vec, + } + let mut ser = FlexbufferSerializer::new(); + let foo1 = Foo { u8s, u16s, u32s, u64s, i8s, i16s, i32s, i64s }; + foo1.serialize(&mut ser).unwrap(); + let r = Reader::get_root(ser.view()).unwrap(); + let foo2 = Foo::deserialize(r).unwrap(); + foo1 == foo2 + } + fn qc_serde_others( + bools: Vec, + strings: Vec, + f32s: Vec, + f64s: Vec + ) -> bool { + #[derive(Serialize, Deserialize, PartialEq)] + struct Foo { + bools: Vec, + strings: Vec, + f32s: Vec, + f64s: Vec, + } + let mut ser = FlexbufferSerializer::new(); + let foo1 = Foo { bools, strings, f32s, f64s }; + foo1.serialize(&mut ser).unwrap(); + let r = Reader::get_root(ser.view()).unwrap(); + let foo2 = Foo::deserialize(r).unwrap(); + foo1 == foo2 + } + fn qc_serde_others2( + bools: Vec, + strings: Vec, + f32s: Vec, + f64s: Vec + ) -> bool { + #[derive(Serialize, Deserialize, PartialEq)] + struct Foo (Vec, Vec, Vec, Vec); + let mut ser = FlexbufferSerializer::new(); + let foo1 = Foo(bools, strings, f32s, f64s); + foo1.serialize(&mut ser).unwrap(); + let r = Reader::get_root(ser.view()).unwrap(); + let foo2 = Foo::deserialize(r).unwrap(); + foo1 == foo2 + } + +} + +#[test] +fn empty_vectors() { + #[derive(PartialEq, Serialize, Deserialize, Default, Debug)] + struct Foo(Vec, Vec); + let foo1 = Foo::default(); + let mut s = FlexbufferSerializer::new(); + foo1.serialize(&mut s).unwrap(); + dbg!(s.view()); + let r = Reader::get_root(s.view()).unwrap(); + let foo2 = Foo::deserialize(r).unwrap(); + assert_eq!(foo1, foo2); +} + +#[test] +fn string() { + let mut builder = Builder::default(); + let mut v = builder.start_vector(); + v.push("foo"); + v.push("barrr"); + v.push("bazzzzzz"); + v.end_vector(); + let r = Reader::get_root(&builder.view()).unwrap().as_vector(); + assert_eq!(r.idx(0).as_str(), "foo"); + assert_eq!(r.idx(1).as_str(), "barrr"); + assert_eq!(r.idx(2).as_str(), "bazzzzzz"); +} + +#[test] +fn store_13() { + let finished = singleton::(13); + let r = Reader::get_root(&finished).unwrap(); + assert_eq!(r.as_i32(), 13); +} +#[test] +fn singleton_vector_uint_4_16bit() { + let mut builder = Builder::default(); + let mut v = builder.start_vector(); + v.push(2u8); + v.push(3u8); + v.push(5u8); + v.end_vector(); + let buf1 = builder.view(); + let buf2 = singleton(&[2u8, 3, 5]); + assert_eq!(buf1, buf2.as_slice()); + + let r = Reader::get_root(&buf1).unwrap().as_vector(); + assert_eq!(r.idx(0).get_u64(), Ok(2)); + assert_eq!(r.idx(1).get_u64(), Ok(3)); + assert_eq!(r.idx(2).get_u64(), Ok(5)); + assert_eq!(r.index(3).unwrap_err(), ReaderError::IndexOutOfBounds); +} +#[test] +fn vector_uint4() { + let mut fxb = Builder::default(); + let mut v = fxb.start_vector(); + v.push(2u8); + v.push(3u8); + v.push(5u8); + v.push(7u8); + v.end_vector(); + let r = Reader::get_root(&fxb.view()).unwrap(); + let v = r.as_vector(); + assert_eq!(v.idx(0).get_u64(), Ok(2)); + assert_eq!(v.idx(1).get_u64(), Ok(3)); + assert_eq!(v.idx(2).get_u64(), Ok(5)); + assert_eq!(v.idx(3).get_u64(), Ok(7)); + assert!(v.index(4).is_err()); + #[cfg(target_endian = "little")] + { + assert_eq!(r.get_slice::().unwrap(), [2, 3, 5, 7]); + } +} +#[test] +fn store_and_read_blob() { + let mut fxb = Builder::default(); + let mut v = fxb.start_vector(); + v.push(Blob(&[1, 2, 3, 4])); + v.push(Blob(&[5, 6, 7])); + v.end_vector(); + + let r = Reader::get_root(&fxb.view()).unwrap().as_vector(); + assert_eq!(r.idx(0).get_blob(), Ok(Blob(&[1, 2, 3, 4]))); + assert_eq!(r.idx(1).get_blob(), Ok(Blob(&[5, 6, 7]))); +} +#[test] +fn map_64bit() { + let mut fxb = Builder::default(); + let mut m = fxb.start_map(); + m.push("a", 257u16); + m.push("b", u64::max_value() - 3); + m.end_map(); + + let r = Reader::get_root(&fxb.view()).unwrap().as_map(); + assert_eq!(r.idx("a").as_u16(), 257); + assert_eq!(r.idx("b").as_u64(), u64::max_value() - 3); +} +#[test] +fn index_map() { + let mut fxb = Builder::default(); + let mut m = fxb.start_map(); + m.push("foo", 17u8); + m.push("bar", 33u16); + m.push("baz", 41u32); + m.end_map(); + + let r = Reader::get_root(fxb.view()).unwrap().as_map(); + assert_eq!(r.idx(0).get_u64(), Ok(33)); + assert_eq!(r.idx(1).get_u64(), Ok(41)); + assert_eq!(r.idx(2).as_u8(), 17); + assert_eq!(r.index(3).unwrap_err(), ReaderError::IndexOutOfBounds); + + assert_eq!(r.idx("bar").as_u64(), 33); + assert_eq!(r.idx("baz").as_u32(), 41); + assert_eq!(r.idx("foo").as_u16(), 17); + assert_eq!(r.index("???").unwrap_err(), ReaderError::KeyNotFound); +} + +#[test] +fn map_strings() { + let mut fxb = Builder::default(); + { + let mut m = fxb.start_map(); + let mut a = m.start_vector("a"); + for &s in ["b", "c", "d", "e"].iter() { + a.push(s); + } + a.end_vector(); + let mut f = m.start_vector("f"); + for &s in ["gh", "ij"].iter() { + f.push(s); + } + } + let r = Reader::get_root(fxb.view()).unwrap().as_map(); + let a = r.idx("a").as_vector(); + + assert_eq!(a.idx(0).as_str(), "b"); + assert_eq!(a.idx(1).as_str(), "c"); + assert_eq!(a.idx(2).as_str(), "d"); + assert_eq!(a.idx(3).as_str(), "e"); + + let f = r.idx("f").as_vector(); + assert_eq!(f.idx(0).as_str(), "gh"); + assert_eq!(f.idx(1).as_str(), "ij"); + + // Defaults to empty string for index errors. + assert_eq!(r.idx("a").as_vector().idx(4).as_str(), ""); + assert_eq!(r.idx("b").as_vector().idx(2).as_str(), ""); + assert_eq!(r.idx("c").as_str(), ""); +} + +#[test] +fn store_u64() { + let finished = singleton(u64::max_value() - 10); + let r = Reader::get_root(&finished).unwrap(); + assert_eq!(r.get_u64(), Ok(u64::max_value() - 10)); +} +#[test] +fn store_indirects() { + let mut b = Builder::default(); + let mut v = b.start_vector(); + v.push(IndirectInt(-42)); + v.push(IndirectUInt(9000)); + v.push(IndirectFloat(3.14)); + v.end_vector(); + let r = Reader::get_root(b.view()).unwrap().as_vector(); + assert_eq!(r.idx(0).get_i64().unwrap(), -42); + assert_eq!(r.idx(1).get_u64().unwrap(), 9000); + assert_eq!(r.idx(2).get_f64().unwrap(), 3.14); +} + +#[derive(Serialize, Deserialize, Debug, PartialEq)] +struct Foo { + a: i8, + b: f64, + c: Vec, + d: String, +} +quickcheck! { + fn serde_foo(a: i8, + b: f64, + c: Vec, + d: String) -> bool { + let mut s = FlexbufferSerializer::new(); + let data = Foo { a, b, c, d }; + data.serialize(&mut s).unwrap(); + + let read = Foo::deserialize(Reader::get_root(s.view()).unwrap()).unwrap(); + data == read + } +} + +#[test] +fn serde_serious() { + #[derive(Debug, PartialEq, Serialize, Deserialize)] + enum MyEnum { + Unit, + NewType([i32; 3]), + Tuple(f32, f64), + Struct { a: u8, b: u16, c: u32 }, + } + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct MyNewType; + + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct MyStruct { + a: u8, + b: u16, + c: u32, + d: u64, + }; + + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct MyUnitStruct(Vec); + + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct MyTupleStruct(MyNewType, MyUnitStruct, MyStruct, Vec); + + let data = MyTupleStruct( + MyNewType, + MyUnitStruct(vec!["Hello".to_string(), "World".to_string()]), + MyStruct { + a: 2, + b: 4, + c: 8, + d: 16, + }, + vec![ + MyEnum::Unit, + MyEnum::NewType([-1, 0, 1]), + MyEnum::Unit, + MyEnum::Tuple(3.14, 2.71), + MyEnum::Struct { + a: 32, + b: 64, + c: 128, + }, + ], + ); + + let mut s = FlexbufferSerializer::new(); + data.serialize(&mut s).unwrap(); + + let reader = Reader::get_root(s.view()).unwrap(); + let read = MyTupleStruct::deserialize(reader).unwrap(); + assert_eq!(data, read); +} + +#[test] +fn iter() { + let mut fxb = Builder::default(); + { + let mut m = fxb.start_map(); + m.push("a", "42"); + m.push("b", 250i64); + m.push("c", 5000u16); + } + let r = Reader::get_root(fxb.view()).unwrap(); + + let v: Vec = r.as_vector().iter().map(|x| x.as_u32()).collect(); + assert_eq!(&v, &[42, 250, 5000]); +} + +#[test] +fn deserialize_newtype_i8() { + #[derive(Deserialize)] + struct Foo(u8); + let data = [13, 4, 1]; + let r = Reader::get_root(&data).unwrap(); + let foo = Foo::deserialize(r).unwrap(); + assert_eq!(foo.0, 13); +} +#[test] +fn deserialize_newtype_str() { + #[derive(Deserialize)] + struct Foo<'a>(&'a str); + let data = [5, b'h', b'e', b'l', b'l', b'o', b'\0', 6, 5 << 2, 1]; + let r = Reader::get_root(&data).unwrap(); + let foo = Foo::deserialize(r).unwrap(); + assert_eq!(foo.0, "hello"); +} +#[test] +#[rustfmt::skip] +fn deserialize_tuple_struct_to_vec_uint4() { + #[derive(Deserialize)] + struct Foo(u8, u16, u32, u64); + let data = [ + 4, 0, 16, 0, 64, 0, 0, 1, // Data + 8, // Vector offset. + 23 << 2 | 1, // (VectorUInt4, W16 - referring to data). + 1, // Root width W8 - referring to vector. + ]; + let r = Reader::get_root(&data).unwrap(); + let foo = Foo::deserialize(r).unwrap(); + assert_eq!(foo.0, 4); + assert_eq!(foo.1, 16); + assert_eq!(foo.2, 64); + assert_eq!(foo.3, 256); + + let data = [ + 1, 2, 3, 4, // The vector. + 4, // Root data (offset). + 23 << 2, // Root type: VectorUInt4, W8. + 1, // Root width: W8. + ]; + let r = Reader::get_root(&data).unwrap(); + let foo = Foo::deserialize(r).unwrap(); + assert_eq!(foo.0, 1); + assert_eq!(foo.1, 2); + assert_eq!(foo.2, 3); + assert_eq!(foo.3, 4); +} diff --git a/tests/rust_usage_test/tests/integration_test.rs b/tests/rust_usage_test/tests/integration_test.rs index ae5c23432..5e5afcbd6 100644 --- a/tests/rust_usage_test/tests/integration_test.rs +++ b/tests/rust_usage_test/tests/integration_test.rs @@ -15,9 +15,18 @@ * limitations under the License. */ +#[macro_use] extern crate quickcheck; - extern crate flatbuffers; +extern crate flexbuffers; +extern crate rand; +extern crate serde; +#[macro_use] +extern crate serde_derive; +#[macro_use] +extern crate quickcheck_derive; + +mod flexbuffers_tests; #[allow(dead_code, unused_imports)] #[path = "../../include_test/include_test1_generated.rs"] @@ -32,6 +41,11 @@ pub mod include_test2_generated; mod monster_test_generated; pub use monster_test_generated::my_game; +#[rustfmt::skip] // TODO: Use standard rust formatting and remove dead code. +#[allow(dead_code)] +mod flatbuffers_tests { +use super::*; + // Include simple random number generator to ensure results will be the // same across platforms. // http://en.wikipedia.org/wiki/Park%E2%80%93Miller_random_number_generator @@ -2907,3 +2921,4 @@ fn load_file(filename: &str) -> Result, std::io::Error> { f.read_to_end(&mut buf)?; Ok(buf) } +}