Rust Flexbuffers (#5669)

* Cargo clippy lints

* more lints

* more lints

* Restored a doc comment

* Comment on float eps-eq and adjusted casting

* Rust Flexbuffers

* more serde tests, removed some unsafe

* Redid serde to be map-like and Reader is Display

* Moved iter from Reader to VectorReader

* Serious quickcheck + bugs

* wvo api

* Made types smaller for a reasonable speedup

* redid reading in a way that's a bit faster.

Profiling shows the rust slowdown as building +10%, reading +20%

* src/bin are developer binaries in rust

* Root and Map width are not packed

* key null check is debug only + doc changes

* BuilderOptions

* Documentation

* Documentation

* Moved tests to rust_usage_test

* Moved rust flexbuffers samples to Flatbuffers/samples

* Fixed RustTest

* Fixed for Rust 1.37.0

* Upgraded to rust 1_40_0

* fixed a little-endian-only feature in a test

* 1.40.0

* fixed some benchmarks for bigendian

* Updated .bat file

* misspelling

* Gold Flexbuffer test.

* Serialize,Deserialize, std::error::Error for Errors.

* Undo rustfmt in integration_test.rs

* from_slice instead of from_vec

* Added comments to unsafe blocks

* expanded on comment

* bump

Co-authored-by: CasperN <cneo@google.com>

rust/flexbuffers/src/builder/map.rs

@@ -0,0 +1,118 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{Builder, Pushable, Value, VectorBuilder};
+
+/// Builds a Flexbuffer map, returned by a [Builder](struct.Builder.html).
+///
+/// ## Side effect when dropped:
+/// When this is dropped, or `end_map` is called, the map is
+/// commited to the buffer. If this map is the root of the flexbuffer, then the
+/// root is written and the flexbuffer is complete.
+/// ## Panics:
+/// -  Duplicate keys will result in a panic in both debug and release mode.
+/// -  Keys with internal nulls results in a panic in debug mode and result in silent truncaction
+///    in release mode.
+pub struct MapBuilder<'a> {
+    pub(super) builder: &'a mut Builder,
+    // If the root is this map then start == None. Otherwise start is the
+    // number of values in the 'values stack' before adding this map.
+    pub(super) start: Option<usize>,
+}
+impl<'a> MapBuilder<'a> {
+    /// Push `p` onto this map with key `key`.
+    /// This will panic (in debug mode) if `key` contains internal nulls.
+    #[inline]
+    pub fn push<P: Pushable>(&mut self, key: &str, p: P) {
+        self.builder.push_key(key);
+        self.builder.push(p);
+    }
+    /// Starts a nested vector that will be pushed onto this map
+    /// with key `key` when it is dropped.
+    ///
+    /// This will panic (in debug mode) if `key` contains internal nulls.
+    #[inline]
+    pub fn start_vector(&mut self, key: &str) -> VectorBuilder {
+        // Push the key that refers to this nested vector.
+        self.builder.push_key(key);
+        // Nested vector.
+        let start = Some(self.builder.values.len());
+        VectorBuilder {
+            builder: &mut self.builder,
+            start,
+        }
+    }
+    /// Starts a nested map which that will be pushed onto this map
+    /// with key `key` when it is dropped.
+    ///
+    /// This will panic (in debug mode) if `key` contains internal nulls.
+    #[inline]
+    pub fn start_map(&mut self, key: &str) -> MapBuilder {
+        // Push the key that refers to this nested vector.
+        self.builder.push_key(key);
+        // Nested map.
+        let start = Some(self.builder.values.len());
+        MapBuilder {
+            builder: &mut self.builder,
+            start,
+        }
+    }
+    /// `end_map` sorts the map by key and writes it to the buffer. This happens anyway
+    /// when the map builder is dropped.
+    #[inline]
+    pub fn end_map(self) {}
+}
+impl<'a> Drop for MapBuilder<'a> {
+    #[inline]
+    fn drop(&mut self) {
+        self.builder.end_map_or_vector(true, self.start);
+    }
+}
+
+// Read known keys / strings as iterators over bytes -- skipping utf8 validation and strlen.
+pub(super) fn get_key(buffer: &[u8], address: usize) -> impl Iterator<Item = &u8> {
+    buffer[address..].iter().take_while(|&&b| b != b'\0')
+}
+
+// `values` is assumed to be of the format [key1, value1, ..., keyN, valueN].
+// The keys refer to cstrings in `buffer`. When this function returns,
+// `values` is sorted in place by key.
+pub(super) fn sort_map_by_keys(values: &mut [Value], buffer: &[u8]) {
+    debug_assert_eq!(values.len() % 2, 0);
+    debug_assert!(values.iter().step_by(2).all(Value::is_key));
+    let raw_pairs = values.as_mut_ptr() as *mut [Value; 2];
+    let pairs_len = values.len() / 2;
+    // Unsafe code needed to treat the slice as key-value pairs when sorting in place. This is
+    // preferred over custom sorting or adding another dependency. By construction, this part
+    // of the values stack must be alternating (key, value) pairs. The public API must not be
+    // able to trigger the above debug_assets that protect this unsafe usage.
+    let pairs: &mut [[Value; 2]] =
+        unsafe { std::slice::from_raw_parts_mut(raw_pairs, pairs_len) };
+    #[rustfmt::skip]
+    pairs.sort_unstable_by(|[key1, _], [key2, _]| {
+        if let Value::Key(a1) = *key1 {
+            if let Value::Key(a2) = *key2 {
+                let s1 = get_key(buffer, a1);
+                let s2 = get_key(buffer, a2);
+                let ord = s1.cmp(s2);
+                if ord == std::cmp::Ordering::Equal {
+                    let dup: String = get_key(buffer, a1).map(|&b| b as char).collect();
+                    panic!("Duplicated key in map {:?}", dup);
+                }
+                return ord;
+            }
+        }
+        unreachable!();
+    });
+}

rust/flexbuffers/src/builder/mod.rs

@@ -0,0 +1,404 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::bitwidth::{align, BitWidth};
+mod value;
+use crate::FlexBufferType;
+use std::cmp::max;
+use value::{find_vector_type, store_value, Value};
+mod map;
+mod push;
+mod ser;
+mod vector;
+use map::sort_map_by_keys;
+pub use map::MapBuilder;
+pub use push::Pushable;
+pub use ser::{Error, FlexbufferSerializer};
+pub use vector::VectorBuilder;
+
+macro_rules! push_slice {
+    ($push_name: ident, $scalar: ty, $Val: ident, $new_vec: ident) => {
+        fn $push_name<T, S>(&mut self, xs: S)
+        where
+            T: Into<$scalar> + Copy,
+            S: AsRef<[T]>
+        {
+            let mut value = Value::$new_vec(xs.as_ref().len());
+            let mut width = xs.as_ref()
+                .iter()
+                .map(|x| BitWidth::from((*x).into()))
+                .max()
+                .unwrap_or_default();
+            if !value.is_fixed_length_vector() {
+                let length = Value::UInt(xs.as_ref().len() as u64);
+                width = std::cmp::max(width, length.width_or_child_width());
+                align(&mut self.buffer, width);
+                store_value(&mut self.buffer, length, width);
+            } else {
+                align(&mut self.buffer, width);
+            }
+            let address = self.buffer.len();
+            for &x in xs.as_ref().iter() {
+                store_value(&mut self.buffer, Value::$Val(x.into()), width);
+            }
+            value.set_address_or_panic(address);
+            value.set_child_width_or_panic(width);
+            self.values.push(value);
+        }
+    }
+}
+macro_rules! push_indirect {
+    ($push_name: ident, $scalar: ty, $Direct: ident, $Indirect: ident) => {
+        fn $push_name<T: Into<$scalar>>(&mut self, x: T) {
+            let x = Value::$Direct(x.into());
+            let child_width = x.width_or_child_width();
+            let address = self.buffer.len();
+            store_value(&mut self.buffer, x, child_width);
+            self.values.push(
+                Value::Reference {
+                    address,
+                    child_width,
+                    fxb_type: FlexBufferType::$Indirect,
+                }
+            );
+        }
+    }
+}
+
+bitflags! {
+    /// Options for sharing data within a flexbuffer.
+    ///
+    /// These increase serialization time but decrease the size of the resulting buffer. By
+    /// default, `SHARE_KEYS`. You may wish to turn on `SHARE_STRINGS` if you know your data has
+    /// many duplicate strings or `SHARE_KEY_VECTORS` if your data has many maps with identical
+    /// keys.
+    ///
+    /// ## Not Yet Implemented
+    /// - `SHARE_STRINGS`
+    /// - `SHARE_KEY_VECTORS`
+    pub struct BuilderOptions: u8 {
+        const SHARE_NONE = 0;
+        const SHARE_KEYS = 1;
+        const SHARE_STRINGS = 2;
+        const SHARE_KEYS_AND_STRINGS = 3;
+        const SHARE_KEY_VECTORS = 4;
+        const SHARE_ALL = 7;
+    }
+}
+impl Default for BuilderOptions {
+    fn default() -> Self {
+        Self::SHARE_KEYS
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+// Address of a Key inside of the buffer.
+struct CachedKey(usize);
+
+/// **Use this struct to build a Flexbuffer.**
+///
+/// Flexbuffers may only have a single root value, which may be constructed
+/// with  one of the following functions.
+/// * `build_singleton` will push 1 value to the buffer and serialize it as the root.
+/// * `start_vector` returns a `VectorBuilder`, into which many (potentially
+/// heterogenous) values can be pushed. The vector itself is the root and is serialized
+/// when the `VectorBuilder` is dropped (or `end` is called).
+/// * `start_map` returns a `MapBuilder`, which is similar to a `VectorBuilder` except
+/// every value must be pushed with an associated key. The map is serialized when the
+/// `MapBuilder` is dropped (or `end` is called).
+///
+/// These functions reset and overwrite the Builder which means, while there are no
+/// active `MapBuilder` or `VectorBuilder`, the internal buffer is empty or contains a
+/// finished Flexbuffer. The internal buffer is accessed with `view`.
+#[derive(Debug, Clone)]
+pub struct Builder {
+    buffer: Vec<u8>,
+    values: Vec<Value>,
+    key_pool: Option<Vec<CachedKey>>,
+}
+impl Default for Builder {
+    fn default() -> Self {
+        let opts = Default::default();
+        Builder::new(opts)
+    }
+}
+
+impl<'a> Builder {
+    pub fn new(opts: BuilderOptions) -> Self {
+        let key_pool = if opts.contains(BuilderOptions::SHARE_KEYS) {
+            Some(vec![])
+        } else {
+            None
+        };
+        Builder {
+            key_pool,
+            values: Vec::new(),
+            buffer: Vec::new(),
+        }
+    }
+    /// Shows the internal flexbuffer. It will either be empty or populated with the most
+    /// recently built flexbuffer.
+    pub fn view(&self) -> &[u8] {
+        &self.buffer
+    }
+    /// Returns the internal buffer, replacing it with a new vector. The returned buffer will
+    /// either be empty or populated with the most recently built flexbuffer.
+    pub fn take_buffer(&mut self) -> Vec<u8> {
+        let mut b = Vec::new();
+        std::mem::swap(&mut self.buffer, &mut b);
+        b
+    }
+    /// Resets the internal state. Automatically called before building a new flexbuffer.
+    pub fn reset(&mut self) {
+        self.buffer.clear();
+        self.values.clear();
+        if let Some(pool) = self.key_pool.as_mut() {
+            pool.clear();
+        }
+    }
+    fn push_key(&mut self, key: &str) {
+        debug_assert!(
+            key.bytes().all(|b| b != b'\0'),
+            "Keys must not have internal nulls."
+        );
+        // Search key pool if there is one.
+        let found = self.key_pool.as_ref().map(|pool| {
+            pool.binary_search_by(|&CachedKey(addr)| {
+                let old_key = map::get_key(&self.buffer, addr);
+                old_key.cloned().cmp(key.bytes())
+            })
+        });
+        let address = if let Some(Ok(idx)) = found {
+            // Found key in key pool.
+            self.key_pool.as_ref().unwrap()[idx].0
+        } else {
+            // Key not in pool (or no pool).
+            let address = self.buffer.len();
+            self.buffer.extend_from_slice(key.as_bytes());
+            self.buffer.push(b'\0');
+            address
+        };
+        if let Some(Err(idx)) = found {
+            // Insert into key pool.
+            let pool = self.key_pool.as_mut().unwrap();
+            pool.insert(idx, CachedKey(address));
+        }
+        self.values.push(Value::Key(address));
+    }
+    fn push_uint<T: Into<u64>>(&mut self, x: T) {
+        self.values.push(Value::UInt(x.into()));
+    }
+    fn push_int<T: Into<i64>>(&mut self, x: T) {
+        self.values.push(Value::Int(x.into()));
+    }
+    fn push_float<T: Into<f64>>(&mut self, x: T) {
+        self.values.push(Value::Float(x.into()));
+    }
+    fn push_null(&mut self) {
+        self.values.push(Value::Null);
+    }
+    fn push_bool(&mut self, x: bool) {
+        self.values.push(Value::Bool(x));
+    }
+    fn store_blob(&mut self, xs: &[u8]) -> Value {
+        let length = Value::UInt(xs.len() as u64);
+        let width = length.width_or_child_width();
+        align(&mut self.buffer, width);
+        store_value(&mut self.buffer, length, width);
+        let address = self.buffer.len();
+        self.buffer.extend_from_slice(xs);
+        Value::Reference {
+            fxb_type: FlexBufferType::Blob,
+            address,
+            child_width: width,
+        }
+    }
+    fn push_str(&mut self, x: &str) {
+        let mut string = self.store_blob(x.as_bytes());
+        self.buffer.push(b'\0');
+        string.set_fxb_type_or_panic(FlexBufferType::String);
+        self.values.push(string);
+    }
+    fn push_blob(&mut self, x: &[u8]) {
+        let blob = self.store_blob(x);
+        self.values.push(blob);
+    }
+    fn push_bools(&mut self, xs: &[bool]) {
+        let length = Value::UInt(xs.len() as u64);
+        let width = length.width_or_child_width();
+        align(&mut self.buffer, width);
+        store_value(&mut self.buffer, length, width);
+        let address = self.buffer.len();
+        for &b in xs.iter() {
+            self.buffer.push(b as u8);
+            for _ in 0..width as u8 {
+                self.buffer.push(0); // Well this seems wasteful.
+            }
+        }
+        self.values.push(Value::Reference {
+            fxb_type: FlexBufferType::VectorBool,
+            address,
+            child_width: width,
+        });
+    }
+
+    push_slice!(push_uints, u64, UInt, new_uint_vector);
+    push_slice!(push_ints, i64, Int, new_int_vector);
+    push_slice!(push_floats, f64, Float, new_float_vector);
+    push_indirect!(push_indirect_int, i64, Int, IndirectInt);
+    push_indirect!(push_indirect_uint, u64, UInt, IndirectUInt);
+    push_indirect!(push_indirect_float, f64, Float, IndirectFloat);
+
+    /// Resets the builder and starts a new flexbuffer with a vector at the root.
+    /// The exact Flexbuffer vector type is dynamically inferred.
+    pub fn start_vector(&'a mut self) -> VectorBuilder<'a> {
+        self.reset();
+        VectorBuilder {
+            builder: self,
+            start: None,
+        }
+    }
+    /// Resets the builder and builds a new flexbuffer with a map at the root.
+    pub fn start_map(&'a mut self) -> MapBuilder<'a> {
+        self.reset();
+        MapBuilder {
+            builder: self,
+            start: None,
+        }
+    }
+    /// Resets the builder and builds a new flexbuffer with the pushed value at the root.
+    pub fn build_singleton<P: Pushable>(&mut self, p: P) {
+        self.reset();
+        p.push_to_builder(self);
+        let root = self.values.pop().unwrap();
+        store_root(&mut self.buffer, root);
+    }
+    fn push<P: Pushable>(&mut self, p: P) {
+        p.push_to_builder(self);
+    }
+    /// Stores the values past `previous_end` as a map or vector depending on `is_map`.
+    /// If `previous_end` is None then this was a root map / vector and the last value
+    /// is stored as the root.
+    fn end_map_or_vector(&mut self, is_map: bool, previous_end: Option<usize>) {
+        let split = previous_end.unwrap_or(0);
+        let value = if is_map {
+            let key_vals = &mut self.values[split..];
+            sort_map_by_keys(key_vals, &self.buffer);
+            let key_vector = store_vector(&mut self.buffer, key_vals, StoreOption::MapKeys);
+            store_vector(&mut self.buffer, key_vals, StoreOption::Map(key_vector))
+        } else {
+            store_vector(&mut self.buffer, &self.values[split..], StoreOption::Vector)
+        };
+        self.values.truncate(split);
+        if previous_end.is_some() {
+            self.values.push(value);
+        } else {
+            store_root(&mut self.buffer, value);
+        }
+    }
+}
+
+/// Builds a Flexbuffer with the single pushed value as the root.
+pub fn singleton<P: Pushable>(p: P) -> Vec<u8> {
+    let mut b = Builder::default();
+    b.build_singleton(p);
+    let Builder { buffer, .. } = b;
+    buffer
+}
+
+/// Stores the root value, root type and root width.
+/// This should be called to finish the Flexbuffer.
+fn store_root(buffer: &mut Vec<u8>, root: Value) {
+    let root_width = root.width_in_vector(buffer.len(), 0);
+    align(buffer, root_width);
+    store_value(buffer, root, root_width);
+    buffer.push(root.packed_type(root_width));
+    buffer.push(root_width.n_bytes() as u8);
+}
+
+pub enum StoreOption {
+    Vector,
+    Map(Value),
+    MapKeys,
+}
+/// Writes a Flexbuffer Vector or Map.
+/// StoreOption::Map(Keys) must be a Value::Key or this will panic.
+// #[inline(always)]
+pub fn store_vector(buffer: &mut Vec<u8>, values: &[Value], opt: StoreOption) -> Value {
+    let (skip, stride) = match opt {
+        StoreOption::Vector => (0, 1),
+        StoreOption::MapKeys => (0, 2),
+        StoreOption::Map(_) => (1, 2),
+    };
+    let iter_values = || values.iter().skip(skip).step_by(stride);
+
+    // Figure out vector type and how long is the prefix.
+    let mut result = if let StoreOption::Map(_) = opt {
+        Value::new_map()
+    } else {
+        find_vector_type(iter_values())
+    };
+    let length_slot = if !result.is_fixed_length_vector() {
+        let length = iter_values().count();
+        Some(Value::UInt(length as u64))
+    } else {
+        None
+    };
+    // Measure required width and align to it.
+    let mut width = BitWidth::W8;
+    if let StoreOption::Map(keys) = opt {
+        width = max(width, keys.width_in_vector(buffer.len(), 0))
+    }
+    if let Some(l) = length_slot {
+        width = max(width, l.width_or_child_width());
+    }
+    let prefix_length = result.prefix_length();
+    for (i, &val) in iter_values().enumerate() {
+        width = max(width, val.width_in_vector(buffer.len(), i + prefix_length));
+    }
+    align(buffer, width);
+    #[allow(deprecated)]
+    {
+        debug_assert_ne!(
+            result.fxb_type(),
+            FlexBufferType::VectorString,
+            "VectorString is deprecated and cannot be written.\
+             (https://github.com/google/flatbuffers/issues/5627)"
+        );
+    }
+    // Write Prefix.
+    if let StoreOption::Map(keys) = opt {
+        let key_width = Value::UInt(keys.width_or_child_width().n_bytes() as u64);
+        store_value(buffer, keys, width);
+        store_value(buffer, key_width, width);
+    }
+    if let Some(len) = length_slot {
+        store_value(buffer, len, width);
+    }
+    // Write data.
+    let address = buffer.len();
+    for &v in iter_values() {
+        store_value(buffer, v, width);
+    }
+    // Write types
+    if result.is_typed_vector_or_map() {
+        for v in iter_values() {
+            buffer.push(v.packed_type(width));
+        }
+    }
+    // Return Value representing this Vector.
+    result.set_address_or_panic(address);
+    result.set_child_width_or_panic(width);
+    result
+}

rust/flexbuffers/src/builder/push.rs

@@ -0,0 +1,167 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::builder::Builder;
+use crate::private::Sealed;
+use crate::{Blob, IndirectFloat, IndirectInt, IndirectUInt};
+
+impl<'a> Sealed for Blob<'a> {}
+impl Sealed for () {}
+
+// TODO: String interning
+// TODO: Pushable for Map types?
+
+/// Types that implement the Pushable trait can be written into a Flexbuffer.
+///
+/// All Rust's standard numbers, `u8, u16, u32, u64, i8, i16, i32, i64, f32, f64`,
+/// can all be pushed. They are  `FlexBufferType::{UInt, Int, Float}`.
+/// Flexbuffers chooses the smallest width that can represent the given number.
+/// Strings can pe pushed, they become `FlexBufferType::String` and are stored
+/// with both a length and null terminator.
+///
+/// * For convenience and speed push typed vectors using rust arrays and slices.
+/// Doing so will immediately serialize the data, skipping the `Builder`'s
+/// internal cache.
+///
+/// * Pushable cannot not be implemented by any downstream crates.
+pub trait Pushable: Sealed + Sized {
+    fn push_to_builder(self, _: &mut Builder) {}
+}
+
+impl Pushable for () {
+    fn push_to_builder(self, builder: &mut Builder) {
+        builder.push_null();
+    }
+}
+impl<'a> Pushable for Blob<'a> {
+    fn push_to_builder(self, builder: &mut Builder) {
+        builder.push_blob(self.0);
+    }
+}
+
+macro_rules! forward_to_builder {
+    ($T: ty, $method: ident) => {
+        impl Sealed for $T {}
+        impl Pushable for $T {
+            fn push_to_builder(self, builder: &mut Builder) {
+                builder.$method(self);
+            }
+        }
+    };
+    ($T: ty, $method: ident, $asT: ty) => {
+        impl Sealed for $T {}
+        impl Pushable for $T {
+            fn push_to_builder(self, builder: &mut Builder) {
+                builder.$method(self as $asT);
+            }
+        }
+    };
+}
+forward_to_builder!(&str, push_str);
+forward_to_builder!(bool, push_bool);
+forward_to_builder!(u8, push_uint);
+forward_to_builder!(u16, push_uint);
+forward_to_builder!(u32, push_uint);
+forward_to_builder!(u64, push_uint);
+forward_to_builder!(i8, push_int);
+forward_to_builder!(i16, push_int);
+forward_to_builder!(i32, push_int);
+forward_to_builder!(i64, push_int);
+forward_to_builder!(f32, push_float);
+forward_to_builder!(f64, push_float);
+forward_to_builder!(&[u8], push_uints);
+forward_to_builder!(&[u16], push_uints);
+forward_to_builder!(&[u32], push_uints);
+forward_to_builder!(&[u64], push_uints);
+forward_to_builder!(&[i8], push_ints);
+forward_to_builder!(&[i16], push_ints);
+forward_to_builder!(&[i32], push_ints);
+forward_to_builder!(&[i64], push_ints);
+forward_to_builder!(&[f32], push_floats);
+forward_to_builder!(&[f64], push_floats);
+forward_to_builder!(&[bool], push_bools);
+forward_to_builder!(&Vec<u8>, push_uints);
+forward_to_builder!(&Vec<u16>, push_uints);
+forward_to_builder!(&Vec<u32>, push_uints);
+forward_to_builder!(&Vec<u64>, push_uints);
+forward_to_builder!(&Vec<i8>, push_ints);
+forward_to_builder!(&Vec<i16>, push_ints);
+forward_to_builder!(&Vec<i32>, push_ints);
+forward_to_builder!(&Vec<i64>, push_ints);
+forward_to_builder!(&Vec<f32>, push_floats);
+forward_to_builder!(&Vec<f64>, push_floats);
+forward_to_builder!(&Vec<bool>, push_bools);
+
+macro_rules! impl_indirects {
+    ($Indirect: ident, $method: ident) => {
+        impl Sealed for $Indirect {}
+        impl Pushable for $Indirect {
+            fn push_to_builder(self, builder: &mut Builder) {
+                builder.$method(self.0);
+            }
+        }
+    };
+}
+impl_indirects!(IndirectInt, push_indirect_int);
+impl_indirects!(IndirectUInt, push_indirect_uint);
+impl_indirects!(IndirectFloat, push_indirect_float);
+
+macro_rules! impl_arrays {
+    ($num: expr) => {
+        forward_to_builder!(&[u8; $num], push_uints, &[u8]);
+        forward_to_builder!(&[u16; $num], push_uints, &[u16]);
+        forward_to_builder!(&[u32; $num], push_uints, &[u32]);
+        forward_to_builder!(&[u64; $num], push_uints, &[u64]);
+        forward_to_builder!(&[i8; $num], push_ints, &[i8]);
+        forward_to_builder!(&[i16; $num], push_ints, &[i16]);
+        forward_to_builder!(&[i32; $num], push_ints, &[i32]);
+        forward_to_builder!(&[i64; $num], push_ints, &[i64]);
+        forward_to_builder!(&[f32; $num], push_floats, &[f32]);
+        forward_to_builder!(&[f64; $num], push_floats, &[f64]);
+        forward_to_builder!(&[bool; $num], push_bools, &[bool]);
+    };
+}
+impl_arrays!(0);
+impl_arrays!(1);
+impl_arrays!(2);
+impl_arrays!(3);
+impl_arrays!(4);
+impl_arrays!(5);
+impl_arrays!(6);
+// impl_arrays!(7);
+// impl_arrays!(8);
+// impl_arrays!(9);
+// impl_arrays!(10);
+// impl_arrays!(11);
+// impl_arrays!(12);
+// impl_arrays!(13);
+// impl_arrays!(14);
+// impl_arrays!(15);
+// impl_arrays!(16);
+// impl_arrays!(17);
+// impl_arrays!(18);
+// impl_arrays!(19);
+// impl_arrays!(20);
+// impl_arrays!(21);
+// impl_arrays!(22);
+// impl_arrays!(23);
+// impl_arrays!(24);
+// impl_arrays!(25);
+// impl_arrays!(26);
+// impl_arrays!(27);
+// impl_arrays!(28);
+// impl_arrays!(29);
+// impl_arrays!(30);
+// impl_arrays!(31);
+// impl_arrays!(32);

rust/flexbuffers/src/builder/ser.rs

@@ -0,0 +1,530 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::Builder;
+use serde::ser;
+use serde::ser::*;
+use std::fmt::Display;
+
+// This struct internally tracks the nested vectors representing
+// nested structs and such.
+// TODO: Add an option field names in a map.
+/// Flexbuffer Serializer. This should be used to serialize structs.
+#[derive(Debug, Default)]
+pub struct FlexbufferSerializer {
+    builder: Builder,
+    nesting: Vec<Option<usize>>,
+}
+impl FlexbufferSerializer {
+    pub fn new() -> Self {
+        Self::default()
+    }
+    pub fn view(&self) -> &[u8] {
+        self.builder.view()
+    }
+    pub fn take_buffer(&mut self) -> Vec<u8> {
+        self.builder.take_buffer()
+    }
+    fn finish_if_not_nested(&mut self) -> Result<(), Error> {
+        if self.nesting.is_empty() {
+            assert_eq!(self.builder.values.len(), 1);
+            let root = self.builder.values.pop().unwrap();
+            super::store_root(&mut self.builder.buffer, root);
+        }
+        Ok(())
+    }
+    fn start_vector(&mut self) {
+        let previous_end = if self.nesting.is_empty() {
+            None
+        } else {
+            Some(self.builder.values.len())
+        };
+        self.nesting.push(previous_end);
+    }
+    fn start_map(&mut self) {
+        let previous_end = if self.nesting.is_empty() {
+            None
+        } else {
+            Some(self.builder.values.len())
+        };
+        self.nesting.push(previous_end);
+    }
+    fn end_vector(&mut self) -> Result<(), Error> {
+        let previous_end = self.nesting.pop().unwrap();
+        self.builder.end_map_or_vector(false, previous_end);
+        Ok(())
+    }
+    fn end_map(&mut self) -> Result<(), Error> {
+        let previous_end = self.nesting.pop().unwrap();
+        self.builder.end_map_or_vector(true, previous_end);
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+/// Errors that may happen with Serde.
+pub enum Error {
+    /// Only `str` and `String` can be serialized as keys in serde maps.
+    KeyMustBeString,
+    Serde(String),
+}
+
+impl std::fmt::Display for Error {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
+        write!(f, "{:?}", self)
+    }
+}
+impl std::error::Error for Error {}
+impl ser::Error for Error {
+    fn custom<T>(msg: T) -> Self
+    where
+        T: Display,
+    {
+        Self::Serde(format!("{}", msg))
+    }
+}
+impl<'a> ser::SerializeSeq for &mut FlexbufferSerializer {
+    type Ok = ();
+    type Error = Error;
+    fn serialize_element<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
+    where
+        T: Serialize,
+    {
+        value.serialize(&mut **self)
+    }
+    fn end(self) -> Result<Self::Ok, Self::Error> {
+        self.end_vector()
+    }
+}
+// This is unlike a flexbuffers map which requires CString like keys.
+// Its implemented as alternating keys and values (hopefully).
+impl<'a> ser::SerializeMap for &'a mut FlexbufferSerializer {
+    type Ok = ();
+    type Error = Error;
+    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
+    where
+        T: Serialize,
+    {
+        key.serialize(MapKeySerializer(&mut **self))
+    }
+    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
+    where
+        T: Serialize,
+    {
+        value.serialize(&mut **self)
+    }
+    fn end(self) -> Result<Self::Ok, Self::Error> {
+        self.end_map()
+    }
+}
+impl<'a> ser::SerializeTuple for &mut FlexbufferSerializer {
+    type Ok = ();
+    type Error = Error;
+    fn serialize_element<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
+    where
+        T: Serialize,
+    {
+        value.serialize(&mut **self)
+    }
+    fn end(self) -> Result<Self::Ok, Self::Error> {
+        self.end_vector()
+    }
+}
+impl<'a> ser::SerializeTupleStruct for &mut FlexbufferSerializer {
+    type Ok = ();
+    type Error = Error;
+    fn serialize_field<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
+    where
+        T: Serialize,
+    {
+        value.serialize(&mut **self)
+    }
+    fn end(self) -> Result<Self::Ok, Self::Error> {
+        self.end_vector()
+    }
+}
+impl<'a> ser::SerializeStruct for &mut FlexbufferSerializer {
+    type Ok = ();
+    type Error = Error;
+    fn serialize_field<T: ?Sized>(
+        &mut self,
+        key: &'static str,
+        value: &T,
+    ) -> Result<(), Self::Error>
+    where
+        T: Serialize,
+    {
+        self.builder.push_key(key);
+        value.serialize(&mut **self)
+    }
+    fn end(self) -> Result<Self::Ok, Self::Error> {
+        self.end_map()
+    }
+}
+impl<'a> ser::SerializeTupleVariant for &mut FlexbufferSerializer {
+    type Ok = ();
+    type Error = Error;
+    fn serialize_field<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
+    where
+        T: Serialize,
+    {
+        value.serialize(&mut **self)
+    }
+    fn end(self) -> Result<Self::Ok, Self::Error> {
+        self.end_vector()?;
+        self.end_map()
+    }
+}
+impl<'a> ser::SerializeStructVariant for &mut FlexbufferSerializer {
+    type Ok = ();
+    type Error = Error;
+    fn serialize_field<T: ?Sized>(
+        &mut self,
+        key: &'static str,
+        value: &T,
+    ) -> Result<(), Self::Error>
+    where
+        T: Serialize,
+    {
+        self.builder.push_key(key);
+        value.serialize(&mut **self)
+    }
+    fn end(self) -> Result<Self::Ok, Self::Error> {
+        self.end_map()?;
+        self.end_map()
+    }
+    // TODO: skip field?
+}
+
+impl<'a> ser::Serializer for &'a mut FlexbufferSerializer {
+    type SerializeSeq = &'a mut FlexbufferSerializer;
+    type SerializeTuple = &'a mut FlexbufferSerializer;
+    type SerializeTupleStruct = &'a mut FlexbufferSerializer;
+    type SerializeTupleVariant = &'a mut FlexbufferSerializer;
+    type SerializeMap = &'a mut FlexbufferSerializer;
+    type SerializeStruct = &'a mut FlexbufferSerializer;
+    type SerializeStructVariant = &'a mut FlexbufferSerializer;
+    type Ok = ();
+    type Error = Error;
+    fn serialize_bool(self, v: bool) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(v);
+        self.finish_if_not_nested()
+    }
+    fn serialize_i8(self, v: i8) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(v);
+        self.finish_if_not_nested()
+    }
+    fn serialize_i16(self, v: i16) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(v);
+        self.finish_if_not_nested()
+    }
+    fn serialize_i32(self, v: i32) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(v);
+        self.finish_if_not_nested()
+    }
+    fn serialize_i64(self, v: i64) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(v);
+        self.finish_if_not_nested()
+    }
+    fn serialize_u8(self, v: u8) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(v);
+        self.finish_if_not_nested()
+    }
+    fn serialize_u16(self, v: u16) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(v);
+        self.finish_if_not_nested()
+    }
+    fn serialize_u32(self, v: u32) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(v);
+        self.finish_if_not_nested()
+    }
+    fn serialize_u64(self, v: u64) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(v);
+        self.finish_if_not_nested()
+    }
+    fn serialize_f32(self, v: f32) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(v);
+        self.finish_if_not_nested()
+    }
+    fn serialize_f64(self, v: f64) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(v);
+        self.finish_if_not_nested()
+    }
+    fn serialize_char(self, v: char) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(v as u8);
+        self.finish_if_not_nested()
+    }
+    fn serialize_str(self, v: &str) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(v);
+        self.finish_if_not_nested()
+    }
+    fn serialize_bytes(self, v: &[u8]) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(v);
+        self.finish_if_not_nested()
+    }
+    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(());
+        self.finish_if_not_nested()
+    }
+    fn serialize_some<T: ?Sized>(self, t: &T) -> Result<Self::Ok, Self::Error>
+    where
+        T: Serialize,
+    {
+        t.serialize(self)
+    }
+    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(());
+        self.finish_if_not_nested()
+    }
+    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(());
+        self.finish_if_not_nested()
+    }
+    fn serialize_unit_variant(
+        self,
+        _name: &'static str,
+        _variant_index: u32,
+        variant: &'static str,
+    ) -> Result<Self::Ok, Self::Error> {
+        self.builder.push(variant);
+        self.finish_if_not_nested()
+    }
+    fn serialize_newtype_struct<T: ?Sized>(
+        self,
+        _name: &'static str,
+        value: &T,
+    ) -> Result<Self::Ok, Self::Error>
+    where
+        T: Serialize,
+    {
+        value.serialize(self)
+    }
+    fn serialize_newtype_variant<T: ?Sized>(
+        self,
+        _name: &'static str,
+        _variant_index: u32,
+        variant: &'static str,
+        value: &T,
+    ) -> Result<Self::Ok, Self::Error>
+    where
+        T: Serialize,
+    {
+        self.start_map();
+        self.builder.push_key(variant);
+        value.serialize(&mut *self)?;
+        self.end_map()
+    }
+    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
+        self.start_vector();
+        Ok(self)
+    }
+    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
+        self.start_vector();
+        Ok(self)
+    }
+    fn serialize_tuple_struct(
+        self,
+        _name: &'static str,
+        _len: usize,
+    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
+        self.start_map();
+        Ok(self)
+    }
+    fn serialize_tuple_variant(
+        self,
+        _name: &'static str,
+        _variant_index: u32,
+        variant: &'static str,
+        _len: usize,
+    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
+        self.start_map();
+        self.builder.push_key(variant);
+        self.start_vector();
+        Ok(self)
+    }
+    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
+        self.start_map();
+        Ok(self)
+    }
+    fn serialize_struct(
+        self,
+        _name: &'static str,
+        _len: usize,
+    ) -> Result<Self::SerializeStruct, Self::Error> {
+        self.start_map();
+        Ok(self)
+    }
+    fn serialize_struct_variant(
+        self,
+        _name: &'static str,
+        _variant_index: u32,
+        variant: &'static str,
+        _len: usize,
+    ) -> Result<Self::SerializeStructVariant, Self::Error> {
+        self.start_map();
+        self.builder.push_key(variant);
+        self.start_map();
+        Ok(self)
+    }
+}
+
+fn key_must_be_a_string<T>() -> Result<T, Error> {
+    Err(Error::KeyMustBeString)
+}
+struct MapKeySerializer<'a>(&'a mut FlexbufferSerializer);
+impl<'a> Serializer for MapKeySerializer<'a> {
+    type Ok = ();
+    type Error = Error;
+    #[inline]
+    fn serialize_str(self, value: &str) -> Result<(), Error> {
+        self.0.builder.push_key(value);
+        Ok(())
+    }
+    #[inline]
+    fn serialize_unit_variant(
+        self,
+        _name: &'static str,
+        _variant_index: u32,
+        variant: &'static str,
+    ) -> Result<(), Error> {
+        self.0.builder.push_key(variant);
+        Ok(())
+    }
+    #[inline]
+    fn serialize_newtype_struct<T: ?Sized>(
+        self,
+        _name: &'static str,
+        value: &T,
+    ) -> Result<(), Error>
+    where
+        T: Serialize,
+    {
+        value.serialize(self)
+    }
+    type SerializeSeq = Impossible<(), Error>;
+    type SerializeTuple = Impossible<(), Error>;
+    type SerializeTupleStruct = Impossible<(), Error>;
+    type SerializeTupleVariant = Impossible<(), Error>;
+    type SerializeMap = Impossible<(), Error>;
+    type SerializeStruct = Impossible<(), Error>;
+    type SerializeStructVariant = Impossible<(), Error>;
+
+    fn serialize_bool(self, _value: bool) -> Result<(), Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_i8(self, _value: i8) -> Result<(), Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_i16(self, _value: i16) -> Result<(), Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_i32(self, _value: i32) -> Result<(), Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_i64(self, _value: i64) -> Result<(), Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_u8(self, _value: u8) -> Result<(), Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_u16(self, _value: u16) -> Result<(), Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_u32(self, _value: u32) -> Result<(), Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_u64(self, _value: u64) -> Result<(), Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_f32(self, _value: f32) -> Result<(), Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_f64(self, _value: f64) -> Result<(), Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_char(self, _value: char) -> Result<(), Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_bytes(self, _value: &[u8]) -> Result<(), Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_unit(self) -> Result<(), Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_unit_struct(self, _name: &'static str) -> Result<(), Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_newtype_variant<T: ?Sized>(
+        self,
+        _name: &'static str,
+        _variant_index: u32,
+        _variant: &'static str,
+        _value: &T,
+    ) -> Result<(), Error>
+    where
+        T: Serialize,
+    {
+        key_must_be_a_string()
+    }
+    fn serialize_none(self) -> Result<(), Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<(), Error>
+    where
+        T: Serialize,
+    {
+        key_must_be_a_string()
+    }
+    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_tuple_struct(
+        self,
+        _name: &'static str,
+        _len: usize,
+    ) -> Result<Self::SerializeTupleStruct, Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_tuple_variant(
+        self,
+        _name: &'static str,
+        _variant_index: u32,
+        _variant: &'static str,
+        _len: usize,
+    ) -> Result<Self::SerializeTupleVariant, Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_struct(
+        self,
+        _name: &'static str,
+        _len: usize,
+    ) -> Result<Self::SerializeStruct, Error> {
+        key_must_be_a_string()
+    }
+    fn serialize_struct_variant(
+        self,
+        _name: &'static str,
+        _variant_index: u32,
+        _variant: &'static str,
+        _len: usize,
+    ) -> Result<Self::SerializeStructVariant, Error> {
+        key_must_be_a_string()
+    }
+}

rust/flexbuffers/src/builder/value.rs

@@ -0,0 +1,306 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use byteorder::{LittleEndian, WriteBytesExt};
+
+use crate::bitwidth::BitWidth;
+use crate::bitwidth::BitWidth::*;
+use crate::flexbuffer_type::FlexBufferType;
+use crate::flexbuffer_type::FlexBufferType::*;
+
+/// Internal representation of FlexBuffer Types and Data before writing.
+/// These get placed on the builder's stack and are eventually commited.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum Value {
+    // Inline types
+    Null,
+    Int(i64),
+    UInt(u64),
+    Float(f64),
+    Bool(bool),
+    /// Null termintated, c_string. Only used with `Map`s.
+    Key(usize),
+    /// The other ~20 or so types.
+    Reference {
+        address: usize,
+        child_width: BitWidth,
+        fxb_type: FlexBufferType,
+    },
+}
+
+macro_rules! new_typed_vector {
+    ($name: ident, $v2: ident, $v3: ident, $v4: ident, $vn: ident) => {
+        /// Returns a typed vector, fixed length if possible.
+        /// Address and child width are zero initialized and must be set.
+        pub fn $name(n: usize) -> Value {
+            let address = 0;
+            let child_width = W8;
+            match n {
+                2 => Value::Reference {
+                    address,
+                    child_width,
+                    fxb_type: $v2,
+                },
+                3 => Value::Reference {
+                    address,
+                    child_width,
+                    fxb_type: $v3,
+                },
+                4 => Value::Reference {
+                    address,
+                    child_width,
+                    fxb_type: $v4,
+                },
+                _ => Value::Reference {
+                    address,
+                    child_width,
+                    fxb_type: $vn,
+                },
+            }
+        }
+    };
+}
+
+impl Value {
+    pub fn new_vector() -> Self {
+        Value::Reference {
+            address: 0,
+            child_width: W8,
+            fxb_type: Vector,
+        }
+    }
+    pub fn new_map() -> Self {
+        Value::Reference {
+            address: 0,
+            child_width: W8,
+            fxb_type: Map,
+        }
+    }
+    new_typed_vector!(
+        new_int_vector,
+        VectorInt2,
+        VectorInt3,
+        VectorInt4,
+        VectorInt
+    );
+    new_typed_vector!(
+        new_uint_vector,
+        VectorUInt2,
+        VectorUInt3,
+        VectorUInt4,
+        VectorUInt
+    );
+    new_typed_vector!(
+        new_float_vector,
+        VectorFloat2,
+        VectorFloat3,
+        VectorFloat4,
+        VectorFloat
+    );
+    pub fn fxb_type(&self) -> FlexBufferType {
+        match *self {
+            Value::Null => Null,
+            Value::Int(_) => Int,
+            Value::UInt(_) => UInt,
+            Value::Float(_) => Float,
+            Value::Bool(_) => Bool,
+            Value::Key(_) => Key,
+            Value::Reference { fxb_type, .. } => fxb_type,
+        }
+    }
+    pub fn is_fixed_length_vector(&self) -> bool {
+        self.fxb_type().is_fixed_length_vector()
+    }
+    pub fn is_inline(&self) -> bool {
+        self.fxb_type().is_inline()
+    }
+    pub fn is_reference(&self) -> bool {
+        !self.is_inline()
+    }
+    pub fn is_key(&self) -> bool {
+        match self {
+            Value::Key(_) => true,
+            _ => false,
+        }
+    }
+    pub fn is_typed_vector_or_map(&self) -> bool {
+        if let Value::Reference { fxb_type, .. } = self {
+            fxb_type.is_heterogenous()
+        } else {
+            false
+        }
+    }
+    pub fn prefix_length(&self) -> usize {
+        if self.is_fixed_length_vector() || self.is_inline() {
+            return 0;
+        }
+        if let Value::Reference { fxb_type, .. } = self {
+            if *fxb_type == Map {
+                return 3;
+            }
+        }
+        1
+    }
+    pub fn set_fxb_type_or_panic(&mut self, new_type: FlexBufferType) {
+        if let Value::Reference { fxb_type, .. } = self {
+            *fxb_type = new_type;
+        } else {
+            panic!("`set_fxb_type_or_panic` called on {:?}", self)
+        }
+    }
+    pub fn set_child_width_or_panic(&mut self, new_width: BitWidth) {
+        if let Value::Reference { child_width, .. } = self {
+            *child_width = new_width;
+        } else {
+            panic!("`set_child_width_or_panic` called on {:?}", self);
+        }
+    }
+    pub fn get_address(&self) -> Option<usize> {
+        if let Value::Reference { address, .. } | Value::Key(address) = self {
+            Some(*address)
+        } else {
+            None
+        }
+    }
+    pub fn set_address_or_panic(&mut self, new_address: usize) {
+        if let Value::Reference { address, .. } | Value::Key(address) = self {
+            *address = new_address;
+        } else {
+            panic!("`set_address_or_panic` called on {:?}", self);
+        }
+    }
+    /// For inline types - the width of the value to be stored.
+    /// For reference types, the width of the referred.
+    /// Note Key types always refer to 8 bit data.
+    pub fn width_or_child_width(&self) -> BitWidth {
+        match *self {
+            Value::Int(x) => x.into(),
+            Value::UInt(x) => x.into(),
+            Value::Float(x) => x.into(),
+            Value::Key(_) | Value::Bool(_) | Value::Null => W8,
+            Value::Reference { child_width, .. } => child_width,
+        }
+    }
+    pub fn relative_address(self, written_at: usize) -> Option<Value> {
+        self.get_address().map(|address| {
+            let offset = written_at
+                .checked_sub(address)
+                .expect("Error: References may only refer backwards in buffer.");
+            Value::UInt(offset as u64)
+        })
+    }
+    /// Computes the minimum required width of `value` when stored in a vector
+    /// starting at `vector_start` at index `idx` (this index includes the prefix).
+    /// `Value::Reference{..}` variants require location information because
+    /// offsets are relative.
+    pub fn width_in_vector(self, vector_start: usize, idx: usize) -> BitWidth {
+        match self {
+            Value::Bool(_) => W8,
+            Value::Null => W8,
+            Value::Int(x) => x.into(),
+            Value::UInt(x) => x.into(),
+            Value::Float(x) => x.into(),
+            _ => {
+                debug_assert!(self.is_reference());
+                for &width in BitWidth::iter() {
+                    let bytes = width as usize + 1;
+                    let alignment = (bytes - vector_start % bytes) % bytes;
+                    let written_at = vector_start + alignment + idx * bytes;
+                    // This match must always succeed.
+                    if let Some(Value::UInt(offset)) = self.relative_address(written_at) {
+                        if BitWidth::from(offset) == width {
+                            return width;
+                        }
+                    }
+                }
+                unreachable!()
+            }
+        }
+    }
+    pub fn packed_type(self, parent_width: BitWidth) -> u8 {
+        let width = if self.is_inline() {
+            std::cmp::max(parent_width, self.width_or_child_width())
+        } else {
+            self.width_or_child_width()
+        };
+        (self.fxb_type() as u8) << 2 | width as u8
+    }
+}
+
+pub fn find_vector_type<'a, T>(mut values: T) -> Value
+where
+    T: std::iter::Iterator<Item = &'a Value>,
+{
+    let first = values.next();
+    if first.is_none() {
+        return Value::new_vector();
+    }
+    let mut len = 1;
+    let init = first.unwrap().fxb_type();
+    for v in values {
+        if v.fxb_type() != init {
+            return Value::new_vector();
+        }
+        len += 1;
+    }
+    let vector_type = match init {
+        Bool => VectorBool,
+        UInt => return Value::new_uint_vector(len),
+        Int => return Value::new_int_vector(len),
+        Float => return Value::new_float_vector(len),
+        Key => VectorKey,
+        // Note that VectorString is deprecated for writing
+        _ => return Value::new_vector(),
+    };
+    Value::Reference {
+        address: 0,
+        child_width: W8,
+        fxb_type: vector_type,
+    }
+}
+
+#[inline]
+pub fn store_value(buffer: &mut Vec<u8>, mut value: Value, width: BitWidth) {
+    // Remap to number types.
+    use Value::*;
+    if let Some(offset) = value.relative_address(buffer.len()) {
+        value = offset;
+    } else {
+        value = match value {
+            Bool(x) => UInt(x.into()),
+            Null => UInt(0), // Should this be 0 bytes?
+            _ => value,
+        }
+    }
+    let write_result = match (value, width) {
+        (UInt(x), W8) => buffer.write_u8(x as u8),
+        (UInt(x), W16) => buffer.write_u16::<LittleEndian>(x as u16),
+        (UInt(x), W32) => buffer.write_u32::<LittleEndian>(x as u32),
+        (UInt(x), W64) => buffer.write_u64::<LittleEndian>(x),
+        (Int(x), W8) => buffer.write_i8(x as i8),
+        (Int(x), W16) => buffer.write_i16::<LittleEndian>(x as i16),
+        (Int(x), W32) => buffer.write_i32::<LittleEndian>(x as i32),
+        (Int(x), W64) => buffer.write_i64::<LittleEndian>(x),
+        (Float(x), W32) => buffer.write_f32::<LittleEndian>(x as f32),
+        (Float(x), W64) => buffer.write_f64::<LittleEndian>(x),
+        (Float(_), _) => unreachable!("Error: Flatbuffers does not support 8 and 16 bit floats."),
+        _ => unreachable!("Variant not considered: {:?}", value),
+    };
+    write_result.unwrap_or_else(|err| {
+        panic!(
+            "Error writing value {:?} with width {:?}: {:?}",
+            value, width, err
+        )
+    });
+}

rust/flexbuffers/src/builder/vector.rs

@@ -0,0 +1,65 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{Builder, MapBuilder, Pushable};
+
+/// Builds a Flexbuffer vector, returned by a [Builder](struct.Builder.html).
+///
+/// ## Side effect when dropped:
+/// When this is dropped, or `end_vector` is called, the vector is
+/// commited to the buffer. If this vector is the root of the flexbuffer, then the
+/// root is written and the flexbuffer is complete. The FlexBufferType of this vector
+/// is determined by the pushed values when this is dropped. The most compact vector type is
+/// automatically chosen.
+pub struct VectorBuilder<'a> {
+    pub(crate) builder: &'a mut Builder,
+    // If the root is this vector then start == None. Otherwise start is the
+    // number of values in the 'values stack' before adding this vector.
+    pub(crate) start: Option<usize>,
+}
+impl<'a> VectorBuilder<'a> {
+    /// Pushes `p` onto the vector.
+    #[inline]
+    pub fn push<P: Pushable>(&mut self, p: P) {
+        self.builder.push(p);
+    }
+    /// Starts a nested vector that will be pushed onto this vector when it is dropped.
+    #[inline]
+    pub fn start_vector(&mut self) -> VectorBuilder {
+        let start = Some(self.builder.values.len());
+        VectorBuilder {
+            builder: &mut self.builder,
+            start,
+        }
+    }
+    /// Starts a nested map that will be pushed onto this vector when it is dropped.
+    #[inline]
+    pub fn start_map(&mut self) -> MapBuilder {
+        let start = Some(self.builder.values.len());
+        MapBuilder {
+            builder: &mut self.builder,
+            start,
+        }
+    }
+    /// `end_vector` determines the type of the vector and writes it to the buffer.
+    /// This will happen automatically if the VectorBuilder is dropped.
+    #[inline]
+    pub fn end_vector(self) {}
+}
+impl<'a> Drop for VectorBuilder<'a> {
+    #[inline]
+    fn drop(&mut self) {
+        self.builder.end_map_or_vector(false, self.start);
+    }
+}