Rust Flexbuffers (#5669)

* Cargo clippy lints

* more lints

* more lints

* Restored a doc comment

* Comment on float eps-eq and adjusted casting

* Rust Flexbuffers

* more serde tests, removed some unsafe

* Redid serde to be map-like and Reader is Display

* Moved iter from Reader to VectorReader

* Serious quickcheck + bugs

* wvo api

* Made types smaller for a reasonable speedup

* redid reading in a way that's a bit faster.

Profiling shows the rust slowdown as building +10%, reading +20%

* src/bin are developer binaries in rust

* Root and Map width are not packed

* key null check is debug only + doc changes

* BuilderOptions

* Documentation

* Documentation

* Moved tests to rust_usage_test

* Moved rust flexbuffers samples to Flatbuffers/samples

* Fixed RustTest

* Fixed for Rust 1.37.0

* Upgraded to rust 1_40_0

* fixed a little-endian-only feature in a test

* 1.40.0

* fixed some benchmarks for bigendian

* Updated .bat file

* misspelling

* Gold Flexbuffer test.

* Serialize,Deserialize, std::error::Error for Errors.

* Undo rustfmt in integration_test.rs

* from_slice instead of from_vec

* Added comments to unsafe blocks

* expanded on comment

* bump

Co-authored-by: CasperN <cneo@google.com>

tests/rust_usage_test/bin/flatbuffers_alloc_check.rs

@@ -44,7 +44,15 @@ pub use monster_test_generated::my_game;
 // verbatim from the test suite:
 fn create_serialized_example_with_generated_code(builder: &mut flatbuffers::FlatBufferBuilder) {
     let mon = {
-        let _ = builder.create_vector_of_strings(&["these", "unused", "strings", "check", "the", "create_vector_of_strings", "function"]);
+        let _ = builder.create_vector_of_strings(&[
+            "these",
+            "unused",
+            "strings",
+            "check",
+            "the",
+            "create_vector_of_strings",
+            "function",
+        ]);
 
         let s0 = builder.create_string("test1");
         let s1 = builder.create_string("test2");
@@ -52,21 +60,36 @@ fn create_serialized_example_with_generated_code(builder: &mut flatbuffers::Flat
 
         // can't inline creation of this Vec3 because we refer to it by reference, so it must live
         // long enough to be used by MonsterArgs.
-        let pos = my_game::example::Vec3::new(1.0, 2.0, 3.0, 3.0, my_game::example::Color::Green, &my_game::example::Test::new(5i16, 6i8));
+        let pos = my_game::example::Vec3::new(
+            1.0,
+            2.0,
+            3.0,
+            3.0,
+            my_game::example::Color::Green,
+            &my_game::example::Test::new(5i16, 6i8),
+        );
 
-        let args = my_game::example::MonsterArgs{
+        let args = my_game::example::MonsterArgs {
             hp: 80,
             mana: 150,
             name: Some(builder.create_string("MyMonster")),
             pos: Some(&pos),
             test_type: my_game::example::Any::Monster,
-            test: Some(my_game::example::Monster::create(builder, &my_game::example::MonsterArgs{
-                name: Some(fred_name),
-                ..Default::default()
-            }).as_union_value()),
+            test: Some(
+                my_game::example::Monster::create(
+                    builder,
+                    &my_game::example::MonsterArgs {
+                        name: Some(fred_name),
+                        ..Default::default()
+                    },
+                )
+                .as_union_value(),
+            ),
             inventory: Some(builder.create_vector_direct(&[0u8, 1, 2, 3, 4][..])),
-            test4: Some(builder.create_vector_direct(&[my_game::example::Test::new(10, 20),
-            my_game::example::Test::new(30, 40)])),
+            test4: Some(builder.create_vector_direct(&[
+                my_game::example::Test::new(10, 20),
+                my_game::example::Test::new(30, 40),
+            ])),
             testarrayofstring: Some(builder.create_vector(&[s0, s1])),
             ..Default::default()
         };
@@ -137,8 +160,13 @@ fn main() {
 
             let test4 = m.test4().unwrap();
             assert_eq!(test4.len(), 2);
-            assert_eq!(i32::from(test4[0].a()) + i32::from(test4[1].a()) +
-                       i32::from(test4[0].b()) + i32::from(test4[1].b()), 100);
+            assert_eq!(
+                i32::from(test4[0].a())
+                    + i32::from(test4[1].a())
+                    + i32::from(test4[0].b())
+                    + i32::from(test4[1].b()),
+                100
+            );
 
             let testarrayofstring = m.testarrayofstring().unwrap();
             assert_eq!(testarrayofstring.len(), 2);

tests/rust_usage_test/bin/flexbuffers_alloc_check.rs

@@ -0,0 +1,138 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+extern crate flexbuffers;
+
+use flexbuffers::*;
+use std::alloc::{GlobalAlloc, Layout, System};
+
+/// We take over the Rust allocator to count allocations. This is super not thread safe.
+static mut NUM_ALLOCS: usize = 0;
+fn current_allocs() -> usize {
+    unsafe { NUM_ALLOCS }
+}
+struct TrackingAllocator;
+unsafe impl GlobalAlloc for TrackingAllocator {
+    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+        NUM_ALLOCS += 1;
+        System.alloc(layout)
+    }
+    unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
+        System.dealloc(ptr, layout)
+    }
+}
+#[global_allocator]
+static T: TrackingAllocator = TrackingAllocator;
+
+/// Make some example data
+fn make_monster(mut monster: MapBuilder) {
+    monster.push("type", "great orc");
+    monster.push("age", 100u8);
+    monster.push("name", "Mr. Orc");
+    monster.push("coins", &[1, 25, 50, 100, 250]);
+    monster.push("color", &[255u8, 0, 0, 0]);
+    {
+        let mut weapons = monster.start_vector("weapons");
+        {
+            let mut hammer = weapons.start_map();
+            hammer.push("name", "hammer");
+            hammer.push("damage type", "crush");
+            hammer.push("damage", 20);
+        }
+        {
+            let mut axe = weapons.start_map();
+            axe.push("name", "Great Axe");
+            axe.push("damage type", "slash");
+            axe.push("damage", 30);
+        }
+    }
+    {
+        let mut sounds = monster.start_vector("sounds");
+        sounds.push("grr");
+        sounds.push("rawr");
+        sounds.push("muahaha");
+    }
+    // TODO(cneo): Directly pushing string slices has alloc.
+}
+
+// Read back the data from make_monster.
+fn validate_monster(flexbuffer: &[u8]) {
+    let r = Reader::get_root(flexbuffer).unwrap().as_map();
+
+    assert_eq!(r.idx("type").as_str(), "great orc");
+    assert_eq!(r.idx("age").as_u8(), 100);
+    assert_eq!(r.idx("name").as_str(), "Mr. Orc");
+
+    let coins = r.idx("coins").as_vector();
+    for (i, &c) in [1, 25, 50, 100, 250].iter().enumerate() {
+        assert_eq!(coins.idx(i).as_u16(), c);
+    }
+    let color = r.idx("color").as_vector();
+    for (i, &c) in [255, 0, 0, 0].iter().enumerate() {
+        assert_eq!(color.idx(i).as_i32(), c);
+    }
+    let weapons = r.idx("weapons").as_vector();
+    assert_eq!(weapons.len(), 2);
+
+    let hammer = weapons.idx(0).as_map();
+    assert_eq!(hammer.idx("name").as_str(), "hammer");
+    assert_eq!(hammer.idx("damage type").as_str(), "crush");
+    assert_eq!(hammer.idx("damage").as_u64(), 20);
+
+    let axe = weapons.idx(1).as_map();
+    assert_eq!(axe.idx("name").as_str(), "Great Axe");
+    assert_eq!(axe.idx("damage type").as_str(), "slash");
+    assert_eq!(axe.idx("damage").as_u64(), 30);
+
+    let sounds = r.idx("sounds").as_vector();
+    for (i, &s) in ["grr", "rawr", "muahaha"].iter().enumerate() {
+        assert_eq!(sounds.idx(i).as_str(), s);
+    }
+}
+
+// This is in a separate binary than tests because taking over the global allocator is not
+// hermetic and not thread safe.
+fn main() {
+    let start_up = current_allocs();
+
+    // Let's build a flexbuffer from a new (cold) flexbuffer builder.
+    let mut builder = Builder::default();
+    make_monster(builder.start_map());
+    let after_warmup = current_allocs();
+
+    // The builder makes some allocations while warming up.
+    assert!(after_warmup > start_up);
+    assert!(after_warmup < start_up + 20);
+
+    // A warm builder should make no allocations.
+    make_monster(builder.start_map());
+    assert_eq!(after_warmup, current_allocs());
+
+    // Nor should a reader.
+    validate_monster(builder.view());
+    assert_eq!(after_warmup, current_allocs());
+
+    // Do it again just for kicks.
+    make_monster(builder.start_map());
+    validate_monster(builder.view());
+    assert_eq!(after_warmup, current_allocs());
+
+    let final_allocs = current_allocs(); // dbg! does allocate.
+    dbg!(start_up, after_warmup, final_allocs);
+}
+
+#[test]
+fn no_extra_allocations() {
+    main()
+}

tests/rust_usage_test/bin/monster_example.rs

@@ -13,7 +13,6 @@ pub mod include_test2_generated;
 mod monster_test_generated;
 pub use monster_test_generated::my_game;
 
-
 use std::io::Read;
 
 fn main() {
@@ -22,7 +21,7 @@ fn main() {
     f.read_to_end(&mut buf).expect("file reading failed");
 
     let monster = my_game::example::get_root_as_monster(&buf[..]);
-    println!("{}", monster.hp());     // `80`
-    println!("{}", monster.mana());   // default value of `150`
+    println!("{}", monster.hp()); // `80`
+    println!("{}", monster.mana()); // default value of `150`
     println!("{:?}", monster.name()); // Some("MyMonster")
 }