BigfootDev/flatbuffers
1
0
Fork 1
mirror of https://github.com/google/flatbuffers.git synced 2026-06-29 03:40:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rust Flexbuffers (#5669)

Browse Source 
* Cargo clippy lints

* more lints

* more lints

* Restored a doc comment

* Comment on float eps-eq and adjusted casting

* Rust Flexbuffers

* more serde tests, removed some unsafe

* Redid serde to be map-like and Reader is Display

* Moved iter from Reader to VectorReader

* Serious quickcheck + bugs

* wvo api

* Made types smaller for a reasonable speedup

* redid reading in a way that's a bit faster.

Profiling shows the rust slowdown as building +10%, reading +20%

* src/bin are developer binaries in rust

* Root and Map width are not packed

* key null check is debug only + doc changes

* BuilderOptions

* Documentation

* Documentation

* Moved tests to rust_usage_test

* Moved rust flexbuffers samples to Flatbuffers/samples

* Fixed RustTest

* Fixed for Rust 1.37.0

* Upgraded to rust 1_40_0

* fixed a little-endian-only feature in a test

* 1.40.0

* fixed some benchmarks for bigendian

* Updated .bat file

* misspelling

* Gold Flexbuffer test.

* Serialize,Deserialize, std::error::Error for Errors.

* Undo rustfmt in integration_test.rs

* from_slice instead of from_vec

* Added comments to unsafe blocks

* expanded on comment

* bump

Co-authored-by: CasperN <cneo@google.com>
This commit is contained in:
Casper
2020-05-07 14:11:26 -07:00
committed by GitHub
parent 870ecbc09a
commit 8be05f6bd4
38 changed files with 5515 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
tests/rust_usage_test/benches/benchmarks.rs Normal file
View File

@@ -0,0 +1,41 @@
/*
* Copyright 2020 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#[macro_use]
extern crate bencher;
extern crate flatbuffers;
extern crate flexbuffers;
mod flatbuffers_benchmarks;
mod flexbuffers_benchmarks;
#[allow(dead_code, unused_imports)]
#[path = "../../include_test/include_test1_generated.rs"]
pub mod include_test1_generated;
#[allow(dead_code, unused_imports)]
#[path = "../../include_test/sub/include_test2_generated.rs"]
pub mod include_test2_generated;
#[allow(dead_code, unused_imports)]
#[path = "../../monster_test_generated.rs"]
mod monster_test_generated;
pub use monster_test_generated::my_game;
benchmark_main!(
flatbuffers_benchmarks::benches,
flexbuffers_benchmarks::benches
);

94
tests/rust_usage_test/benches/flatbuffers_benchmarks.rs
View File

@@ -14,12 +14,8 @@
* limitations under the License.
*/
#[macro_use]
extern crate bencher;
use bencher::Bencher;
extern crate flatbuffers;
#[allow(dead_code, unused_imports)]
#[path = "../../include_test/include_test1_generated.rs"]
pub mod include_test1_generated;
@@ -63,42 +59,69 @@ fn create_canonical_buffer_then_reset(bench: &mut Bencher) {
}
#[inline(always)]
fn create_serialized_example_with_generated_code(builder: &mut flatbuffers::FlatBufferBuilder, finish: bool) -> usize{
fn create_serialized_example_with_generated_code(
builder: &mut flatbuffers::FlatBufferBuilder,
finish: bool,
) -> usize {
let s0 = builder.create_string("test1");
let s1 = builder.create_string("test2");
let t0_name = builder.create_string("Barney");
let t1_name = builder.create_string("Fred");
let t2_name = builder.create_string("Wilma");
let t0 = my_game::example::Monster::create(builder, &my_game::example::MonsterArgs{
hp: 1000,
name: Some(t0_name),
..Default::default()
});
let t1 = my_game::example::Monster::create(builder, &my_game::example::MonsterArgs{
name: Some(t1_name),
..Default::default()
});
let t2 = my_game::example::Monster::create(builder, &my_game::example::MonsterArgs{
name: Some(t2_name),
..Default::default()
});
let t0 = my_game::example::Monster::create(
builder,
&my_game::example::MonsterArgs {
hp: 1000,
name: Some(t0_name),
..Default::default()
},
);
let t1 = my_game::example::Monster::create(
builder,
&my_game::example::MonsterArgs {
name: Some(t1_name),
..Default::default()
},
);
let t2 = my_game::example::Monster::create(
builder,
&my_game::example::MonsterArgs {
name: Some(t2_name),
..Default::default()
},
);
let mon = {
let name = builder.create_string("MyMonster");
let fred_name = builder.create_string("Fred");
let inventory = builder.create_vector_direct(&[0u8, 1, 2, 3, 4]);
let test4 = builder.create_vector_direct(&[my_game::example::Test::new(10, 20),
my_game::example::Test::new(30, 40)]);
let pos = my_game::example::Vec3::new(1.0, 2.0, 3.0, 3.0, my_game::example::Color::Green, &my_game::example::Test::new(5i16, 6i8));
let args = my_game::example::MonsterArgs{
let test4 = builder.create_vector_direct(&[
my_game::example::Test::new(10, 20),
my_game::example::Test::new(30, 40),
]);
let pos = my_game::example::Vec3::new(
1.0,
2.0,
3.0,
3.0,
my_game::example::Color::Green,
&my_game::example::Test::new(5i16, 6i8),
);
let args = my_game::example::MonsterArgs {
hp: 80,
mana: 150,
name: Some(name),
pos: Some(&pos),
test_type: my_game::example::Any::Monster,
test: Some(my_game::example::Monster::create(builder, &my_game::example::MonsterArgs{
name: Some(fred_name),
..Default::default()
}).as_union_value()),
test: Some(
my_game::example::Monster::create(
builder,
&my_game::example::MonsterArgs {
name: Some(fred_name),
..Default::default()
},
)
.as_union_value(),
),
inventory: Some(inventory),
test4: Some(test4),
testarrayofstring: Some(builder.create_vector(&[s0, s1])),
@@ -155,7 +178,7 @@ fn traverse_serialized_example_with_generated_code(bytes: &[u8]) {
}
fn create_string_10(bench: &mut Bencher) {
let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1<<20);
let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1 << 20);
let mut i = 0;
bench.iter(|| {
builder.create_string("foobarbaz"); // zero-terminated -> 10 bytes
@@ -170,7 +193,7 @@ fn create_string_10(bench: &mut Bencher) {
}
fn create_string_100(bench: &mut Bencher) {
let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1<<20);
let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1 << 20);
let s_owned = (0..99).map(|_| "x").collect::<String>();
let s: &str = &s_owned;
@@ -188,7 +211,7 @@ fn create_string_100(bench: &mut Bencher) {
}
fn create_byte_vector_100_naive(bench: &mut Bencher) {
let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1<<20);
let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1 << 20);
let v_owned = (0u8..100).map(|i| i).collect::<Vec<u8>>();
let v: &[u8] = &v_owned;
@@ -206,7 +229,7 @@ fn create_byte_vector_100_naive(bench: &mut Bencher) {
}
fn create_byte_vector_100_optimal(bench: &mut Bencher) {
let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1<<20);
let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1 << 20);
let v_owned = (0u8..100).map(|i| i).collect::<Vec<u8>>();
let v: &[u8] = &v_owned;
@@ -223,5 +246,12 @@ fn create_byte_vector_100_optimal(bench: &mut Bencher) {
bench.bytes = v.len() as u64;
}
benchmark_group!(benches, create_byte_vector_100_naive, create_byte_vector_100_optimal, traverse_canonical_buffer, create_canonical_buffer_then_reset, create_string_10, create_string_100);
benchmark_main!(benches);
benchmark_group!(
benches,
create_byte_vector_100_naive,
create_byte_vector_100_optimal,
traverse_canonical_buffer,
create_canonical_buffer_then_reset,
create_string_10,
create_string_100
);

295
tests/rust_usage_test/benches/flexbuffers_benchmarks.rs Normal file
View File

@@ -0,0 +1,295 @@
// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use bencher::Bencher;
use flexbuffers::*;
fn push_vec_u64_to_map(b: &mut Bencher) {
let va = vec![u64::max_value() - 10; 512];
let vb = vec![u64::max_value() - 20; 512];
let vc = vec![u64::max_value() - 30; 512];
let mut n = 0;
b.iter(|| {
let mut fxb = Builder::default();
let mut m = fxb.start_map();
let mut ma = m.start_vector("a");
for &a in va.iter() {
ma.push(a);
}
ma.end_vector();
let mut mb = m.start_vector("b");
for &b in vb.iter() {
mb.push(b);
}
mb.end_vector();
let mut mc = m.start_vector("c");
for &c in vc.iter() {
mc.push(c);
}
mc.end_vector();
m.end_map();
n = fxb.view().len();
});
b.bytes = n as u64;
}
fn push_vec_u64_to_map_reused(b: &mut Bencher) {
let va = vec![u64::max_value() - 10; 512];
let vb = vec![u64::max_value() - 20; 512];
let vc = vec![u64::max_value() - 30; 512];
let mut fxb = Builder::default();
let mut n = 0;
let mut go = || {
let mut m = fxb.start_map();
let mut ma = m.start_vector("a");
for &a in va.iter() {
ma.push(a);
}
ma.end_vector();
let mut mb = m.start_vector("b");
for &b in vb.iter() {
mb.push(b);
}
mb.end_vector();
let mut mc = m.start_vector("c");
for &c in vc.iter() {
mc.push(c);
}
mc.end_vector();
m.end_map();
n = fxb.view().len();
};
go(); // warm up allocations.
b.iter(go);
b.bytes = n as u64;
}
fn push_vec_u64_to_map_direct(b: &mut Bencher) {
let va = vec![u64::max_value() - 10; 512];
let vb = vec![u64::max_value() - 20; 512];
let vc = vec![u64::max_value() - 30; 512];
let mut n = 0;
b.iter(|| {
let mut fxb = Builder::default();
let mut m = fxb.start_map();
m.push("a", &va);
m.push("b", &vb);
m.push("c", &vc);
m.end_map();
n = fxb.view().len();
});
b.bytes = n as u64;
}
fn push_vec_u64_to_map_direct_reused(b: &mut Bencher) {
let va = vec![u64::max_value() - 10; 512];
let vb = vec![u64::max_value() - 20; 512];
let vc = vec![u64::max_value() - 30; 512];
let mut n = 0;
let mut fxb = Builder::default();
let mut go = || {
let mut m = fxb.start_map();
m.push("a", &va);
m.push("b", &vb);
m.push("c", &vc);
m.end_map();
n = fxb.view().len();
};
go(); // warm up allocations.
b.iter(go);
b.bytes = n as u64;
}
fn push_vec_without_indirect(b: &mut Bencher) {
let mut builder = Builder::default();
let mut n = 0;
let mut go = || {
let mut b = builder.start_vector();
for i in 0..1024u16 {
b.push(i);
}
b.push(i64::max_value());
b.end_vector();
n = builder.view().len();
};
go(); // warm up allocations.
b.iter(go);
b.bytes = n as u64;
}
// This isn't actually faster than the alternative but it is a lot smaller.
// Based on the above benchmarks a lot of time is stuck in the `values` stack.
fn push_vec_with_indirect(b: &mut Bencher) {
let mut builder = Builder::default();
let mut n = 0;
let mut go = || {
let mut b = builder.start_vector();
for i in 0..1024u16 {
b.push(i);
}
b.push(IndirectInt(i64::max_value()));
b.end_vector();
n = builder.view().len();
};
go(); // warm up allocations.
b.iter(go);
b.bytes = n as u64;
}
fn example_map<'a>(m: &mut MapBuilder<'a>) {
m.push("some_ints", &[256; 5]);
m.push("some_uints", &[256u16; 5]);
m.push("some_floats", &[256f32; 5]);
m.push("some_strings", "muahahahahaha");
}
fn hundred_maps(b: &mut Bencher) {
let mut builder = Builder::default();
let mut n = 0;
let mut go = || {
let mut v = builder.start_vector();
for _ in 0..100 {
example_map(&mut v.start_map());
}
v.end_vector();
n = builder.view().len();
};
go(); // Warm up allocations.
b.iter(go);
b.bytes = n as u64;
}
fn hundred_maps_pooled(b: &mut Bencher) {
let mut builder = Builder::default();
let mut n = 0;
let mut go = || {
let mut v = builder.start_vector();
for _ in 0..100 {
example_map(&mut v.start_map());
}
v.end_vector();
n = builder.view().len();
};
go(); // Warm up allocations.
b.iter(go);
b.bytes = n as u64;
}
fn make_monster(mut monster: MapBuilder) {
monster.push("type", "great orc");
monster.push("age", 100u8);
monster.push("name", "Mr. Orc");
monster.push("coins", &[1, 25, 50, 100, 250]);
monster.push("color", &[255u8, 0, 0, 0]);
{
let mut weapons = monster.start_vector("weapons");
{
let mut hammer = weapons.start_map();
hammer.push("name", "hammer");
hammer.push("damage type", "crush");
hammer.push("damage", 20);
}
{
let mut axe = weapons.start_map();
axe.push("name", "Great Axe");
axe.push("damage type", "slash");
axe.push("damage", 30);
}
}
{
let mut sounds = monster.start_vector("sounds");
sounds.push("grr");
sounds.push("rawr");
sounds.push("muahaha");
}
}
fn serialize_monsters(b: &mut Bencher) {
let mut builder = Builder::default();
let mut n = 0;
let mut go = || {
let mut monsters = builder.start_vector();
for _ in 0..100 {
make_monster(monsters.start_map())
}
monsters.end_vector();
n = builder.view().len();
};
go(); // Warm up allocations.
b.iter(go);
b.bytes = n as u64;
}
fn validate_monster(r: MapReader) {
assert_eq!(r.idx("type").as_str(), "great orc");
assert_eq!(r.idx("age").as_u8(), 100);
assert_eq!(r.idx("name").as_str(), "Mr. Orc");
assert!(r
.idx("coins")
.as_vector()
.iter()
.map(|c| c.as_i16())
.eq([1, 25, 50, 100, 250].iter().cloned()));
assert!(r
.idx("color")
.as_vector()
.iter()
.map(|c| c.as_u8())
.eq([255, 0, 0, 0].iter().cloned()));
let weapons = r.idx("weapons").as_vector();
assert_eq!(weapons.len(), 2);
let hammer = weapons.idx(0).as_map();
assert_eq!(hammer.idx("name").as_str(), "hammer");
assert_eq!(hammer.idx("damage type").as_str(), "crush");
assert_eq!(hammer.idx("damage").as_u64(), 20);
let axe = weapons.idx(1).as_map();
assert_eq!(axe.idx("name").as_str(), "Great Axe");
assert_eq!(axe.idx("damage type").as_str(), "slash");
assert_eq!(axe.idx("damage").as_u64(), 30);
assert!(r
.idx("sounds")
.as_vector()
.iter()
.map(|s| s.as_str())
.eq(["grr", "rawr", "muahaha"].iter().cloned()));
}
fn read_monsters(b: &mut Bencher) {
let mut builder = Builder::default();
let mut monsters = builder.start_vector();
for _ in 0..100 {
make_monster(monsters.start_map());
}
monsters.end_vector();
b.bytes = builder.view().len() as u64;
let go = || {
let r = Reader::get_root(builder.view()).unwrap().as_vector();
assert_eq!(r.len(), 100);
for i in 0..100 {
validate_monster(r.idx(i).as_map());
}
};
b.iter(go);
}
benchmark_group!(
benches,
push_vec_u64_to_map,
push_vec_u64_to_map_reused,
push_vec_u64_to_map_direct,
push_vec_u64_to_map_direct_reused,
push_vec_without_indirect,
push_vec_with_indirect,
hundred_maps,
hundred_maps_pooled,
serialize_monsters,
read_monsters,
);
benchmark_main!(benches);