Port FlatBuffers to Rust (#4898)

This is a port of FlatBuffers to Rust. It provides code generation and a
runtime library derived from the C++ implementation. It utilizes the
Rust type system to provide safe and fast traversal of FlatBuffers data.

There are 188 tests, including many fuzz tests of roundtrips for various
serialization scenarios. Initial benchmarks indicate that the canonical
example payload can be written in ~700ns, and traversed in ~100ns.

Rustaceans may be interested in the Follow, Push, and SafeSliceAccess
traits. These traits lift traversals, reads, writes, and slice accesses
into the type system, providing abstraction with no runtime penalty.
This commit is contained in:
Robert
2018-09-02 17:05:50 -07:00
committed by rw
parent e7578548a5
commit 3c54fd964b
46 changed files with 9998 additions and 61 deletions

285
tests/rust_usage_test/Cargo.lock generated Normal file
View File

@@ -0,0 +1,285 @@
[[package]]
name = "aho-corasick"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "atty"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)",
"termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "bencher"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "bitflags"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "cfg-if"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "env_logger"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "flatbuffers"
version = "0.1.0"
dependencies = [
"smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "fuchsia-zircon"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
"fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "fuchsia-zircon-sys"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "humantime"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "lazy_static"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "libc"
version = "0.2.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "log"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "memchr"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "quick-error"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "quickcheck"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rand"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "redox_syscall"
version = "0.1.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "redox_termios"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "regex"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"aho-corasick 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
"memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"regex-syntax 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
"thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
"utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "regex-syntax"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rust_usage_test"
version = "0.1.0"
dependencies = [
"bencher 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"flatbuffers 0.1.0",
"quickcheck 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "smallvec"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "termcolor"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "termion"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)",
"redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
"redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "thread_local"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "ucd-util"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "unreachable"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "utf8-ranges"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "void"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "winapi"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "wincolor"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[metadata]
"checksum aho-corasick 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f0ba20154ea1f47ce2793322f049c5646cc6d0fa9759d5f333f286e507bf8080"
"checksum atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4a1aa4c24c0718a250f0681885c1af91419d242f29eb8f2ab28502d80dbd1"
"checksum bencher 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "7dfdb4953a096c551ce9ace855a604d702e6e62d77fac690575ae347571717f5"
"checksum bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d0c54bb8f454c567f21197eefcdbf5679d0bd99f2ddbe52e84c77061952e6789"
"checksum cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "efe5c877e17a9c717a0bf3613b2709f723202c4e4675cc8f12926ded29bcb17e"
"checksum env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0e6e40ebb0e66918a37b38c7acab4e10d299e0463fe2af5d29b9cc86710cfd2a"
"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
"checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e"
"checksum lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e6412c5e2ad9584b0b8e979393122026cdd6d2a80b933f890dcd694ddbe73739"
"checksum libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" = "b685088df2b950fccadf07a7187c8ef846a959c142338a48f9dc0b94517eb5f1"
"checksum log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "61bd98ae7f7b754bc53dca7d44b604f733c6bba044ea6f41bc8d89272d8161d2"
"checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d"
"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0"
"checksum quickcheck 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c01babc5ffd48a2a83744b3024814bb46dfd4f2a4705ccb44b1b60e644fdcab7"
"checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5"
"checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1"
"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
"checksum regex 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "13c93d55961981ba9226a213b385216f83ab43bd6ac53ab16b2eeb47e337cf4e"
"checksum regex-syntax 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05b06a75f5217880fc5e905952a42750bf44787e56a6c6d6852ed0992f5e1d54"
"checksum smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "153ffa32fd170e9944f7e0838edf824a754ec4c1fc64746fcc9fe1f8fa602e5d"
"checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83"
"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"
"checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963"
"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d"
"checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56"
"checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122"
"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
"checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd"
"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
"checksum wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb06499a3a4d44302791052df005d5232b927ed1a9658146d842165c4de7767"

View File

@@ -0,0 +1,22 @@
[package]
name = "rust_usage_test"
version = "0.1.0"
authors = ["Robert Winslow <hello@rwinslow.com>", "FlatBuffers Maintainers"]
[dependencies]
flatbuffers = { path = "../../rust/flatbuffers" }
[[bin]]
name = "monster_example"
path = "bin/monster_example.rs"
[dev-dependencies]
quickcheck = "0.6"
# TODO(rw): look into moving to criterion.rs
bencher = "0.1.5"
[[bench]]
# setup for bencher
name = "flatbuffers_benchmarks"
harness = false

View File

@@ -0,0 +1,218 @@
/*
* Copyright 2018 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#[macro_use]
extern crate bencher;
use bencher::Bencher;
extern crate flatbuffers;
#[path = "../../monster_test_generated.rs"]
mod monster_test_generated;
pub use monster_test_generated::my_game;
fn traverse_canonical_buffer(bench: &mut Bencher) {
let owned_data = {
let mut builder = &mut flatbuffers::FlatBufferBuilder::new();
create_serialized_example_with_generated_code(&mut builder, true);
builder.finished_data().to_vec()
};
let data = &owned_data[..];
let n = data.len() as u64;
bench.iter(|| {
traverse_serialized_example_with_generated_code(data);
});
bench.bytes = n;
}
fn create_canonical_buffer_then_reset(bench: &mut Bencher) {
let mut builder = &mut flatbuffers::FlatBufferBuilder::new();
// warmup
create_serialized_example_with_generated_code(&mut builder, true);
let n = builder.finished_data().len() as u64;
builder.reset();
bench.iter(|| {
let _ = create_serialized_example_with_generated_code(&mut builder, true);
builder.reset();
});
bench.bytes = n;
}
#[inline(always)]
fn create_serialized_example_with_generated_code(builder: &mut flatbuffers::FlatBufferBuilder, finish: bool) -> usize{
let s0 = builder.create_string("test1");
let s1 = builder.create_string("test2");
let t0_name = builder.create_string("Barney");
let t1_name = builder.create_string("Fred");
let t2_name = builder.create_string("Wilma");
let t0 = my_game::example::Monster::create(builder, &my_game::example::MonsterArgs{
hp: 1000,
name: Some(t0_name),
..Default::default()
});
let t1 = my_game::example::Monster::create(builder, &my_game::example::MonsterArgs{
name: Some(t1_name),
..Default::default()
});
let t2 = my_game::example::Monster::create(builder, &my_game::example::MonsterArgs{
name: Some(t2_name),
..Default::default()
});
let mon = {
let name = builder.create_string("MyMonster");
let fred_name = builder.create_string("Fred");
let inventory = builder.create_vector_direct(&[0u8, 1, 2, 3, 4]);
let test4 = builder.create_vector_direct(&[my_game::example::Test::new(10, 20),
my_game::example::Test::new(30, 40)]);
let pos = my_game::example::Vec3::new(1.0, 2.0, 3.0, 3.0, my_game::example::Color::Green, &my_game::example::Test::new(5i16, 6i8));
let args = my_game::example::MonsterArgs{
hp: 80,
mana: 150,
name: Some(name),
pos: Some(&pos),
test_type: my_game::example::Any::Monster,
test: Some(my_game::example::Monster::create(builder, &my_game::example::MonsterArgs{
name: Some(fred_name),
..Default::default()
}).as_union_value()),
inventory: Some(inventory),
test4: Some(test4),
testarrayofstring: Some(builder.create_vector(&[s0, s1])),
testarrayoftables: Some(builder.create_vector(&[t0, t1, t2])),
..Default::default()
};
my_game::example::Monster::create(builder, &args)
};
if finish {
my_game::example::finish_monster_buffer(builder, mon);
}
builder.finished_data().len()
// make it do some work
// if builder.finished_data().len() == 0 { panic!("bad benchmark"); }
}
#[inline(always)]
fn blackbox<T>(t: T) -> T {
// encapsulate this in case we need to turn it into a noop
bencher::black_box(t)
}
#[inline(always)]
fn traverse_serialized_example_with_generated_code(bytes: &[u8]) {
let m = my_game::example::get_root_as_monster(bytes);
blackbox(m.hp());
blackbox(m.mana());
blackbox(m.name());
let pos = m.pos().unwrap();
blackbox(pos.x());
blackbox(pos.y());
blackbox(pos.z());
blackbox(pos.test1());
blackbox(pos.test2());
let pos_test3 = pos.test3();
blackbox(pos_test3.a());
blackbox(pos_test3.b());
blackbox(m.test_type());
let table2 = m.test().unwrap();
let monster2 = my_game::example::Monster::init_from_table(table2);
blackbox(monster2.name());
blackbox(m.inventory());
blackbox(m.test4());
let testarrayoftables = m.testarrayoftables().unwrap();
blackbox(testarrayoftables.get(0).hp());
blackbox(testarrayoftables.get(0).name());
blackbox(testarrayoftables.get(1).name());
blackbox(testarrayoftables.get(2).name());
let testarrayofstring = m.testarrayofstring().unwrap();
blackbox(testarrayofstring.get(0));
blackbox(testarrayofstring.get(1));
}
fn create_string_10(bench: &mut Bencher) {
let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1<<20);
let mut i = 0;
bench.iter(|| {
builder.create_string("foobarbaz"); // zero-terminated -> 10 bytes
i += 1;
if i == 10000 {
builder.reset();
i = 0;
}
});
bench.bytes = 10;
}
fn create_string_100(bench: &mut Bencher) {
let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1<<20);
let s_owned = (0..99).map(|_| "x").collect::<String>();
let s: &str = &s_owned;
let mut i = 0;
bench.iter(|| {
builder.create_string(s); // zero-terminated -> 100 bytes
i += 1;
if i == 1000 {
builder.reset();
i = 0;
}
});
bench.bytes = s.len() as u64;
}
fn create_byte_vector_100_naive(bench: &mut Bencher) {
let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1<<20);
let v_owned = (0u8..100).map(|i| i).collect::<Vec<u8>>();
let v: &[u8] = &v_owned;
let mut i = 0;
bench.iter(|| {
builder.create_vector(v); // zero-terminated -> 100 bytes
i += 1;
if i == 10000 {
builder.reset();
i = 0;
}
});
bench.bytes = v.len() as u64;
}
fn create_byte_vector_100_optimal(bench: &mut Bencher) {
let builder = &mut flatbuffers::FlatBufferBuilder::new_with_capacity(1<<20);
let v_owned = (0u8..100).map(|i| i).collect::<Vec<u8>>();
let v: &[u8] = &v_owned;
let mut i = 0;
bench.iter(|| {
builder.create_vector_direct(v);
i += 1;
if i == 10000 {
builder.reset();
i = 0;
}
});
bench.bytes = v.len() as u64;
}
benchmark_group!(benches, create_byte_vector_100_naive, create_byte_vector_100_optimal, traverse_canonical_buffer, create_canonical_buffer_then_reset, create_string_10, create_string_100);
benchmark_main!(benches);

View File

@@ -0,0 +1,18 @@
extern crate flatbuffers;
#[path = "../../monster_test_generated.rs"]
mod monster_test_generated;
pub use monster_test_generated::my_game;
use std::io::Read;
fn main() {
let mut f = std::fs::File::open("../monsterdata_test.mon").unwrap();
let mut buf = Vec::new();
f.read_to_end(&mut buf).expect("file reading failed");
let monster = my_game::example::get_root_as_monster(&buf[..]);
println!("{}", monster.hp()); // `80`
println!("{}", monster.mana()); // default value of `150`
println!("{:?}", monster.name()); // Some("MyMonster")
}

File diff suppressed because it is too large Load Diff