diff --git a/docs/README.md b/docs/README.md index 3ba063c52..b4695a2eb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,6 +14,7 @@ Install: ``` pip install mkdocs-material +pip install mkdocs-redirects ``` Build and Serve: diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index cec8dd088..a161ae3d1 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -54,10 +54,40 @@ extra: link: https://twitter.com/dbaileychess plugins: + # Use redirects to update links from the original docs to the new ones. + # # https://github.com/mkdocs/mkdocs-redirects - redirects: + # Note the .html are suffixed with .md to avoid warnings. Got from + # https://github.com/mkdocs/mkdocs-redirects/issues/51#issuecomment-2408548029 redirect_maps: - 'flatbuffers_guide_use_cpp.html.md': 'cpp.md' + 'flatbuffers_guide_building.html.md': 'building.md' + 'flatbuffers_guide_tutorial.html.md': 'tutorial.md' + 'flatbuffers_guide_using_schema_compiler.html.md': 'flatc.md' + 'flatbuffers_guide_writing_schema.html.md': 'schema.md' + 'flatbuffers_guide_use_c.html.md': 'languages/c.md' + 'flatbuffers_guide_use_cpp.html.md': 'languages/cpp.md' + 'flatbuffers_guide_use_c-sharp.html.md': 'languages/c_sharp.md' + 'flatbuffers_guide_use_dart.html.md': 'languages/dart.md' + 'flatbuffers_guide_use_go.html.md': 'languages/go.md' + 'flatbuffers_guide_use_java.html.md': 'languages/java.md' + 'flatbuffers_guide_use_javascript.html.md': 'languages/javascript.md' + 'flatbuffers_guide_use_lobster.html.md': 'languages/lobster.md' + 'flatbuffers_guide_use_lua.html.md': 'languages/lua.md' + 'flatbuffers_guide_use_php.html.md': 'languages/php.md' + 'flatbuffers_guide_use_python.html.md': 'languages/python.md' + 'flatbuffers_guide_use_rust.html.md': 'languages/rust.md' + 'flatbuffers_guide_use_swift.html.md': 'languages/swift.md' + 'flatbuffers_guide_use_typescript.html.md': 'languages/typescript.md' + 'flatbuffers_grpc_guide_use_cpp.html.md' : "languages/cpp.md#grpc" + 'flatbuffers_support.html.md': 'support.md' + 'flatbuffers_white_paper.html.md': 'white_paper.md' + 'flatbuffers_grammar.html.md': 'grammar.md' + 'flatbuffers_internals.html.md': 'internals.md' + 'intermediate_representation.html.md': 'intermediate_representation.md' + 'flatbuffers_benchmarks.html.md': 'benchmarks.md' + 'flexbuffers.html.md': 'flexbuffers.md' + 'contributing.html.md': 'contributing.md' markdown_extensions: @@ -94,7 +124,26 @@ nav: - Evolution: "evolution.md" - Grammar: "grammar.md" - Language Guides: - - C++: "cpp.md" + - C: "languages/c.md" + - C++: "languages/cpp.md" + - C#: "languages/c_sharp.md" + - Dart: "languages/dart.md" + - Go: "languages/go.md" + - Java: "languages/java.md" + - JavasScript: "languages/javascript.md" + - Lobster: "languages/lobster.md" + - Lua: "languages/lua.md" + - PHP: "languages/php.md" + - Python: "languages/python.md" + - Rust: "languages/rust.md" + - Swift: "languages/swift.md" + - TypeScript: "languages/typescript.md" + - Supported Configurations: "support.md" + - White Paper: "white_paper.md" - Advanced: + - FlatBuffers Internals: "internals.md" + - Intermediate Representation: "intermediate_representation.md" - Annotating Buffers (.afb): "annotation.md" + - Benchmarks: "benchmarks.md" + - FlexBuffers (Schema-less version): "flexbuffers.md" - Contributing: "contributing.md" diff --git a/docs/source/benchmarks.md b/docs/source/benchmarks.md new file mode 100644 index 000000000..848f4e39b --- /dev/null +++ b/docs/source/benchmarks.md @@ -0,0 +1,63 @@ +C++ Benchmarks {#flatbuffers_benchmarks} +========== + +Comparing against other serialization solutions, running on Windows 7 +64bit. We use the LITE runtime for Protocol Buffers (less code / lower +overhead), Rapid JSON (one of the fastest C++ JSON parsers around), +and pugixml, also one of the fastest XML parsers. + +We also compare against code that doesn't use a serialization library +at all (the column "Raw structs"), which is what you get if you write +hardcoded code that just writes structs. This is the fastest possible, +but of course is not cross platform nor has any kind of forwards / +backwards compatibility. + +We compare against Flatbuffers with the binary wire format (as +intended), and also with JSON as the wire format with the optional JSON +parser (which, using a schema, parses JSON into a binary buffer that can +then be accessed as before). + +The benchmark object is a set of about 10 objects containing an array, 4 +strings, and a large variety of int/float scalar values of all sizes, +meant to be representative of game data, e.g. a scene format. + +| | FlatBuffers (binary) | Protocol Buffers LITE | Rapid JSON | FlatBuffers (JSON) | pugixml | Raw structs | +|--------------------------------------------------------|-----------------------|-----------------------|-----------------------|------------------------| ----------------------| ----------------------| +| Decode + Traverse + Dealloc (1 million times, seconds) | 0.08 | 302 | 583 | 105 | 196 | 0.02 | +| Decode / Traverse / Dealloc (breakdown) | 0 / 0.08 / 0 | 220 / 0.15 / 81 | 294 / 0.9 / 287 | 70 / 0.08 / 35 | 41 / 3.9 / 150 | 0 / 0.02 / 0 | +| Encode (1 million times, seconds) | 3.2 | 185 | 650 | 169 | 273 | 0.15 | +| Wire format size (normal / zlib, bytes) | 344 / 220 | 228 / 174 | 1475 / 322 | 1029 / 298 | 1137 / 341 | 312 / 187 | +| Memory needed to store decoded wire (bytes / blocks) | 0 / 0 | 760 / 20 | 65689 / 4 | 328 / 1 | 34194 / 3 | 0 / 0 | +| Transient memory allocated during decode (KB) | 0 | 1 | 131 | 4 | 34 | 0 | +| Generated source code size (KB) | 4 | 61 | 0 | 4 | 0 | 0 | +| Field access in handwritten traversal code | typed accessors | typed accessors | manual error checking | typed accessors | manual error checking | typed but no safety | +| Library source code (KB) | 15 | some subset of 3800 | 87 | 43 | 327 | 0 | + +### Some other serialization systems we compared against but did not benchmark (yet), in rough order of applicability: + +- Cap'n'Proto promises to reduce Protocol Buffers much like FlatBuffers does, + though with a more complicated binary encoding and less flexibility (no + optional fields to allow deprecating fields or serializing with missing + fields for which defaults exist). + It currently also isn't fully cross-platform portable (lack of VS support). +- msgpack: has very minimal forwards/backwards compatibility support when used + with the typed C++ interface. Also lacks VS2010 support. +- Thrift: very similar to Protocol Buffers, but appears to be less efficient, + and have more dependencies. +- YAML: a superset of JSON and otherwise very similar. Used by e.g. Unity. +- C# comes with built-in serialization functionality, as used by Unity also. + Being tied to the language, and having no automatic versioning support + limits its applicability. +- Project Anarchy (the free mobile engine by Havok) comes with a serialization + system, that however does no automatic versioning (have to code around new + fields manually), is very much tied to the rest of the engine, and works + without a schema to generate code (tied to your C++ class definition). + +### Code for benchmarks + +Code for these benchmarks sits in `benchmarks/` in git branch `benchmarks`. +It sits in its own branch because it has submodule dependencies that the main +project doesn't need, and the code standards do not meet those of the main +project. Please read `benchmarks/cpp/README.txt` before working with the code. + +
diff --git a/docs/source/cpp.md b/docs/source/cpp.md deleted file mode 100644 index a9bb0265c..000000000 --- a/docs/source/cpp.md +++ /dev/null @@ -1,2 +0,0 @@ -# Language Guide: C++ - diff --git a/docs/source/flatc.md b/docs/source/flatc.md index f977eb503..6fcbf8f50 100644 --- a/docs/source/flatc.md +++ b/docs/source/flatc.md @@ -85,4 +85,219 @@ list of `FILES...`. This will generate a `mydata.json` file. +### Additional options + +- `-o PATH` : Output all generated files to PATH (either absolute, or + relative to the current directory). If omitted, PATH will be the + current directory. PATH should end in your systems path separator, + e.g. `/` or `\`. + +- `-I PATH` : when encountering `include` statements, attempt to load the + files from this path. Paths will be tried in the order given, and if all + fail (or none are specified) it will try to load relative to the path of + the schema file being parsed. + +- `-M` : Print make rules for generated files. + +- `--strict-json` : Require & generate strict JSON (field names are enclosed + in quotes, no trailing commas in tables/vectors). By default, no quotes are + required/generated, and trailing commas are allowed. + +- `--allow-non-utf8` : Pass non-UTF-8 input through parser and emit nonstandard + \x escapes in JSON. (Default is to raise parse error on non-UTF-8 input.) + +- `--natural-utf8` : Output strings with UTF-8 as human-readable strings. + By default, UTF-8 characters are printed as \uXXXX escapes." + +- `--defaults-json` : Output fields whose value is equal to the default value + when writing JSON text. + +- `--no-prefix` : Don't prefix enum values in generated C++ by their enum + type. + +- `--scoped-enums` : Use C++11 style scoped and strongly typed enums in + generated C++. This also implies `--no-prefix`. + +- `--no-emit-min-max-enum-values` : Disable generation of MIN and MAX + enumerated values for scoped enums and prefixed enums. + +- `--gen-includes` : (deprecated), this is the default behavior. + If the original behavior is required (no include + statements) use `--no-includes.` + +- `--no-includes` : Don't generate include statements for included schemas the + generated file depends on (C++ / Python). + +- `--gen-mutable` : Generate additional non-const accessors for mutating + FlatBuffers in-place. + +- `--gen-onefile` : Generate single output file for C#, Go, and Python. + +- `--gen-name-strings` : Generate type name functions for C++. + +- `--gen-object-api` : Generate an additional object-based API. This API is + more convenient for object construction and mutation than the base API, + at the cost of efficiency (object allocation). Recommended only to be used + if other options are insufficient. + +- `--gen-compare` : Generate operator== for object-based API types. + +- `--gen-nullable` : Add Clang \_Nullable for C++ pointer. or @Nullable for Java. + +- `--gen-generated` : Add @Generated annotation for Java. + +- `--gen-jvmstatic` : Add @JvmStatic annotation for Kotlin methods + in companion object for interop from Java to Kotlin. + +- `--gen-all` : Generate not just code for the current schema files, but + for all files it includes as well. If the language uses a single file for + output (by default the case for C++ and JS), all code will end up in + this one file. + +- `--cpp-include` : Adds an #include in generated file + +- `--cpp-ptr-type T` : Set object API pointer type (default std::unique_ptr) + +- `--cpp-str-type T` : Set object API string type (default std::string) + T::c_str(), T::length() and T::empty() must be supported. + The custom type also needs to be constructible from std::string (see the + --cpp-str-flex-ctor option to change this behavior). + +- `--cpp-str-flex-ctor` : Don't construct custom string types by passing + std::string from Flatbuffers, but (char* + length). This allows efficient + construction of custom string types, including zero-copy construction. + +- `--no-cpp-direct-copy` : Don't generate direct copy methods for C++ + object-based API. + +- `--cpp-std CPP_STD` : Generate a C++ code using features of selected C++ standard. + Supported `CPP_STD` values: + * `c++0x` - generate code compatible with old compilers (VS2010), + * `c++11` - use C++11 code generator (default), + * `c++17` - use C++17 features in generated code (experimental). + +- `--object-prefix` : Customise class prefix for C++ object-based API. + +- `--object-suffix` : Customise class suffix for C++ object-based API. + +- `--go-namespace` : Generate the overrided namespace in Golang. + +- `--go-import` : Generate the overrided import for flatbuffers in Golang. + (default is "github.com/google/flatbuffers/go"). + +- `--raw-binary` : Allow binaries without a file_indentifier to be read. + This may crash flatc given a mismatched schema. + +- `--size-prefixed` : Input binaries are size prefixed buffers. + +- `--proto`: Expect input files to be .proto files (protocol buffers). + Output the corresponding .fbs file. + Currently supports: `package`, `message`, `enum`, nested declarations, + `import` (use `-I` for paths), `extend`, `oneof`, `group`. + Does not support, but will skip without error: `option`, `service`, + `extensions`, and most everything else. + +- `--oneof-union` : Translate .proto oneofs to flatbuffer unions. + +- `--grpc` : Generate GRPC interfaces for the specified languages. + +- `--schema`: Serialize schemas instead of JSON (use with -b). This will + output a binary version of the specified schema that itself corresponds + to the reflection/reflection.fbs schema. Loading this binary file is the + basis for reflection functionality. + +- `--bfbs-comments`: Add doc comments to the binary schema files. + +- `--conform FILE` : Specify a schema the following schemas should be + an evolution of. Gives errors if not. Useful to check if schema + modifications don't break schema evolution rules. + +- `--conform-includes PATH` : Include path for the schema given with + `--conform PATH`. + +- `--filename-suffix SUFFIX` : The suffix appended to the generated + file names. Default is '\_generated'. + +- `--filename-ext EXTENSION` : The extension appended to the generated + file names. Default is language-specific (e.g. "h" for C++). This + should not be used when multiple languages are specified. + +- `--include-prefix PATH` : Prefix this path to any generated include + statements. + +- `--keep-prefix` : Keep original prefix of schema include statement. + +- `--reflect-types` : Add minimal type reflection to code generation. + +- `--reflect-names` : Add minimal type/name reflection. + +- `--root-type T` : Select or override the default root_type. + +- `--require-explicit-ids` : When parsing schemas, require explicit ids (id: x). + +- `--force-defaults` : Emit default values in binary output from JSON. + +- `--force-empty` : When serializing from object API representation, force + strings and vectors to empty rather than null. + +- `--force-empty-vectors` : When serializing from object API representation, force + vectors to empty rather than null. + +- `--flexbuffers` : Used with "binary" and "json" options, it generates + data using schema-less FlexBuffers. + +- `--no-warnings` : Inhibit all warning messages. + +- `--cs-global-alias` : Prepend `global::` to all user generated csharp classes and structs. + +- `--json-nested-bytes` : Allow a nested_flatbuffer field to be parsed as a + vector of bytes in JSON, which is unsafe unless checked by a verifier + afterwards. + +- `--python-no-type-prefix-suffix` : Skip emission of Python functions that are prefixed + with typenames + +- `--python-typing` : Generate Python type annotations + +Additional gRPC options: + +- `--grpc-filename-suffix`: `[C++]` An optional suffix for the generated + files' names. For example, compiling gRPC for C++ with + `--grpc-filename-suffix=.fbs` will generate `{name}.fbs.h` and + `{name}.fbs.cc` files. + +- `--grpc-additional-header`: `[C++]` Additional headers to include in the + generated files. + +- `--grpc-search-path`: `[C++]` An optional prefix for the gRPC runtime path. + For example, compiling gRPC for C++ with `--grpc-search-path=some/path` will + generate the following includes: + + ```cpp + #include "some/path/grpcpp/impl/codegen/async_stream.h" + #include "some/path/grpcpp/impl/codegen/async_unary_call.h" + #include "some/path/grpcpp/impl/codegen/method_handler.h" + ... + ``` + +- `--grpc-use-system-headers`: `[C++]` Whether to generate `#include
` + instead of `#include "header.h"` for all headers when compiling gRPC for + C++. For example, compiling gRPC for C++ with `--grpc-use-system-headers` + will generate the following includes: + + ```cpp + #include + #include + #include + ... + ``` + + NOTE: This option can be negated with `--no-grpc-use-system-headers`. + +- `--grpc-python-typed-handlers`: `[Python]` Whether to generate the typed + handlers that use the generated Python classes instead of raw bytes for + requests/responses. + +NOTE: short-form options for generators are deprecated, use the long form +whenever possible. diff --git a/docs/source/flexbuffers.md b/docs/source/flexbuffers.md new file mode 100644 index 000000000..974dd693f --- /dev/null +++ b/docs/source/flexbuffers.md @@ -0,0 +1,204 @@ +FlexBuffers {#flexbuffers} +========== + +FlatBuffers was designed around schemas, because when you want maximum +performance and data consistency, strong typing is helpful. + +There are however times when you want to store data that doesn't fit a +schema, because you can't know ahead of time what all needs to be stored. + +For this, FlatBuffers has a dedicated format, called FlexBuffers. +This is a binary format that can be used in conjunction +with FlatBuffers (by storing a part of a buffer in FlexBuffers +format), or also as its own independent serialization format. + +While it loses the strong typing, you retain the most unique advantage +FlatBuffers has over other serialization formats (schema-based or not): +FlexBuffers can also be accessed without parsing / copying / object allocation. +This is a huge win in efficiency / memory friendly-ness, and allows unique +use cases such as mmap-ing large amounts of free-form data. + +FlexBuffers' design and implementation allows for a very compact encoding, +combining automatic pooling of strings with automatic sizing of containers to +their smallest possible representation (8/16/32/64 bits). Many values and +offsets can be encoded in just 8 bits. While a schema-less representation is +usually more bulky because of the need to be self-descriptive, FlexBuffers +generates smaller binaries for many cases than regular FlatBuffers. + +FlexBuffers is still slower than regular FlatBuffers though, so we recommend to +only use it if you need it. + + +# Usage in C++ + +Include the header `flexbuffers.h`, which in turn depends on `flatbuffers.h` +and `util.h`. + +To create a buffer: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} +flexbuffers::Builder fbb; +fbb.Int(13); +fbb.Finish(); +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You create any value, followed by `Finish`. Unlike FlatBuffers which requires +the root value to be a table, here any value can be the root, including a lonely +int value. + +You can now access the `std::vector` that contains the encoded value +as `fbb.GetBuffer()`. Write it, send it, or store it in a parent FlatBuffer. In +this case, the buffer is just 3 bytes in size. + +To read this value back, you could just say: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} +auto root = flexbuffers::GetRoot(my_buffer); +int64_t i = root.AsInt64(); +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +FlexBuffers stores ints only as big as needed, so it doesn't differentiate +between different sizes of ints. You can ask for the 64 bit version, +regardless of what you put in. In fact, since you demand to read the root +as an int, if you supply a buffer that actually contains a float, or a +string with numbers in it, it will convert it for you on the fly as well, +or return 0 if it can't. If instead you actually want to know what is inside +the buffer before you access it, you can call `root.GetType()` or `root.IsInt()` +etc. + +Here's a slightly more complex value you could write instead of `fbb.Int` above: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} +fbb.Map([&]() { + fbb.Vector("vec", [&]() { + fbb.Int(-100); + fbb.String("Fred"); + fbb.IndirectFloat(4.0f); + }); + fbb.UInt("foo", 100); +}); +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This stores the equivalent of the JSON value +`{ vec: [ -100, "Fred", 4.0 ], foo: 100 }`. The root is a dictionary that has +just two key-value pairs, with keys `vec` and `foo`. Unlike FlatBuffers, it +actually has to store these keys in the buffer (which it does only once if +you store multiple such objects, by pooling key values), but also unlike +FlatBuffers it has no restriction on the keys (fields) that you use. + +The map constructor uses a C++11 Lambda to group its children, but you can +also use more conventional start/end calls if you prefer. + +The first value in the map is a vector. You'll notice that unlike FlatBuffers, +you can use mixed types. There is also a `TypedVector` variant that only +allows a single type, and uses a bit less memory. + +`IndirectFloat` is an interesting feature that allows you to store values +by offset rather than inline. Though that doesn't make any visible change +to the user, the consequence is that large values (especially doubles or +64 bit ints) that occur more than once can be shared (see ReuseValue). +Another use case is inside of vectors, where the largest element makes +up the size of all elements (e.g. a single double forces all elements to +64bit), so storing a lot of small integers together with a double is more efficient if the double is indirect. + +Accessing it: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} +auto map = flexbuffers::GetRoot(my_buffer).AsMap(); +map.size(); // 2 +auto vec = map["vec"].AsVector(); +vec.size(); // 3 +vec[0].AsInt64(); // -100; +vec[1].AsString().c_str(); // "Fred"; +vec[1].AsInt64(); // 0 (Number parsing failed). +vec[2].AsDouble(); // 4.0 +vec[2].AsString().IsTheEmptyString(); // true (Wrong Type). +vec[2].AsString().c_str(); // "" (This still works though). +vec[2].ToString().c_str(); // "4" (Or have it converted). +map["foo"].AsUInt8(); // 100 +map["unknown"].IsNull(); // true +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +# Usage in Java + +Java implementation follows the C++ one, closely. + +For creating the equivalent of the same JSON `{ vec: [ -100, "Fred", 4.0 ], foo: 100 }`, +one could use the following code: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.java} +FlexBuffersBuilder builder = new FlexBuffersBuilder(ByteBuffer.allocate(512), + FlexBuffersBuilder.BUILDER_FLAG_SHARE_KEYS_AND_STRINGS); +int smap = builder.startMap(); +int svec = builder.startVector(); +builder.putInt(-100); +builder.putString("Fred"); +builder.putFloat(4.0); +builder.endVector("vec", svec, false, false); +builder.putInt("foo", 100); +builder.endMap(null, smap); +ByteBuffer bb = builder.finish(); +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Similarly, to read the data, just: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.java} +FlexBuffers.Map map = FlexBuffers.getRoot(bb).asMap(); +map.size(); // 2 +FlexBuffers.Vector vec = map.get("vec").asVector(); +vec.size(); // 3 +vec.get(0).asLong(); // -100; +vec.get(1).asString(); // "Fred"; +vec.get(1).asLong(); // 0 (Number parsing failed). +vec.get(2).asFloat(); // 4.0 +vec.get(2).asString().isEmpty(); // true (Wrong Type). +vec.get(2).asString(); // "" (This still works though). +vec.get(2).toString(); // "4.0" (Or have it converted). +map.get("foo").asUInt(); // 100 +map.get("unknown").isNull(); // true +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +# Binary encoding + +A description of how FlexBuffers are encoded is in the +[internals](@ref flatbuffers_internals) document. + + +# Nesting inside a FlatBuffer + +You can mark a field as containing a FlexBuffer, e.g. + + a:[ubyte] (flexbuffer); + +A special accessor will be generated that allows you to access the root value +directly, e.g. `a_flexbuffer_root().AsInt64()`. + + +# Efficiency tips + +* Vectors generally are a lot more efficient than maps, so prefer them over maps + when possible for small objects. Instead of a map with keys `x`, `y` and `z`, + use a vector. Better yet, use a typed vector. Or even better, use a fixed + size typed vector. +* Maps are backwards compatible with vectors, and can be iterated as such. + You can iterate either just the values (`map.Values()`), or in parallel with + the keys vector (`map.Keys()`). If you intend + to access most or all elements, this is faster than looking up each element + by key, since that involves a binary search of the key vector. +* When possible, don't mix values that require a big bit width (such as double) + in a large vector of smaller values, since all elements will take on this + width. Use `IndirectDouble` when this is a possibility. Note that + integers automatically use the smallest width possible, i.e. if you ask + to serialize an int64_t whose value is actually small, you will use less + bits. Doubles are represented as floats whenever possible losslessly, but + this is only possible for few values. + Since nested vectors/maps are stored over offsets, they typically don't + affect the vector width. +* To store large arrays of byte data, use a blob. If you'd use a typed + vector, the bit width of the size field may make it use more space than + expected, and may not be compatible with `memcpy`. + Similarly, large arrays of (u)int16_t may be better off stored as a + binary blob if their size could exceed 64k elements. + Construction and use are otherwise similar to strings. diff --git a/docs/source/intermediate_representation.md b/docs/source/intermediate_representation.md new file mode 100644 index 000000000..f4eb07539 --- /dev/null +++ b/docs/source/intermediate_representation.md @@ -0,0 +1,35 @@ +# Flatbuffers Intermediate Representation {#intermediate_representation} + +We use [reflection.fbs](https://github.com/google/flatbuffers/blob/master/reflection/reflection.fbs) +as our intermediate representation. `flatc` parses `.fbs` files, checks them for +errors and stores the resulting data in this IR, outputting `.bfbs` files. +Since this IR is a Flatbuffer, you can load and use it at runtime for runtime +reflection purposes. + +There are some quirks: +- Tables and Structs are serialized as `Object`s. +- Unions and Enums are serialized as `Enum`s. +- It is the responsibility of the code generator to check the `advanced_features` + field of `Schema`. These mark the presence of new, backwards incompatible, + schema features. Code generators must error if generating a schema with + unrecognized advanced features. +- Filenames are relative to a "project root" denoted by "//" in the path. This + may be specified in flatc with `--bfbs-filenames=$PROJECT_ROOT`, or it will be + inferred to be the directory containing the first provided schema file. + + +## Invocation +You can invoke it like so +```{.sh} +flatc -b --schema ${your_fbs_files} +``` +This generates `.bfbs` (binary flatbuffer schema) files. + +Some information is not included by default. See the `--bfbs-filenames` and +`--bfbs-comments` flags. These may be necessary for code-generators, so they can +add documentation and maybe name generated files (depending on the generator). + + +TODO(cneo): Flags to output bfbs as flexbuffers or json. + +TODO(cneo): Tutorial for building a flatc plugin. diff --git a/docs/source/internals.md b/docs/source/internals.md new file mode 100644 index 000000000..e53ce9a78 --- /dev/null +++ b/docs/source/internals.md @@ -0,0 +1,466 @@ +FlatBuffer Internals {#flatbuffers_internals} +==================== + +This section is entirely optional for the use of FlatBuffers. In normal +usage, you should never need the information contained herein. If you're +interested however, it should give you more of an appreciation of why +FlatBuffers is both efficient and convenient. + +### Format components + +A FlatBuffer is a binary file and in-memory format consisting mostly of +scalars of various sizes, all aligned to their own size. Each scalar is +also always represented in little-endian format, as this corresponds to +all commonly used CPUs today. FlatBuffers will also work on big-endian +machines, but will be slightly slower because of additional +byte-swap intrinsics. + +It is assumed that the following conditions are met, to ensure +cross-platform interoperability: +- The binary `IEEE-754` format is used for floating-point numbers. +- The `two's complemented` representation is used for signed integers. +- The endianness is the same for floating-point numbers as for integers. + +On purpose, the format leaves a lot of details about where exactly +things live in memory undefined, e.g. fields in a table can have any +order, and objects to some extent can be stored in many orders. This is +because the format doesn't need this information to be efficient, and it +leaves room for optimization and extension (for example, fields can be +packed in a way that is most compact). Instead, the format is defined in +terms of offsets and adjacency only. This may mean two different +implementations may produce different binaries given the same input +values, and this is perfectly valid. + +### Format identification + +The format also doesn't contain information for format identification +and versioning, which is also by design. FlatBuffers is a statically typed +system, meaning the user of a buffer needs to know what kind of buffer +it is. FlatBuffers can of course be wrapped inside other containers +where needed, or you can use its union feature to dynamically identify +multiple possible sub-objects stored. Additionally, it can be used +together with the schema parser if full reflective capabilities are +desired. + +Versioning is something that is intrinsically part of the format (the +optionality / extensibility of fields), so the format itself does not +need a version number (it's a meta-format, in a sense). We're hoping +that this format can accommodate all data needed. If format breaking +changes are ever necessary, it would become a new kind of format rather +than just a variation. + +### Offsets + +The most important and generic offset type (see `flatbuffers.h`) is +`uoffset_t`, which is currently always a `uint32_t`, and is used to +refer to all tables/unions/strings/vectors (these are never stored +in-line). 32bit is +intentional, since we want to keep the format binary compatible between +32 and 64bit systems, and a 64bit offset would bloat the size for almost +all uses. A version of this format with 64bit (or 16bit) offsets is easy to set +when needed. Unsigned means they can only point in one direction, which +typically is forward (towards a higher memory location). Any backwards +offsets will be explicitly marked as such. + +The format starts with an `uoffset_t` to the root table in the buffer. + +We have two kinds of objects, structs and tables. + +### Structs + +These are the simplest, and as mentioned, intended for simple data that +benefits from being extra efficient and doesn't need versioning / +extensibility. They are always stored inline in their parent (a struct, +table, or vector) for maximum compactness. Structs define a consistent +memory layout where all components are aligned to their size, and +structs aligned to their largest scalar member. This is done independent +of the alignment rules of the underlying compiler to guarantee a cross +platform compatible layout. This layout is then enforced in the generated +code. + +### Tables + +Unlike structs, these are not stored in inline in their parent, but are +referred to by offset. + +They start with an `soffset_t` to a vtable. This is a signed version of +`uoffset_t`, since vtables may be stored anywhere relative to the object. +This offset is subtracted (not added) from the object start to arrive at +the vtable start. This offset is followed by all the +fields as aligned scalars (or offsets). Unlike structs, not all fields +need to be present. There is no set order and layout. A table may contain +field offsets that point to the same value if the user explicitly +serializes the same offset twice. + +To be able to access fields regardless of these uncertainties, we go +through a vtable of offsets. Vtables are shared between any objects that +happen to have the same vtable values. + +The elements of a vtable are all of type `voffset_t`, which is +a `uint16_t`. The first element is the size of the vtable in bytes, +including the size element. The second one is the size of the object, in bytes +(including the vtable offset). This size could be used for streaming, to know +how many bytes to read to be able to access all *inline* fields of the object. +The remaining elements are the N offsets, where N is the amount of fields +declared in the schema when the code that constructed this buffer was +compiled (thus, the size of the table is N + 2). + +All accessor functions in the generated code for tables contain the +offset into this table as a constant. This offset is checked against the +first field (the number of elements), to protect against newer code +reading older data. If this offset is out of range, or the vtable entry +is 0, that means the field is not present in this object, and the +default value is return. Otherwise, the entry is used as offset to the +field to be read. + +### Unions + +Unions are encoded as the combination of two fields: an enum representing the +union choice and the offset to the actual element. FlatBuffers reserves the +enumeration constant `NONE` (encoded as 0) to mean that the union field is not +set. + +### Strings and Vectors + +Strings are simply a vector of bytes, and are always +null-terminated. Vectors are stored as contiguous aligned scalar +elements prefixed by a 32bit element count (not including any +null termination). Neither is stored inline in their parent, but are referred to +by offset. A vector may consist of more than one offset pointing to the same +value if the user explicitly serializes the same offset twice. + +### Construction + +The current implementation constructs these buffers backwards (starting +at the highest memory address of the buffer), since +that significantly reduces the amount of bookkeeping and simplifies the +construction API. + +### Code example + +Here's an example of the code that gets generated for the `samples/monster.fbs`. +What follows is the entire file, broken up by comments: + + // automatically generated, do not modify + + #include "flatbuffers/flatbuffers.h" + + namespace MyGame { + namespace Sample { + +Nested namespace support. + + enum { + Color_Red = 0, + Color_Green = 1, + Color_Blue = 2, + }; + + inline const char **EnumNamesColor() { + static const char *names[] = { "Red", "Green", "Blue", nullptr }; + return names; + } + + inline const char *EnumNameColor(int e) { return EnumNamesColor()[e]; } + +Enums and convenient reverse lookup. + + enum { + Any_NONE = 0, + Any_Monster = 1, + }; + + inline const char **EnumNamesAny() { + static const char *names[] = { "NONE", "Monster", nullptr }; + return names; + } + + inline const char *EnumNameAny(int e) { return EnumNamesAny()[e]; } + +Unions share a lot with enums. + + struct Vec3; + struct Monster; + +Predeclare all data types since circular references between types are allowed +(circular references between object are not, though). + + FLATBUFFERS_MANUALLY_ALIGNED_STRUCT(4) Vec3 { + private: + float x_; + float y_; + float z_; + + public: + Vec3(float x, float y, float z) + : x_(flatbuffers::EndianScalar(x)), y_(flatbuffers::EndianScalar(y)), z_(flatbuffers::EndianScalar(z)) {} + + float x() const { return flatbuffers::EndianScalar(x_); } + float y() const { return flatbuffers::EndianScalar(y_); } + float z() const { return flatbuffers::EndianScalar(z_); } + }; + FLATBUFFERS_STRUCT_END(Vec3, 12); + +These ugly macros do a couple of things: they turn off any padding the compiler +might normally do, since we add padding manually (though none in this example), +and they enforce alignment chosen by FlatBuffers. This ensures the layout of +this struct will look the same regardless of compiler and platform. Note that +the fields are private: this is because these store little endian scalars +regardless of platform (since this is part of the serialized data). +`EndianScalar` then converts back and forth, which is a no-op on all current +mobile and desktop platforms, and a single machine instruction on the few +remaining big endian platforms. + + struct Monster : private flatbuffers::Table { + const Vec3 *pos() const { return GetStruct(4); } + int16_t mana() const { return GetField(6, 150); } + int16_t hp() const { return GetField(8, 100); } + const flatbuffers::String *name() const { return GetPointer(10); } + const flatbuffers::Vector *inventory() const { return GetPointer *>(14); } + int8_t color() const { return GetField(16, 2); } + }; + +Tables are a bit more complicated. A table accessor struct is used to point at +the serialized data for a table, which always starts with an offset to its +vtable. It derives from `Table`, which contains the `GetField` helper functions. +GetField takes a vtable offset, and a default value. It will look in the vtable +at that offset. If the offset is out of bounds (data from an older version) or +the vtable entry is 0, the field is not present and the default is returned. +Otherwise, it uses the entry as an offset into the table to locate the field. + + struct MonsterBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_pos(const Vec3 *pos) { fbb_.AddStruct(4, pos); } + void add_mana(int16_t mana) { fbb_.AddElement(6, mana, 150); } + void add_hp(int16_t hp) { fbb_.AddElement(8, hp, 100); } + void add_name(flatbuffers::Offset name) { fbb_.AddOffset(10, name); } + void add_inventory(flatbuffers::Offset> inventory) { fbb_.AddOffset(14, inventory); } + void add_color(int8_t color) { fbb_.AddElement(16, color, 2); } + MonsterBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); } + flatbuffers::Offset Finish() { return flatbuffers::Offset(fbb_.EndTable(start_, 7)); } + }; + +`MonsterBuilder` is the base helper struct to construct a table using a +`FlatBufferBuilder`. You can add the fields in any order, and the `Finish` +call will ensure the correct vtable gets generated. + + inline flatbuffers::Offset CreateMonster(flatbuffers::FlatBufferBuilder &_fbb, + const Vec3 *pos, int16_t mana, + int16_t hp, + flatbuffers::Offset name, + flatbuffers::Offset> inventory, + int8_t color) { + MonsterBuilder builder_(_fbb); + builder_.add_inventory(inventory); + builder_.add_name(name); + builder_.add_pos(pos); + builder_.add_hp(hp); + builder_.add_mana(mana); + builder_.add_color(color); + return builder_.Finish(); + } + +`CreateMonster` is a convenience function that calls all functions in +`MonsterBuilder` above for you. Note that if you pass values which are +defaults as arguments, it will not actually construct that field, so +you can probably use this function instead of the builder class in +almost all cases. + + inline const Monster *GetMonster(const void *buf) { return flatbuffers::GetRoot(buf); } + +This function is only generated for the root table type, to be able to +start traversing a FlatBuffer from a raw buffer pointer. + + }; // namespace MyGame + }; // namespace Sample + +### Encoding example. + +Below is a sample encoding for the following JSON corresponding to the above +schema: + + { pos: { x: 1, y: 2, z: 3 }, name: "fred", hp: 50 } + +Resulting in this binary buffer: + + // Start of the buffer: + uint32_t 20 // Offset to the root table. + + // Start of the vtable. Not shared in this example, but could be: + uint16_t 16 // Size of table, starting from here. + uint16_t 22 // Size of object inline data. + uint16_t 4, 0, 20, 16, 0, 0 // Offsets to fields from start of (root) table, 0 for not present. + + // Start of the root table: + int32_t 16 // Offset to vtable used (default negative direction) + float 1, 2, 3 // the Vec3 struct, inline. + uint32_t 8 // Offset to the name string. + int16_t 50 // hp field. + int16_t 0 // Padding for alignment. + + // Start of name string: + uint32_t 4 // Length of string. + int8_t 'f', 'r', 'e', 'd', 0, 0, 0, 0 // Text + 0 termination + padding. + +Note that this not the only possible encoding, since the writer has some +flexibility in which of the children of root object to write first (though in +this case there's only one string), and what order to write the fields in. +Different orders may also cause different alignments to happen. + +### Additional reading. + +The author of the C language implementation has made a similar +[document](https://github.com/dvidelabs/flatcc/blob/master/doc/binary-format.md#flatbuffers-binary-format) +that may further help clarify the format. + +# FlexBuffers + +The [schema-less](@ref flexbuffers) version of FlatBuffers have their +own encoding, detailed here. + +It shares many properties mentioned above, in that all data is accessed +over offsets, all scalars are aligned to their own size, and +all data is always stored in little endian format. + +One difference is that FlexBuffers are built front to back, so children are +stored before parents, and the root of the data starts at the last byte. + +Another difference is that scalar data is stored with a variable number of bits +(8/16/32/64). The current width is always determined by the *parent*, i.e. if +the scalar sits in a vector, the vector determines the bit width for all +elements at once. Selecting the minimum bit width for a particular vector is +something the encoder does automatically and thus is typically of no concern +to the user, though being aware of this feature (and not sticking a double in +the same vector as a bunch of byte sized elements) is helpful for efficiency. + +Unlike FlatBuffers there is only one kind of offset, and that is an unsigned +integer indicating the number of bytes in a negative direction from the address +of itself (where the offset is stored). + +### Vectors + +The representation of the vector is at the core of how FlexBuffers works (since +maps are really just a combination of 2 vectors), so it is worth starting there. + +As mentioned, a vector is governed by a single bit width (supplied by its +parent). This includes the size field. For example, a vector that stores the +integer values `1, 2, 3` is encoded as follows: + + uint8_t 3, 1, 2, 3, 4, 4, 4 + +The first `3` is the size field, and is placed before the vector (an offset +from the parent to this vector points to the first element, not the size +field, so the size field is effectively at index -1). +Since this is an untyped vector `SL_VECTOR`, it is followed by 3 type +bytes (one per element of the vector), which are always following the vector, +and are always a uint8_t even if the vector is made up of bigger scalars. + +A vector may include more than one offset pointing to the same value if the +user explicitly serializes the same offset twice. + +### Types + +A type byte is made up of 2 components (see flexbuffers.h for exact values): + +* 2 lower bits representing the bit-width of the child (8, 16, 32, 64). + This is only used if the child is accessed over an offset, such as a child + vector. It is ignored for inline types. +* 6 bits representing the actual type (see flexbuffers.h). + +Thus, in this example `4` means 8 bit child (value 0, unused, since the value is +in-line), type `SL_INT` (value 1). + +### Typed Vectors + +These are like the Vectors above, but omit the type bytes. The type is instead +determined by the vector type supplied by the parent. Typed vectors are only +available for a subset of types for which these savings can be significant, +namely inline signed/unsigned integers (`TYPE_VECTOR_INT` / `TYPE_VECTOR_UINT`), +floats (`TYPE_VECTOR_FLOAT`), and keys (`TYPE_VECTOR_KEY`, see below). + +Additionally, for scalars, there are fixed length vectors of sizes 2 / 3 / 4 +that don't store the size (`TYPE_VECTOR_INT2` etc.), for an additional savings +in space when storing common vector or color data. + +### Scalars + +FlexBuffers supports integers (`TYPE_INT` and `TYPE_UINT`) and floats +(`TYPE_FLOAT`), available in the bit-widths mentioned above. They can be stored +both inline and over an offset (`TYPE_INDIRECT_*`). + +The offset version is useful to encode costly 64bit (or even 32bit) quantities +into vectors / maps of smaller sizes, and to share / repeat a value multiple +times. + +### Booleans and Nulls + +Booleans (`TYPE_BOOL`) and nulls (`TYPE_NULL`) are encoded as inlined unsigned integers. + +### Blobs, Strings and Keys. + +A blob (`TYPE_BLOB`) is encoded similar to a vector, with one difference: the +elements are always `uint8_t`. The parent bit width only determines the width of +the size field, allowing blobs to be large without the elements being large. + +Strings (`TYPE_STRING`) are similar to blobs, except they have an additional 0 +termination byte for convenience, and they MUST be UTF-8 encoded (since an +accessor in a language that does not support pointers to UTF-8 data may have to +convert them to a native string type). + +A "Key" (`TYPE_KEY`) is similar to a string, but doesn't store the size +field. They're so named because they are used with maps, which don't care +for the size, and can thus be even more compact. Unlike strings, keys cannot +contain bytes of value 0 as part of their data (size can only be determined by +`strlen`), so while you can use them outside the context of maps if you so +desire, you're usually better off with strings. + +### Maps + +A map (`TYPE_MAP`) is like an (untyped) vector, but with 2 prefixes before the +size field: + +| index | field | +| ----: | :----------------------------------------------------------- | +| -3 | An offset to the keys vector (may be shared between tables). | +| -2 | Byte width of the keys vector. | +| -1 | Size (from here on it is compatible with `TYPE_VECTOR`) | +| 0 | Elements. | +| Size | Types. | + +Since a map is otherwise the same as a vector, it can be iterated like +a vector (which is probably faster than lookup by key). + +The keys vector is a typed vector of keys. Both the keys and corresponding +values *have* to be stored in sorted order (as determined by `strcmp`), such +that lookups can be made using binary search. + +The reason the key vector is a separate structure from the value vector is +such that it can be shared between multiple value vectors, and also to +allow it to be treated as its own individual vector in code. + +An example map { foo: 13, bar: 14 } would be encoded as: + + 0 : uint8_t 'b', 'a', 'r', 0 + 4 : uint8_t 'f', 'o', 'o', 0 + 8 : uint8_t 2 // key vector of size 2 + // key vector offset points here + 9 : uint8_t 9, 6 // offsets to bar_key and foo_key + 11: uint8_t 2, 1 // offset to key vector, and its byte width + 13: uint8_t 2 // value vector of size + // value vector offset points here + 14: uint8_t 14, 13 // values + 16: uint8_t 4, 4 // types + +### The root + +As mentioned, the root starts at the end of the buffer. +The last uint8_t is the width in bytes of the root (normally the parent +determines the width, but the root has no parent). The uint8_t before this is +the type of the root, and the bytes before that are the root value (of the +number of bytes specified by the last byte). + +So for example, the integer value `13` as root would be: + + uint8_t 13, 4, 1 // Value, type, root byte width. + diff --git a/docs/source/languages/c.md b/docs/source/languages/c.md new file mode 100644 index 000000000..bd1ec159d --- /dev/null +++ b/docs/source/languages/c.md @@ -0,0 +1,224 @@ +Use in C {#flatbuffers_guide_use_c} +========== + +The C language binding exists in a separate project named [FlatCC](https://github.com/dvidelabs/flatcc). + +The `flatcc` C schema compiler can generate code offline as well as +online via a C library. It can also generate buffer verifiers and fast +JSON parsers, printers. + +Great care has been taken to ensure compatibility with the main `flatc` +project. + +## General Documention + +- [Tutorial](@ref flatbuffers_guide_tutorial) - select C as language + when scrolling down +- [FlatCC Guide](https://github.com/dvidelabs/flatcc#flatcc-flatbuffers-in-c-for-c) +- [The C Builder Interface](https://github.com/dvidelabs/flatcc/blob/master/doc/builder.md#the-builder-interface) +- [The Monster Sample in C](https://github.com/dvidelabs/flatcc/blob/master/samples/monster/monster.c) +- [GitHub](https://github.com/dvidelabs/flatcc) + + +## Supported Platforms + +- Ubuntu (clang / gcc, ninja / gnu make) +- OS-X (clang / gcc, ninja / gnu make) +- Windows MSVC 2010, 2013, 2015 + +CI builds recent versions of gcc, clang and MSVC on OS-X, Ubuntu, and +Windows, and occasionally older compiler versions. See main project [Status](https://github.com/dvidelabs/flatcc#status). + +Other platforms may well work, including Centos, but are not tested +regularly. + +The monster sample project was specifically written for C99 in order to +follow the C++ version and for that reason it will not work with MSVC +2010. + +## Modular Object Creation + +In the tutorial we used the call `Monster_create_as_root` to create the +root buffer object since this is easier in simple use cases. Sometimes +we need more modularity so we can reuse a function to create nested +tables and root tables the same way. For this we need the +`flatcc_builder_buffer_create_call`. It is best to keep `flatcc_builder` +calls isolated at the top driver level, so we get: + +
+~~~{.c} + ns(Monster_ref_t) create_orc(flatcc_builder_t *B) + { + // ... same as in the tutorial. + return s(Monster_create(B, ...)); + } + + void create_monster_buffer() + { + uint8_t *buf; + size_t size; + flatcc_builder_t builder, *B; + + // Initialize the builder object. + B = &builder; + flatcc_builder_init(B); + // Only use `buffer_create` without `create/start/end_as_root`. + flatcc_builder_buffer_create(create_orc(B)); + // Allocate and copy buffer to user memory. + buf = flatcc_builder_finalize_buffer(B, &size); + // ... write the buffer to disk or network, or something. + + free(buf); + flatcc_builder_clear(B); + } +~~~ +
+ +The same principle applies with `start/end` vs `start/end_as_root` in +the top-down approach. + + +## Top Down Example + +The tutorial uses a bottom up approach. In C it is also possible to use +a top-down approach by starting and ending objects nested within each +other. In the tutorial there is no deep nesting, so the difference is +limited, but it shows the idea: + +
+
+~~~{.c} + uint8_t treasure[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + size_t treasure_count = c_vec_len(treasure); + ns(Weapon_ref_t) axe; + + // NOTE: if we use end_as_root, we MUST also start as root. + ns(Monster_start_as_root(B)); + ns(Monster_pos_create(B, 1.0f, 2.0f, 3.0f)); + ns(Monster_hp_add(B, 300)); + ns(Monster_mana_add(B, 150)); + // We use create_str instead of add because we have no existing string reference. + ns(Monster_name_create_str(B, "Orc")); + // Again we use create because we no existing vector object, only a C-array. + ns(Monster_inventory_create(B, treasure, treasure_count)); + ns(Monster_color_add(B, ns(Color_Red))); + if (1) { + ns(Monster_weapons_start(B)); + ns(Monster_weapons_push_create(B, flatbuffers_string_create_str(B, "Sword"), 3)); + // We reuse the axe object later. Note that we dereference a pointer + // because push always returns a short-term pointer to the stored element. + // We could also have created the axe object first and simply pushed it. + axe = *ns(Monster_weapons_push_create(B, flatbuffers_string_create_str(B, "Axe"), 5)); + ns(Monster_weapons_end(B)); + } else { + // We can have more control with the table elements added to a vector: + // + ns(Monster_weapons_start(B)); + ns(Monster_weapons_push_start(B)); + ns(Weapon_name_create_str(B, "Sword")); + ns(Weapon_damage_add(B, 3)); + ns(Monster_weapons_push_end(B)); + ns(Monster_weapons_push_start(B)); + ns(Monster_weapons_push_start(B)); + ns(Weapon_name_create_str(B, "Axe")); + ns(Weapon_damage_add(B, 5)); + axe = *ns(Monster_weapons_push_end(B)); + ns(Monster_weapons_end(B)); + } + // Unions can get their type by using a type-specific add/create/start method. + ns(Monster_equipped_Weapon_add(B, axe)); + + ns(Monster_end_as_root(B)); +~~~ +
+ + +## Basic Reflection + +The C-API does support reading binary schema (.bfbs) +files via code generated from the `reflection.fbs` schema, and an +[example usage](https://github.com/dvidelabs/flatcc/tree/master/samples/reflection) +shows how to use this. The reflection schema files are pre-generated +in the [runtime distribution](https://github.com/dvidelabs/flatcc/tree/master/include/flatcc/reflection). + + +## Mutations and Reflection + +The C-API does not support mutating reflection like C++ does, nor does +the reader interface support mutating scalars (and it is generally +unsafe to do so even after verification). + +The generated reader interface supports sorting vectors in-place after +casting them to a mutating type because it is not practical to do so +while building a buffer. This is covered in the builder documentation. +The reflection example makes use of this feature to look up objects by +name. + +It is possible to build new buffers using complex objects from existing +buffers as source. This can be very efficient due to direct copy +semantics without endian conversion or temporary stack allocation. + +Scalars, structs and strings can be used as source, as well vectors of +these. + +It is currently not possible to use an existing table or vector of table +as source, but it would be possible to add support for this at some +point. + + +## Namespaces + +The `FLATBUFFERS_WRAP_NAMESPACE` approach used in the tutorial is convenient +when each function has a very long namespace prefix. But it isn't always +the best approach. If the namespace is absent, or simple and +informative, we might as well use the prefix directly. The +[reflection example](https://github.com/dvidelabs/flatcc/blob/master/samples/reflection/bfbs2json.c) +mentioned above uses this approach. + + +## Checking for Present Members + +Not all languages support testing if a field is present, but in C we can +elaborate the reader section of the tutorial with tests for this. Recall +that `mana` was set to the default value `150` and therefore shouldn't +be present. + +
+~~~{.c} + int hp_present = ns(Monster_hp_is_present(monster)); // 1 + int mana_present = ns(Monster_mana_is_present(monster)); // 0 +~~~ +
+ +## Alternative ways to add a Union + +In the tutorial we used a single call to add a union. Here we show +different ways to accomplish the same thing. The last form is rarely +used, but is the low-level way to do it. It can be used to group small +values together in the table by adding type and data at different +points in time. + +
+~~~{.c} + ns(Equipment_union_ref_t) equipped = ns(Equipment_as_Weapon(axe)); + ns(Monster_equipped_add(B, equipped)); + // or alternatively + ns(Monster_equipped_Weapon_add(B, axe); + // or alternatively + ns(Monster_equipped_add_type(B, ns(Equipment_Weapon)); + ns(Monster_equipped_add_member(B, axe)); +~~~ +
+ +## Why not integrate with the `flatc` tool? + +[It was considered how the C code generator could be integrated into the +`flatc` tool](https://github.com/dvidelabs/flatcc/issues/1), but it +would either require that the standalone C implementation of the schema +compiler was dropped, or it would lead to excessive code duplication, or +a complicated intermediate representation would have to be invented. +Neither of these alternatives are very attractive, and it isn't a big +deal to use the `flatcc` tool instead of `flatc` given that the +FlatBuffers C runtime library needs to be made available regardless. + + diff --git a/docs/source/languages/c_sharp.md b/docs/source/languages/c_sharp.md new file mode 100644 index 000000000..398a7259d --- /dev/null +++ b/docs/source/languages/c_sharp.md @@ -0,0 +1,265 @@ +Use in C\# {#flatbuffers_guide_use_c-sharp} +============== + +## Before you get started + +Before diving into the FlatBuffers usage in C#, it should be noted that +the [Tutorial](@ref flatbuffers_guide_tutorial) page has a complete guide to +general FlatBuffers usage in all of the supported languages (including C#). +This page is designed to cover the nuances of FlatBuffers usage, +specific to C#. + +You should also have read the [Building](@ref flatbuffers_guide_building) +documentation to build `flatc` and should be familiar with +[Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler) and +[Writing a schema](@ref flatbuffers_guide_writing_schema). + +## FlatBuffers C# code location + +The code for the FlatBuffers C# library can be found at +`flatbuffers/net/FlatBuffers`. You can browse the library on the +[FlatBuffers GitHub page](https://github.com/google/flatbuffers/tree/master/net/ +FlatBuffers). + +## Building the FlatBuffers C# library + +The `FlatBuffers.csproj` project contains multitargeting for .NET Standard 2.1, +.NET 6 and .NET 8. + +You can build for a specific framework target when using the cross-platform +[.NET Core SDK](https://dotnet.microsoft.com/download) by adding the `-f` +command line option: + +~~~{.sh} + dotnet build -f netstandard2.1 "FlatBuffers.csproj" +~~~ + +The `FlatBuffers.csproj` project also provides support for defining various +conditional compilation symbols (see "Conditional compilation symbols" section +below) using the `-p` command line option: + +~~~{.sh} + dotnet build -f netstandard2.1 -p:ENABLE_SPAN_T=true -p:UNSAFE_BYTEBUFFER=true "FlatBuffers.csproj" +~~~ + +## Testing the FlatBuffers C# library + +The code to test the libraries can be found at `flatbuffers/tests`. + +The test code for C# is located in the [FlatBuffers.Test](https://github.com/ +google/flatbuffers/tree/master/tests/FlatBuffers.Test) subfolder. To run the +tests, open `FlatBuffers.Test.csproj` in [Visual Studio]( +https://www.visualstudio.com), and compile/run the project. + +Optionally, you can run this using [Mono](http://www.mono-project.com/) instead. +Once you have installed Mono, you can run the tests from the command line +by running the following commands from inside the `FlatBuffers.Test` folder: + +~~~{.sh} + mcs *.cs ../MyGame/Example/*.cs ../../net/FlatBuffers/*.cs + mono Assert.exe +~~~ + +## Using the FlatBuffers C# library + +*Note: See [Tutorial](@ref flatbuffers_guide_tutorial) for a more in-depth +example of how to use FlatBuffers in C#.* + +FlatBuffers supports reading and writing binary FlatBuffers in C#. + +To use FlatBuffers in your own code, first generate C# classes from your +schema with the `--csharp` option to `flatc`. +Then you can include both FlatBuffers and the generated code to read +or write a FlatBuffer. + +For example, here is how you would read a FlatBuffer binary file in C#: +First, import the library and generated code. Then, you read a FlatBuffer binary +file into a `byte[]`. You then turn the `byte[]` into a `ByteBuffer`, which you +pass to the `GetRootAsMyRootType` function: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cs} + using MyGame.Example; + using Google.FlatBuffers; + + // This snippet ignores exceptions for brevity. + byte[] data = File.ReadAllBytes("monsterdata_test.mon"); + + ByteBuffer bb = new ByteBuffer(data); + Monster monster = Monster.GetRootAsMonster(bb); +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Now you can access the data from the `Monster monster`: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cs} + short hp = monster.Hp; + Vec3 pos = monster.Pos; +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +C# code naming follows standard C# style with PascalCasing identifiers, +e.g. `GetRootAsMyRootType`. Also, values (except vectors and unions) are +available as properties instead of parameterless accessor methods. +The performance-enhancing methods to which you can pass an already created +object are prefixed with `Get`, e.g.: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cs} + // property + var pos = monster.Pos; + + // method filling a preconstructed object + var preconstructedPos = new Vec3(); + monster.GetPos(preconstructedPos); +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +## Storing dictionaries in a FlatBuffer + +FlatBuffers doesn't support dictionaries natively, but there is support to +emulate their behavior with vectors and binary search, which means you +can have fast lookups directly from a FlatBuffer without having to unpack +your data into a `Dictionary` or similar. + +To use it: +- Designate one of the fields in a table as the "key" field. You do this + by setting the `key` attribute on this field, e.g. + `name:string (key)`. + You may only have one key field, and it must be of string or scalar type. +- Write out tables of this type as usual, collect their offsets in an + array. +- Instead of calling standard generated method, + e.g.: `Monster.createTestarrayoftablesVector`, + call `CreateSortedVectorOfMonster` in C# + which will first sort all offsets such that the tables they refer to + are sorted by the key field, then serialize it. +- Now when you're accessing the FlatBuffer, you can use + the `ByKey` accessor to access elements of the vector, e.g.: + `monster.TestarrayoftablesByKey("Frodo")` in C#, + which returns an object of the corresponding table type, + or `null` if not found. + `ByKey` performs a binary search, so should have a similar + speed to `Dictionary`, though may be faster because of better caching. + `ByKey` only works if the vector has been sorted, it will + likely not find elements if it hasn't been sorted. + +## Buffer verification + +As mentioned in [C++ Usage](@ref flatbuffers_guide_use_cpp) buffer +accessor functions do not verify buffer offsets at run-time. +If it is necessary, you can optionally use a buffer verifier before you +access the data. This verifier will check all offsets, all sizes of +fields, and null termination of strings to ensure that when a buffer +is accessed, all reads will end up inside the buffer. + +Each root type will have a verification function generated for it, +e.g. `Monster.VerifyMonster`. This can be called as shown: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cs} + var ok = Monster.VerifyMonster(buf); +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +if `ok` is true, the buffer is safe to read. + +For a more detailed control of verification `MonsterVerify.Verify` +for `Monster` type can be used: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cs} + # Sequence of calls + FlatBuffers.Verifier verifier = new FlatBuffers.Verifier(buf); + var ok = verifier.VerifyBuffer("MONS", false, MonsterVerify.Verify); + + # Or single line call + var ok = new FlatBuffers.Verifier(bb).setStringCheck(true).\ + VerifyBuffer("MONS", false, MonsterVerify.Verify); + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +if `ok` is true, the buffer is safe to read. + +A second parameter of `verifyBuffer` specifies whether buffer content is +size prefixed or not. In the example above, the buffer is assumed to not include +size prefix (`false`). + +Verifier supports options that can be set using appropriate fluent methods: +* SetMaxDepth - limit the nesting depth. Default: 1000000 +* SetMaxTables - total amount of tables the verifier may encounter. Default: 64 +* SetAlignmentCheck - check content alignment. Default: True +* SetStringCheck - check if strings contain termination '0' character. Default: true + + +## Text parsing + +There currently is no support for parsing text (Schema's and JSON) directly +from C#, though you could use the C++ parser through native call +interfaces available to each language. Please see the +C++ documentation for more on text parsing. + +## Object based API + +FlatBuffers is all about memory efficiency, which is why its base API is written +around using as little as possible of it. This does make the API clumsier +(requiring pre-order construction of all data, and making mutation harder). + +For times when efficiency is less important a more convenient object based API +can be used (through `--gen-object-api`) that is able to unpack & pack a +FlatBuffer into objects and standard `System.Collections.Generic` containers, +allowing for convenient construction, access and mutation. + +To use: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cs} + // Deserialize from buffer into object. + MonsterT monsterobj = GetMonster(flatbuffer).UnPack(); + + // Update object directly like a C# class instance. + Console.WriteLine(monsterobj.Name); + monsterobj.Name = "Bob"; // Change the name. + + // Serialize into new flatbuffer. + FlatBufferBuilder fbb = new FlatBufferBuilder(1); + fbb.Finish(Monster.Pack(fbb, monsterobj).Value); +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +### Json Serialization + +An additional feature of the object API is the ability to allow you to +serialize & deserialize a JSON text. +To use Json Serialization, add `--cs-gen-json-serializer` option to `flatc` and +add `Newtonsoft.Json` nuget package to csproj. This requires explicitly setting +the `--gen-object-api` option as well. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cs} + // Deserialize MonsterT from json + string jsonText = File.ReadAllText(@"Resources/monsterdata_test.json"); + MonsterT mon = MonsterT.DeserializeFromJson(jsonText); + + // Serialize MonsterT to json + string jsonText2 = mon.SerializeToJson(); +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Limitation + * `hash` attribute currently not supported. +* NuGet package Dependency + * [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) + +## Conditional compilation symbols + +There are three conditional compilation symbols that have an impact on +performance/features of the C# `ByteBuffer` implementation. + +* `UNSAFE_BYTEBUFFER` + + This will use unsafe code to manipulate the underlying byte array. This can + yield a reasonable performance increase. + +* `BYTEBUFFER_NO_BOUNDS_CHECK` + + This will disable the bounds check asserts to the byte array. This can yield a + small performance gain in normal code. + +* `ENABLE_SPAN_T` + + This will enable reading and writing blocks of memory with a `Span` instead + of just `T[]`. You can also enable writing directly to shared memory or other + types of memory by providing a custom implementation of `ByteBufferAllocator`. + `ENABLE_SPAN_T` also requires `UNSAFE_BYTEBUFFER` to be defined, or .NET + Standard 2.1. + +Using `UNSAFE_BYTEBUFFER` and `BYTEBUFFER_NO_BOUNDS_CHECK` together can yield a +performance gain of ~15% for some operations, however doing so is potentially +dangerous. Do so at your own risk! + +
diff --git a/docs/source/languages/cpp.md b/docs/source/languages/cpp.md new file mode 100644 index 000000000..e694a5961 --- /dev/null +++ b/docs/source/languages/cpp.md @@ -0,0 +1,672 @@ +# Language Guide: C++ + +## Before you get started + +Before diving into the FlatBuffers usage in C++, it should be noted that +the [Tutorial](@ref flatbuffers_guide_tutorial) page has a complete guide +to general FlatBuffers usage in all of the supported languages (including C++). +This page is designed to cover the nuances of FlatBuffers usage, specific to +C++. + +#### Prerequisites + +This page assumes you have written a FlatBuffers schema and compiled it +with the Schema Compiler. If you have not, please see +[Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler) +and [Writing a schema](@ref flatbuffers_guide_writing_schema). + +Assuming you wrote a schema, say `mygame.fbs` (though the extension doesn't +matter), you've generated a C++ header called `mygame_generated.h` using the +compiler (e.g. `flatc -c mygame.fbs`), you can now start using this in +your program by including the header. As noted, this header relies on +`flatbuffers/flatbuffers.h`, which should be in your include path. + +## FlatBuffers C++ library code location + +The code for the FlatBuffers C++ library can be found at +`flatbuffers/include/flatbuffers`. You can browse the library code on the +[FlatBuffers GitHub page](https://github.com/google/flatbuffers/tree/master/include/flatbuffers). + +## Testing the FlatBuffers C++ library + +The code to test the C++ library can be found at `flatbuffers/tests`. +The test code itself is located in +[test.cpp](https://github.com/google/flatbuffers/blob/master/tests/test.cpp). + +This test file is built alongside `flatc`. To review how to build the project, +please read the [Building](@ref flatbuffers_guide_building) documentation. + +To run the tests, execute `flattests` from the root `flatbuffers/` directory. +For example, on [Linux](https://en.wikipedia.org/wiki/Linux), you would simply +run: `./flattests`. + +## Using the FlatBuffers C++ library + +*Note: See [Tutorial](@ref flatbuffers_guide_tutorial) for a more in-depth +example of how to use FlatBuffers in C++.* + +FlatBuffers supports both reading and writing FlatBuffers in C++. + +To use FlatBuffers in your code, first generate the C++ classes from your +schema with the `--cpp` option to `flatc`. Then you can include both FlatBuffers +and the generated code to read or write FlatBuffers. + +For example, here is how you would read a FlatBuffer binary file in C++: +First, include the library and generated code. Then read the file into +a `char *` array, which you pass to `GetMonster()`. + +```cpp + #include "flatbuffers/flatbuffers.h" + #include "monster_test_generate.h" + #include // C++ header file for printing + #include // C++ header file for file access + + + std::ifstream infile; + infile.open("monsterdata_test.mon", std::ios::binary | std::ios::in); + infile.seekg(0,std::ios::end); + int length = infile.tellg(); + infile.seekg(0,std::ios::beg); + char *data = new char[length]; + infile.read(data, length); + infile.close(); + + auto monster = GetMonster(data); +``` + +`monster` is of type `Monster *`, and points to somewhere *inside* your +buffer (root object pointers are not the same as `buffer_pointer` \!). +If you look in your generated header, you'll see it has +convenient accessors for all fields, e.g. `hp()`, `mana()`, etc: + +```cpp + std::cout << "hp : " << monster->hp() << std::endl; // '80' + std::cout << "mana : " << monster->mana() << std::endl; // default value of '150' + std::cout << "name : " << monster->name()->c_str() << std::endl; // "MyMonster" +``` + +*Note: That we never stored a `mana` value, so it will return the default.* + +The following attributes are supported: + +- `shared` (on a field): For string fields, this enables the usage of string + pooling (i.e. `CreateSharedString`) as default serialization behavior. + + Specifically, `CreateXxxDirect` functions and `Pack` functions for object + based API (see below) will use `CreateSharedString` to create strings. + +## Object based API + +FlatBuffers is all about memory efficiency, which is why its base API is written +around using as little as possible of it. This does make the API clumsier +(requiring pre-order construction of all data, and making mutation harder). + +For times when efficiency is less important a more convenient object based API +can be used (through `--gen-object-api`) that is able to unpack & pack a +FlatBuffer into objects and standard STL containers, allowing for convenient +construction, access and mutation. + +To use: + +```cpp + // Autogenerated class from table Monster. + MonsterT monsterobj; + + // Deserialize from buffer into object. + GetMonster(flatbuffer)->UnPackTo(&monsterobj); + + // Update object directly like a C++ class instance. + cout << monsterobj.name; // This is now a std::string! + monsterobj.name = "Bob"; // Change the name. + + // Serialize into new flatbuffer. + FlatBufferBuilder fbb; + fbb.Finish(Monster::Pack(fbb, &monsterobj)); +``` + +The following attributes are specific to the object-based API code generation: + +- `native_inline` (on a field): Because FlatBuffer tables and structs are + optionally present in a given buffer, they are best represented as pointers + (specifically std::unique_ptrs) in the native class since they can be null. + This attribute changes the member declaration to use the type directly + rather than wrapped in a unique_ptr. + +- `native_default("value")` (on a field): For members that are declared + "native_inline", the value specified with this attribute will be included + verbatim in the class constructor initializer list for this member. + +- `native_custom_alloc("custom_allocator")` (on a table or struct): When using the + object-based API all generated NativeTables that are allocated when unpacking + your flatbuffer will use "custom allocator". The allocator is also used by + any std::vector that appears in a table defined with `native_custom_alloc`. + This can be used to provide allocation from a pool for example, for faster + unpacking when using the object-based API. + +Minimal Example: + +schema: + +```cpp + table mytable(native_custom_alloc:"custom_allocator") { + ... + } +``` + +with `custom_allocator` defined before `flatbuffers.h` is included, as: + +```cpp + template struct custom_allocator : public std::allocator { + + typedef T *pointer; + + template + struct rebind { + typedef custom_allocator other; + }; + + pointer allocate(const std::size_t n) { + return std::allocator::allocate(n); + } + + void deallocate(T* ptr, std::size_t n) { + return std::allocator::deallocate(ptr,n); + } + + custom_allocator() throw() {} + + template + custom_allocator(const custom_allocator&) throw() {} + }; +``` + +- `native_type("type")` (on a struct): In some cases, a more optimal C++ data +type exists for a given struct. For example, the following schema: + +```cpp + struct Vec2 { + x: float; + y: float; + } +``` + +generates the following Object-Based API class: + +```cpp + struct Vec2T : flatbuffers::NativeTable { + float x; + float y; + }; +``` + +However, it can be useful to instead use a user-defined C++ type since it +can provide more functionality, eg. + +```cpp + struct vector2 { + float x = 0, y = 0; + vector2 operator+(vector2 rhs) const { ... } + vector2 operator-(vector2 rhs) const { ... } + float length() const { ... } + // etc. + }; +``` + +The `native_type` attribute will replace the usage of the generated class +with the given type. So, continuing with the example, the generated +code would use `vector2` in place of `Vec2T` for all generated code of +the Object-Based API. + +However, because the `native_type` is unknown to flatbuffers, the user must +provide the following functions to aide in the serialization process: + +```cpp + namespace flatbuffers { + Vec2 Pack(const vector2& obj); + vector2 UnPack(const Vec2& obj); + } +``` + +- `native_type_pack_name("name")` (on a struct when `native_type` is + specified, too): when you want to use the same `native_type` multiple times + (e. g. with different precision) you must make the names of the Pack/UnPack + functions unique, otherwise you will run into compile errors. This attribute + appends a name to the expected Pack/UnPack functions. So when you + specify `native_type_pack_name("Vec2")` in the above example you now need to + implement these serialization functions instead: + +```cpp + namespace flatbuffers { + Vec2 PackVec2(const vector2& obj); + vector2 UnPackVec2(const Vec2& obj); + } +``` + +Finally, the following top-level attributes: + +- `native_include("path")` (at file level): Because the `native_type` attribute + can be used to introduce types that are unknown to flatbuffers, it may be + necessary to include "external" header files in the generated code. This + attribute can be used to directly add an #include directive to the top of + the generated code that includes the specified path directly. + +- `force_align`: this attribute may not be respected in the object API, + depending on the aligned of the allocator used with `new`. + +## External references + +An additional feature of the object API is the ability to allow you to load +multiple independent FlatBuffers, and have them refer to eachothers objects +using hashes which are then represented as typed pointers in the object API. + +To make this work have a field in the objects you want to referred to which is +using the string hashing feature (see `hash` attribute in the +[schema](@ref flatbuffers_guide_writing_schema) documentation). Then you have +a similar hash in the field referring to it, along with a `cpp_type` +attribute specifying the C++ type this will refer to (this can be any C++ +type, and will get a `*` added). + +Then, in JSON or however you create these buffers, make sure they use the +same string (or hash). + +When you call `UnPack` (or `Create`), you'll need a function that maps from +hash to the object (see `resolver_function_t` for details). + +## Using different pointer types + +By default the object tree is built out of `std::unique_ptr`, but you can +influence this either globally (using the `--cpp-ptr-type` argument to +`flatc`) or per field (using the `cpp_ptr_type` attribute) to by any smart +pointer type (`my_ptr`), or by specifying `naked` as the type to get `T *` +pointers. Unlike the smart pointers, naked pointers do not manage memory for +you, so you'll have to manage their lifecycles manually. To reference the +pointer type specified by the `--cpp-ptr-type` argument to `flatc` from a +flatbuffer field set the `cpp_ptr_type` attribute to `default_ptr_type`. + +## Using different string type + +By default the object tree is built out of `std::string`, but you can +influence this either globally (using the `--cpp-str-type` argument to +`flatc`) or per field using the `cpp_str_type` attribute. + +The type must support `T::c_str()`, `T::length()` and `T::empty()` as member functions. + +Further, the type must be constructible from std::string, as by default a +std::string instance is constructed and then used to initialize the custom +string type. This behavior impedes efficient and zero-copy construction of +custom string types; the `--cpp-str-flex-ctor` argument to `flatc` or the +per field attribute `cpp_str_flex_ctor` can be used to change this behavior, +so that the custom string type is constructed by passing the pointer and +length of the FlatBuffers String. The custom string class will require a +constructor in the following format: `custom_str_class(const char *, size_t)`. +Please note that the character array is not guaranteed to be NULL terminated, +you should always use the provided size to determine end of string. + +## Reflection (& Resizing) + +There is experimental support for reflection in FlatBuffers, allowing you to +read and write data even if you don't know the exact format of a buffer, and +even allows you to change sizes of strings and vectors in-place. + +The way this works is very elegant; there is actually a FlatBuffer schema that +describes schemas (\!) which you can find in `reflection/reflection.fbs`. +The compiler, `flatc`, can write out any schemas it has just parsed as a binary +FlatBuffer, corresponding to this meta-schema. + +Loading in one of these binary schemas at runtime allows you traverse any +FlatBuffer data that corresponds to it without knowing the exact format. You +can query what fields are present, and then read/write them after. + +For convenient field manipulation, you can include the header +`flatbuffers/reflection.h` which includes both the generated code from the meta +schema, as well as a lot of helper functions. + +And example of usage, for the time being, can be found in +`test.cpp/ReflectionTest()`. + +## Mini Reflection + +A more limited form of reflection is available for direct inclusion in +generated code, which doesn't do any (binary) schema access at all. It was designed +to keep the overhead of reflection as low as possible (on the order of 2-6 +bytes per field added to your executable), but doesn't contain all the +information the (binary) schema contains. + +You add this information to your generated code by specifying `--reflect-types` +(or instead `--reflect-names` if you also want field / enum names). + +You can now use this information, for example to print a FlatBuffer to text: + + auto s = flatbuffers::FlatBufferToString(flatbuf, MonsterTypeTable()); + +`MonsterTypeTable()` is declared in the generated code for each type. The +string produced is very similar to the JSON produced by the `Parser` based +text generator. + +You'll need `flatbuffers/minireflect.h` for this functionality. In there is also +a convenient visitor/iterator so you can write your own output / functionality +based on the mini reflection tables without having to know the FlatBuffers or +reflection encoding. + +## Storing maps / dictionaries in a FlatBuffer + +FlatBuffers doesn't support maps natively, but there is support to +emulate their behavior with vectors and binary search, which means you +can have fast lookups directly from a FlatBuffer without having to unpack +your data into a `std::map` or similar. + +To use it: +- Designate one of the fields in a table as they "key" field. You do this + by setting the `key` attribute on this field, e.g. + `name:string (key)`. + You may only have one key field, and it must be of string or scalar type. +- Write out tables of this type as usual, collect their offsets in an + array or vector. +- Instead of `CreateVector`, call `CreateVectorOfSortedTables`, + which will first sort all offsets such that the tables they refer to + are sorted by the key field, then serialize it. +- Now when you're accessing the FlatBuffer, you can use `Vector::LookupByKey` + instead of just `Vector::Get` to access elements of the vector, e.g.: + `myvector->LookupByKey("Fred")`, which returns a pointer to the + corresponding table type, or `nullptr` if not found. + `LookupByKey` performs a binary search, so should have a similar speed to + `std::map`, though may be faster because of better caching. `LookupByKey` + only works if the vector has been sorted, it will likely not find elements + if it hasn't been sorted. + +## Direct memory access + +As you can see from the above examples, all elements in a buffer are +accessed through generated accessors. This is because everything is +stored in little endian format on all platforms (the accessor +performs a swap operation on big endian machines), and also because +the layout of things is generally not known to the user. + +For structs, layout is deterministic and guaranteed to be the same +across platforms (scalars are aligned to their +own size, and structs themselves to their largest member), and you +are allowed to access this memory directly by using `sizeof()` and +`memcpy` on the pointer to a struct, or even an array of structs. + +To compute offsets to sub-elements of a struct, make sure they +are a structs themselves, as then you can use the pointers to +figure out the offset without having to hardcode it. This is +handy for use of arrays of structs with calls like `glVertexAttribPointer` +in OpenGL or similar APIs. + +It is important to note is that structs are still little endian on all +machines, so only use tricks like this if you can guarantee you're not +shipping on a big endian machine (an `assert(FLATBUFFERS_LITTLEENDIAN)` +would be wise). + +## Access of untrusted buffers + +The generated accessor functions access fields over offsets, which is +very quick. These offsets are not verified at run-time, so a malformed +buffer could cause a program to crash by accessing random memory. + +When you're processing large amounts of data from a source you know (e.g. +your own generated data on disk), this is acceptable, but when reading +data from the network that can potentially have been modified by an +attacker, this is undesirable. + +For this reason, you can optionally use a buffer verifier before you +access the data. This verifier will check all offsets, all sizes of +fields, and null termination of strings to ensure that when a buffer +is accessed, all reads will end up inside the buffer. + +Each root type will have a verification function generated for it, +e.g. for `Monster`, you can call: + +```cpp + bool ok = VerifyMonsterBuffer(Verifier(buf, len)); +``` + +if `ok` is true, the buffer is safe to read. + +Besides untrusted data, this function may be useful to call in debug +mode, as extra insurance against data being corrupted somewhere along +the way. + +While verifying a buffer isn't "free", it is typically faster than +a full traversal (since any scalar data is not actually touched), +and since it may cause the buffer to be brought into cache before +reading, the actual overhead may be even lower than expected. + +In specialized cases where a denial of service attack is possible, +the verifier has two additional constructor arguments that allow +you to limit the nesting depth and total amount of tables the +verifier may encounter before declaring the buffer malformed. The default is +`Verifier(buf, len, 64 /* max depth */, 1000000, /* max tables */)` which +should be sufficient for most uses. + +## Text & schema parsing + +Using binary buffers with the generated header provides a super low +overhead use of FlatBuffer data. There are, however, times when you want +to use text formats, for example because it interacts better with source +control, or you want to give your users easy access to data. + +Another reason might be that you already have a lot of data in JSON +format, or a tool that generates JSON, and if you can write a schema for +it, this will provide you an easy way to use that data directly. + +(see the schema documentation for some specifics on the JSON format +accepted). + +Schema evolution compatibility for the JSON format follows the same rules as the binary format (JSON formatted data will be forwards/backwards compatible with schemas that evolve in a compatible way). + +There are two ways to use text formats: + +#### Using the compiler as a conversion tool + +This is the preferred path, as it doesn't require you to add any new +code to your program, and is maximally efficient since you can ship with +binary data. The disadvantage is that it is an extra step for your +users/developers to perform, though you might be able to automate it. + + flatc -b myschema.fbs mydata.json + +This will generate the binary file `mydata_wire.bin` which can be loaded +as before. + +#### Making your program capable of loading text directly + +This gives you maximum flexibility. You could even opt to support both, +i.e. check for both files, and regenerate the binary from text when +required, otherwise just load the binary. + +This option is currently only available for C++, or Java through JNI. + +As mentioned in the section "Building" above, this technique requires +you to link a few more files into your program, and you'll want to include +`flatbuffers/idl.h`. + +Load text (either a schema or json) into an in-memory buffer (there is a +convenient `LoadFile()` utility function in `flatbuffers/util.h` if you +wish). Construct a parser: + +```cpp + flatbuffers::Parser parser; +``` + +Now you can parse any number of text files in sequence: + +```cpp + parser.Parse(text_file.c_str()); +``` + +This works similarly to how the command-line compiler works: a sequence +of files parsed by the same `Parser` object allow later files to +reference definitions in earlier files. Typically this means you first +load a schema file (which populates `Parser` with definitions), followed +by one or more JSON files. + +As optional argument to `Parse`, you may specify a null-terminated list of +include paths. If not specified, any include statements try to resolve from +the current directory. + +If there were any parsing errors, `Parse` will return `false`, and +`Parser::error_` contains a human readable error string with a line number +etc, which you should present to the creator of that file. + +After each JSON file, the `Parser::fbb` member variable is the +`FlatBufferBuilder` that contains the binary buffer version of that +file, that you can access as described above. + +`samples/sample_text.cpp` is a code sample showing the above operations. + +## Threading + +Reading a FlatBuffer does not touch any memory outside the original buffer, +and is entirely read-only (all const), so is safe to access from multiple +threads even without synchronisation primitives. + +Creating a FlatBuffer is not thread safe. All state related to building +a FlatBuffer is contained in a FlatBufferBuilder instance, and no memory +outside of it is touched. To make this thread safe, either do not +share instances of FlatBufferBuilder between threads (recommended), or +manually wrap it in synchronisation primitives. There's no automatic way to +accomplish this, by design, as we feel multithreaded construction +of a single buffer will be rare, and synchronisation overhead would be costly. + +## Advanced union features + +The C++ implementation currently supports vectors of unions (i.e. you can +declare a field as `[T]` where `T` is a union type instead of a table type). It +also supports structs and strings in unions, besides tables. + +For an example of these features, see `tests/union_vector`, and +`UnionVectorTest` in `test.cpp`. + +Since these features haven't been ported to other languages yet, if you +choose to use them, you won't be able to use these buffers in other languages +(`flatc` will refuse to compile a schema that uses these features). + +These features reduce the amount of "table wrapping" that was previously +needed to use unions. + +To use scalars, simply wrap them in a struct. + +## Depth limit of nested objects and stack-overflow control +The parser of Flatbuffers schema or json-files is kind of recursive parser. +To avoid stack-overflow problem the parser has a built-in limiter of +recursion depth. Number of nested declarations in a schema or number of +nested json-objects is limited. By default, this depth limit set to `64`. +It is possible to override this limit with `FLATBUFFERS_MAX_PARSING_DEPTH` +definition. This definition can be helpful for testing purposes or embedded +applications. For details see [build](@ref flatbuffers_guide_building) of +CMake-based projects. + +## Dependence from C-locale {#flatbuffers_locale_cpp} +The Flatbuffers [grammar](@ref flatbuffers grammar) uses ASCII +character set for identifiers, alphanumeric literals, reserved words. + +Internal implementation of the Flatbuffers depends from functions which +depend from C-locale: `strtod()` or `strtof()`, for example. +The library expects the dot `.` symbol as the separator of an integer +part from the fractional part of a float number. +Another separator symbols (`,` for example) will break the compatibility +and may lead to an error while parsing a Flatbuffers schema or a json file. + +The Standard C locale is a global resource, there is only one locale for +the entire application. Some modern compilers and platforms have +locale-independent or locale-narrow functions `strtof_l`, `strtod_l`, +`strtoll_l`, `strtoull_l` to resolve this dependency. +These functions use specified locale rather than the global or per-thread +locale instead. They are part of POSIX-2008 but not part of the C/C++ +standard library, therefore, may be missing on some platforms. +The Flatbuffers library try to detect these functions at configuration and +compile time: +- CMake `"CMakeLists.txt"`: + - Check existence of `strtol_l` and `strtod_l` in the ``. +- Compile-time `"/include/base.h"`: + - `_MSC_VER >= 1900`: MSVC2012 or higher if build with MSVC. + - `_XOPEN_SOURCE>=700`: POSIX-2008 if build with GCC/Clang. + +After detection, the definition `FLATBUFFERS_LOCALE_INDEPENDENT` will be +set to `0` or `1`. +To override or stop this detection use CMake `-DFLATBUFFERS_LOCALE_INDEPENDENT={0|1}` +or predefine `FLATBUFFERS_LOCALE_INDEPENDENT` symbol. + +To test the compatibility of the Flatbuffers library with +a specific locale use the environment variable `FLATBUFFERS_TEST_LOCALE`: +```sh +>FLATBUFFERS_TEST_LOCALE="" ./flattests +>FLATBUFFERS_TEST_LOCALE="ru_RU.CP1251" ./flattests +``` + +## Support of floating-point numbers +The Flatbuffers library assumes that a C++ compiler and a CPU are +compatible with the `IEEE-754` floating-point standard. +The schema and json parser may fail if `fast-math` or `/fp:fast` mode is active. + +### Support of hexadecimal and special floating-point numbers +According to the [grammar](@ref flatbuffers_grammar) `fbs` and `json` files +may use hexadecimal and special (`NaN`, `Inf`) floating-point literals. +The Flatbuffers uses `strtof` and `strtod` functions to parse floating-point +literals. The Flatbuffers library has a code to detect a compiler compatibility +with the literals. If necessary conditions are met the preprocessor constant +`FLATBUFFERS_HAS_NEW_STRTOD` will be set to `1`. +The support of floating-point literals will be limited at compile time +if `FLATBUFFERS_HAS_NEW_STRTOD` constant is less than `1`. +In this case, schemas with hexadecimal or special literals cannot be used. + +### Comparison of floating-point NaN values +The floating-point `NaN` (`not a number`) is special value which +representing an undefined or unrepresentable value. +`NaN` may be explicitly assigned to variables, typically as a representation +for missing values or may be a result of a mathematical operation. +The `IEEE-754` defines two kind of `NaNs`: +- Quiet NaNs, or `qNaNs`. +- Signaling NaNs, or `sNaNs`. + +According to the `IEEE-754`, a comparison with `NaN` always returns +an unordered result even when compared with itself. As a result, a whole +Flatbuffers object will be not equal to itself if has one or more `NaN`. +Flatbuffers scalar fields that have the default value are not actually stored +in the serialized data but are generated in code (see [Writing a schema](@ref flatbuffers_guide_writing_schema)). +Scalar fields with `NaN` defaults break this behavior. +If a schema has a lot of `NaN` defaults the Flatbuffers can override +the unordered comparison by the ordered: `(NaN==NaN)->true`. +This ordered comparison is enabled when compiling a program with the symbol +`FLATBUFFERS_NAN_DEFAULTS` defined. +Additional computations added by `FLATBUFFERS_NAN_DEFAULTS` are very cheap +if GCC or Clang used. These compilers have a compile-time implementation +of `isnan` checking which MSVC does not. + +## gRPC + +### Before you get started + +Before diving into the FlatBuffers gRPC usage in C++, you should already be +familiar with the following: + +- FlatBuffers as a serialization format +- [gRPC](http://www.grpc.io/docs/) usage + +### Using the FlatBuffers gRPC C++ library + +NOTE: The examples below are also in the `grpc/samples/greeter` directory. + +We will illustrate usage with the following schema: + +``` title="grpc/samples/greeter/greeter.fbs" +--8<-- "https://raw.githubusercontent.com/google/flatbuffers/refs/heads/master/grpc/samples/greeter/greeter.fbs" +``` + +When we run `flatc`, we pass in the `--grpc` option and generage an additional +`greeter.grpc.fb.h` and `greeter.grpc.fb.cc`. + +Example server code looks like this: + +``` title="grpc/samples/greeter/server.cpp" +--8<-- "https://raw.githubusercontent.com/google/flatbuffers/refs/heads/master/grpc/samples/greeter/server.cpp" +``` + +Example client code looks like this: + +``` title="grpc/samples/greeter/client.cpp" +--8<-- "https://raw.githubusercontent.com/google/flatbuffers/refs/heads/master/grpc/samples/greeter/client.cpp" +``` + diff --git a/docs/source/languages/dart.md b/docs/source/languages/dart.md new file mode 100644 index 000000000..515a64411 --- /dev/null +++ b/docs/source/languages/dart.md @@ -0,0 +1,131 @@ +Use in Dart {#flatbuffers_guide_use_dart} +=========== + +## Before you get started + +Before diving into the FlatBuffers usage in Dart, it should be noted that +the [Tutorial](@ref flatbuffers_guide_tutorial) page has a complete guide +to general FlatBuffers usage in all of the supported languages (including Dart). +This page is designed to cover the nuances of FlatBuffers usage, specific to +Dart. + +You should also have read the [Building](@ref flatbuffers_guide_building) +documentation to build `flatc` and should be familiar with +[Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler) and +[Writing a schema](@ref flatbuffers_guide_writing_schema). + +## FlatBuffers Dart library code location + +The code for the FlatBuffers Dart library can be found at +`flatbuffers/dart`. You can browse the library code on the [FlatBuffers +GitHub page](https://github.com/google/flatbuffers/tree/master/dart). + +## Testing the FlatBuffers Dart library + +The code to test the Dart library can be found at `flatbuffers/tests`. +The test code itself is located in [dart_test.dart](https://github.com/google/ +flatbuffers/blob/master/tests/dart_test.dart). + +To run the tests, use the [DartTest.sh](https://github.com/google/flatbuffers/ +blob/master/tests/DartTest.sh) shell script. + +*Note: The shell script requires the [Dart SDK](https://www.dartlang.org/tools/sdk) +to be installed.* + +## Using the FlatBuffers Dart library + +*Note: See [Tutorial](@ref flatbuffers_guide_tutorial) for a more in-depth +example of how to use FlatBuffers in Dart.* + +FlatBuffers supports reading and writing binary FlatBuffers in Dart. + +To use FlatBuffers in your own code, first generate Dart classes from your +schema with the `--dart` option to `flatc`. Then you can include both FlatBuffers +and the generated code to read or write a FlatBuffer. + +For example, here is how you would read a FlatBuffer binary file in Dart: First, +include the library and generated code. Then read a FlatBuffer binary file into +a `List`, which you pass to the factory constructor for `Monster`: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.dart} +import 'dart:io' as io; + +import 'package:flat_buffers/flat_buffers.dart' as fb; +import './monster_my_game.sample_generated.dart' as myGame; + +List data = await new io.File('monster.dat').readAsBytes(); +var monster = new myGame.Monster(data); +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Now you can access values like this: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.dart} +var hp = monster.hp; +var pos = monster.pos; +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +## Differences from the Dart SDK Front End flat_buffers + +The work in this repository is signfiicantly based on the implementation used +internally by the Dart SDK in the front end/analyzer package. Several +significant changes have been made. + +1. Support for packed boolean lists has been removed. This is not standard + in other implementations and is not compatible with them. Do note that, + like in the JavaScript implementation, __null values in boolean lists + will be treated as false__. It is also still entirely possible to pack data + in a single scalar field, but that would have to be done on the application + side. +2. The SDK implementation supports enums with regular Dart enums, which + works if enums are always indexed at 1; however, FlatBuffers does not + require that. This implementation uses specialized enum-like classes to + ensure proper mapping from FlatBuffers to Dart and other platforms. +3. The SDK implementation does not appear to support FlatBuffer structs or + vectors of structs - it treated everything as a built-in scalar or a table. + This implementation treats structs in a way that is compatible with other + non-Dart implementations, and properly handles vectors of structs. Many of + the methods prefixed with 'low' have been prepurposed to support this. +4. The SDK implementation treats int64 and uint64 as float64s. This + implementation does not. This may cause problems with JavaScript + compatibility - however, it should be possible to use the JavaScript + implementation, or to do a customized implementation that treats all 64 bit + numbers as floats. Supporting the Dart VM and Flutter was a more important + goal of this implementation. Support for 16 bit integers was also added. +5. The code generation in this offers an "ObjectBuilder", which generates code + very similar to the SDK classes that consume FlatBuffers, as well as Builder + classes, which produces code which more closely resembles the builders in + other languages. The ObjectBuilder classes are easier to use, at the cost of + additional references allocated. + +## Text Parsing + +There currently is no support for parsing text (Schema's and JSON) directly +from Dart, though you could use the C++ parser through Dart Native Extensions. +Please see the C++ documentation for more on text parsing (note that this is +not currently an option in Flutter - follow [this issue](https://github.com/flutter/flutter/issues/7053) +for the latest). + +## Object based API + +FlatBuffers is all about memory efficiency, which is why its base API is written +around using as little as possible of it. This does make the API clumsier +(requiring pre-order construction of all data, and making mutation harder). + +For times when efficiency is less important a more convenient object based API +can be used (through `--gen-object-api`) that is able to unpack & pack a FlatBuffer +into objects and lists, allowing for convenient construction, access and mutation. + +To use: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.dart} + // Deserialize from buffer into object. + MonsterT monster = Monster(flatbuffer).unpack(); + + // Update object directly like a Dart class instance. + print(monster.Name); + monster.Name = "Bob"; // Change the name. + + // Serialize into new flatbuffer. + final fbb = Builder(); + fbb.Finish(monster.pack(fbb)); +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/languages/go.md b/docs/source/languages/go.md new file mode 100644 index 000000000..e2aa115e8 --- /dev/null +++ b/docs/source/languages/go.md @@ -0,0 +1,99 @@ +Use in Go {#flatbuffers_guide_use_go} +========= + +## Before you get started + +Before diving into the FlatBuffers usage in Go, it should be noted that +the [Tutorial](@ref flatbuffers_guide_tutorial) page has a complete guide +to general FlatBuffers usage in all of the supported languages (including Go). +This page is designed to cover the nuances of FlatBuffers usage, specific to +Go. + +You should also have read the [Building](@ref flatbuffers_guide_building) +documentation to build `flatc` and should be familiar with +[Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler) and +[Writing a schema](@ref flatbuffers_guide_writing_schema). + +## FlatBuffers Go library code location + +The code for the FlatBuffers Go library can be found at +`flatbuffers/go`. You can browse the library code on the [FlatBuffers +GitHub page](https://github.com/google/flatbuffers/tree/master/go). + +## Testing the FlatBuffers Go library + +The code to test the Go library can be found at `flatbuffers/tests`. +The test code itself is located in [go_test.go](https://github.com/google/ +flatbuffers/blob/master/tests/go_test.go). + +To run the tests, use the [GoTest.sh](https://github.com/google/flatbuffers/ +blob/master/tests/GoTest.sh) shell script. + +*Note: The shell script requires [Go](https://golang.org/doc/install) to +be installed.* + +## Using the FlatBuffers Go library + +*Note: See [Tutorial](@ref flatbuffers_guide_tutorial) for a more in-depth +example of how to use FlatBuffers in Go.* + +FlatBuffers supports reading and writing binary FlatBuffers in Go. + +To use FlatBuffers in your own code, first generate Go classes from your +schema with the `--go` option to `flatc`. Then you can include both FlatBuffers +and the generated code to read or write a FlatBuffer. + +For example, here is how you would read a FlatBuffer binary file in Go: First, +include the library and generated code. Then read a FlatBuffer binary file into +a `[]byte`, which you pass to the `GetRootAsMonster` function: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.go} + import ( + example "MyGame/Example" + flatbuffers "github.com/google/flatbuffers/go" + + "os" + ) + + buf, err := os.ReadFile("monster.dat") + // handle err + monster := example.GetRootAsMonster(buf, 0) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Now you can access values like this: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.go} + hp := monster.Hp() + pos := monster.Pos(nil) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +In some cases it's necessary to modify values in an existing FlatBuffer in place (without creating a copy). For this reason, scalar fields of a Flatbuffer table or struct can be mutated. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.go} + monster := example.GetRootAsMonster(buf, 0) + + // Set table field. + if ok := monster.MutateHp(10); !ok { + panic("failed to mutate Hp") + } + + // Set struct field. + monster.Pos().MutateZ(4) + + // This mutation will fail because the mana field is not available in + // the buffer. It should be set when creating the buffer. + if ok := monster.MutateMana(20); !ok { + panic("failed to mutate Hp") + } +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The term `mutate` is used instead of `set` to indicate that this is a special use case. All mutate functions return a boolean value which is false if the field we're trying to mutate is not available in the buffer. + +## Text Parsing + +There currently is no support for parsing text (Schema's and JSON) directly +from Go, though you could use the C++ parser through cgo. Please see the +C++ documentation for more on text parsing. + +
diff --git a/docs/source/languages/java.md b/docs/source/languages/java.md new file mode 100644 index 000000000..30ba06133 --- /dev/null +++ b/docs/source/languages/java.md @@ -0,0 +1,114 @@ +Use in Java {#flatbuffers_guide_use_java} +============== + +## Before you get started + +Before diving into the FlatBuffers usage in Java, it should be noted that +the [Tutorial](@ref flatbuffers_guide_tutorial) page has a complete guide to +general FlatBuffers usage in all of the supported languages (including Java). +This page is designed to cover the nuances of FlatBuffers usage, +specific to Java. + +You should also have read the [Building](@ref flatbuffers_guide_building) +documentation to build `flatc` and should be familiar with +[Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler) and +[Writing a schema](@ref flatbuffers_guide_writing_schema). + +## FlatBuffers Java code location + +The code for the FlatBuffers Java library can be found at +`flatbuffers/java/com/google/flatbuffers`. You can browse the library on the +[FlatBuffers GitHub page](https://github.com/google/flatbuffers/tree/master/ +java/com/google/flatbuffers). + +## Testing the FlatBuffers Java libraries + +The code to test the libraries can be found at `flatbuffers/tests`. + +The test code for Java is located in [JavaTest.java](https://github.com/google +/flatbuffers/blob/master/tests/JavaTest.java). + +To run the tests, use either [JavaTest.sh](https://github.com/google/ +flatbuffers/blob/master/tests/JavaTest.sh) or [JavaTest.bat](https://github.com/ +google/flatbuffers/blob/master/tests/JavaTest.bat), depending on your operating +system. + +*Note: These scripts require that [Java](https://www.oracle.com/java/index.html) +is installed.* + +## Using the FlatBuffers Java library + +*Note: See [Tutorial](@ref flatbuffers_guide_tutorial) for a more in-depth +example of how to use FlatBuffers in Java.* + +FlatBuffers supports reading and writing binary FlatBuffers in Java. + +To use FlatBuffers in your own code, first generate Java classes from your +schema with the `--java` option to `flatc`. +Then you can include both FlatBuffers and the generated code to read +or write a FlatBuffer. + +For example, here is how you would read a FlatBuffer binary file in Java: +First, import the library and generated code. Then, you read a FlatBuffer binary +file into a `byte[]`. You then turn the `byte[]` into a `ByteBuffer`, which you +pass to the `getRootAsMyRootType` function: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.java} + import MyGame.Example.*; + import com.google.flatbuffers.FlatBufferBuilder; + + // This snippet ignores exceptions for brevity. + File file = new File("monsterdata_test.mon"); + RandomAccessFile f = new RandomAccessFile(file, "r"); + byte[] data = new byte[(int)f.length()]; + f.readFully(data); + f.close(); + + ByteBuffer bb = ByteBuffer.wrap(data); + Monster monster = Monster.getRootAsMonster(bb); +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Now you can access the data from the `Monster monster`: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.java} + short hp = monster.hp(); + Vec3 pos = monster.pos(); +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +## Storing dictionaries in a FlatBuffer + +FlatBuffers doesn't support dictionaries natively, but there is support to +emulate their behavior with vectors and binary search, which means you +can have fast lookups directly from a FlatBuffer without having to unpack +your data into a `Dictionary` or similar. + +To use it: +- Designate one of the fields in a table as the "key" field. You do this + by setting the `key` attribute on this field, e.g. + `name:string (key)`. + You may only have one key field, and it must be of string or scalar type. +- Write out tables of this type as usual, collect their offsets in an + array. +- Instead of calling standard generated method, + e.g.: `Monster.createTestarrayoftablesVector`, + call `createSortedVectorOfTables` (from the `FlatBufferBuilder` object). + which will first sort all offsets such that the tables they refer to + are sorted by the key field, then serialize it. +- Now when you're accessing the FlatBuffer, you can use + the `ByKey` accessor to access elements of the vector, e.g.: + `monster.testarrayoftablesByKey("Frodo")`. + which returns an object of the corresponding table type, + or `null` if not found. + `ByKey` performs a binary search, so should have a similar + speed to `Dictionary`, though may be faster because of better caching. + `ByKey` only works if the vector has been sorted, it will + likely not find elements if it hasn't been sorted. + +## Text parsing + +There currently is no support for parsing text (Schema's and JSON) directly +from Java, though you could use the C++ parser through native call +interfaces available to each language. Please see the +C++ documentation for more on text parsing. + +
diff --git a/docs/source/languages/javascript.md b/docs/source/languages/javascript.md new file mode 100644 index 000000000..64764e216 --- /dev/null +++ b/docs/source/languages/javascript.md @@ -0,0 +1,93 @@ +Use in JavaScript {#flatbuffers_guide_use_javascript} +================= + +## Before you get started + +Before diving into the FlatBuffers usage in JavaScript, it should be noted that +the [Tutorial](@ref flatbuffers_guide_tutorial) page has a complete guide to +general FlatBuffers usage in all of the supported languages +(including JavaScript). This page is specifically designed to cover the nuances +of FlatBuffers usage in JavaScript. + +You should also have read the [Building](@ref flatbuffers_guide_building) +documentation to build `flatc` and should be familiar with +[Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler) and +[Writing a schema](@ref flatbuffers_guide_writing_schema). + +## FlatBuffers JavaScript library code location + +The generated code for the FlatBuffers JavaScript library can be found at +https://www.npmjs.com/package/flatbuffers. To use it from sources: + +1. Run `npm run compile` from the main folder to generate JS files from TS. +1. In your project, install it as a normal dependency, using the flatbuffers +folder as the source. + +## Using the FlatBuffers JavaScript library + +*Note: See [Tutorial](@ref flatbuffers_guide_tutorial) for a more in-depth +example of how to use FlatBuffers.* + +Due to the complexity related with large amounts of JS flavors and module types, +native JS support has been replaced in 2.0 by transpilation from TypeScript. + +Please look at [TypeScript usage](@ref flatbuffers_guide_use_typescript) and +transpile your sources to desired JS flavor. The minimal steps to get up and +running with JS are: + +1. Generate TS files from `*.fbs` by using the `--ts` option. +1. Transpile resulting TS files to desired JS flavor using `tsc` (see + https://www.typescriptlang.org/download for installation instructions). + +~~~{.js} + // Note: These require functions are an example - use your desired module flavor. + var fs = require('fs'); + + var flatbuffers = require('../flatbuffers').flatbuffers; + var MyGame = require('./monster_generated').MyGame; + + var data = new Uint8Array(fs.readFileSync('monster.dat')); + var buf = new flatbuffers.ByteBuffer(data); + + var monster = MyGame.Example.Monster.getRootAsMonster(buf); + + //--------------------------------------------------------------------------// + + // Note: This code is an example of browser-based HTML/JavaScript. See above + // for the code using JavaScript module loaders (e.g. Node.js). + + + + + // Open the HTML file in a browser and select "monster.dat" from with the + // field. + +~~~ + +Now you can access values like this: + +~~~{.js} + var hp = monster.hp(); + var pos = monster.pos(); +~~~ + +## Text parsing FlatBuffers in JavaScript + +There currently is no support for parsing text (Schema's and JSON) directly +from JavaScript. diff --git a/docs/source/languages/lobster.md b/docs/source/languages/lobster.md new file mode 100644 index 000000000..723966be7 --- /dev/null +++ b/docs/source/languages/lobster.md @@ -0,0 +1,85 @@ +Use in Lobster {#flatbuffers_guide_use_lobster} +============== + +## Before you get started + +Before diving into the FlatBuffers usage in Lobster, it should be noted that the +[Tutorial](@ref flatbuffers_guide_tutorial) page has a complete guide to general +FlatBuffers usage in all of the supported languages (including Lobster). This +page is designed to cover the nuances of FlatBuffers usage, specific to +Lobster. + +You should also have read the [Building](@ref flatbuffers_guide_building) +documentation to build `flatc` and should be familiar with +[Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler) and +[Writing a schema](@ref flatbuffers_guide_writing_schema). + +## FlatBuffers Lobster library code location + +The code for the FlatBuffers Lobster library can be found at +`flatbuffers/lobster`. You can browse the library code on the +[FlatBuffers GitHub page](https://github.com/google/flatbuffers/tree/master/ +lobster). + +## Testing the FlatBuffers Lobster library + +The code to test the Lobster library can be found at `flatbuffers/tests`. +The test code itself is located in [lobstertest.lobster](https://github.com/google/ +flatbuffers/blob/master/tests/lobstertest.lobster). + +To run the tests, run `lobster lobstertest.lobster`. To obtain Lobster itself, +go to the [Lobster homepage](http://strlen.com/lobster) or +[github](https://github.com/aardappel/lobster) to learn how to build it for your +platform. + +## Using the FlatBuffers Lobster library + +*Note: See [Tutorial](@ref flatbuffers_guide_tutorial) for a more in-depth +example of how to use FlatBuffers in Lobster.* + +There is support for both reading and writing FlatBuffers in Lobster. + +To use FlatBuffers in your own code, first generate Lobster classes from your +schema with the `--lobster` option to `flatc`. Then you can include both +FlatBuffers and the generated code to read or write a FlatBuffer. + +For example, here is how you would read a FlatBuffer binary file in Lobster: +First, import the library and the generated code. Then read a FlatBuffer binary +file into a string, which you pass to the `GetRootAsMonster` function: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.lobster} + include "monster_generated.lobster" + + let fb = read_file("monsterdata_test.mon") + assert fb + let monster = MyGame_Example_GetRootAsMonster(fb) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Now you can access values like this: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.lobster} + let hp = monster.hp + let pos = monster.pos +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As you can see, even though `hp` and `pos` are functions that access FlatBuffer +data in-place in the string buffer, they appear as field accesses. + +## Speed + +Using FlatBuffers in Lobster should be relatively fast, as the implementation +makes use of native support for writing binary values, and access of vtables. +Both generated code and the runtime library are therefore small and fast. + +Actual speed will depend on whether you use Lobster as bytecode VM or compiled to +C++. + +## Text Parsing + +Lobster has full support for parsing JSON into FlatBuffers, or generating +JSON from FlatBuffers. See `samples/sample_test.lobster` for an example. + +This uses the C++ parser and generator underneath, so should be both fast and +conformant. + +
diff --git a/docs/source/languages/lua.md b/docs/source/languages/lua.md new file mode 100644 index 000000000..43c370f5d --- /dev/null +++ b/docs/source/languages/lua.md @@ -0,0 +1,81 @@ +Use in Lua {#flatbuffers_guide_use_lua} +============= + +## Before you get started + +Before diving into the FlatBuffers usage in Lua, it should be noted that the +[Tutorial](@ref flatbuffers_guide_tutorial) page has a complete guide to general +FlatBuffers usage in all of the supported languages (including Lua). This +page is designed to cover the nuances of FlatBuffers usage, specific to +Lua. + +You should also have read the [Building](@ref flatbuffers_guide_building) +documentation to build `flatc` and should be familiar with +[Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler) and +[Writing a schema](@ref flatbuffers_guide_writing_schema). + +## FlatBuffers Lua library code location + +The code for the FlatBuffers Lua library can be found at +`flatbuffers/lua`. You can browse the library code on the +[FlatBuffers GitHub page](https://github.com/google/flatbuffers/tree/master/lua). + +## Testing the FlatBuffers Lua library + +The code to test the Lua library can be found at `flatbuffers/tests`. +The test code itself is located in [luatest.lua](https://github.com/google/ +flatbuffers/blob/master/tests/luatest.lua). + +To run the tests, use the [LuaTest.sh](https://github.com/google/flatbuffers/ +blob/master/tests/LuaTest.sh) shell script. + +*Note: This script requires [Lua 5.3](https://www.lua.org/) and +[LuaJIT](http://luajit.org/) to be installed.* + +## Using the FlatBuffers Lua library + +*Note: See [Tutorial](@ref flatbuffers_guide_tutorial) for a more in-depth +example of how to use FlatBuffers in Lua.* + +There is support for both reading and writing FlatBuffers in Lua. + +To use FlatBuffers in your own code, first generate Lua classes from your +schema with the `--lua` option to `flatc`. Then you can include both +FlatBuffers and the generated code to read or write a FlatBuffer. + +For example, here is how you would read a FlatBuffer binary file in Lua: +First, require the module and the generated code. Then read a FlatBuffer binary +file into a `string`, which you pass to the `GetRootAsMonster` function: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.lua} + -- require the library + local flatbuffers = require("flatbuffers") + + -- require the generated code + local monster = require("MyGame.Sample.Monster") + + -- read the flatbuffer from a file into a string + local f = io.open('monster.dat', 'rb') + local buf = f:read('*a') + f:close() + + -- parse the flatbuffer to get an instance to the root monster + local monster1 = monster.GetRootAsMonster(buf, 0) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Now you can access values like this: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.lua} + -- use the : notation to access member data + local hp = monster1:Hp() + local pos = monster1:Pos() +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +## Text Parsing + +There currently is no support for parsing text (Schema's and JSON) directly +from Lua, though you could use the C++ parser through SWIG or ctypes. Please +see the C++ documentation for more on text parsing. + +
diff --git a/docs/source/languages/php.md b/docs/source/languages/php.md new file mode 100644 index 000000000..cdff44954 --- /dev/null +++ b/docs/source/languages/php.md @@ -0,0 +1,89 @@ +Use in PHP {#flatbuffers_guide_use_php} +========== + +## Before you get started + +Before diving into the FlatBuffers usage in PHP, it should be noted that +the [Tutorial](@ref flatbuffers_guide_tutorial) page has a complete guide to +general FlatBuffers usage in all of the supported languages +(including PHP). This page is specifically designed to cover the nuances of +FlatBuffers usage in PHP. + +You should also have read the [Building](@ref flatbuffers_guide_building) +documentation to build `flatc` and should be familiar with +[Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler) and +[Writing a schema](@ref flatbuffers_guide_writing_schema). + +## FlatBuffers PHP library code location + +The code for FlatBuffers PHP library can be found at `flatbuffers/php`. You +can browse the library code on the [FlatBuffers +GitHub page](https://github.com/google/flatbuffers/tree/master/php). + +## Testing the FlatBuffers JavaScript library + +The code to test the PHP library can be found at `flatbuffers/tests`. +The test code itself is located in [phpTest.php](https://github.com/google/ +flatbuffers/blob/master/tests/phpTest.php). + +You can run the test with `php phpTest.php` from the command line. + +*Note: The PHP test file requires +[PHP](http://php.net/manual/en/install.php) to be installed.* + +## Using theFlatBuffers PHP library + +*Note: See [Tutorial](@ref flatbuffers_guide_tutorial) for a more in-depth +example of how to use FlatBuffers in PHP.* + +FlatBuffers supports both reading and writing FlatBuffers in PHP. + +To use FlatBuffers in your own code, first generate PHP classes from your schema +with the `--php` option to `flatc`. Then you can include both FlatBuffers and +the generated code to read or write a FlatBuffer. + +For example, here is how you would read a FlatBuffer binary file in PHP: +First, include the library and generated code (using the PSR `autoload` +function). Then you can read a FlatBuffer binary file, which you +pass the contents of to the `GetRootAsMonster` function: + +~~~{.php} + // It is recommended that your use PSR autoload when using FlatBuffers in PHP. + // Here is an example: + function __autoload($class_name) { + // The last segment of the class name matches the file name. + $class = substr($class_name, strrpos($class_name, "\\") + 1); + $root_dir = join(DIRECTORY_SEPARATOR, array(dirname(dirname(__FILE__)))); // `flatbuffers` root. + + // Contains the `*.php` files for the FlatBuffers library and the `flatc` generated files. + $paths = array(join(DIRECTORY_SEPARATOR, array($root_dir, "php")), + join(DIRECTORY_SEPARATOR, array($root_dir, "tests", "MyGame", "Example"))); + foreach ($paths as $path) { + $file = join(DIRECTORY_SEPARATOR, array($path, $class . ".php")); + if (file_exists($file)) { + require($file); + break; + } + } + + // Read the contents of the FlatBuffer binary file. + $filename = "monster.dat"; + $handle = fopen($filename, "rb"); + $contents = $fread($handle, filesize($filename)); + fclose($handle); + + // Pass the contents to `GetRootAsMonster`. + $monster = \MyGame\Example\Monster::GetRootAsMonster($contents); +~~~ + +Now you can access values like this: + +~~~{.php} + $hp = $monster->GetHp(); + $pos = $monster->GetPos(); +~~~ + +## Text Parsing + +There currently is no support for parsing text (Schema's and JSON) directly +from PHP. diff --git a/docs/source/languages/python.md b/docs/source/languages/python.md new file mode 100644 index 000000000..f338cda43 --- /dev/null +++ b/docs/source/languages/python.md @@ -0,0 +1,100 @@ +Use in Python {#flatbuffers_guide_use_python} +============= + +## Before you get started + +Before diving into the FlatBuffers usage in Python, it should be noted that the +[Tutorial](@ref flatbuffers_guide_tutorial) page has a complete guide to general +FlatBuffers usage in all of the supported languages (including Python). This +page is designed to cover the nuances of FlatBuffers usage, specific to +Python. + +You should also have read the [Building](@ref flatbuffers_guide_building) +documentation to build `flatc` and should be familiar with +[Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler) and +[Writing a schema](@ref flatbuffers_guide_writing_schema). + +## FlatBuffers Python library code location + +The code for the FlatBuffers Python library can be found at +`flatbuffers/python/flatbuffers`. You can browse the library code on the +[FlatBuffers GitHub page](https://github.com/google/flatbuffers/tree/master/ +python). + +## Testing the FlatBuffers Python library + +The code to test the Python library can be found at `flatbuffers/tests`. +The test code itself is located in [py_test.py](https://github.com/google/ +flatbuffers/blob/master/tests/py_test.py). + +To run the tests, use the [PythonTest.sh](https://github.com/google/flatbuffers/ +blob/master/tests/PythonTest.sh) shell script. + +*Note: This script requires [python](https://www.python.org/) to be +installed.* + +## Using the FlatBuffers Python library + +*Note: See [Tutorial](@ref flatbuffers_guide_tutorial) for a more in-depth +example of how to use FlatBuffers in Python.* + +There is support for both reading and writing FlatBuffers in Python. + +To use FlatBuffers in your own code, first generate Python classes from your +schema with the `--python` option to `flatc`. Then you can include both +FlatBuffers and the generated code to read or write a FlatBuffer. + +For example, here is how you would read a FlatBuffer binary file in Python: +First, import the library and the generated code. Then read a FlatBuffer binary +file into a `bytearray`, which you pass to the `GetRootAsMonster` function: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.py} + import MyGame.Example as example + import flatbuffers + + buf = open('monster.dat', 'rb').read() + buf = bytearray(buf) + monster = example.GetRootAsMonster(buf, 0) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Now you can access values like this: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.py} + hp = monster.Hp() + pos = monster.Pos() +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +## Support for Numpy arrays + +The Flatbuffers python library also has support for accessing scalar +vectors as numpy arrays. This can be orders of magnitude faster than +iterating over the vector one element at a time, and is particularly +useful when unpacking large nested flatbuffers. The generated code for +a scalar vector will have a method `AsNumpy()`. In the +case of the Monster example, you could access the inventory vector +like this: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.py} + inventory = monster.InventoryAsNumpy() + # inventory is a numpy array of type np.dtype('uint8') +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +instead of + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.py} + inventory = [] + for i in range(monster.InventoryLength()): + inventory.append(int(monster.Inventory(i))) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Numpy is not a requirement. If numpy is not installed on your system, +then attempting to access one of the `*asNumpy()` methods will result +in a `NumpyRequiredForThisFeature` exception. + +## Text Parsing + +There currently is no support for parsing text (Schema's and JSON) directly +from Python, though you could use the C++ parser through SWIG or ctypes. Please +see the C++ documentation for more on text parsing. + +
diff --git a/docs/source/languages/rust.md b/docs/source/languages/rust.md new file mode 100644 index 000000000..2a162588a --- /dev/null +++ b/docs/source/languages/rust.md @@ -0,0 +1,186 @@ +Use in Rust {#flatbuffers_guide_use_rust} +========== + +## Before you get started + +Before diving into the FlatBuffers usage in Rust, it should be noted that +the [Tutorial](@ref flatbuffers_guide_tutorial) page has a complete guide +to general FlatBuffers usage in all of the supported languages (including Rust). +This page is designed to cover the nuances of FlatBuffers usage, specific to +Rust. + +#### Prerequisites + +This page assumes you have written a FlatBuffers schema and compiled it +with the Schema Compiler. If you have not, please see +[Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler) +and [Writing a schema](@ref flatbuffers_guide_writing_schema). + +Assuming you wrote a schema, say `mygame.fbs` (though the extension doesn't +matter), you've generated a Rust file called `mygame_generated.rs` using the +compiler (e.g. `flatc --rust mygame.fbs` or via helpers listed in "Useful +tools created by others" section bellow), you can now start using this in +your program by including the file. As noted, this header relies on the crate +`flatbuffers`, which should be in your include `Cargo.toml`. + +## FlatBuffers Rust library code location + +The code for the FlatBuffers Rust library can be found at +`flatbuffers/rust`. You can browse the library code on the +[FlatBuffers GitHub page](https://github.com/google/flatbuffers/tree/master/rust). + +## Testing the FlatBuffers Rust library + +The code to test the Rust library can be found at `flatbuffers/tests/rust_usage_test`. +The test code itself is located in +[integration_test.rs](https://github.com/google/flatbuffers/blob/master/tests/rust_usage_test/tests/integration_test.rs) + +This test file requires `flatc` to be present. To review how to build the project, +please read the [Building](@ref flatbuffers_guide_building) documentation. + +To run the tests, execute `RustTest.sh` from the `flatbuffers/tests` directory. +For example, on [Linux](https://en.wikipedia.org/wiki/Linux), you would simply +run: `cd tests && ./RustTest.sh`. + +*Note: The shell script requires [Rust](https://www.rust-lang.org) to +be installed.* + +## Using the FlatBuffers Rust library + +*Note: See [Tutorial](@ref flatbuffers_guide_tutorial) for a more in-depth +example of how to use FlatBuffers in Rust.* + +FlatBuffers supports both reading and writing FlatBuffers in Rust. + +To use FlatBuffers in your code, first generate the Rust modules from your +schema with the `--rust` option to `flatc`. Then you can import both FlatBuffers +and the generated code to read or write FlatBuffers. + +For example, here is how you would read a FlatBuffer binary file in Rust: +First, include the library and generated code. Then read the file into +a `u8` vector, which you pass, as a byte slice, to `root_as_monster()`. + +This full example program is available in the Rust test suite: +[monster_example.rs](https://github.com/google/flatbuffers/blob/master/tests/rust_usage_test/bin/monster_example.rs) + +It can be run by `cd`ing to the `rust_usage_test` directory and executing: `cargo run monster_example`. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.rs} + extern crate flatbuffers; + + #[allow(dead_code, unused_imports)] + #[path = "../../monster_test_generated.rs"] + mod monster_test_generated; + pub use monster_test_generated::my_game; + + use std::io::Read; + + fn main() { + let mut f = std::fs::File::open("../monsterdata_test.mon").unwrap(); + let mut buf = Vec::new(); + f.read_to_end(&mut buf).expect("file reading failed"); + + let monster = my_game::example::root_as_monster(&buf[..]); +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +`monster` is of type `Monster`, and points to somewhere *inside* your +buffer (root object pointers are not the same as `buffer_pointer` !). +If you look in your generated header, you'll see it has +convenient accessors for all fields, e.g. `hp()`, `mana()`, etc: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.rs} + println!("{}", monster.hp()); // `80` + println!("{}", monster.mana()); // default value of `150` + println!("{:?}", monster.name()); // Some("MyMonster") + } +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +*Note: That we never stored a `mana` value, so it will return the default.* + +## Direct memory access + +As you can see from the above examples, all elements in a buffer are +accessed through generated accessors. This is because everything is +stored in little endian format on all platforms (the accessor +performs a swap operation on big endian machines), and also because +the layout of things is generally not known to the user. + +For structs, layout is deterministic and guaranteed to be the same +across platforms (scalars are aligned to their +own size, and structs themselves to their largest member), and you +are allowed to access this memory directly by using `safe_slice` +on the reference to a struct, or even an array of structs. + +To compute offsets to sub-elements of a struct, make sure they +are structs themselves, as then you can use the pointers to +figure out the offset without having to hardcode it. This is +handy for use of arrays of structs with calls like `glVertexAttribPointer` +in OpenGL or similar APIs. + +It is important to note is that structs are still little endian on all +machines, so the functions to enable tricks like this are only exposed on little +endian machines. If you also ship on big endian machines, using an +`#[cfg(target_endian = "little")]` attribute would be wise or your code will not +compile. + +The special function `safe_slice` is implemented on Vector objects that are +represented in memory the same way as they are represented on the wire. This +function is always available on vectors of struct, bool, u8, and i8. It is +conditionally-compiled on little-endian systems for all the remaining scalar +types. + +The FlatBufferBuilder function `create_vector_direct` is implemented for all +types that are endian-safe to write with a `memcpy`. It is the write-equivalent +of `safe_slice`. + +## Access of untrusted buffers + +The safe Rust functions to interpret a slice as a table (`root`, +`size_prefixed_root`, `root_with_opts`, and `size_prefixed_root_with_opts`) +verify the data first. This has some performance cost, but is intended to be +safe for use on flatbuffers from untrusted sources. There are corresponding +`unsafe` versions with names ending in `_unchecked` which skip this +verification, and may access arbitrary memory. + +The generated accessor functions access fields over offsets, which is +very quick. The current implementation uses these to access memory without any +further bounds checking. All of the safe Rust APIs ensure the verifier is run +over these flatbuffers before accessing them. + +When you're processing large amounts of data from a source you know (e.g. +your own generated data on disk), the `_unchecked` versions are acceptable, but +when reading data from the network that can potentially have been modified by an +attacker, it is desirable to use the safe versions which use the verifier. + +## Threading + +Reading a FlatBuffer does not touch any memory outside the original buffer, +and is entirely read-only (all immutable), so is safe to access from multiple +threads even without synchronisation primitives. + +Creating a FlatBuffer is not thread safe. All state related to building +a FlatBuffer is contained in a FlatBufferBuilder instance, and no memory +outside of it is touched. To make this thread safe, either do not +share instances of FlatBufferBuilder between threads (recommended), or +manually wrap it in synchronisation primitives. There's no automatic way to +accomplish this, by design, as we feel multithreaded construction +of a single buffer will be rare, and synchronisation overhead would be costly. + +Unlike most other languages, in Rust these properties are exposed to and +enforced by the type system. `flatbuffers::Table` and the generated table types +are `Send + Sync`, indicating they may be freely shared across threads and data +may be accessed from any thread which receives a const (aka shared) reference. +There are no functions which require a mutable (aka exclusive) reference, which +means all the available functions may be called like this. +`flatbuffers::FlatBufferBuilder` is also `Send + Sync`, but all of the mutating +functions require a mutable (aka exclusive) reference which can only be created +when no other references to the `FlatBufferBuilder` exist, and may not be copied +within the same thread, let alone to a second thread. + +## Useful tools created by others + +* [flatc-rust](https://github.com/frol/flatc-rust) - FlatBuffers compiler +(flatc) as API for transparent `.fbs` to `.rs` code-generation via Cargo +build scripts integration. + +
diff --git a/docs/source/languages/swift.md b/docs/source/languages/swift.md new file mode 100644 index 000000000..c6116f6ae --- /dev/null +++ b/docs/source/languages/swift.md @@ -0,0 +1,97 @@ +Use in Swift {#flatbuffers_guide_use_swift} +========= + +## Before you get started + +Before diving into the FlatBuffers usage in Swift, it should be noted that +the [Tutorial](@ref flatbuffers_guide_tutorial) page has a complete guide +to general FlatBuffers usage in all of the supported languages (including Swift). +This page is designed to cover the nuances of FlatBuffers usage, specific to +Swift. + +You should also have read the [Building](@ref flatbuffers_guide_building) +documentation to build `flatc` and should be familiar with +[Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler) and +[Writing a schema](@ref flatbuffers_guide_writing_schema). + +## FlatBuffers Swift library code location + +The code for the FlatBuffers Swift library can be found at +`flatbuffers/swift`. You can browse the library code on the [FlatBuffers +GitHub page](https://github.com/google/flatbuffers/tree/master/swift). + +## Testing the FlatBuffers Swift library + +The code to test the Swift library can be found at `flatbuffers/tests/swift/tests`. +The test code itself is located in [flatbuffers/tests/swift/tests](https://github.com/google/flatbuffers/blob/master/tests/swift/tests). + +To run the tests, use the [SwiftTest.sh](https://github.com/google/flatbuffers/blob/master/tests/swift/tests/SwiftTest.sh) shell script. + +*Note: The shell script requires [Swift](https://swift.org) to +be installed.* + +## Using the FlatBuffers Swift library + +*Note: See [Tutorial](@ref flatbuffers_guide_tutorial) for a more in-depth +example of how to use FlatBuffers in Swift.* + +FlatBuffers supports reading and writing binary FlatBuffers in Swift. + +To use FlatBuffers in your own code, first generate Swift structs from your +schema with the `--swift` option to `flatc`. Then include FlatBuffers using `SPM` in +by adding the path to `FlatBuffers/swift` into it. The generated code should also be +added to xcode or the path of the package you will be using. Note: sometimes xcode cant +and wont see the generated files, so it's better that you copy them to xcode. + +For example, here is how you would read a FlatBuffer binary file in Swift: First, +include the library and copy thegenerated code. Then read a FlatBuffer binary file or +a data object from the server, which you can pass into the `GetRootAsMonster` function. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.swift} + import FlatBuffers + + typealias Monster1 = MyGame.Sample.Monster + typealias Vec3 = MyGame.Sample.Vec3 + + let path = FileManager.default.currentDirectoryPath + let url = URL(fileURLWithPath: path, isDirectory: true).appendingPathComponent("monsterdata_test").appendingPathExtension("mon") + guard let data = try? Data(contentsOf: url) else { return } + + let monster = Monster.getRootAsMonster(bb: ByteBuffer(data: data)) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Now you can access values like this: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.swift} + let hp = monster.hp + let pos = monster.pos // uses native swift structs + let pos = monster.mutablePos // uses flatbuffers structs +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +In some cases it's necessary to modify values in an existing FlatBuffer in place (without creating a copy). For this reason, scalar fields of a Flatbuffer table or struct can be mutated. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.swift} + var byteBuffer = ByteBuffer(bytes: data) + // Get an accessor to the root object inside the buffer. + let monster: Monster = try! getCheckedRoot(byteBuffer: &byteBuffer) + // let monster: Monster = getRoot(byteBuffer: &byteBuffer) + + if !monster.mutate(hp: 10) { + fatalError("couldn't mutate") + } + // mutate a struct field using flatbuffers struct + // DONT use monster.pos to mutate since swift copy on write + // will not mutate the value in the buffer + let vec = monster.mutablePos.mutate(z: 4) + + // This mutation will fail because the mana field is not available in + // the buffer. It should be set when creating the buffer. + if !monster.mutate(mana: 20) { + fatalError("couldn't mutate") + } +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The term `mutate` is used instead of `set` to indicate that this is a special use case. All mutate functions return a boolean value which is false if the field we're trying to mutate is not available in the buffer. + +
diff --git a/docs/source/languages/typescript.md b/docs/source/languages/typescript.md new file mode 100644 index 000000000..0dd1da2eb --- /dev/null +++ b/docs/source/languages/typescript.md @@ -0,0 +1,96 @@ +Use in TypeScript {#flatbuffers_guide_use_typescript} +================= + +## Before you get started + +Before diving into the FlatBuffers usage in TypeScript, it should be noted that +the [Tutorial](@ref flatbuffers_guide_tutorial) page has a complete guide to +general FlatBuffers usage in all of the supported languages +(including TypeScript). This page is specifically designed to cover the nuances +of FlatBuffers usage in TypeScript. + +You should also have read the [Building](@ref flatbuffers_guide_building) +documentation to build `flatc` and should be familiar with +[Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler) and +[Writing a schema](@ref flatbuffers_guide_writing_schema). + +## FlatBuffers TypeScript library code location + +The code for the FlatBuffers TypeScript library can be found at +https://www.npmjs.com/package/flatbuffers. + +## Testing the FlatBuffers TypeScript library + +To run the tests, use the [TypeScriptTest.py](https://github.com/google/ +flatbuffers/blob/master/tests/TypeScriptTest.py) Python3 script. + +*Note: The TypeScript test file requires [Node.js](https://nodejs.org/en/).* + +## Using the FlatBuffers TypeScript library + +*Note: See [Tutorial](@ref flatbuffers_guide_tutorial) for a more in-depth +example of how to use FlatBuffers in TypeScript.* + +FlatBuffers supports both reading and writing FlatBuffers in TypeScript. + +To use FlatBuffers in your own code, first generate TypeScript classes from your +schema with the `--ts` option to `flatc`. Then you can include both FlatBuffers +and the generated code to read or write a FlatBuffer. + +For example, here is how you would read a FlatBuffer binary file in TypeScript: +First, include the library and generated code. Then read the file into an +`Uint8Array`. Make a `flatbuffers.ByteBuffer` out of the `Uint8Array`, and pass +the ByteBuffer to the `getRootAsMonster` function. + +~~~{.ts} + import * as flatbuffers from 'flatbuffers'; + + import { MyGame } from './monster_generated'; + + let data = new Uint8Array(fs.readFileSync('monster.dat')); + let buf = new flatbuffers.ByteBuffer(data); + + let monster = MyGame.Example.Monster.getRootAsMonster(buf); +~~~ + +Now you can access values like this: + +~~~{.ts} + let hp = monster.hp(); + let pos = monster.pos(); +~~~ + +## Object based API + +FlatBuffers is all about memory efficiency, which is why its base API is written +around using as little as possible of it. This does make the API clumsier +(requiring pre-order construction of all data, and making mutation harder). + +For times when efficiency is less important a more convenient object based API +can be used (through `--gen-object-api`) that is able to unpack & pack a +FlatBuffer into objects and standard TS types. + +To use: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.ts} + // Autogenerated class from table Monster. + let monsterobj = new MonsterT(); + + // Deserialize from buffer into object. + Monster.getRootAsMonster(flatbuffer).unpackTo(monsterobj); + // or + let monsterobj = Monster.getRootAsMonster(flatbuffer).unpack(); + + // Update object directly like a regular TS class instance. + console.log(monsterobj.name); + monsterobj.name = "Bob"; + + // Serialize into new flatbuffer. + let fbb = new flatbuffers.Builder(1); + Monster.finishMonsterBuffer(fbb, monsterobj.pack(fbb)); +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +## Text parsing FlatBuffers in TypeScript + +There currently is no support for parsing text (Schema's and JSON) directly +from TypeScript. diff --git a/docs/source/support.md b/docs/source/support.md new file mode 100644 index 000000000..3436a6fbd --- /dev/null +++ b/docs/source/support.md @@ -0,0 +1,57 @@ +Platform / Language / Feature support {#flatbuffers_support} +===================================== + +FlatBuffers is actively being worked on, which means that certain platform / +language / feature combinations may not be available yet. + +This page tries to track those issues, to make informed decisions easier. +In general: + + * Languages: language support beyond the ones created by the original + FlatBuffer authors typically depends on community contributions. + * Features: C++ was the first language supported, since our original + target was high performance game development. It thus has the richest + feature set, and is likely most robust. Other languages are catching up + however. + * Platforms: All language implementations are typically portable to most + platforms, unless where noted otherwise. + +NOTE: this table is a start, it needs to be extended. + +Feature | C++ | Java | C# | Go | Python | JS | TS | C | PHP | Dart | Lobster | Rust | Swift +------------------------------ | ------ | ----- | -------- | ----- | ------ | ----- | --- | ------ | --- | ------- | ------- | ------ | ------ +Codegen for all basic features | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | WiP | Yes | Yes | Yes | Yes +JSON parsing | Yes | No | No | No | No | No | No | Yes | No | No | Yes | No | No +Simple mutation | Yes | Yes | Yes | Yes | No | No | No | No | No | No | No | No | Yes +Reflection | Yes | No | No | No | No | No | No | Basic | No | No | No | No | No +Buffer verifier | Yes | No | No | No | No | No | No | Yes | No | No | No | No | No +Native Object API | Yes | No | Yes | Yes | Yes | Yes | Yes | No | No | Yes | No | No | No +Optional Scalars | Yes | Yes | Yes | No | No | Yes | Yes | Yes | No | No | Yes | Yes | Yes +Flexbuffers | Yes | Yes | ? | ? | ? | ? | ? | ? | ? | ? | ? | Yes | ? +Testing: basic | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | ? | Yes | Yes | Yes | Yes +Testing: fuzz | Yes | No | No | Yes | Yes | No | No | No | ? | No | No | Yes | No +Performance: | Superb | Great | Great | Great | Ok | ? | ? | Superb | ? | ? | Great | Superb | Great +Platform: Windows | VS2010 | Yes | Yes | ? | ? | ? | Yes | VS2010 | ? | Yes | Yes | Yes | No +Platform: Linux | GCC282 | Yes | ? | Yes | Yes | ? | Yes | Yes | ? | Yes | Yes | Yes | Yes +Platform: OS X | Xcode4 | ? | ? | ? | Yes | ? | Yes | Yes | ? | Yes | Yes | Yes | Yes +Platform: Android | NDK10d | Yes | ? | ? | ? | ? | ? | ? | ? | Flutter | Yes | ? | No +Platform: iOS | ? | ? | ? | ? | ? | ? | ? | ? | ? | Flutter | Yes | ? | Yes +Engine: Unity | ? | ? | Yes | ? | ? | ? | ? | ? | ? | ? | No | ? | No +Primary authors (github) | aard | aard | ev/js/df | rw | rw | ew/ev | kr | mik | ch | df | aard | rw/cn | mi/mz + +Above | Github username +----- | ----------------------------- +aard | aardappel (previously: gwvo) +ch | chobie +cn | caspern +df | dnfield +ev | evolutional +ew | evanw +js | jonsimantov +kr | krojew +mi | mustiikhalil +mik | mikkelfj +mz | mzaks +rw | rw + +
diff --git a/docs/source/white_paper.md b/docs/source/white_paper.md new file mode 100644 index 000000000..dbad2cb35 --- /dev/null +++ b/docs/source/white_paper.md @@ -0,0 +1,128 @@ +FlatBuffers white paper {#flatbuffers_white_paper} +======================= + +This document tries to shed some light on to the "why" of FlatBuffers, a +new serialization library. + +## Motivation + +Back in the good old days, performance was all about instructions and +cycles. Nowadays, processing units have run so far ahead of the memory +subsystem, that making an efficient application should start and finish +with thinking about memory. How much you use of it. How you lay it out +and access it. How you allocate it. When you copy it. + +Serialization is a pervasive activity in a lot programs, and a common +source of memory inefficiency, with lots of temporary data structures +needed to parse and represent data, and inefficient allocation patterns +and locality. + +If it would be possible to do serialization with no temporary objects, +no additional allocation, no copying, and good locality, this could be +of great value. The reason serialization systems usually don't manage +this is because it goes counter to forwards/backwards compatibility, and +platform specifics like endianness and alignment. + +FlatBuffers is what you get if you try anyway. + +In particular, FlatBuffers focus is on mobile hardware (where memory +size and memory bandwidth is even more constrained than on desktop +hardware), and applications that have the highest performance needs: +games. + +## FlatBuffers + +*This is a summary of FlatBuffers functionality, with some rationale. +A more detailed description can be found in the FlatBuffers +documentation.* + +### Summary + +A FlatBuffer is a binary buffer containing nested objects (structs, +tables, vectors,..) organized using offsets so that the data can be +traversed in-place just like any pointer-based data structure. Unlike +most in-memory data structures however, it uses strict rules of +alignment and endianness (always little) to ensure these buffers are +cross platform. Additionally, for objects that are tables, FlatBuffers +provides forwards/backwards compatibility and general optionality of +fields, to support most forms of format evolution. + +You define your object types in a schema, which can then be compiled to +C++ or Java for low to zero overhead reading & writing. +Optionally, JSON data can be dynamically parsed into buffers. + +### Tables + +Tables are the cornerstone of FlatBuffers, since format evolution is +essential for most applications of serialization. Typically, dealing +with format changes is something that can be done transparently during +the parsing process of most serialization solutions out there. +But a FlatBuffer isn't parsed before it is accessed. + +Tables get around this by using an extra indirection to access fields, +through a *vtable*. Each table comes with a vtable (which may be shared +between multiple tables with the same layout), and contains information +where fields for this particular kind of instance of vtable are stored. +The vtable may also indicate that the field is not present (because this +FlatBuffer was written with an older version of the software, or simply +because the information was not necessary for this instance, or deemed +deprecated), in which case a default value is returned. + +Tables have a low overhead in memory (since vtables are small and +shared) and in access cost (an extra indirection), but provide great +flexibility. Tables may even cost less memory than the equivalent +struct, since fields do not need to be stored when they are equal to +their default. + +FlatBuffers additionally offers "naked" structs, which do not offer +forwards/backwards compatibility, but can be even smaller (useful for +very small objects that are unlikely to change, like e.g. a coordinate +pair or a RGBA color). + +### Schemas + +While schemas reduce some generality (you can't just read any data +without having its schema), they have a lot of upsides: + +- Most information about the format can be factored into the generated + code, reducing memory needed to store data, and time to access it. + +- The strong typing of the data definitions means less error + checking/handling at runtime (less can go wrong). + +- A schema enables us to access a buffer without parsing. + +FlatBuffer schemas are fairly similar to those of the incumbent, +Protocol Buffers, and generally should be readable to those familiar +with the C family of languages. We chose to improve upon the features +offered by .proto files in the following ways: + +- Deprecation of fields instead of manual field id assignment. + Extending an object in a .proto means hunting for a free slot among + the numbers (preferring lower numbers since they have a more compact + representation). Besides being inconvenient, it also makes removing + fields problematic: you either have to keep them, not making it + obvious that this field shouldn't be read/written anymore, and still + generating accessors. Or you remove it, but now you risk that + there's still old data around that uses that field by the time + someone reuses that field id, with nasty consequences. + +- Differentiating between tables and structs (see above). Effectively + all table fields are `optional`, and all struct fields are + `required`. + +- Having a native vector type instead of `repeated`. This gives you a + length without having to collect all items, and in the case of + scalars provides for a more compact representation, and one that + guarantees adjacency. + +- Having a native `union` type instead of using a series of `optional` + fields, all of which must be checked individually. + +- Being able to define defaults for all scalars, instead of having to + deal with their optionality at each access. + +- A parser that can deal with both schemas and data definitions (JSON + compatible) uniformly. + +