mirror of
https://github.com/google/flatbuffers.git
synced 2026-06-02 04:04:19 +00:00
7181d77700
A field with key attribute must always be written on the message so it can be looked up by key. There is a edge case where inserting a key field with same value as default would prevent it to be written on the message and later cannot be found when searched by key.
478 lines
12 KiB
Python
Executable File
478 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# Copyright 2021 Google Inc. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import argparse
|
|
import filecmp
|
|
import glob
|
|
import platform
|
|
import shutil
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
"--flatc",
|
|
help="path of the Flat C compiler relative to the root directory",
|
|
)
|
|
parser.add_argument("--cpp-0x", action="store_true", help="use --cpp-std c++ox")
|
|
parser.add_argument(
|
|
"--skip-monster-extra",
|
|
action="store_true",
|
|
help="skip generating tests involving monster_extra.fbs",
|
|
)
|
|
parser.add_argument(
|
|
"--skip-gen-reflection",
|
|
action="store_true",
|
|
help="skip generating the reflection.fbs files",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
# Get the path where this script is located so we can invoke the script from
|
|
# any directory and have the paths work correctly.
|
|
script_path = Path(__file__).parent.resolve()
|
|
|
|
# Get the root path as an absolute path, so all derived paths are absolute.
|
|
root_path = script_path.parent.absolute()
|
|
|
|
# Get the location of the flatc executable, reading from the first command line
|
|
# argument or defaulting to default names.
|
|
flatc_exe = Path(
|
|
("flatc" if not platform.system() == "Windows" else "flatc.exe")
|
|
if not args.flatc
|
|
else args.flatc
|
|
)
|
|
|
|
# Find and assert flatc compiler is present.
|
|
if root_path in flatc_exe.parents:
|
|
flatc_exe = flatc_exe.relative_to(root_path)
|
|
flatc_path = Path(root_path, flatc_exe)
|
|
assert flatc_path.exists(), "Cannot find the flatc compiler " + str(flatc_path)
|
|
|
|
# Specify the other paths that will be referenced
|
|
tests_path = Path(root_path, "tests")
|
|
swift_code_gen = Path(root_path, "tests/FlatBuffers.Test.Swift/CodeGenerationTests")
|
|
samples_path = Path(root_path, "samples")
|
|
reflection_path = Path(root_path, "reflection")
|
|
|
|
# Execute the flatc compiler with the specified parameters
|
|
def flatc(options, schema, prefix=None, include=None, data=None, cwd=tests_path):
|
|
cmd = [str(flatc_path)] + options
|
|
if prefix:
|
|
cmd += ["-o"] + [prefix]
|
|
if include:
|
|
cmd += ["-I"] + [include]
|
|
cmd += [schema] if isinstance(schema, str) else schema
|
|
if data:
|
|
cmd += [data] if isinstance(data, str) else data
|
|
result = subprocess.run(cmd, cwd=str(cwd), check=True)
|
|
|
|
|
|
# Generate the code for flatbuffers reflection schema
|
|
def flatc_reflection(options, location, target):
|
|
full_options = ["--no-prefix"] + options
|
|
temp_dir = ".tmp"
|
|
flatc(
|
|
full_options,
|
|
prefix=temp_dir,
|
|
schema="reflection.fbs",
|
|
cwd=reflection_path,
|
|
)
|
|
new_reflection_path = Path(reflection_path, temp_dir, target)
|
|
original_reflection_path = Path(root_path, location, target)
|
|
if not filecmp.cmp(str(new_reflection_path), str(original_reflection_path)):
|
|
shutil.rmtree(str(original_reflection_path))
|
|
shutil.move(str(new_reflection_path), str(original_reflection_path))
|
|
shutil.rmtree(str(Path(reflection_path, temp_dir)))
|
|
|
|
|
|
# Glob a pattern relative to file path
|
|
def glob(path, pattern):
|
|
return [str(p) for p in path.glob(pattern)]
|
|
|
|
|
|
# flatc options that are shared
|
|
BASE_OPTS = ["--reflect-names", "--gen-mutable", "--gen-object-api"]
|
|
NO_INCL_OPTS = BASE_OPTS + ["--no-includes"]
|
|
|
|
# Language specific options
|
|
CS_OPTS = ["--csharp", "--cs-gen-json-serializer"]
|
|
CPP_OPTS = [
|
|
"--cpp",
|
|
"--gen-compare",
|
|
"--cpp-ptr-type",
|
|
"flatbuffers::unique_ptr",
|
|
] + (["--cpp-std", "c++0x"] if args.cpp_0x else [])
|
|
|
|
CPP_17_OPTS = NO_INCL_OPTS + [
|
|
"--cpp",
|
|
"--cpp-std",
|
|
"c++17",
|
|
"--cpp-static-reflection",
|
|
"--gen-object-api",
|
|
]
|
|
RUST_OPTS = BASE_OPTS + [
|
|
"--rust",
|
|
"--gen-all",
|
|
"--gen-name-strings",
|
|
"--rust-module-root-file",
|
|
]
|
|
RUST_SERIALIZE_OPTS = BASE_OPTS + [
|
|
"--rust",
|
|
"--gen-all",
|
|
"--gen-name-strings",
|
|
"--rust-serialize",
|
|
"--rust-module-root-file",
|
|
]
|
|
TS_OPTS = ["--ts", "--gen-name-strings"]
|
|
LOBSTER_OPTS = ["--lobster"]
|
|
SWIFT_OPTS = ["--swift", "--gen-json-emit", "--bfbs-filenames", str(tests_path)]
|
|
SWIFT_OPTS_CODE_GEN = [
|
|
"--swift",
|
|
"--gen-json-emit",
|
|
"--bfbs-filenames",
|
|
swift_code_gen
|
|
]
|
|
JAVA_OPTS = ["--java"]
|
|
KOTLIN_OPTS = ["--kotlin"]
|
|
PHP_OPTS = ["--php"]
|
|
DART_OPTS = ["--dart"]
|
|
PYTHON_OPTS = ["--python"]
|
|
BINARY_OPTS = ["-b", "--schema", "--bfbs-comments", "--bfbs-builtins"]
|
|
|
|
# Basic Usage
|
|
|
|
flatc(
|
|
NO_INCL_OPTS
|
|
+ CPP_OPTS
|
|
+ CS_OPTS
|
|
+ TS_OPTS
|
|
+ [
|
|
"--binary",
|
|
"--java",
|
|
"--kotlin",
|
|
"--dart",
|
|
"--go",
|
|
"--lobster",
|
|
"--php",
|
|
],
|
|
schema="monster_test.fbs",
|
|
include="include_test",
|
|
data="monsterdata_test.json",
|
|
)
|
|
|
|
flatc(
|
|
["--lua", "--bfbs-filenames", str(tests_path)],
|
|
schema="monster_test.fbs",
|
|
include="include_test",
|
|
)
|
|
|
|
flatc(
|
|
NO_INCL_OPTS + CPP_OPTS + ["--grpc"],
|
|
schema="monster_test.fbs",
|
|
include="include_test",
|
|
data="monsterdata_test.json",
|
|
)
|
|
|
|
flatc(
|
|
RUST_OPTS,
|
|
schema="monster_test.fbs",
|
|
include="include_test",
|
|
prefix="monster_test",
|
|
data="monsterdata_test.json",
|
|
)
|
|
|
|
flatc(
|
|
RUST_SERIALIZE_OPTS,
|
|
schema="monster_test.fbs",
|
|
include="include_test",
|
|
prefix="monster_test_serialize",
|
|
data="monsterdata_test.json",
|
|
)
|
|
|
|
flatc(
|
|
options=BASE_OPTS + ["--python"],
|
|
schema="monster_test.fbs",
|
|
include="include_test",
|
|
data="monsterdata_test.json",
|
|
)
|
|
|
|
flatc(
|
|
options=BASE_OPTS + ["--python", "--gen-onefile"],
|
|
schema="monster_test.fbs",
|
|
include="include_test",
|
|
data="monsterdata_test.json",
|
|
)
|
|
|
|
# For Rust we currently generate two independent schemas, with namespace_test2
|
|
# duplicating the types in namespace_test1
|
|
flatc(
|
|
RUST_OPTS,
|
|
prefix="namespace_test",
|
|
schema=[
|
|
"namespace_test/namespace_test1.fbs",
|
|
"namespace_test/namespace_test2.fbs",
|
|
],
|
|
)
|
|
|
|
flatc(
|
|
BASE_OPTS + CPP_OPTS + CS_OPTS + TS_OPTS + JAVA_OPTS + KOTLIN_OPTS + PHP_OPTS,
|
|
prefix="union_vector",
|
|
schema="union_vector/union_vector.fbs",
|
|
)
|
|
|
|
flatc(
|
|
BASE_OPTS + TS_OPTS + ["--gen-name-strings", "--gen-mutable"],
|
|
include="include_test",
|
|
schema="monster_test.fbs",
|
|
)
|
|
|
|
flatc(
|
|
BASE_OPTS + TS_OPTS + ["-b"],
|
|
include="include_test",
|
|
schema="monster_test.fbs",
|
|
data="unicode_test.json",
|
|
)
|
|
|
|
flatc(
|
|
BASE_OPTS + TS_OPTS + ["--gen-name-strings"],
|
|
prefix="union_vector",
|
|
schema="union_vector/union_vector.fbs",
|
|
)
|
|
|
|
flatc(
|
|
RUST_OPTS,
|
|
prefix="include_test1",
|
|
include="include_test",
|
|
schema="include_test/include_test1.fbs",
|
|
)
|
|
|
|
flatc(
|
|
RUST_OPTS,
|
|
prefix="include_test2",
|
|
include="include_test",
|
|
schema="include_test/sub/include_test2.fbs",
|
|
)
|
|
|
|
flatc(
|
|
BINARY_OPTS + ["--bfbs-filenames", str(tests_path)],
|
|
include="include_test",
|
|
schema="monster_test.fbs",
|
|
)
|
|
|
|
flatc(
|
|
CPP_OPTS
|
|
+ NO_INCL_OPTS
|
|
+ [
|
|
"--bfbs-comments",
|
|
"--bfbs-builtins",
|
|
"--bfbs-gen-embed",
|
|
"--bfbs-filenames",
|
|
str(tests_path),
|
|
],
|
|
include="include_test",
|
|
schema="monster_test.fbs",
|
|
)
|
|
|
|
flatc(
|
|
BINARY_OPTS + ["--bfbs-filenames", str(tests_path)],
|
|
include="include_test",
|
|
schema="arrays_test.fbs",
|
|
)
|
|
|
|
flatc(
|
|
["--jsonschema", "--schema"],
|
|
include="include_test",
|
|
schema="monster_test.fbs",
|
|
)
|
|
|
|
if not args.skip_monster_extra:
|
|
flatc(
|
|
CPP_OPTS + CS_OPTS + NO_INCL_OPTS + JAVA_OPTS + KOTLIN_OPTS + PYTHON_OPTS,
|
|
schema="monster_extra.fbs",
|
|
data="monsterdata_extra.json",
|
|
)
|
|
|
|
flatc(
|
|
DART_OPTS + ["--gen-object-api"],
|
|
schema="monster_extra.fbs",
|
|
)
|
|
|
|
flatc(
|
|
CPP_OPTS + CS_OPTS + NO_INCL_OPTS + JAVA_OPTS + ["--jsonschema", "--scoped-enums"],
|
|
schema="arrays_test.fbs",
|
|
)
|
|
|
|
flatc(
|
|
RUST_OPTS,
|
|
prefix="arrays_test",
|
|
schema="arrays_test.fbs",
|
|
)
|
|
|
|
flatc(
|
|
BASE_OPTS + PYTHON_OPTS,
|
|
schema="arrays_test.fbs",
|
|
)
|
|
|
|
|
|
# Optional Scalars
|
|
optional_scalars_schema = "optional_scalars.fbs"
|
|
flatc(["--java", "--kotlin", "--lobster", "--ts"], schema=optional_scalars_schema)
|
|
|
|
flatc(["--csharp", "--gen-object-api"], schema=optional_scalars_schema)
|
|
|
|
flatc(RUST_OPTS, prefix="optional_scalars", schema=optional_scalars_schema)
|
|
|
|
flatc(NO_INCL_OPTS + CPP_OPTS, schema=optional_scalars_schema)
|
|
|
|
# Type / field collsion
|
|
type_field_collsion_schema = "type_field_collsion.fbs"
|
|
|
|
flatc(["--csharp", "--gen-object-api"], schema=type_field_collsion_schema)
|
|
|
|
# Generate string/vector default code for tests
|
|
flatc(RUST_OPTS, prefix="more_defaults", schema="more_defaults.fbs")
|
|
|
|
# Generate the schema evolution tests
|
|
flatc(
|
|
CPP_OPTS + ["--scoped-enums"],
|
|
prefix="evolution_test",
|
|
schema=glob(tests_path, "evolution_test/evolution_v*.fbs"),
|
|
)
|
|
|
|
# Generate the keywords tests
|
|
flatc(BASE_OPTS + CS_OPTS, schema="keyword_test.fbs")
|
|
flatc(RUST_OPTS, prefix="keyword_test", schema="keyword_test.fbs")
|
|
flatc(
|
|
BASE_OPTS + CS_OPTS + ["--cs-global-alias", "--gen-onefile"],
|
|
prefix="nested_namespace_test",
|
|
schema=glob(tests_path, "nested_namespace_test/nested_namespace_test*.fbs"),
|
|
)
|
|
flatc(BASE_OPTS + DART_OPTS, prefix="../dart/test/", schema="keyword_test.fbs")
|
|
|
|
# Field key lookup with default value test
|
|
dictionary_lookup_schema = "dictionary_lookup.fbs"
|
|
flatc(["--java"], schema=dictionary_lookup_schema)
|
|
|
|
# Swift Tests
|
|
swift_prefix = "FlatBuffers.Test.Swift/Tests/FlatBuffers.Test.SwiftTests"
|
|
flatc(
|
|
SWIFT_OPTS + BASE_OPTS + ["--grpc"],
|
|
schema="monster_test.fbs",
|
|
include="include_test",
|
|
prefix=swift_prefix,
|
|
)
|
|
flatc(
|
|
SWIFT_OPTS + BASE_OPTS,
|
|
schema="union_vector/union_vector.fbs",
|
|
prefix=swift_prefix,
|
|
)
|
|
flatc(SWIFT_OPTS, schema="optional_scalars.fbs", prefix=swift_prefix)
|
|
flatc(
|
|
SWIFT_OPTS + ["--gen-object-api"],
|
|
schema="more_defaults.fbs",
|
|
prefix=swift_prefix,
|
|
)
|
|
flatc(
|
|
SWIFT_OPTS + BASE_OPTS,
|
|
schema="MutatingBool.fbs",
|
|
prefix=swift_prefix,
|
|
)
|
|
|
|
flatc(
|
|
SWIFT_OPTS_CODE_GEN + BASE_OPTS + ["--grpc", "--swift-implementation-only"],
|
|
schema="test_import.fbs",
|
|
cwd=swift_code_gen
|
|
)
|
|
|
|
flatc(
|
|
SWIFT_OPTS_CODE_GEN + NO_INCL_OPTS + ["--grpc"],
|
|
schema="test_no_include.fbs",
|
|
cwd=swift_code_gen
|
|
)
|
|
|
|
# --filename-suffix and --filename-ext tests
|
|
flatc(
|
|
CPP_OPTS + NO_INCL_OPTS + ["--filename-suffix", "_suffix", "--filename-ext", "hpp"],
|
|
include="include_test",
|
|
schema="monster_test.fbs",
|
|
)
|
|
orig_monster_file = Path(tests_path, "monster_test_generated.h")
|
|
new_monster_file = Path(tests_path, "monster_test_suffix.hpp")
|
|
assert new_monster_file.exists(), "filename suffix option did not produce a file"
|
|
assert filecmp.cmp(
|
|
str(orig_monster_file), str(new_monster_file)
|
|
), "filename suffix option did not produce identical results"
|
|
new_monster_file.unlink()
|
|
|
|
# Flag c++17 requires Clang6, GCC7, MSVC2017 (_MSC_VER >= 1914) or higher.
|
|
cpp_17_prefix = "cpp17/generated_cpp17"
|
|
flatc(
|
|
CPP_17_OPTS,
|
|
schema="monster_test.fbs",
|
|
include="include_test",
|
|
prefix=cpp_17_prefix,
|
|
)
|
|
flatc(
|
|
CPP_17_OPTS,
|
|
schema="optional_scalars.fbs",
|
|
prefix=cpp_17_prefix,
|
|
)
|
|
flatc(
|
|
CPP_17_OPTS,
|
|
schema="union_vector/union_vector.fbs",
|
|
prefix=cpp_17_prefix,
|
|
)
|
|
|
|
# Sample files
|
|
samples_schema = "monster.fbs"
|
|
flatc(BASE_OPTS + CPP_OPTS + LOBSTER_OPTS, schema=samples_schema, cwd=samples_path)
|
|
flatc(RUST_OPTS, prefix="rust_generated", schema=samples_schema, cwd=samples_path)
|
|
flatc(
|
|
BINARY_OPTS + ["--bfbs-filenames", str(samples_path)],
|
|
schema=samples_schema,
|
|
cwd=samples_path,
|
|
)
|
|
|
|
# Reflection
|
|
|
|
# Skip generating the reflection if told too, as we run this script after
|
|
# building flatc which uses the reflection_generated.h itself.
|
|
if not args.skip_gen_reflection:
|
|
# C++ Reflection
|
|
flatc_reflection(
|
|
["-c", "--cpp-std", "c++0x"], "include/flatbuffers", "reflection_generated.h"
|
|
)
|
|
|
|
# Python Reflection
|
|
flatc_reflection(["-p"], "python/flatbuffers", "reflection")
|
|
|
|
# Annotation
|
|
|
|
|
|
def flatc_annotate(schema, include=None, data=None, cwd=tests_path):
|
|
cmd = [str(flatc_path)]
|
|
if include:
|
|
cmd += ["-I"] + [include]
|
|
cmd += ["--annotate", schema]
|
|
if data:
|
|
cmd += [data] if isinstance(data, str) else data
|
|
subprocess.run(cmd, cwd=str(cwd), check=True)
|
|
|
|
|
|
flatc_annotate(
|
|
schema="monster_test.fbs", include="include_test", data="monsterdata_test.mon"
|
|
)
|