BigfootDev/flatbuffers
1
0
Fork 1
mirror of https://github.com/google/flatbuffers.git synced 2026-06-01 19:58:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

[gRPC] Update the code generator for Python to produce typed handlers (#8326)

Browse Source 
* Move `namer.h` and `idl_namer.h` to `include/codegen` so they can be reused from `grpc` dirqectory.

* [gRPC] Update the Python generator to produce typed handlers and Python stubs if requested.

* [gRPC] Document the newly added compiler flags.
This commit is contained in:
Anton Bobukh
2024-06-18 16:02:57 -07:00
committed by GitHub
parent dafd2f1f29
commit fb9afbafc7
22 changed files with 1177 additions and 665 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@@ -183,6 +183,8 @@ set(FlatBuffers_Compiler_SRCS
src/bfbs_gen_lua.h
src/bfbs_gen_nim.h
src/bfbs_namer.h
include/codegen/idl_namer.h
include/codegen/namer.h
include/codegen/python.h
include/codegen/python.cc
include/flatbuffers/code_generators.h

43
docs/source/Compiler.md
View File

@@ -96,10 +96,10 @@ Additional options:
- `--scoped-enums` : Use C++11 style scoped and strongly typed enums in
generated C++. This also implies `--no-prefix`.
- `--no-emit-min-max-enum-values` : Disable generation of MIN and MAX
enumerated values for scoped enums and prefixed enums.
- `--gen-includes` : (deprecated), this is the default behavior.
If the original behavior is required (no include
statements) use `--no-includes.`
@@ -238,5 +238,44 @@ Additional options:
- `--python-typing` : Generate Python type annotations
Additional gRPC options:
- `--grpc-filename-suffix`: `[C++]` An optional suffix for the generated
files' names. For example, compiling gRPC for C++ with
`--grpc-filename-suffix=.fbs` will generate `{name}.fbs.h` and
`{name}.fbs.cc` files.
- `--grpc-additional-header`: `[C++]` Additional headers to include in the
generated files.
- `--grpc-search-path`: `[C++]` An optional prefix for the gRPC runtime path.
For example, compiling gRPC for C++ with `--grpc-search-path=some/path` will
generate the following includes:
```cpp
#include "some/path/grpcpp/impl/codegen/async_stream.h"
#include "some/path/grpcpp/impl/codegen/async_unary_call.h"
#include "some/path/grpcpp/impl/codegen/method_handler.h"
...
```
- `--grpc-use-system-headers`: `[C++]` Whether to generate `#include <header>`
instead of `#include "header.h"` for all headers when compiling gRPC for
C++. For example, compiling gRPC for C++ with `--grpc-use-system-headers`
will generate the following includes:
```cpp
#include <some/path/grpcpp/impl/codegen/async_stream.h>
#include <some/path/grpcpp/impl/codegen/async_unary_call.h>
#include <some/path/grpcpp/impl/codegen/method_handler.h>
...
```
NOTE: This option can be negated with `--no-grpc-use-system-headers`.
- `--grpc-python-typed-handlers`: `[Python]` Whether to generate the typed
handlers that use the generated Python classes instead of raw bytes for
requests/responses.
NOTE: short-form options for generators are deprecated, use the long form
whenever possible.

2
grpc/src/compiler/BUILD.bazel
View File

@@ -95,6 +95,8 @@ cc_library(
visibility = ["//visibility:private"],
deps = [
"//:flatbuffers",
"//include/codegen:namer",
"//include/codegen:python",
],
)

469
grpc/src/compiler/python_generator.cc
View File

@@ -16,136 +16,365 @@
*
*/
#include <map>
#include <sstream>
#include "flatbuffers/util.h"
#include "src/compiler/python_generator.h"
namespace grpc_python_generator {
#include <algorithm>
#include <cstddef>
#include <map>
#include <set>
#include <sstream>
#include <string>
#include "codegen/idl_namer.h"
#include "codegen/namer.h"
#include "codegen/python.h"
#include "flatbuffers/idl.h"
#include "flatbuffers/util.h"
namespace flatbuffers {
namespace python {
namespace grpc {
namespace {
static grpc::string GenerateMethodType(const grpc_generator::Method *method) {
if (method->NoStreaming())
return "unary_unary";
if (method->ServerStreaming())
return "unary_stream";
if (method->ClientStreaming())
return "stream_unary";
return "stream_stream";
bool ClientStreaming(const RPCCall *method) {
const Value *val = method->attributes.Lookup("streaming");
return val != nullptr && (val->constant == "client" || val->constant == "bidi");
}
grpc::string GenerateMethodInput(const grpc_generator::Method *method) {
if (method->NoStreaming() || method->ServerStreaming())
return "self, request, context";
return "self, request_iterator, context";
bool ServerStreaming(const RPCCall *method) {
const Value *val = method->attributes.Lookup("streaming");
return val != nullptr && (val->constant == "server" || val->constant == "bidi");
}
void GenerateStub(const grpc_generator::Service *service,
grpc_generator::Printer *printer,
std::map<grpc::string, grpc::string> *dictonary) {
auto vars = *dictonary;
printer->Print(vars, "class $ServiceName$Stub(object):\n");
printer->Indent();
printer->Print("\"\"\" Interface exported by the server. \"\"\"");
printer->Print("\n\n");
printer->Print("def __init__(self, channel):\n");
printer->Indent();
printer->Print("\"\"\" Constructor. \n\n");
printer->Print("Args: \n");
printer->Print("channel: A grpc.Channel. \n");
printer->Print("\"\"\"\n\n");
for (int j = 0; j < service->method_count(); j++) {
auto method = service->method(j);
vars["MethodName"] = method->name();
vars["MethodType"] = GenerateMethodType(&*method);
printer->Print(vars, "self.$MethodName$ = channel.$MethodType$(\n");
printer->Indent();
printer->Print(vars, "\"/$PATH$$ServiceName$/$MethodName$\"\n");
printer->Print(")\n");
printer->Outdent();
printer->Print("\n");
void FormatImports(std::stringstream &ss, const Imports &imports) {
std::set<std::string> modules;
std::map<std::string, std::set<std::string>> names_by_module;
for (const Import &import : imports.imports) {
if (import.IsLocal()) continue; // skip all local imports
if (import.name == "") {
modules.insert(import.module);
} else {
names_by_module[import.module].insert(import.name);
}
}
printer->Outdent();
printer->Outdent();
printer->Print("\n");
}
void GenerateServicer(const grpc_generator::Service *service,
grpc_generator::Printer *printer,
std::map<grpc::string, grpc::string> *dictonary) {
auto vars = *dictonary;
printer->Print(vars, "class $ServiceName$Servicer(object):\n");
printer->Indent();
printer->Print("\"\"\" Interface exported by the server. \"\"\"");
printer->Print("\n\n");
for (int j = 0; j < service->method_count(); j++) {
auto method = service->method(j);
vars["MethodName"] = method->name();
vars["MethodInput"] = GenerateMethodInput(&*method);
printer->Print(vars, "def $MethodName$($MethodInput$):\n");
printer->Indent();
printer->Print("context.set_code(grpc.StatusCode.UNIMPLEMENTED)\n");
printer->Print("context.set_details('Method not implemented!')\n");
printer->Print("raise NotImplementedError('Method not implemented!')\n");
printer->Outdent();
printer->Print("\n\n");
for (const std::string &module : modules) {
ss << "import " << module << '\n';
}
printer->Outdent();
printer->Print("\n");
}
void GenerateRegister(const grpc_generator::Service *service,
grpc_generator::Printer *printer,
std::map<grpc::string, grpc::string> *dictonary) {
auto vars = *dictonary;
printer->Print(vars, "def add_$ServiceName$Servicer_to_server(servicer, server):\n");
printer->Indent();
printer->Print("rpc_method_handlers = {\n");
printer->Indent();
for (int j = 0; j < service->method_count(); j++) {
auto method = service->method(j);
vars["MethodName"] = method->name();
vars["MethodType"] = GenerateMethodType(&*method);
printer->Print(vars, "'$MethodName$': grpc.$MethodType$_rpc_method_handler(\n");
printer->Indent();
printer->Print(vars, "servicer.$MethodName$\n");
printer->Outdent();
printer->Print("),\n");
ss << '\n';
for (const auto &import : names_by_module) {
ss << "from " << import.first << " import ";
size_t i = 0;
for (const std::string &name : import.second) {
if (i > 0) ss << ", ";
ss << name;
++i;
}
ss << '\n';
}
printer->Outdent();
printer->Print("}\n");
printer->Print(vars, "generic_handler = grpc.method_handlers_generic_handler(\n");
printer->Indent();
printer->Print(vars, "'$PATH$$ServiceName$', rpc_method_handlers)\n");
printer->Outdent();
printer->Print("server.add_generic_rpc_handlers((generic_handler,))");
printer->Outdent();
printer->Print("\n");
}
} // namespace
grpc::string Generate(grpc_generator::File *file,
const grpc_generator::Service *service) {
grpc::string output;
std::map<grpc::string, grpc::string> vars;
vars["PATH"] = file->package();
if (!file->package().empty()) { vars["PATH"].append("."); }
vars["ServiceName"] = service->name();
auto printer = file->CreatePrinter(&output);
GenerateStub(service, &*printer, &vars);
GenerateServicer(service, &*printer, &vars);
GenerateRegister(service, &*printer, &vars);
return output;
ss << "\n\n";
}
} // namespace grpc_python_generator
bool SaveStub(const std::string &filename, const Imports &imports,
const std::string &content) {
std::stringstream ss;
ss << "# Generated by the gRPC FlatBuffers compiler. DO NOT EDIT!\n"
<< '\n'
<< "from __future__ import annotations\n"
<< '\n';
FormatImports(ss, imports);
ss << content << '\n';
EnsureDirExists(StripFileName(filename));
return flatbuffers::SaveFile(filename.c_str(), ss.str(), false);
}
bool SaveService(const std::string &filename, const Imports &imports,
const std::string &content) {
std::stringstream ss;
ss << "# Generated by the gRPC FlatBuffers compiler. DO NOT EDIT!\n" << '\n';
FormatImports(ss, imports);
ss << content << '\n';
EnsureDirExists(StripFileName(filename));
return flatbuffers::SaveFile(filename.c_str(), ss.str(), false);
}
class BaseGenerator {
protected:
BaseGenerator(const Parser &parser, const Namer::Config &config,
const std::string &path, const Version &version)
: parser_{parser},
namer_{WithFlagOptions(config, parser.opts, path), Keywords(version)},
version_{version} {}
protected:
std::string ModuleForFile(const std::string &file) const {
std::string module = parser_.opts.include_prefix + StripExtension(file) +
parser_.opts.filename_suffix;
std::replace(module.begin(), module.end(), '/', '.');
return module;
}
template <typename T>
std::string ModuleFor(const T *def) const {
if (parser_.opts.one_file) return ModuleForFile(def->file);
return namer_.NamespacedType(*def);
}
const Parser &parser_;
const IdlNamer namer_;
const Version version_;
};
class StubGenerator : public BaseGenerator {
public:
StubGenerator(const Parser &parser, const std::string &path,
const Version &version)
: BaseGenerator(parser, kStubConfig, path, version) {}
bool Generate() {
Imports imports;
std::stringstream stub;
for (const ServiceDef *service : parser_.services_.vec) {
Generate(stub, service, &imports);
}
std::string filename =
namer_.config_.output_path +
StripPath(StripExtension(parser_.file_being_parsed_)) + "_grpc" +
parser_.opts.grpc_filename_suffix + namer_.config_.filename_extension;
return SaveStub(filename, imports, stub.str());
}
private:
void Generate(std::stringstream &ss, const ServiceDef *service,
Imports *imports) {
imports->Import("grpc");
ss << "class " << service->name << "Stub(object):\n"
<< " def __init__(self, channel: grpc.Channel) -> None: ...\n";
for (const RPCCall *method : service->calls.vec) {
std::string request = "bytes";
std::string response = "bytes";
if (parser_.opts.grpc_python_typed_handlers) {
request = namer_.Type(*method->request);
response = namer_.Type(*method->response);
imports->Import(ModuleFor(method->request), request);
imports->Import(ModuleFor(method->response), response);
}
ss << " def " << method->name << "(self, ";
if (ClientStreaming(method)) {
imports->Import("typing");
ss << "request_iterator: typing.Iterator[" << request << "]";
} else {
ss << "request: " << request;
}
ss << ") -> ";
if (ServerStreaming(method)) {
imports->Import("typing");
ss << "typing.Iterator[" << response << "]";
} else {
ss << response;
}
ss << ": ...\n";
}
ss << "\n\n";
ss << "class " << service->name << "Servicer(object):\n";
for (const RPCCall *method : service->calls.vec) {
std::string request = "bytes";
std::string response = "bytes";
if (parser_.opts.grpc_python_typed_handlers) {
request = namer_.Type(*method->request);
response = namer_.Type(*method->response);
imports->Import(ModuleFor(method->request), request);
imports->Import(ModuleFor(method->response), response);
}
ss << " def " << method->name << "(self, ";
if (ClientStreaming(method)) {
imports->Import("typing");
ss << "request_iterator: typing.Iterator[" << request << "]";
} else {
ss << "request: " << request;
}
ss << ", context: grpc.ServicerContext) -> ";
if (ServerStreaming(method)) {
imports->Import("typing");
ss << "typing.Iterator[" << response << "]";
} else {
ss << response;
}
ss << ": ...\n";
}
ss << '\n'
<< '\n'
<< "def add_" << service->name
<< "Servicer_to_server(servicer: " << service->name
<< "Servicer, server: grpc.Server) -> None: ...\n";
}
};
class ServiceGenerator : public BaseGenerator {
public:
ServiceGenerator(const Parser &parser, const std::string &path,
const Version &version)
: BaseGenerator(parser, kConfig, path, version) {}
bool Generate() {
Imports imports;
std::stringstream ss;
imports.Import("flatbuffers");
if (parser_.opts.grpc_python_typed_handlers) {
ss << "def _serialize_to_bytes(table):\n"
<< " buf = table._tab.Bytes\n"
<< " n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0)\n"
<< " if table._tab.Pos != n:\n"
<< " raise ValueError('must be a top-level table')\n"
<< " return bytes(buf)\n"
<< '\n'
<< '\n';
}
for (const ServiceDef *service : parser_.services_.vec) {
GenerateStub(ss, service, &imports);
GenerateServicer(ss, service, &imports);
GenerateRegister(ss, service, &imports);
}
std::string filename =
namer_.config_.output_path +
StripPath(StripExtension(parser_.file_being_parsed_)) + "_grpc" +
parser_.opts.grpc_filename_suffix + namer_.config_.filename_extension;
return SaveService(filename, imports, ss.str());
}
private:
void GenerateStub(std::stringstream &ss, const ServiceDef *service,
Imports *imports) {
ss << "class " << service->name << "Stub";
if (version_.major != 3) ss << "(object)";
ss << ":\n"
<< " '''Interface exported by the server.'''\n"
<< '\n'
<< " def __init__(self, channel):\n"
<< " '''Constructor.\n"
<< '\n'
<< " Args:\n"
<< " channel: A grpc.Channel.\n"
<< " '''\n"
<< '\n';
for (const RPCCall *method : service->calls.vec) {
std::string response = namer_.Type(*method->response);
imports->Import(ModuleFor(method->response), response);
ss << " self." << method->name << " = channel."
<< (ClientStreaming(method) ? "stream" : "unary") << "_"
<< (ServerStreaming(method) ? "stream" : "unary") << "(\n"
<< " method='/"
<< service->defined_namespace->GetFullyQualifiedName(service->name)
<< "/" << method->name << "'";
if (parser_.opts.grpc_python_typed_handlers) {
ss << ",\n"
<< " request_serializer=_serialize_to_bytes,\n"
<< " response_deserializer=" << response << ".GetRootAs";
}
ss << ")\n\n";
}
ss << '\n';
}
void GenerateServicer(std::stringstream &ss, const ServiceDef *service,
Imports *imports) {
imports->Import("grpc");
ss << "class " << service->name << "Servicer";
if (version_.major != 3) ss << "(object)";
ss << ":\n"
<< " '''Interface exported by the server.'''\n"
<< '\n';
for (const RPCCall *method : service->calls.vec) {
const std::string request_param =
ClientStreaming(method) ? "request_iterator" : "request";
ss << " def " << method->name << "(self, " << request_param
<< ", context):\n"
<< " context.set_code(grpc.StatusCode.UNIMPLEMENTED)\n"
<< " context.set_details('Method not implemented!')\n"
<< " raise NotImplementedError('Method not implemented!')\n"
<< '\n';
}
ss << '\n';
}
void GenerateRegister(std::stringstream &ss, const ServiceDef *service,
Imports *imports) {
imports->Import("grpc");
ss << "def add_" << service->name
<< "Servicer_to_server(servicer, server):\n"
<< " rpc_method_handlers = {\n";
for (const RPCCall *method : service->calls.vec) {
std::string request = namer_.Type(*method->request);
imports->Import(ModuleFor(method->request), request);
ss << " '" << method->name << "': grpc."
<< (ClientStreaming(method) ? "stream" : "unary") << "_"
<< (ServerStreaming(method) ? "stream" : "unary")
<< "_rpc_method_handler(\n"
<< " servicer." << method->name;
if (parser_.opts.grpc_python_typed_handlers) {
ss << ",\n"
<< " request_deserializer=" << request << ".GetRootAs,\n"
<< " response_serializer=_serialize_to_bytes";
}
ss << "),\n";
}
ss << " }\n"
<< '\n'
<< " generic_handler = grpc.method_handlers_generic_handler(\n"
<< " '"
<< service->defined_namespace->GetFullyQualifiedName(service->name)
<< "', rpc_method_handlers)\n"
<< '\n'
<< " server.add_generic_rpc_handlers((generic_handler,))\n"
<< '\n';
}
};
} // namespace
bool Generate(const Parser &parser, const std::string &path,
const Version &version) {
ServiceGenerator generator{parser, path, version};
return generator.Generate();
}
bool GenerateStub(const Parser &parser, const std::string &path,
const Version &version) {
StubGenerator generator{parser, path, version};
return generator.Generate();
}
} // namespace grpc
} // namespace python
} // namespace flatbuffers

19
grpc/src/compiler/python_generator.h
View File

@@ -19,14 +19,21 @@
#ifndef GRPC_INTERNAL_COMPILER_PYTHON_GENERATOR_H
#define GRPC_INTERNAL_COMPILER_PYTHON_GENERATOR_H
#include <utility>
#include <string>
#include "src/compiler/schema_interface.h"
#include "codegen/python.h"
#include "flatbuffers/idl.h"
namespace grpc_python_generator {
namespace flatbuffers {
namespace python {
namespace grpc {
bool Generate(const Parser &parser, const std::string &path,
const Version &version);
grpc::string Generate(grpc_generator::File *file,
const grpc_generator::Service *service);
} // namespace grpc_python_generator
bool GenerateStub(const Parser &parser, const std::string &path,
const Version &version);
} // namespace grpc
} // namespace python
} // namespace flatbuffers
#endif // GRPC_INTERNAL_COMPILER_PYTHON_GENERATOR_H

17
include/codegen/BUILD.bazel
View File

@@ -15,10 +15,25 @@ filegroup(
visibility = ["//visibility:public"],
)
cc_library(
name = "namer",
hdrs = [
"idl_namer.h",
"namer.h",
],
strip_include_prefix = "/include",
visibility = ["//:__subpackages__"],
deps = ["//:runtime_cc"],
)
cc_library(
name = "python",
srcs = ["python.cc"],
hdrs = ["python.h"],
strip_include_prefix = "/include",
visibility = ["//src:__subpackages__"],
visibility = [
"//grpc:__subpackages__",
"//src:__subpackages__",
],
deps = [":namer"],
)

179
include/codegen/idl_namer.h Normal file
View File

@@ -0,0 +1,179 @@
#ifndef FLATBUFFERS_INCLUDE_CODEGEN_IDL_NAMER_H_
#define FLATBUFFERS_INCLUDE_CODEGEN_IDL_NAMER_H_
#include "codegen/namer.h"
#include "flatbuffers/idl.h"
namespace flatbuffers {
// Provides Namer capabilities to types defined in the flatbuffers IDL.
class IdlNamer : public Namer {
public:
explicit IdlNamer(Config config, std::set<std::string> keywords)
: Namer(config, std::move(keywords)) {}
using Namer::Constant;
using Namer::Directories;
using Namer::Field;
using Namer::File;
using Namer::Function;
using Namer::Method;
using Namer::Namespace;
using Namer::NamespacedType;
using Namer::ObjectType;
using Namer::Type;
using Namer::Variable;
using Namer::Variant;
std::string Constant(const FieldDef &d) const { return Constant(d.name); }
// Types are always structs or enums so we can only expose these two
// overloads.
std::string Type(const StructDef &d) const { return Type(d.name); }
std::string Type(const EnumDef &d) const { return Type(d.name); }
std::string Function(const Definition &s) const { return Function(s.name); }
std::string Function(const std::string& prefix, const Definition &s) const {
return Function(prefix + s.name);
}
std::string Field(const FieldDef &s) const { return Field(s.name); }
std::string Field(const FieldDef &d, const std::string &s) const {
return Field(d.name + "_" + s);
}
std::string Variable(const FieldDef &s) const { return Variable(s.name); }
std::string Variable(const StructDef &s) const { return Variable(s.name); }
std::string Variant(const EnumVal &s) const { return Variant(s.name); }
std::string EnumVariant(const EnumDef &e, const EnumVal &v) const {
return Type(e) + config_.enum_variant_seperator + Variant(v);
}
std::string ObjectType(const StructDef &d) const {
return ObjectType(d.name);
}
std::string ObjectType(const EnumDef &d) const { return ObjectType(d.name); }
std::string Method(const FieldDef &d, const std::string &suffix) const {
return Method(d.name, suffix);
}
std::string Method(const std::string &prefix, const StructDef &d) const {
return Method(prefix, d.name);
}
std::string Method(const std::string &prefix, const FieldDef &d) const {
return Method(prefix, d.name);
}
std::string Method(const std::string &prefix, const FieldDef &d,
const std::string &suffix) const {
return Method(prefix, d.name, suffix);
}
std::string Namespace(const struct Namespace &ns) const {
return Namespace(ns.components);
}
std::string NamespacedEnumVariant(const EnumDef &e, const EnumVal &v) const {
return NamespacedString(e.defined_namespace, EnumVariant(e, v));
}
std::string NamespacedType(const Definition &def) const {
return NamespacedString(def.defined_namespace, Type(def.name));
}
std::string NamespacedObjectType(const Definition &def) const {
return NamespacedString(def.defined_namespace, ObjectType(def.name));
}
std::string Directories(const struct Namespace &ns,
SkipDir skips = SkipDir::None,
Case input_case = Case::kUpperCamel) const {
return Directories(ns.components, skips, input_case);
}
// Legacy fields do not really follow the usual config and should be
// considered for deprecation.
std::string LegacyRustNativeVariant(const EnumVal &v) const {
return ConvertCase(EscapeKeyword(v.name), Case::kUpperCamel);
}
std::string LegacyRustFieldOffsetName(const FieldDef &field) const {
return "VT_" + ConvertCase(EscapeKeyword(field.name), Case::kAllUpper);
}
std::string LegacyRustUnionTypeOffsetName(const FieldDef &field) const {
return "VT_" + ConvertCase(EscapeKeyword(field.name + "_type"), Case::kAllUpper);
}
std::string LegacySwiftVariant(const EnumVal &ev) const {
auto name = ev.name;
if (isupper(name.front())) {
std::transform(name.begin(), name.end(), name.begin(), CharToLower);
}
return EscapeKeyword(ConvertCase(name, Case::kLowerCamel));
}
// Also used by Kotlin, lol.
std::string LegacyJavaMethod2(const std::string &prefix, const StructDef &sd,
const std::string &suffix) const {
return prefix + sd.name + suffix;
}
std::string LegacyKotlinVariant(EnumVal &ev) const {
// Namer assumes the input case is snake case which is wrong...
return ConvertCase(EscapeKeyword(ev.name), Case::kLowerCamel);
}
// Kotlin methods escapes keywords after case conversion but before
// prefixing and suffixing.
std::string LegacyKotlinMethod(const std::string &prefix, const FieldDef &d,
const std::string &suffix) const {
return prefix + ConvertCase(EscapeKeyword(d.name), Case::kUpperCamel) +
suffix;
}
std::string LegacyKotlinMethod(const std::string &prefix, const StructDef &d,
const std::string &suffix) const {
return prefix + ConvertCase(EscapeKeyword(d.name), Case::kUpperCamel) +
suffix;
}
// This is a mix of snake case and keep casing, when Ts should be using
// lower camel case.
std::string LegacyTsMutateMethod(const FieldDef& d) {
return "mutate_" + d.name;
}
std::string LegacyRustUnionTypeMethod(const FieldDef &d) {
// assert d is a union
// d should convert case but not escape keywords due to historical reasons
return ConvertCase(d.name, config_.fields, Case::kLowerCamel) + "_type";
}
private:
std::string NamespacedString(const struct Namespace *ns,
const std::string &str) const {
std::string ret;
if (ns != nullptr) { ret += Namespace(ns->components); }
if (!ret.empty()) ret += config_.namespace_seperator;
return ret + str;
}
};
// This is a temporary helper function for code generators to call until all
// flag-overriding logic into flatc.cpp
inline Namer::Config WithFlagOptions(const Namer::Config &input,
const IDLOptions &opts,
const std::string &path) {
Namer::Config result = input;
result.object_prefix = opts.object_prefix;
result.object_suffix = opts.object_suffix;
result.output_path = path;
result.filename_suffix = opts.filename_suffix;
return result;
}
} // namespace flatbuffers
#endif // FLATBUFFERS_INCLUDE_CODEGEN_IDL_NAMER_H_

270
include/codegen/namer.h Normal file
View File

@@ -0,0 +1,270 @@
#ifndef FLATBUFFERS_INCLUDE_CODEGEN_NAMER_H_
#define FLATBUFFERS_INCLUDE_CODEGEN_NAMER_H_
#include "flatbuffers/util.h"
namespace flatbuffers {
// Options for Namer::File.
enum class SkipFile {
None = 0,
Suffix = 1,
Extension = 2,
SuffixAndExtension = 3,
};
inline SkipFile operator&(SkipFile a, SkipFile b) {
return static_cast<SkipFile>(static_cast<int>(a) & static_cast<int>(b));
}
// Options for Namer::Directories
enum class SkipDir {
None = 0,
// Skip prefixing the -o $output_path.
OutputPath = 1,
// Skip trailing path seperator.
TrailingPathSeperator = 2,
OutputPathAndTrailingPathSeparator = 3,
};
inline SkipDir operator&(SkipDir a, SkipDir b) {
return static_cast<SkipDir>(static_cast<int>(a) & static_cast<int>(b));
}
// `Namer` applies style configuration to symbols in generated code. It manages
// casing, escapes keywords, and object API naming.
// TODO: Refactor all code generators to use this.
class Namer {
public:
struct Config {
// Symbols in code.
// Case style for flatbuffers-defined types.
// e.g. `class TableA {}`
Case types;
// Case style for flatbuffers-defined constants.
// e.g. `uint64_t ENUM_A_MAX`;
Case constants;
// Case style for flatbuffers-defined methods.
// e.g. `class TableA { int field_a(); }`
Case methods;
// Case style for flatbuffers-defined functions.
// e.g. `TableA* get_table_a_root()`;
Case functions;
// Case style for flatbuffers-defined fields.
// e.g. `struct Struct { int my_field; }`
Case fields;
// Case style for flatbuffers-defined variables.
// e.g. `int my_variable = 2`
Case variables;
// Case style for flatbuffers-defined variants.
// e.g. `enum class Enum { MyVariant, }`
Case variants;
// Seperator for qualified enum names.
// e.g. `Enum::MyVariant` uses `::`.
std::string enum_variant_seperator;
// Configures, when formatting code, whether symbols are checked against
// keywords and escaped before or after case conversion. It does not make
// sense to do so before, but its legacy behavior. :shrug:
// TODO(caspern): Deprecate.
enum class Escape {
BeforeConvertingCase,
AfterConvertingCase,
};
Escape escape_keywords;
// Namespaces
// e.g. `namespace my_namespace {}`
Case namespaces;
// The seperator between namespaces in a namespace path.
std::string namespace_seperator;
// Object API.
// Native versions flatbuffers types have this prefix.
// e.g. "" (it's usually empty string)
std::string object_prefix;
// Native versions flatbuffers types have this suffix.
// e.g. "T"
std::string object_suffix;
// Keywords.
// Prefix used to escape keywords. It is usually empty string.
std::string keyword_prefix;
// Suffix used to escape keywords. It is usually "_".
std::string keyword_suffix;
// Files.
// Case style for filenames. e.g. `foo_bar_generated.rs`
Case filenames;
// Case style for directories, e.g. `output_files/foo_bar/baz/`
Case directories;
// The directory within which we will generate files.
std::string output_path;
// Suffix for generated file names, e.g. "_generated".
std::string filename_suffix;
// Extension for generated files, e.g. ".cpp" or ".rs".
std::string filename_extension;
};
Namer(Config config, std::set<std::string> keywords)
: config_(config), keywords_(std::move(keywords)) {}
virtual ~Namer() {}
template<typename T> std::string Method(const T &s) const {
return Method(s.name);
}
virtual std::string Method(const std::string &pre,
const std::string &mid,
const std::string &suf) const {
return Format(pre + "_" + mid + "_" + suf, config_.methods);
}
virtual std::string Method(const std::string &pre,
const std::string &suf) const {
return Format(pre + "_" + suf, config_.methods);
}
virtual std::string Method(const std::string &s) const {
return Format(s, config_.methods);
}
virtual std::string Constant(const std::string &s) const {
return Format(s, config_.constants);
}
virtual std::string Function(const std::string &s) const {
return Format(s, config_.functions);
}
virtual std::string Variable(const std::string &s) const {
return Format(s, config_.variables);
}
template<typename T>
std::string Variable(const std::string &p, const T &s) const {
return Format(p + "_" + s.name, config_.variables);
}
virtual std::string Variable(const std::string &p,
const std::string &s) const {
return Format(p + "_" + s, config_.variables);
}
virtual std::string Namespace(const std::string &s) const {
return Format(s, config_.namespaces);
}
virtual std::string Namespace(const std::vector<std::string> &ns) const {
std::string result;
for (auto it = ns.begin(); it != ns.end(); it++) {
if (it != ns.begin()) result += config_.namespace_seperator;
result += Namespace(*it);
}
return result;
}
virtual std::string NamespacedType(const std::vector<std::string> &ns,
const std::string &s) const {
return (ns.empty() ? "" : (Namespace(ns) + config_.namespace_seperator)) +
Type(s);
}
// Returns `filename` with the right casing, suffix, and extension.
virtual std::string File(const std::string &filename,
SkipFile skips = SkipFile::None) const {
const bool skip_suffix = (skips & SkipFile::Suffix) != SkipFile::None;
const bool skip_ext = (skips & SkipFile::Extension) != SkipFile::None;
return ConvertCase(filename, config_.filenames, Case::kUpperCamel) +
(skip_suffix ? "" : config_.filename_suffix) +
(skip_ext ? "" : config_.filename_extension);
}
template<typename T>
std::string File(const T &f, SkipFile skips = SkipFile::None) const {
return File(f.name, skips);
}
// Formats `directories` prefixed with the output_path and joined with the
// right seperator. Output path prefixing and the trailing separator may be
// skiped using `skips`.
// Callers may want to use `EnsureDirExists` with the result.
// input_case is used to tell how to modify namespace. e.g. kUpperCamel will
// add a underscode between case changes, so MyGame turns into My_Game
// (depending also on the output_case).
virtual std::string Directories(const std::vector<std::string> &directories,
SkipDir skips = SkipDir::None,
Case input_case = Case::kUpperCamel) const {
const bool skip_output_path =
(skips & SkipDir::OutputPath) != SkipDir::None;
const bool skip_trailing_seperator =
(skips & SkipDir::TrailingPathSeperator) != SkipDir::None;
std::string result = skip_output_path ? "" : config_.output_path;
for (auto d = directories.begin(); d != directories.end(); d++) {
result += ConvertCase(*d, config_.directories, input_case);
result.push_back(kPathSeparator);
}
if (skip_trailing_seperator && !result.empty()) result.pop_back();
return result;
}
virtual std::string EscapeKeyword(const std::string &name) const {
if (keywords_.find(name) == keywords_.end()) {
return name;
} else {
return config_.keyword_prefix + name + config_.keyword_suffix;
}
}
virtual std::string Type(const std::string &s) const {
return Format(s, config_.types);
}
virtual std::string Type(const std::string &t, const std::string &s) const {
return Format(t + "_" + s, config_.types);
}
virtual std::string ObjectType(const std::string &s) const {
return config_.object_prefix + Type(s) + config_.object_suffix;
}
virtual std::string Field(const std::string &s) const {
return Format(s, config_.fields);
}
virtual std::string Variant(const std::string &s) const {
return Format(s, config_.variants);
}
virtual std::string Format(const std::string &s, Case casing) const {
if (config_.escape_keywords == Config::Escape::BeforeConvertingCase) {
return ConvertCase(EscapeKeyword(s), casing, Case::kLowerCamel);
} else {
return EscapeKeyword(ConvertCase(s, casing, Case::kLowerCamel));
}
}
// Denamespaces a string (e.g. The.Quick.Brown.Fox) by returning the last part
// after the `delimiter` (Fox) and placing the rest in `namespace_prefix`
// (The.Quick.Brown).
virtual std::string Denamespace(const std::string &s,
std::string &namespace_prefix,
const char delimiter = '.') const {
const size_t pos = s.find_last_of(delimiter);
if (pos == std::string::npos) {
namespace_prefix = "";
return s;
}
namespace_prefix = s.substr(0, pos);
return s.substr(pos + 1);
}
// Same as above, but disregards the prefix.
virtual std::string Denamespace(const std::string &s,
const char delimiter = '.') const {
std::string prefix;
return Denamespace(s, prefix, delimiter);
}
const Config config_;
const std::set<std::string> keywords_;
};
} // namespace flatbuffers
#endif // FLATBUFFERS_INCLUDE_CODEGEN_NAMER_H_

48
include/codegen/python.h
View File

@@ -6,8 +6,56 @@
#include <string>
#include <vector>
#include "codegen/namer.h"
namespace flatbuffers {
namespace python {
static const Namer::Config kConfig = {
/*types=*/Case::kKeep,
/*constants=*/Case::kScreamingSnake,
/*methods=*/Case::kUpperCamel,
/*functions=*/Case::kUpperCamel,
/*fields=*/Case::kLowerCamel,
/*variable=*/Case::kLowerCamel,
/*variants=*/Case::kKeep,
/*enum_variant_seperator=*/".",
/*escape_keywords=*/Namer::Config::Escape::AfterConvertingCase,
/*namespaces=*/Case::kKeep, // Packages in python.
/*namespace_seperator=*/".",
/*object_prefix=*/"",
/*object_suffix=*/"T",
/*keyword_prefix=*/"",
/*keyword_suffix=*/"_",
/*filenames=*/Case::kKeep,
/*directories=*/Case::kKeep,
/*output_path=*/"",
/*filename_suffix=*/"",
/*filename_extension=*/".py",
};
static const Namer::Config kStubConfig = {
/*types=*/Case::kKeep,
/*constants=*/Case::kScreamingSnake,
/*methods=*/Case::kUpperCamel,
/*functions=*/Case::kUpperCamel,
/*fields=*/Case::kLowerCamel,
/*variables=*/Case::kLowerCamel,
/*variants=*/Case::kKeep,
/*enum_variant_seperator=*/".",
/*escape_keywords=*/Namer::Config::Escape::AfterConvertingCase,
/*namespaces=*/Case::kKeep, // Packages in python.
/*namespace_seperator=*/".",
/*object_prefix=*/"",
/*object_suffix=*/"T",
/*keyword_prefix=*/"",
/*keyword_suffix=*/"_",
/*filenames=*/Case::kKeep,
/*directories=*/Case::kKeep,
/*output_path=*/"",
/*filename_suffix=*/"",
/*filename_extension=*/".pyi",
};
// `Version` represent a Python version.
//
// The zero value (i.e. `Version{}`) represents both Python2 and Python3.

6
include/flatbuffers/idl.h
View File

@@ -784,6 +784,9 @@ struct IDLOptions {
std::string grpc_search_path;
std::vector<std::string> grpc_additional_headers;
/******************************* Python gRPC ********************************/
bool grpc_python_typed_handlers;
IDLOptions()
: gen_jvmstatic(false),
use_flexbuffers(false),
@@ -857,7 +860,8 @@ struct IDLOptions {
set_empty_strings_to_null(true),
set_empty_vectors_to_null(true),
grpc_filename_suffix(".fb"),
grpc_use_system_headers(true) {}
grpc_use_system_headers(true),
grpc_python_typed_handlers(false) {}
};
// This encapsulates where the parser is in the current source file.

2
src/BUILD.bazel
View File

@@ -89,6 +89,7 @@ cc_library(
visibility = ["//:__pkg__"],
deps = [
":flatbuffers",
"//include/codegen:namer",
],
)
@@ -155,6 +156,7 @@ cc_library(
"//grpc/src/compiler:python_generator",
"//grpc/src/compiler:swift_generator",
"//grpc/src/compiler:ts_generator",
"//include/codegen:namer",
"//include/codegen:python",
],
)

8
src/flatc.cpp
View File

@@ -268,6 +268,8 @@ const static FlatCOption flatc_options[] = {
{ "", "grpc-use-system-headers", "",
"Use <> for headers included from the generated code." },
{ "", "grpc-search-path", "PATH", "Prefix to any gRPC includes." },
{ "", "grpc-python-typed-handlers", "",
"The handlers will use the generated classes rather than raw bytes." },
};
auto cmp = [](FlatCOption a, FlatCOption b) { return a.long_opt < b.long_opt; };
@@ -720,6 +722,12 @@ FlatCOptions FlatCompiler::ParseFromCommandLineArguments(int argc,
} else if (arg == "--no-grpc-use-system-headers" ||
arg == "--grpc-use-system-headers=false") {
opts.grpc_use_system_headers = false;
} else if (arg == "--grpc-python-typed-handlers" ||
arg == "--grpc-python-typed-handlers=true") {
opts.grpc_python_typed_handlers = true;
} else if (arg == "--no-grpc-python-typed-handlers" ||
arg == "--grpc-python-typed-handlers=false") {
opts.grpc_python_typed_handlers = false;
} else {
if (arg == "--proto") { opts.proto_mode = true; }

54
src/idl_gen_grpc.cpp
View File

@@ -435,47 +435,8 @@ bool GenerateJavaGRPC(const Parser &parser, const std::string &path,
return JavaGRPCGenerator(parser, path, file_name).generate();
}
class PythonGRPCGenerator : public flatbuffers::BaseGenerator {
private:
CodeWriter code_;
public:
PythonGRPCGenerator(const Parser &parser, const std::string &filename)
: BaseGenerator(parser, "", filename, "", "" /*Unused*/, "swift") {}
bool generate() {
code_.Clear();
code_ +=
"# Generated by the gRPC Python protocol compiler plugin. "
"DO NOT EDIT!\n";
code_ += "import grpc\n";
FlatBufFile file(parser_, file_name_, FlatBufFile::kLanguagePython);
for (int i = 0; i < file.service_count(); i++) {
auto service = file.service(i);
code_ += grpc_python_generator::Generate(&file, service.get());
}
const auto final_code = code_.ToString();
const auto filename = GenerateFileName();
return SaveFile(filename.c_str(), final_code, false);
}
std::string GenerateFileName() {
std::string namespace_dir;
auto &namespaces = parser_.namespaces_.back()->components;
for (auto it = namespaces.begin(); it != namespaces.end(); ++it) {
if (it != namespaces.begin()) namespace_dir += kPathSeparator;
namespace_dir += *it;
}
std::string grpc_py_filename = namespace_dir;
if (!namespace_dir.empty()) grpc_py_filename += kPathSeparator;
return grpc_py_filename + file_name_ + "_grpc_fb.py";
}
};
bool GeneratePythonGRPC(const Parser &parser, const std::string & /*path*/,
const std::string &file_name) {
bool GeneratePythonGRPC(const Parser &parser, const std::string &path,
const std::string & /*file_name*/) {
int nservices = 0;
for (auto it = parser.services_.vec.begin(); it != parser.services_.vec.end();
++it) {
@@ -483,7 +444,16 @@ bool GeneratePythonGRPC(const Parser &parser, const std::string & /*path*/,
}
if (!nservices) return true;
return PythonGRPCGenerator(parser, file_name).generate();
flatbuffers::python::Version version{parser.opts.python_version};
if (!version.IsValid()) return false;
if (!flatbuffers::python::grpc::Generate(parser, path, version)) {
return false;
}
if (parser.opts.python_typing) {
return flatbuffers::python::grpc::GenerateStub(parser, path, version);
}
return true;
}
class SwiftGRPCGenerator : public flatbuffers::BaseGenerator {

50
src/idl_gen_python.cpp
View File

@@ -29,12 +29,12 @@
#include <utility>
#include <vector>
#include "codegen/idl_namer.h"
#include "codegen/python.h"
#include "flatbuffers/code_generators.h"
#include "flatbuffers/flatbuffers.h"
#include "flatbuffers/idl.h"
#include "flatbuffers/util.h"
#include "idl_namer.h"
namespace flatbuffers {
namespace python {
@@ -44,52 +44,6 @@ namespace {
typedef std::pair<std::string, std::string> ImportMapEntry;
typedef std::set<ImportMapEntry> ImportMap;
static Namer::Config PythonDefaultConfig() {
return { /*types=*/Case::kKeep,
/*constants=*/Case::kScreamingSnake,
/*methods=*/Case::kUpperCamel,
/*functions=*/Case::kUpperCamel,
/*fields=*/Case::kLowerCamel,
/*variable=*/Case::kLowerCamel,
/*variants=*/Case::kKeep,
/*enum_variant_seperator=*/".",
/*escape_keywords=*/Namer::Config::Escape::AfterConvertingCase,
/*namespaces=*/Case::kKeep, // Packages in python.
/*namespace_seperator=*/".",
/*object_prefix=*/"",
/*object_suffix=*/"T",
/*keyword_prefix=*/"",
/*keyword_suffix=*/"_",
/*filenames=*/Case::kKeep,
/*directories=*/Case::kKeep,
/*output_path=*/"",
/*filename_suffix=*/"",
/*filename_extension=*/".py" };
}
static Namer::Config kStubConfig = {
/*types=*/Case::kKeep,
/*constants=*/Case::kScreamingSnake,
/*methods=*/Case::kUpperCamel,
/*functions=*/Case::kUpperCamel,
/*fields=*/Case::kLowerCamel,
/*variables=*/Case::kLowerCamel,
/*variants=*/Case::kKeep,
/*enum_variant_seperator=*/".",
/*escape_keywords=*/Namer::Config::Escape::AfterConvertingCase,
/*namespaces=*/Case::kKeep, // Packages in python.
/*namespace_seperator=*/".",
/*object_prefix=*/"",
/*object_suffix=*/"T",
/*keyword_prefix=*/"",
/*keyword_suffix=*/"_",
/*filenames=*/Case::kKeep,
/*directories=*/Case::kKeep,
/*output_path=*/"",
/*filename_suffix=*/"",
/*filename_extension=*/".pyi",
};
// Hardcode spaces per indentation.
static const CommentConfig def_comment = { nullptr, "#", nullptr };
static const std::string Indent = " ";
@@ -662,7 +616,7 @@ class PythonGenerator : public BaseGenerator {
: BaseGenerator(parser, path, file_name, "" /* not used */,
"" /* not used */, "py"),
float_const_gen_("float('nan')", "float('inf')", "float('-inf')"),
namer_(WithFlagOptions(PythonDefaultConfig(), parser.opts, path),
namer_(WithFlagOptions(kConfig, parser.opts, path),
Keywords(version)) {}
// Most field accessors need to retrieve and test the field offset first,

181
src/idl_namer.h
View File

@@ -1,179 +1,6 @@
#ifndef FLATBUFFERS_IDL_NAMER
#define FLATBUFFERS_IDL_NAMER
#ifndef FLATBUFFERS_IDL_NAMER_H_
#define FLATBUFFERS_IDL_NAMER_H_
#include "flatbuffers/idl.h"
#include "namer.h"
#include "codegen/idl_namer.h"
namespace flatbuffers {
// Provides Namer capabilities to types defined in the flatbuffers IDL.
class IdlNamer : public Namer {
public:
explicit IdlNamer(Config config, std::set<std::string> keywords)
: Namer(config, std::move(keywords)) {}
using Namer::Constant;
using Namer::Directories;
using Namer::Field;
using Namer::File;
using Namer::Function;
using Namer::Method;
using Namer::Namespace;
using Namer::NamespacedType;
using Namer::ObjectType;
using Namer::Type;
using Namer::Variable;
using Namer::Variant;
std::string Constant(const FieldDef &d) const { return Constant(d.name); }
// Types are always structs or enums so we can only expose these two
// overloads.
std::string Type(const StructDef &d) const { return Type(d.name); }
std::string Type(const EnumDef &d) const { return Type(d.name); }
std::string Function(const Definition &s) const { return Function(s.name); }
std::string Function(const std::string& prefix, const Definition &s) const {
return Function(prefix + s.name);
}
std::string Field(const FieldDef &s) const { return Field(s.name); }
std::string Field(const FieldDef &d, const std::string &s) const {
return Field(d.name + "_" + s);
}
std::string Variable(const FieldDef &s) const { return Variable(s.name); }
std::string Variable(const StructDef &s) const { return Variable(s.name); }
std::string Variant(const EnumVal &s) const { return Variant(s.name); }
std::string EnumVariant(const EnumDef &e, const EnumVal &v) const {
return Type(e) + config_.enum_variant_seperator + Variant(v);
}
std::string ObjectType(const StructDef &d) const {
return ObjectType(d.name);
}
std::string ObjectType(const EnumDef &d) const { return ObjectType(d.name); }
std::string Method(const FieldDef &d, const std::string &suffix) const {
return Method(d.name, suffix);
}
std::string Method(const std::string &prefix, const StructDef &d) const {
return Method(prefix, d.name);
}
std::string Method(const std::string &prefix, const FieldDef &d) const {
return Method(prefix, d.name);
}
std::string Method(const std::string &prefix, const FieldDef &d,
const std::string &suffix) const {
return Method(prefix, d.name, suffix);
}
std::string Namespace(const struct Namespace &ns) const {
return Namespace(ns.components);
}
std::string NamespacedEnumVariant(const EnumDef &e, const EnumVal &v) const {
return NamespacedString(e.defined_namespace, EnumVariant(e, v));
}
std::string NamespacedType(const Definition &def) const {
return NamespacedString(def.defined_namespace, Type(def.name));
}
std::string NamespacedObjectType(const Definition &def) const {
return NamespacedString(def.defined_namespace, ObjectType(def.name));
}
std::string Directories(const struct Namespace &ns,
SkipDir skips = SkipDir::None,
Case input_case = Case::kUpperCamel) const {
return Directories(ns.components, skips, input_case);
}
// Legacy fields do not really follow the usual config and should be
// considered for deprecation.
std::string LegacyRustNativeVariant(const EnumVal &v) const {
return ConvertCase(EscapeKeyword(v.name), Case::kUpperCamel);
}
std::string LegacyRustFieldOffsetName(const FieldDef &field) const {
return "VT_" + ConvertCase(EscapeKeyword(field.name), Case::kAllUpper);
}
std::string LegacyRustUnionTypeOffsetName(const FieldDef &field) const {
return "VT_" + ConvertCase(EscapeKeyword(field.name + "_type"), Case::kAllUpper);
}
std::string LegacySwiftVariant(const EnumVal &ev) const {
auto name = ev.name;
if (isupper(name.front())) {
std::transform(name.begin(), name.end(), name.begin(), CharToLower);
}
return EscapeKeyword(ConvertCase(name, Case::kLowerCamel));
}
// Also used by Kotlin, lol.
std::string LegacyJavaMethod2(const std::string &prefix, const StructDef &sd,
const std::string &suffix) const {
return prefix + sd.name + suffix;
}
std::string LegacyKotlinVariant(EnumVal &ev) const {
// Namer assumes the input case is snake case which is wrong...
return ConvertCase(EscapeKeyword(ev.name), Case::kLowerCamel);
}
// Kotlin methods escapes keywords after case conversion but before
// prefixing and suffixing.
std::string LegacyKotlinMethod(const std::string &prefix, const FieldDef &d,
const std::string &suffix) const {
return prefix + ConvertCase(EscapeKeyword(d.name), Case::kUpperCamel) +
suffix;
}
std::string LegacyKotlinMethod(const std::string &prefix, const StructDef &d,
const std::string &suffix) const {
return prefix + ConvertCase(EscapeKeyword(d.name), Case::kUpperCamel) +
suffix;
}
// This is a mix of snake case and keep casing, when Ts should be using
// lower camel case.
std::string LegacyTsMutateMethod(const FieldDef& d) {
return "mutate_" + d.name;
}
std::string LegacyRustUnionTypeMethod(const FieldDef &d) {
// assert d is a union
// d should convert case but not escape keywords due to historical reasons
return ConvertCase(d.name, config_.fields, Case::kLowerCamel) + "_type";
}
private:
std::string NamespacedString(const struct Namespace *ns,
const std::string &str) const {
std::string ret;
if (ns != nullptr) { ret += Namespace(ns->components); }
if (!ret.empty()) ret += config_.namespace_seperator;
return ret + str;
}
};
// This is a temporary helper function for code generators to call until all
// flag-overriding logic into flatc.cpp
inline Namer::Config WithFlagOptions(const Namer::Config &input,
const IDLOptions &opts,
const std::string &path) {
Namer::Config result = input;
result.object_prefix = opts.object_prefix;
result.object_suffix = opts.object_suffix;
result.output_path = path;
result.filename_suffix = opts.filename_suffix;
return result;
}
} // namespace flatbuffers
#endif // FLATBUFFERS_IDL_NAMER
#endif // FLATBUFFERS_IDL_NAMER_H_

272
src/namer.h
View File

@@ -1,270 +1,6 @@
#ifndef FLATBUFFERS_NAMER
#define FLATBUFFERS_NAMER
#ifndef FLATBUFFERS_NAMER_H_
#define FLATBUFFERS_NAMER_H_
#include "flatbuffers/util.h"
#include "codegen/namer.h"
namespace flatbuffers {
// Options for Namer::File.
enum class SkipFile {
None = 0,
Suffix = 1,
Extension = 2,
SuffixAndExtension = 3,
};
inline SkipFile operator&(SkipFile a, SkipFile b) {
return static_cast<SkipFile>(static_cast<int>(a) & static_cast<int>(b));
}
// Options for Namer::Directories
enum class SkipDir {
None = 0,
// Skip prefixing the -o $output_path.
OutputPath = 1,
// Skip trailing path seperator.
TrailingPathSeperator = 2,
OutputPathAndTrailingPathSeparator = 3,
};
inline SkipDir operator&(SkipDir a, SkipDir b) {
return static_cast<SkipDir>(static_cast<int>(a) & static_cast<int>(b));
}
// `Namer` applies style configuration to symbols in generated code. It manages
// casing, escapes keywords, and object API naming.
// TODO: Refactor all code generators to use this.
class Namer {
public:
struct Config {
// Symbols in code.
// Case style for flatbuffers-defined types.
// e.g. `class TableA {}`
Case types;
// Case style for flatbuffers-defined constants.
// e.g. `uint64_t ENUM_A_MAX`;
Case constants;
// Case style for flatbuffers-defined methods.
// e.g. `class TableA { int field_a(); }`
Case methods;
// Case style for flatbuffers-defined functions.
// e.g. `TableA* get_table_a_root()`;
Case functions;
// Case style for flatbuffers-defined fields.
// e.g. `struct Struct { int my_field; }`
Case fields;
// Case style for flatbuffers-defined variables.
// e.g. `int my_variable = 2`
Case variables;
// Case style for flatbuffers-defined variants.
// e.g. `enum class Enum { MyVariant, }`
Case variants;
// Seperator for qualified enum names.
// e.g. `Enum::MyVariant` uses `::`.
std::string enum_variant_seperator;
// Configures, when formatting code, whether symbols are checked against
// keywords and escaped before or after case conversion. It does not make
// sense to do so before, but its legacy behavior. :shrug:
// TODO(caspern): Deprecate.
enum class Escape {
BeforeConvertingCase,
AfterConvertingCase,
};
Escape escape_keywords;
// Namespaces
// e.g. `namespace my_namespace {}`
Case namespaces;
// The seperator between namespaces in a namespace path.
std::string namespace_seperator;
// Object API.
// Native versions flatbuffers types have this prefix.
// e.g. "" (it's usually empty string)
std::string object_prefix;
// Native versions flatbuffers types have this suffix.
// e.g. "T"
std::string object_suffix;
// Keywords.
// Prefix used to escape keywords. It is usually empty string.
std::string keyword_prefix;
// Suffix used to escape keywords. It is usually "_".
std::string keyword_suffix;
// Files.
// Case style for filenames. e.g. `foo_bar_generated.rs`
Case filenames;
// Case style for directories, e.g. `output_files/foo_bar/baz/`
Case directories;
// The directory within which we will generate files.
std::string output_path;
// Suffix for generated file names, e.g. "_generated".
std::string filename_suffix;
// Extension for generated files, e.g. ".cpp" or ".rs".
std::string filename_extension;
};
Namer(Config config, std::set<std::string> keywords)
: config_(config), keywords_(std::move(keywords)) {}
virtual ~Namer() {}
template<typename T> std::string Method(const T &s) const {
return Method(s.name);
}
virtual std::string Method(const std::string &pre,
const std::string &mid,
const std::string &suf) const {
return Format(pre + "_" + mid + "_" + suf, config_.methods);
}
virtual std::string Method(const std::string &pre,
const std::string &suf) const {
return Format(pre + "_" + suf, config_.methods);
}
virtual std::string Method(const std::string &s) const {
return Format(s, config_.methods);
}
virtual std::string Constant(const std::string &s) const {
return Format(s, config_.constants);
}
virtual std::string Function(const std::string &s) const {
return Format(s, config_.functions);
}
virtual std::string Variable(const std::string &s) const {
return Format(s, config_.variables);
}
template<typename T>
std::string Variable(const std::string &p, const T &s) const {
return Format(p + "_" + s.name, config_.variables);
}
virtual std::string Variable(const std::string &p,
const std::string &s) const {
return Format(p + "_" + s, config_.variables);
}
virtual std::string Namespace(const std::string &s) const {
return Format(s, config_.namespaces);
}
virtual std::string Namespace(const std::vector<std::string> &ns) const {
std::string result;
for (auto it = ns.begin(); it != ns.end(); it++) {
if (it != ns.begin()) result += config_.namespace_seperator;
result += Namespace(*it);
}
return result;
}
virtual std::string NamespacedType(const std::vector<std::string> &ns,
const std::string &s) const {
return (ns.empty() ? "" : (Namespace(ns) + config_.namespace_seperator)) +
Type(s);
}
// Returns `filename` with the right casing, suffix, and extension.
virtual std::string File(const std::string &filename,
SkipFile skips = SkipFile::None) const {
const bool skip_suffix = (skips & SkipFile::Suffix) != SkipFile::None;
const bool skip_ext = (skips & SkipFile::Extension) != SkipFile::None;
return ConvertCase(filename, config_.filenames, Case::kUpperCamel) +
(skip_suffix ? "" : config_.filename_suffix) +
(skip_ext ? "" : config_.filename_extension);
}
template<typename T>
std::string File(const T &f, SkipFile skips = SkipFile::None) const {
return File(f.name, skips);
}
// Formats `directories` prefixed with the output_path and joined with the
// right seperator. Output path prefixing and the trailing separator may be
// skiped using `skips`.
// Callers may want to use `EnsureDirExists` with the result.
// input_case is used to tell how to modify namespace. e.g. kUpperCamel will
// add a underscode between case changes, so MyGame turns into My_Game
// (depending also on the output_case).
virtual std::string Directories(const std::vector<std::string> &directories,
SkipDir skips = SkipDir::None,
Case input_case = Case::kUpperCamel) const {
const bool skip_output_path =
(skips & SkipDir::OutputPath) != SkipDir::None;
const bool skip_trailing_seperator =
(skips & SkipDir::TrailingPathSeperator) != SkipDir::None;
std::string result = skip_output_path ? "" : config_.output_path;
for (auto d = directories.begin(); d != directories.end(); d++) {
result += ConvertCase(*d, config_.directories, input_case);
result.push_back(kPathSeparator);
}
if (skip_trailing_seperator && !result.empty()) result.pop_back();
return result;
}
virtual std::string EscapeKeyword(const std::string &name) const {
if (keywords_.find(name) == keywords_.end()) {
return name;
} else {
return config_.keyword_prefix + name + config_.keyword_suffix;
}
}
virtual std::string Type(const std::string &s) const {
return Format(s, config_.types);
}
virtual std::string Type(const std::string &t, const std::string &s) const {
return Format(t + "_" + s, config_.types);
}
virtual std::string ObjectType(const std::string &s) const {
return config_.object_prefix + Type(s) + config_.object_suffix;
}
virtual std::string Field(const std::string &s) const {
return Format(s, config_.fields);
}
virtual std::string Variant(const std::string &s) const {
return Format(s, config_.variants);
}
virtual std::string Format(const std::string &s, Case casing) const {
if (config_.escape_keywords == Config::Escape::BeforeConvertingCase) {
return ConvertCase(EscapeKeyword(s), casing, Case::kLowerCamel);
} else {
return EscapeKeyword(ConvertCase(s, casing, Case::kLowerCamel));
}
}
// Denamespaces a string (e.g. The.Quick.Brown.Fox) by returning the last part
// after the `delimiter` (Fox) and placing the rest in `namespace_prefix`
// (The.Quick.Brown).
virtual std::string Denamespace(const std::string &s,
std::string &namespace_prefix,
const char delimiter = '.') const {
const size_t pos = s.find_last_of(delimiter);
if (pos == std::string::npos) {
namespace_prefix = "";
return s;
}
namespace_prefix = s.substr(0, pos);
return s.substr(pos + 1);
}
// Same as above, but disregards the prefix.
virtual std::string Denamespace(const std::string &s,
const char delimiter = '.') const {
std::string prefix;
return Denamespace(s, prefix, delimiter);
}
const Config config_;
const std::set<std::string> keywords_;
};
} // namespace flatbuffers
#endif // FLATBUFFERS_NAMER
#endif // FLATBUFFERS_NAMER_H_

1
tests/PythonTest.sh
View File

@@ -27,6 +27,7 @@ ${test_dir}/../flatc -p -o ${gen_code_path} -I include_test monster_test.fbs --g
${test_dir}/../flatc -p -o ${gen_code_path} -I include_test monster_extra.fbs --gen-object-api --python-typing --gen-compare
${test_dir}/../flatc -p -o ${gen_code_path} -I include_test arrays_test.fbs --gen-object-api --python-typing
${test_dir}/../flatc -p -o ${gen_code_path} -I include_test nested_union_test.fbs --gen-object-api --python-typing
${test_dir}/../flatc -p -o ${gen_code_path} -I include_test service_test.fbs --grpc --grpc-python-typed-handlers --python-typing --no-python-gen-numpy --gen-onefile
# Syntax: run_tests <interpreter> <benchmark vtable dedupes>
# <benchmark read count> <benchmark build count>

11
tests/service_test.fbs Normal file
View File

@@ -0,0 +1,11 @@
namespace example;
table HelloRequest {}
table HelloResponse {}
rpc_service HelloService {
Hello(HelloRequest):HelloResponse;
StreamClient(HelloRequest):HelloResponse (streaming: "client");
StreamServer(HelloRequest):HelloResponse (streaming: "server");
Stream(HelloRequest):HelloResponse (streaming: "bidi");
}

58
tests/service_test_generated.py Normal file
View File

@@ -0,0 +1,58 @@
# automatically generated by the FlatBuffers compiler, do not modify
# namespace: example
import flatbuffers
from typing import Any
class HelloRequest(object):
__slots__ = ['_tab']
@classmethod
def GetRootAs(cls, buf, offset: int = 0):
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
x = HelloRequest()
x.Init(buf, n + offset)
return x
@classmethod
def GetRootAsHelloRequest(cls, buf, offset=0):
"""This method is deprecated. Please switch to GetRootAs."""
return cls.GetRootAs(buf, offset)
# HelloRequest
def Init(self, buf: bytes, pos: int):
self._tab = flatbuffers.table.Table(buf, pos)
def HelloRequestStart(builder: flatbuffers.Builder):
builder.StartObject(0)
def HelloRequestEnd(builder: flatbuffers.Builder) -> int:
return builder.EndObject()
class HelloResponse(object):
__slots__ = ['_tab']
@classmethod
def GetRootAs(cls, buf, offset: int = 0):
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
x = HelloResponse()
x.Init(buf, n + offset)
return x
@classmethod
def GetRootAsHelloResponse(cls, buf, offset=0):
"""This method is deprecated. Please switch to GetRootAs."""
return cls.GetRootAs(buf, offset)
# HelloResponse
def Init(self, buf: bytes, pos: int):
self._tab = flatbuffers.table.Table(buf, pos)
def HelloResponseStart(builder: flatbuffers.Builder):
builder.StartObject(0)
def HelloResponseEnd(builder: flatbuffers.Builder) -> int:
return builder.EndObject()

26
tests/service_test_generated.pyi Normal file
View File

@@ -0,0 +1,26 @@
from __future__ import annotations
import flatbuffers
import flatbuffers
import typing
uoffset: typing.TypeAlias = flatbuffers.number_types.UOffsetTFlags.py_type
class HelloRequest(object):
@classmethod
def GetRootAs(cls, buf: bytes, offset: int) -> HelloRequest: ...
@classmethod
def GetRootAsHelloRequest(cls, buf: bytes, offset: int) -> HelloRequest: ...
def Init(self, buf: bytes, pos: int) -> None: ...
def HelloRequestStart(builder: flatbuffers.Builder) -> None: ...
def HelloRequestEnd(builder: flatbuffers.Builder) -> uoffset: ...
class HelloResponse(object):
@classmethod
def GetRootAs(cls, buf: bytes, offset: int) -> HelloResponse: ...
@classmethod
def GetRootAsHelloResponse(cls, buf: bytes, offset: int) -> HelloResponse: ...
def Init(self, buf: bytes, pos: int) -> None: ...
def HelloResponseStart(builder: flatbuffers.Builder) -> None: ...
def HelloResponseEnd(builder: flatbuffers.Builder) -> uoffset: ...

97
tests/service_test_grpc.fb.py Normal file
View File

@@ -0,0 +1,97 @@
# Generated by the gRPC FlatBuffers compiler. DO NOT EDIT!
import flatbuffers
import grpc
from service_test_generated import HelloRequest, HelloResponse
def _serialize_to_bytes(table):
buf = table._tab.Bytes
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0)
if table._tab.Pos != n:
raise ValueError('must be a top-level table')
return bytes(buf)
class HelloServiceStub(object):
'''Interface exported by the server.'''
def __init__(self, channel):
'''Constructor.
Args:
channel: A grpc.Channel.
'''
self.Hello = channel.unary_unary(
method='/example.HelloService/Hello',
request_serializer=_serialize_to_bytes,
response_deserializer=HelloResponse.GetRootAs)
self.StreamClient = channel.stream_unary(
method='/example.HelloService/StreamClient',
request_serializer=_serialize_to_bytes,
response_deserializer=HelloResponse.GetRootAs)
self.StreamServer = channel.unary_stream(
method='/example.HelloService/StreamServer',
request_serializer=_serialize_to_bytes,
response_deserializer=HelloResponse.GetRootAs)
self.Stream = channel.stream_stream(
method='/example.HelloService/Stream',
request_serializer=_serialize_to_bytes,
response_deserializer=HelloResponse.GetRootAs)
class HelloServiceServicer(object):
'''Interface exported by the server.'''
def Hello(self, request, context):
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def StreamClient(self, request_iterator, context):
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def StreamServer(self, request, context):
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def Stream(self, request_iterator, context):
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def add_HelloServiceServicer_to_server(servicer, server):
rpc_method_handlers = {
'Hello': grpc.unary_unary_rpc_method_handler(
servicer.Hello,
request_deserializer=HelloRequest.GetRootAs,
response_serializer=_serialize_to_bytes),
'StreamClient': grpc.stream_unary_rpc_method_handler(
servicer.StreamClient,
request_deserializer=HelloRequest.GetRootAs,
response_serializer=_serialize_to_bytes),
'StreamServer': grpc.unary_stream_rpc_method_handler(
servicer.StreamServer,
request_deserializer=HelloRequest.GetRootAs,
response_serializer=_serialize_to_bytes),
'Stream': grpc.stream_stream_rpc_method_handler(
servicer.Stream,
request_deserializer=HelloRequest.GetRootAs,
response_serializer=_serialize_to_bytes),
}
generic_handler = grpc.method_handlers_generic_handler(
'example.HelloService', rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler,))

27
tests/service_test_grpc.fb.pyi Normal file
View File

@@ -0,0 +1,27 @@
# Generated by the gRPC FlatBuffers compiler. DO NOT EDIT!
from __future__ import annotations
import grpc
import typing
from service_test_generated import HelloRequest, HelloResponse
class HelloServiceStub(object):
def __init__(self, channel: grpc.Channel) -> None: ...
def Hello(self, request: HelloRequest) -> HelloResponse: ...
def StreamClient(self, request_iterator: typing.Iterator[HelloRequest]) -> HelloResponse: ...
def StreamServer(self, request: HelloRequest) -> typing.Iterator[HelloResponse]: ...
def Stream(self, request_iterator: typing.Iterator[HelloRequest]) -> typing.Iterator[HelloResponse]: ...
class HelloServiceServicer(object):
def Hello(self, request: HelloRequest, context: grpc.ServicerContext) -> HelloResponse: ...
def StreamClient(self, request_iterator: typing.Iterator[HelloRequest], context: grpc.ServicerContext) -> HelloResponse: ...
def StreamServer(self, request: HelloRequest, context: grpc.ServicerContext) -> typing.Iterator[HelloResponse]: ...
def Stream(self, request_iterator: typing.Iterator[HelloRequest], context: grpc.ServicerContext) -> typing.Iterator[HelloResponse]: ...
def add_HelloServiceServicer_to_server(servicer: HelloServiceServicer, server: grpc.Server) -> None: ...