BigfootDev/flatbuffers
1
0
Fork 1
mirror of https://github.com/google/flatbuffers.git synced 2026-06-03 20:31:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

[TS/JS] Entry point per namespace and reworked 1.x compatible single file build (#7510)

Browse Source 
* [TS/JS] Entry point per namespace

* Fix handling of outputpath and array_test

* Attempt to fix generate_code

* Fix cwd for ts in generate_code

* Attempt to fixup bazel and some docs

* Add --ts-flat-files to bazel build to get bundle

* Move to DEFAULT_FLATC_TS_ARGS

* Attempt to add esbuild

* Attempt to use npm instead

* Remove futile attempt to add esbuild

* Attempt to as bazel esbuild

* Shuffle

* Upgrade bazel deps

* Revert failed attempts to get bazel working

* Ignore flatc tests for now

* Add esbuild dependency

* `package.json` Include esbuild

* `WORKSPACE` Add fetching esbuild binary

* Update WORKSPACE

* Unfreeze Lockfile

* Update WORKSPACE

* Update BUILD.bazel

* Rework to suggest instead of running external bundler

* Add esbuild generation to test script

* Prelim bundle test

* Run test JavaScriptTest from flatbuffers 1.x

* Deps upgrade

* Clang format fix

* Revert bazel changes

* Fix newline

* Generate with type declarations

* Handle "empty" root namespace

* Adjust tests for typescript_keywords.ts

* Separate test procedure for old node resolution module output

* Fix rel path for root level re-exports

* Bazel support for esbuild-based flatc

Unfortunately, we lose typing information because the new esbuild method
of generating single files does not generate type information.

The method used here is a bit hack-ish because it relies on parsing the
console output of flatc to figure out what to do.

* Try to fix bazel build for when node isn't present on host

* Auto formatting fixes

* Fix missing generated code

Co-authored-by: Derek Bailey <derekbailey@google.com>
Co-authored-by: James Kuszmaul <jabukuszmaul+collab@gmail.com>
This commit is contained in:
Björn Harrtell
2023-01-21 21:22:22 +01:00
committed by GitHub
parent 1703662285
commit ef76b5ece4
249 changed files with 11509 additions and 15906 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/flatc.cpp
View File

@@ -230,7 +230,10 @@ const static FlatCOption flatc_options[] = {
"Allow a nested_flatbuffer field to be parsed as a vector of bytes "
"in JSON, which is unsafe unless checked by a verifier afterwards." },
{ "", "ts-flat-files", "",
"Only generated one typescript file per .fbs file." },
"Generate a single typescript file per .fbs file. Implies "
"ts_entry_points." },
{ "", "ts-entry-points", "",
"Generate entry point typescript per namespace. Implies gen-all." },
{ "", "annotate", "SCHEMA",
"Annotate the provided BINARY_FILE with the specified SCHEMA file." },
{ "", "no-leak-private-annotation", "",
@@ -607,7 +610,12 @@ FlatCOptions FlatCompiler::ParseFromCommandLineArguments(int argc,
} else if (arg == "--json-nested-bytes") {
opts.json_nested_legacy_flatbuffers = true;
} else if (arg == "--ts-flat-files") {
opts.ts_flat_file = true;
opts.ts_flat_files = true;
opts.ts_entry_points = true;
opts.generate_all = true;
} else if (arg == "--ts-entry-points") {
opts.ts_entry_points = true;
opts.generate_all = true;
} else if (arg == "--ts-no-import-ext") {
opts.ts_no_import_ext = true;
} else if (arg == "--no-leak-private-annotation") {

272
src/idl_gen_ts.cpp
View File

@@ -17,6 +17,7 @@
#include <algorithm>
#include <cassert>
#include <cmath>
#include <iostream>
#include <unordered_map>
#include <unordered_set>
@@ -39,6 +40,14 @@ struct ImportDefinition {
const Definition *dependency = nullptr;
};
struct NsDefinition {
std::string path;
std::string filepath;
std::string symbolic_name;
const Namespace *ns;
std::map<std::string, const Definition *> definitions;
};
Namer::Config TypeScriptDefaultConfig() {
return { /*types=*/Case::kKeep,
/*constants=*/Case::kUnknown,
@@ -102,33 +111,26 @@ class TsGenerator : public BaseGenerator {
generateEnums();
generateStructs();
generateEntry();
if (!generateBundle()) return false;
return true;
}
bool IncludeNamespace() const {
// When generating a single flat file and all its includes, namespaces are
// important to avoid type name clashes.
return parser_.opts.ts_flat_file && parser_.opts.generate_all;
}
std::string GetTypeName(const EnumDef &def, const bool = false,
const bool force_ns_wrap = false) {
if (IncludeNamespace() || force_ns_wrap) {
return namer_.NamespacedType(def);
}
if (force_ns_wrap) { return namer_.NamespacedType(def); }
return namer_.Type(def);
}
std::string GetTypeName(const StructDef &def, const bool object_api = false,
const bool force_ns_wrap = false) {
if (object_api && parser_.opts.generate_object_based_api) {
if (IncludeNamespace() || force_ns_wrap) {
if (force_ns_wrap) {
return namer_.NamespacedObjectType(def);
} else {
return namer_.ObjectType(def);
}
} else {
if (IncludeNamespace() || force_ns_wrap) {
if (force_ns_wrap) {
return namer_.NamespacedType(def);
} else {
return namer_.Type(def);
@@ -144,58 +146,62 @@ class TsGenerator : public BaseGenerator {
std::string code;
if (!parser_.opts.ts_flat_file) {
code += "// " + std::string(FlatBuffersGeneratedWarning()) + "\n\n";
code += "// " + std::string(FlatBuffersGeneratedWarning()) + "\n\n";
for (auto it = bare_imports.begin(); it != bare_imports.end(); it++) {
for (auto it = bare_imports.begin(); it != bare_imports.end(); it++) {
code += it->second.import_statement + "\n";
}
if (!bare_imports.empty()) code += "\n";
for (auto it = imports.begin(); it != imports.end(); it++) {
if (it->second.dependency != &definition) {
code += it->second.import_statement + "\n";
}
if (!bare_imports.empty()) code += "\n";
for (auto it = imports.begin(); it != imports.end(); it++) {
if (it->second.dependency != &definition) {
code += it->second.import_statement + "\n";
}
}
if (!imports.empty()) code += "\n\n";
}
if (!imports.empty()) code += "\n\n";
code += class_code;
if (parser_.opts.ts_flat_file) {
flat_file_ += code;
flat_file_ += "\n";
flat_file_definitions_.insert(&definition);
return true;
} else {
auto dirs = namer_.Directories(*definition.defined_namespace);
EnsureDirExists(dirs);
auto basename = dirs + namer_.File(definition, SkipFile::Suffix);
auto dirs = namer_.Directories(*definition.defined_namespace);
EnsureDirExists(dirs);
auto basename = dirs + namer_.File(definition, SkipFile::Suffix);
return SaveFile(basename.c_str(), code, false);
return SaveFile(basename.c_str(), code, false);
}
void TrackNsDef(const Definition &definition, std::string type_name) {
std::string path;
std::string filepath;
std::string symbolic_name;
if (definition.defined_namespace->components.size() > 0) {
path = namer_.Directories(*definition.defined_namespace,
SkipDir::TrailingPathSeperator);
filepath = path + ".ts";
path = namer_.Directories(*definition.defined_namespace,
SkipDir::OutputPathAndTrailingPathSeparator);
symbolic_name = definition.defined_namespace->components.back();
} else {
auto def_mod_name = namer_.File(definition, SkipFile::SuffixAndExtension);
symbolic_name = file_name_;
filepath = path_ + symbolic_name + ".ts";
}
if (ns_defs_.count(path) == 0) {
NsDefinition nsDef;
nsDef.path = path;
nsDef.filepath = filepath;
nsDef.ns = definition.defined_namespace;
nsDef.definitions.insert(std::make_pair(type_name, &definition));
nsDef.symbolic_name = symbolic_name;
ns_defs_[path] = nsDef;
} else {
ns_defs_[path].definitions.insert(std::make_pair(type_name, &definition));
}
}
private:
IdlNamer namer_;
import_set imports_all_;
// The following three members are used when generating typescript code into a
// single file rather than creating separate files for each type.
// flat_file_ contains the aggregated contents of the file prior to being
// written to disk.
std::string flat_file_;
// flat_file_definitions_ tracks which types have been written to flat_file_.
std::unordered_set<const Definition *> flat_file_definitions_;
// This maps from import names to types to import.
std::map<std::string, std::map<std::string, std::string>>
flat_file_import_declarations_;
// For flat file codegen, tracks whether we need to import the flatbuffers
// library itself (not necessary for files that solely consist of enum
// definitions).
bool import_flatbuffers_lib_ = false;
std::map<std::string, NsDefinition> ns_defs_;
// Generate code for all enums.
void generateEnums() {
@@ -207,8 +213,9 @@ class TsGenerator : public BaseGenerator {
auto &enum_def = **it;
GenEnum(enum_def, &enumcode, imports, false);
GenEnum(enum_def, &enumcode, imports, true);
std::string type_name = GetTypeName(enum_def);
TrackNsDef(enum_def, type_name);
SaveType(enum_def, enumcode, imports, bare_imports);
imports_all_.insert(imports.begin(), imports.end());
}
}
@@ -219,76 +226,101 @@ class TsGenerator : public BaseGenerator {
import_set bare_imports;
import_set imports;
AddImport(bare_imports, "* as flatbuffers", "flatbuffers");
import_flatbuffers_lib_ = true;
auto &struct_def = **it;
std::string declcode;
GenStruct(parser_, struct_def, &declcode, imports);
std::string type_name = GetTypeName(struct_def);
TrackNsDef(struct_def, type_name);
SaveType(struct_def, declcode, imports, bare_imports);
imports_all_.insert(imports.begin(), imports.end());
}
}
// Generate code for a single entry point module.
void generateEntry() {
std::string code =
"// " + std::string(FlatBuffersGeneratedWarning()) + "\n\n";
if (parser_.opts.ts_flat_file) {
if (import_flatbuffers_lib_) {
code += "import * as flatbuffers from 'flatbuffers';\n";
code += "\n";
}
// Only include import statements when not generating all.
if (!parser_.opts.generate_all) {
for (const auto &it : flat_file_import_declarations_) {
// Note that we do end up generating an import for ourselves, which
// should generally be harmless.
// TODO: Make it so we don't generate a self-import; this will also
// require modifying AddImport to ensure that we don't use
// namespace-prefixed names anywhere...
std::string file = it.first;
if (file.empty()) { continue; }
std::string noext = flatbuffers::StripExtension(file);
std::string basename = flatbuffers::StripPath(noext);
std::string include_file = GeneratedFileName(
parser_.opts.include_prefix,
parser_.opts.keep_prefix ? noext : basename, parser_.opts);
// TODO: what is the right behavior when different include flags are
// specified here? Should we always be adding the "./" for a relative
// path or turn it off if --include-prefix is specified, or something
// else?
std::string import_extension = parser_.opts.ts_no_import_ext ? "" : ".js";
std::string include_name =
"./" + flatbuffers::StripExtension(include_file) + import_extension;
code += "import {";
for (const auto &pair : it.second) {
code += namer_.EscapeKeyword(pair.first) + " as " +
namer_.EscapeKeyword(pair.second) + ", ";
}
code.resize(code.size() - 2);
code += "} from '" + include_name + "';\n";
}
code += "\n";
}
std::string code;
code += flat_file_;
const std::string filename =
GeneratedFileName(path_, file_name_, parser_.opts);
SaveFile(filename.c_str(), code, false);
} else {
for (auto it = imports_all_.begin(); it != imports_all_.end(); it++) {
code += it->second.export_statement + "\n";
}
if (imports_all_.empty()) {
// if the file is empty, add an empty export so that tsc doesn't
// complain when running under `--isolatedModules` mode
code += "export {}";
}
const std::string path =
GeneratedFileName(path_, file_name_, parser_.opts);
SaveFile(path.c_str(), code, false);
// add root namespace def if not already existing from defs tracking
std::string root;
if (ns_defs_.count(root) == 0) {
NsDefinition nsDef;
nsDef.path = root;
nsDef.symbolic_name = file_name_;
nsDef.filepath = path_ + file_name_ + ".ts";
nsDef.ns = new Namespace();
ns_defs_[nsDef.path] = nsDef;
}
for (const auto &it : ns_defs_) {
code = "// " + std::string(FlatBuffersGeneratedWarning()) + "\n\n";
// export all definitions in ns entry point module
int export_counter = 0;
for (const auto &def : it.second.definitions) {
std::vector<std::string> rel_components;
// build path for root level vs child level
if (it.second.ns->components.size() > 1)
std::copy(it.second.ns->components.begin() + 1,
it.second.ns->components.end(),
std::back_inserter(rel_components));
else
std::copy(it.second.ns->components.begin(),
it.second.ns->components.end(),
std::back_inserter(rel_components));
auto base_file_name =
namer_.File(*(def.second), SkipFile::SuffixAndExtension);
auto base_name =
namer_.Directories(it.second.ns->components, SkipDir::OutputPath) +
base_file_name;
auto ts_file_path = base_name + ".ts";
auto base_name_rel = std::string("./");
base_name_rel +=
namer_.Directories(rel_components, SkipDir::OutputPath);
base_name_rel += base_file_name;
auto ts_file_path_rel = base_name_rel + ".ts";
auto type_name = def.first;
code += "export { " + type_name + " } from '";
std::string import_extension =
parser_.opts.ts_no_import_ext ? "" : ".js";
code += base_name_rel + import_extension + "';\n";
export_counter++;
}
// re-export child namespace(s) in parent
const auto child_ns_level = it.second.ns->components.size() + 1;
for (const auto &it2 : ns_defs_) {
if (it2.second.ns->components.size() != child_ns_level) continue;
auto ts_file_path = it2.second.path + ".ts";
code += "export * as " + it2.second.symbolic_name + " from './";
std::string rel_path = it2.second.path;
code += rel_path + ".js';\n";
export_counter++;
}
if (export_counter > 0) SaveFile(it.second.filepath.c_str(), code, false);
}
}
bool generateBundle() {
if (parser_.opts.ts_flat_files) {
std::string inputpath;
std::string symbolic_name = file_name_;
inputpath = path_ + file_name_ + ".ts";
std::string bundlepath =
GeneratedFileName(path_, file_name_, parser_.opts);
bundlepath = bundlepath.substr(0, bundlepath.size() - 3) + ".js";
std::string cmd = "esbuild";
cmd += " ";
cmd += inputpath;
// cmd += " --minify";
cmd += " --format=cjs --bundle --outfile=";
cmd += bundlepath;
cmd += " --external:flatbuffers";
std::cout << "Entry point " << inputpath << " generated." << std::endl;
std::cout << "A single file bundle can be created using fx. esbuild with:"
<< std::endl;
std::cout << "> " << cmd << std::endl;
}
return true;
}
// Generate a documentation comment, if available.
@@ -839,28 +871,6 @@ class TsGenerator : public BaseGenerator {
const std::string object_name =
GetTypeName(dependency, /*object_api=*/true, has_name_clash);
if (parser_.opts.ts_flat_file) {
// In flat-file generation, do not attempt to import things from ourselves
// *and* do not wrap namespaces (note that this does override the logic
// above, but since we force all non-self-imports to use namespace-based
// names in flat file generation, it's fine).
if (dependent.file == dependency.file) {
name = import_name;
} else {
const std::string file =
RelativeToRootPath(StripFileName(AbsolutePath(dependent.file)),
dependency.file)
// Strip the leading //
.substr(2);
flat_file_import_declarations_[file][import_name] = name;
if (parser_.opts.generate_object_based_api &&
SupportsObjectAPI<DefinitionT>::value) {
flat_file_import_declarations_[file][import_name + "T"] = object_name;
}
}
}
const std::string symbols_expression = GenSymbolExpression(
dependency, has_name_clash, import_name, name, object_name);