Lua Generator using IR. (#6940)

* initial hack to get new Lua generator into flatc

* Starting to output enum defs for Lua

* Continue to work on table generation for Lua

* Finished basic getter access for Lua

* Added ability to get object by index

* Finished struct builder

* aliased reflection to r

* finish table builder generation

* register requiring files

* better generated header info

* Tying up loose ends

* Updated reflection to handle struct padding

* Addd type sizes to reflection

* Fixed some vector indirect issues

* Lua tests passed

* Misc cleanup

* ci fixes 1

* ci fixes 2

* renaming

* up size of type sizes

* manually ran clang-format-11 -i src/idl_parser.cpp

* fixed some windows casting

* remove stupid auto import

* more static_casting

* remove std

* update other build environments

* remove scoped enums

* replaced std::to_string with NumToString

* more win fixes

* more win fixes

* replaced old lua with new

* removed auto import

* review responses

* more style fixes

* refactor bfbs_gen_len to use code +=

* added consts

* fix lambda capture for windows

* remove unused return type
This commit is contained in:
Derek Bailey
2021-12-02 21:29:19 -08:00
committed by GitHub
parent cffe0c4546
commit 061d61f3f8
43 changed files with 3461 additions and 1725 deletions

View File

@@ -34,6 +34,9 @@ cc_library(
cc_library(
name = "flatc_library",
srcs = [
"bfbs_gen.h",
"bfbs_gen_lua.cpp",
"bfbs_gen_lua.h",
"flatc.cpp",
],
hdrs = [
@@ -50,6 +53,9 @@ cc_library(
cc_library(
name = "flatc",
srcs = [
"bfbs_gen.h",
"bfbs_gen_lua.cpp",
"bfbs_gen_lua.h",
"flatc_main.cpp",
"idl_gen_cpp.cpp",
"idl_gen_csharp.cpp",

224
src/bfbs_gen.h Normal file
View File

@@ -0,0 +1,224 @@
/*
* Copyright 2021 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FLATBUFFERS_BFBS_GEN_H_
#define FLATBUFFERS_BFBS_GEN_H_
#include <cstdint>
#include "flatbuffers/bfbs_generator.h"
#include "flatbuffers/reflection_generated.h"
namespace flatbuffers {
void ForAllEnums(
const flatbuffers::Vector<flatbuffers::Offset<reflection::Enum>> *enums,
std::function<void(const reflection::Enum *)> func) {
for (auto it = enums->cbegin(); it != enums->cend(); ++it) { func(*it); }
}
void ForAllObjects(
const flatbuffers::Vector<flatbuffers::Offset<reflection::Object>> *objects,
std::function<void(const reflection::Object *)> func) {
for (auto it = objects->cbegin(); it != objects->cend(); ++it) { func(*it); }
}
void ForAllEnumValues(const reflection::Enum *enum_def,
std::function<void(const reflection::EnumVal *)> func) {
for (auto it = enum_def->values()->cbegin(); it != enum_def->values()->cend();
++it) {
func(*it);
}
}
void ForAllDocumentation(
const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>
*documentation,
std::function<void(const flatbuffers::String *)> func) {
if (!documentation) { return; }
for (auto it = documentation->cbegin(); it != documentation->cend(); ++it) {
func(*it);
}
}
// Maps the field index into object->fields() to the field's ID (the ith element
// in the return vector).
static std::vector<uint32_t> FieldIdToIndex(const reflection::Object *object) {
std::vector<uint32_t> field_index_by_id;
field_index_by_id.resize(object->fields()->size());
// Create the mapping of field ID to the index into the vector.
for (uint32_t i = 0; i < object->fields()->size(); ++i) {
auto field = object->fields()->Get(i);
field_index_by_id[field->id()] = i;
}
return field_index_by_id;
}
static bool IsStructOrTable(const reflection::BaseType base_type) {
return base_type == reflection::Obj;
}
static bool IsScalar(const reflection::BaseType base_type) {
return base_type >= reflection::UType && base_type <= reflection::Double;
}
static bool IsFloatingPoint(const reflection::BaseType base_type) {
return base_type == reflection::Float || base_type == reflection::Double;
}
static bool IsBool(const reflection::BaseType base_type) {
return base_type == reflection::Bool;
}
static bool IsSingleByte(const reflection::BaseType base_type) {
return base_type >= reflection::UType && base_type <= reflection::UByte;
}
static bool IsVector(const reflection::BaseType base_type) {
return base_type == reflection::Vector;
}
static std::string MakeCamelCase(const std::string &in,
bool uppercase_first = true) {
std::string s;
for (size_t i = 0; i < in.length(); i++) {
if (!i && uppercase_first)
s += static_cast<char>(::toupper(static_cast<unsigned char>(in[0])));
else if (in[i] == '_' && i + 1 < in.length())
s += static_cast<char>(::toupper(static_cast<unsigned char>(in[++i])));
else
s += in[i];
}
return s;
}
static std::string Denamespace(const flatbuffers::String *name,
std::string &ns) {
const size_t pos = name->str().find_last_of('.');
if (pos == std::string::npos) {
ns = "";
return name->str();
}
ns = name->str().substr(0, pos);
return name->str().substr(pos + 1);
}
static std::string Denamespace(const flatbuffers::String *name) {
std::string ns;
return Denamespace(name, ns);
}
// A concrete base Flatbuffer Generator that specific language generators can
// derive from.
class BaseBfbsGenerator : public BfbsGenerator {
public:
virtual ~BaseBfbsGenerator() {}
BaseBfbsGenerator() : schema_(nullptr) {}
virtual GeneratorStatus GenerateFromSchema(
const reflection::Schema *schema) = 0;
//
virtual uint64_t SupportedAdvancedFeatures() const = 0;
// Override of the Generator::generate method that does the initial
// deserialization and verification steps.
GeneratorStatus Generate(const uint8_t *buffer,
int64_t length) FLATBUFFERS_OVERRIDE {
flatbuffers::Verifier verifier(buffer, static_cast<size_t>(length));
if (!reflection::VerifySchemaBuffer(verifier)) {
return FAILED_VERIFICATION;
}
// Store the root schema since there are cases where leaf nodes refer to
// things in the root schema (e.g., indexing the objects).
schema_ = reflection::GetSchema(buffer);
const uint64_t advance_features = schema_->advanced_features();
if (advance_features > SupportedAdvancedFeatures()) {
return FAILED_VERIFICATION;
}
GeneratorStatus status = GenerateFromSchema(schema_);
schema_ = nullptr;
return status;
}
protected:
const reflection::Object *GetObject(const reflection::Type *type) const {
if (type->index() >= 0 && IsStructOrTable(type->base_type())) {
return GetObjectByIndex(type->index());
}
return nullptr;
}
const reflection::Enum *GetEnum(const reflection::Type *type) const {
// TODO(derekbailey): it would be better to have a explicit list of allowed
// base types, instead of negating Obj types.
if (type->index() >= 0 && !IsStructOrTable(type->base_type())) {
return GetEnumByIndex(type->index());
}
return nullptr;
}
// Used to get a object that is reference by index. (e.g.
// reflection::Type::index). Returns nullptr if no object is available.
const reflection::Object *GetObjectByIndex(int32_t index) const {
if (!schema_ || index < 0 ||
index >= static_cast<int32_t>(schema_->objects()->size())) {
return nullptr;
}
return schema_->objects()->Get(index);
}
// Used to get a enum that is reference by index. (e.g.
// reflection::Type::index). Returns nullptr if no enum is available.
const reflection::Enum *GetEnumByIndex(int32_t index) const {
if (!schema_ || index < 0 ||
index >= static_cast<int32_t>(schema_->enums()->size())) {
return nullptr;
}
return schema_->enums()->Get(index);
}
void ForAllFields(const reflection::Object *object, bool reverse,
std::function<void(const reflection::Field *)> func) const {
const std::vector<uint32_t> field_to_id_map = FieldIdToIndex(object);
for (size_t i = 0; i < field_to_id_map.size(); ++i) {
func(object->fields()->Get(
field_to_id_map[reverse ? field_to_id_map.size() - (i + 1) : i]));
}
}
bool IsTable(const reflection::Type *type, bool use_element = false) const {
return !IsStruct(type, use_element);
}
bool IsStruct(const reflection::Type *type, bool use_element = false) const {
const reflection::BaseType base_type =
use_element ? type->element() : type->base_type();
return IsStructOrTable(base_type) &&
GetObjectByIndex(type->index())->is_struct();
}
const reflection::Schema *schema_;
};
} // namespace flatbuffers
#endif // FLATBUFFERS_BFBS_GEN_H_

620
src/bfbs_gen_lua.cpp Normal file
View File

@@ -0,0 +1,620 @@
/*
* Copyright 2021 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "bfbs_gen_lua.h"
#include <cstdint>
#include <map>
#include <memory>
#include <string>
#include <unordered_set>
#include <vector>
// Ensure no includes to flatc internals. bfbs_gen.h and generator.h are OK.
#include "bfbs_gen.h"
#include "flatbuffers/bfbs_generator.h"
// The intermediate representation schema.
#include "flatbuffers/reflection_generated.h"
namespace flatbuffers {
namespace {
// To reduce typing
namespace r = ::reflection;
class LuaBfbsGenerator : public BaseBfbsGenerator {
public:
explicit LuaBfbsGenerator(const std::string &flatc_version)
: BaseBfbsGenerator(),
keywords_(),
requires_(),
current_obj_(nullptr),
current_enum_(nullptr),
flatc_version_(flatc_version) {
static const char *const keywords[] = {
"and", "break", "do", "else", "elseif", "end", "false", "for",
"function", "goto", "if", "in", "local", "nil", "not", "or",
"repeat", "return", "then", "true", "until", "while"
};
keywords_.insert(std::begin(keywords), std::end(keywords));
}
GeneratorStatus GenerateFromSchema(const r::Schema *schema)
FLATBUFFERS_OVERRIDE {
if (!GenerateEnums(schema->enums())) { return FAILED; }
if (!GenerateObjects(schema->objects(), schema->root_table())) {
return FAILED;
}
return OK;
}
uint64_t SupportedAdvancedFeatures() const FLATBUFFERS_OVERRIDE {
return 0xF;
}
protected:
bool GenerateEnums(
const flatbuffers::Vector<flatbuffers::Offset<r::Enum>> *enums) {
ForAllEnums(enums, [&](const r::Enum *enum_def) {
std::string code;
StartCodeBlock(enum_def);
std::string ns;
const std::string enum_name =
NormalizeName(Denamespace(enum_def->name(), ns));
GenerateDocumentation(enum_def->documentation(), "", code);
code += "local " + enum_name + " = {\n";
ForAllEnumValues(enum_def, [&](const reflection::EnumVal *enum_val) {
GenerateDocumentation(enum_val->documentation(), " ", code);
code += " " + NormalizeName(enum_val->name()) + " = " +
NumToString(enum_val->value()) + ",\n";
});
code += "}\n";
code += "\n";
EmitCodeBlock(code, enum_name, ns, enum_def->declaration_file()->str());
});
return true;
}
bool GenerateObjects(
const flatbuffers::Vector<flatbuffers::Offset<r::Object>> *objects,
const r::Object *root_object) {
ForAllObjects(objects, [&](const r::Object *object) {
std::string code;
StartCodeBlock(object);
// Register the main flatbuffers module.
RegisterRequires("flatbuffers", "flatbuffers");
std::string ns;
const std::string object_name =
NormalizeName(Denamespace(object->name(), ns));
GenerateDocumentation(object->documentation(), "", code);
code += "local " + object_name + " = {}\n";
code += "local mt = {}\n";
code += "\n";
code += "function " + object_name + ".New()\n";
code += " local o = {}\n";
code += " setmetatable(o, {__index = mt})\n";
code += " return o\n";
code += "end\n";
code += "\n";
if (object == root_object) {
code += "function " + object_name + ".GetRootAs" + object_name +
"(buf, offset)\n";
code += " if type(buf) == \"string\" then\n";
code += " buf = flatbuffers.binaryArray.New(buf)\n";
code += " end\n";
code += "\n";
code += " local n = flatbuffers.N.UOffsetT:Unpack(buf, offset)\n";
code += " local o = " + object_name + ".New()\n";
code += " o:Init(buf, n + offset)\n";
code += " return o\n";
code += "end\n";
code += "\n";
}
// Generates a init method that receives a pre-existing accessor object,
// so that objects can be reused.
code += "function mt:Init(buf, pos)\n";
code += " self.view = flatbuffers.view.New(buf, pos)\n";
code += "end\n";
code += "\n";
// Create all the field accessors.
ForAllFields(object, /*reverse=*/false, [&](const r::Field *field) {
// Skip writing deprecated fields altogether.
if (field->deprecated()) { return; }
const std::string field_name = NormalizeName(field->name());
const std::string field_name_camel_case = MakeCamelCase(field_name);
const r::BaseType base_type = field->type()->base_type();
// Generate some fixed strings so we don't repeat outselves later.
const std::string getter_signature =
"function mt:" + field_name_camel_case + "()\n";
const std::string offset_prefix = "local o = self.view:Offset(" +
NumToString(field->offset()) + ")\n";
const std::string offset_prefix_2 = "if o ~= 0 then\n";
GenerateDocumentation(field->documentation(), "", code);
if (IsScalar(base_type)) {
code += getter_signature;
if (object->is_struct()) {
// TODO(derekbailey): it would be nice to modify the view:Get to
// just pass in the offset and not have to add it its own
// self.view.pos.
code += " return " + GenerateGetter(field->type()) +
"self.view.pos + " + NumToString(field->offset()) + ")\n";
} else {
// Table accessors
code += " " + offset_prefix;
code += " " + offset_prefix_2;
std::string getter =
GenerateGetter(field->type()) + "self.view.pos + o)";
if (IsBool(base_type)) { getter = "(" + getter + " ~=0)"; }
code += " return " + getter + "\n";
code += " end\n";
code += " return " + DefaultValue(field) + "\n";
}
code += "end\n";
code += "\n";
} else {
switch (base_type) {
case r::String: {
code += getter_signature;
code += " " + offset_prefix;
code += " " + offset_prefix_2;
code += " return " + GenerateGetter(field->type()) +
"self.view.pos + o)\n";
code += " end\n";
code += "end\n";
code += "\n";
break;
}
case r::Obj: {
if (object->is_struct()) {
code += "function mt:" + field_name_camel_case + "(obj)\n";
code += " obj:Init(self.view.bytes, self.view.pos + " +
NumToString(field->offset()) + ")\n";
code += " return obj\n";
code += "end\n";
code += "\n";
} else {
code += getter_signature;
code += " " + offset_prefix;
code += " " + offset_prefix_2;
const r::Object *field_object = GetObject(field->type());
if (!field_object) {
// TODO(derekbailey): this is an error condition. we
// should report it better.
return;
}
code += " local x = " +
std::string(
field_object->is_struct()
? "self.view.pos + o\n"
: "self.view:Indirect(self.view.pos + o)\n");
const std::string require_name = RegisterRequires(field);
code += " local obj = " + require_name + ".New()\n";
code += " obj:Init(self.view.bytes, x)\n";
code += " return obj\n";
code += " end\n";
code += "end\n";
code += "\n";
}
break;
}
case r::Union: {
code += getter_signature;
code += " " + offset_prefix;
code += " " + offset_prefix_2;
code +=
" local obj = "
"flatbuffers.view.New(flatbuffers.binaryArray.New("
"0), 0)\n";
code += " " + GenerateGetter(field->type()) + "obj, o)\n";
code += " return obj\n";
code += " end\n";
code += "end\n";
code += "\n";
break;
}
case r::Array:
case r::Vector: {
const r::BaseType vector_base_type = field->type()->element();
int32_t element_size = field->type()->element_size();
code += "function mt:" + field_name_camel_case + "(j)\n";
code += " " + offset_prefix;
code += " " + offset_prefix_2;
if (IsStructOrTable(vector_base_type)) {
code += " local x = self.view:Vector(o)\n";
code +=
" x = x + ((j-1) * " + NumToString(element_size) + ")\n";
if (IsTable(field->type(), /*use_element=*/true)) {
code += " x = self.view:Indirect(x)\n";
} else {
// Vector of structs are inline, so we need to query the
// size of the struct.
const reflection::Object *obj =
GetObjectByIndex(field->type()->index());
element_size = obj->bytesize();
}
// Include the referenced type, thus we need to make sure
// we set `use_element` to true.
const std::string require_name =
RegisterRequires(field, /*use_element=*/true);
code += " local obj = " + require_name + ".New()\n";
code += " obj:Init(self.view.bytes, x)\n";
code += " return obj\n";
} else {
code += " local a = self.view:Vector(o)\n";
code += " return " + GenerateGetter(field->type()) +
"a + ((j-1) * " + NumToString(element_size) + "))\n";
}
code += " end\n";
// Only generate a default value for those types that are
// supported.
if (!IsStructOrTable(vector_base_type)) {
code +=
" return " +
std::string(vector_base_type == r::String ? "''\n" : "0\n");
}
code += "end\n";
code += "\n";
// If the vector is composed of single byte values, we
// generate a helper function to get it as a byte string in
// Lua.
if (IsSingleByte(vector_base_type)) {
code += "function mt:" + field_name_camel_case +
"AsString(start, stop)\n";
code += " return self.view:VectorAsString(" +
NumToString(field->offset()) + ", start, stop)\n";
code += "end\n";
code += "\n";
}
// We also make a new accessor to query just the length of the
// vector.
code += "function mt:" + field_name_camel_case + "Length()\n";
code += " " + offset_prefix;
code += " " + offset_prefix_2;
code += " return self.view:VectorLen(o)\n";
code += " end\n";
code += " return 0\n";
code += "end\n";
code += "\n";
break;
}
default: {
return;
}
}
}
return;
});
// Create all the builders
if (object->is_struct()) {
code += "function " + object_name + ".Create" + object_name +
"(builder" + GenerateStructBuilderArgs(object) + ")\n";
code += AppendStructBuilderBody(object);
code += " return builder:Offset()\n";
code += "end\n";
code += "\n";
} else {
// Table builders
code += "function " + object_name + ".Start(builder)\n";
code += " builder:StartObject(" +
NumToString(object->fields()->size()) + ")\n";
code += "end\n";
code += "\n";
ForAllFields(object, /*reverse=*/false, [&](const r::Field *field) {
if (field->deprecated()) { return; }
const std::string field_name = NormalizeName(field->name());
code += "function " + object_name + ".Add" +
MakeCamelCase(field_name) + "(builder, " +
MakeCamelCase(field_name, false) + ")\n";
code += " builder:Prepend" + GenerateMethod(field) + "Slot(" +
NumToString(field->id()) + ", " +
MakeCamelCase(field_name, false) + ", " +
DefaultValue(field) + ")\n";
code += "end\n";
code += "\n";
if (IsVector(field->type()->base_type())) {
code += "function " + object_name + ".Start" +
MakeCamelCase(field_name) + "Vector(builder, numElems)\n";
const int32_t element_size = field->type()->element_size();
int32_t alignment = 0;
if (IsStruct(field->type(), /*use_element=*/true)) {
alignment = GetObjectByIndex(field->type()->index())->minalign();
} else {
alignment = element_size;
}
code += " return builder:StartVector(" +
NumToString(element_size) + ", numElems, " +
NumToString(alignment) + ")\n";
code += "end\n";
code += "\n";
}
});
code += "function " + object_name + ".End(builder)\n";
code += " return builder:EndObject()\n";
code += "end\n";
code += "\n";
}
EmitCodeBlock(code, object_name, ns, object->declaration_file()->str());
});
return true;
}
private:
void GenerateDocumentation(
const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>
*documentation,
std::string indent, std::string &code) const {
flatbuffers::ForAllDocumentation(
documentation, [&](const flatbuffers::String *str) {
code += indent + "--" + str->str() + "\n";
});
}
std::string GenerateStructBuilderArgs(const r::Object *object,
std::string prefix = "") const {
std::string signature;
ForAllFields(object, /*reverse=*/false, [&](const r::Field *field) {
if (IsStructOrTable(field->type()->base_type())) {
const r::Object *field_object = GetObject(field->type());
signature += GenerateStructBuilderArgs(
field_object, prefix + NormalizeName(field->name()) + "_");
} else {
signature +=
", " + prefix + MakeCamelCase(NormalizeName(field->name()), false);
}
});
return signature;
}
std::string AppendStructBuilderBody(const r::Object *object,
std::string prefix = "") const {
std::string code;
code += " builder:Prep(" + NumToString(object->minalign()) + ", " +
NumToString(object->bytesize()) + ")\n";
// We need to reverse the order we iterate over, since we build the
// buffer backwards.
ForAllFields(object, /*reverse=*/true, [&](const r::Field *field) {
const int32_t num_padding_bytes = field->padding();
if (num_padding_bytes) {
code += " builder:Pad(" + NumToString(num_padding_bytes) + ")\n";
}
if (IsStructOrTable(field->type()->base_type())) {
const r::Object *field_object = GetObject(field->type());
code += AppendStructBuilderBody(
field_object, prefix + NormalizeName(field->name()) + "_");
} else {
code += " builder:Prepend" + GenerateMethod(field) + "(" + prefix +
MakeCamelCase(NormalizeName(field->name()), false) + ")\n";
}
});
return code;
}
std::string GenerateMethod(const r::Field *field) const {
const r::BaseType base_type = field->type()->base_type();
if (IsScalar(base_type)) { return MakeCamelCase(GenerateType(base_type)); }
if (IsStructOrTable(base_type)) { return "Struct"; }
return "UOffsetTRelative";
}
std::string GenerateGetter(const r::Type *type,
bool element_type = false) const {
switch (element_type ? type->element() : type->base_type()) {
case r::String: return "self.view:String(";
case r::Union: return "self.view:Union(";
case r::Vector: return GenerateGetter(type, true);
default:
return "self.view:Get(flatbuffers.N." +
MakeCamelCase(GenerateType(type, element_type)) + ", ";
}
}
std::string GenerateType(const r::Type *type,
bool element_type = false) const {
const r::BaseType base_type =
element_type ? type->element() : type->base_type();
if (IsScalar(base_type)) { return GenerateType(base_type); }
switch (base_type) {
case r::String: return "string";
case r::Vector: return GenerateGetter(type, true);
case r::Obj: {
const r::Object *obj = GetObject(type);
return NormalizeName(Denamespace(obj->name()));
};
default: return "*flatbuffers.Table";
}
}
std::string GenerateType(const r::BaseType base_type) const {
// Need to override the default naming to match the Lua runtime libraries.
// TODO(derekbailey): make overloads in the runtime libraries to avoid this.
switch (base_type) {
case r::None: return "uint8";
case r::UType: return "uint8";
case r::Byte: return "int8";
case r::UByte: return "uint8";
case r::Short: return "int16";
case r::UShort: return "uint16";
case r::Int: return "int32";
case r::UInt: return "uint32";
case r::Long: return "int64";
case r::ULong: return "uint64";
case r::Float: return "Float32";
case r::Double: return "Float64";
default: return r::EnumNameBaseType(base_type);
}
}
std::string DefaultValue(const r::Field *field) const {
const r::BaseType base_type = field->type()->base_type();
if (IsFloatingPoint(base_type)) {
return NumToString(field->default_real());
}
if (IsBool(base_type)) {
return field->default_integer() ? "true" : "false";
}
if (IsScalar(base_type)) { return NumToString((field->default_integer())); }
// represents offsets
return "0";
}
std::string NormalizeName(const std::string name) const {
return keywords_.find(name) == keywords_.end() ? name : "_" + name;
}
std::string NormalizeName(const flatbuffers::String *name) const {
return NormalizeName(name->str());
}
void StartCodeBlock(const reflection::Enum *enum_def) {
current_enum_ = enum_def;
current_obj_ = nullptr;
requires_.clear();
}
void StartCodeBlock(const reflection::Object *object) {
current_obj_ = object;
current_enum_ = nullptr;
requires_.clear();
}
std::string RegisterRequires(const r::Field *field,
bool use_element = false) {
std::string type_name;
const r::BaseType type =
use_element ? field->type()->element() : field->type()->base_type();
if (IsStructOrTable(type)) {
const r::Object *object = GetObjectByIndex(field->type()->index());
if (object == current_obj_) { return Denamespace(object->name()); }
type_name = object->name()->str();
} else {
const r::Enum *enum_def = GetEnumByIndex(field->type()->index());
if (enum_def == current_enum_) { return Denamespace(enum_def->name()); }
type_name = enum_def->name()->str();
}
// Prefix with double __ to avoid name clashing, since these are defined
// at the top of the file and have lexical scoping. Replace '.' with '_'
// so it can be a legal identifier.
std::string name = "__" + type_name;
std::replace(name.begin(), name.end(), '.', '_');
return RegisterRequires(name, type_name);
}
std::string RegisterRequires(const std::string &local_name,
const std::string &requires_name) {
requires_[local_name] = requires_name;
return local_name;
}
void EmitCodeBlock(const std::string &code_block, const std::string &name,
const std::string &ns,
const std::string &declaring_file) const {
const std::string root_type = schema_->root_table()->name()->str();
const std::string root_file =
schema_->root_table()->declaration_file()->str();
const std::string full_qualified_name = ns.empty() ? name : ns + "." + name;
std::string code = "--[[ " + full_qualified_name + "\n\n";
code +=
" Automatically generated by the FlatBuffers compiler, do not "
"modify.\n";
code += " Or modify. I'm a message, not a cop.\n";
code += "\n";
code += " flatc version: " + flatc_version_ + "\n";
code += "\n";
code += " Declared by : " + declaring_file + "\n";
code += " Rooting type : " + root_type + " (" + root_file + ")\n";
code += "\n--]]\n\n";
if (!requires_.empty()) {
for (auto it = requires_.cbegin(); it != requires_.cend(); ++it) {
code += "local " + it->first + " = require('" + it->second + "')\n";
}
code += "\n";
}
code += code_block;
code += "return " + name;
// Namespaces are '.' deliminted, so replace it with the path separator.
std::string path = ns;
if (path.empty()) {
path = ".";
} else {
std::replace(path.begin(), path.end(), '.', '/');
}
// TODO(derekbailey): figure out a save file without depending on util.h
EnsureDirExists(path);
const std::string file_name = path + "/" + name + ".lua";
SaveFile(file_name.c_str(), code, false);
}
std::unordered_set<std::string> keywords_;
std::map<std::string, std::string> requires_;
const r::Object *current_obj_;
const r::Enum *current_enum_;
const std::string flatc_version_;
};
} // namespace
std::unique_ptr<BfbsGenerator> NewLuaBfbsGenerator(
const std::string &flatc_version) {
return std::unique_ptr<LuaBfbsGenerator>(new LuaBfbsGenerator(flatc_version));
}
} // namespace flatbuffers

33
src/bfbs_gen_lua.h Normal file
View File

@@ -0,0 +1,33 @@
/*
* Copyright 2021 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FLATBUFFERS_BFBS_GEN_LUA_H_
#define FLATBUFFERS_BFBS_GEN_LUA_H_
#include <memory>
#include <string>
#include "flatbuffers/bfbs_generator.h"
namespace flatbuffers {
// Constructs a new Lua Code generator.
std::unique_ptr<BfbsGenerator> NewLuaBfbsGenerator(
const std::string &flatc_version);
} // namespace flatbuffers
#endif // FLATBUFFERS_BFBS_GEN_LUA_H_

View File

@@ -18,6 +18,9 @@
#include <list>
#include "bfbs_gen_lua.h"
#include "flatbuffers/util.h"
namespace flatbuffers {
const char *FLATC_VERSION() { return FLATBUFFERS_VERSION(); }
@@ -205,6 +208,7 @@ int FlatCompiler::Compile(int argc, const char **argv) {
bool raw_binary = false;
bool schema_binary = false;
bool grpc_enabled = false;
bool requires_bfbs = false;
std::vector<std::string> filenames;
std::list<std::string> include_directories_storage;
std::vector<const char *> include_directories;
@@ -405,10 +409,15 @@ int FlatCompiler::Compile(int argc, const char **argv) {
generator_enabled[i] = true;
any_generator = true;
opts.lang_to_generate |= params_.generators[i].lang;
if (params_.generators[i].bfbs_generator) {
opts.binary_schema_comments = true;
requires_bfbs = true;
}
goto found;
}
}
Error("unknown commandline argument: " + arg, true);
found:;
}
} else {
@@ -528,20 +537,42 @@ int FlatCompiler::Compile(int argc, const char **argv) {
parser->file_extension_ = reflection::SchemaExtension();
}
}
std::string filebase =
flatbuffers::StripPath(flatbuffers::StripExtension(filename));
// If one of the generators uses bfbs, serialize the parser and get
// the serialized buffer and length.
const uint8_t *bfbs_buffer = nullptr;
int64_t bfbs_length = 0;
if (requires_bfbs) {
parser->Serialize();
bfbs_buffer = parser->builder_.GetBufferPointer();
bfbs_length = parser->builder_.GetSize();
}
for (size_t i = 0; i < params_.num_generators; ++i) {
if (generator_enabled[i]) {
if (!print_make_rules) {
flatbuffers::EnsureDirExists(output_path);
if ((!params_.generators[i].schema_only ||
(is_schema || is_binary_schema)) &&
!params_.generators[i].generate(*parser.get(), output_path,
filebase)) {
Error(std::string("Unable to generate ") +
params_.generators[i].lang_name + " for " + filebase);
// Prefer bfbs generators if present.
if (params_.generators[i].bfbs_generator) {
const GeneratorStatus status =
params_.generators[i].bfbs_generator->Generate(bfbs_buffer,
bfbs_length);
if (status != OK) {
Error(std::string("Unable to generate ") +
params_.generators[i].lang_name + " for " + filebase +
" using bfbs generator.");
}
} else {
if ((!params_.generators[i].schema_only ||
(is_schema || is_binary_schema)) &&
!params_.generators[i].generate(*parser.get(), output_path,
filebase)) {
Error(std::string("Unable to generate ") +
params_.generators[i].lang_name + " for " + filebase);
}
}
} else {
if (params_.generators[i].make_rule == nullptr) {

View File

@@ -14,6 +14,10 @@
* limitations under the License.
*/
#include <memory>
#include "bfbs_gen_lua.h"
#include "flatbuffers/base.h"
#include "flatbuffers/flatc.h"
#include "flatbuffers/util.h"
@@ -50,59 +54,69 @@ int main(int argc, const char *argv[]) {
// Prevent Appveyor-CI hangs.
flatbuffers::SetupDefaultCRTReportMode();
const std::string flatbuffers_version(flatbuffers::FLATBUFFERS_VERSION());
std::unique_ptr<flatbuffers::BfbsGenerator> bfbs_gen_lua =
flatbuffers::NewLuaBfbsGenerator(flatbuffers_version);
g_program_name = argv[0];
const flatbuffers::FlatCompiler::Generator generators[] = {
{ flatbuffers::GenerateBinary, "-b", "--binary", "binary", false, nullptr,
flatbuffers::IDLOptions::kBinary,
"Generate wire format binaries for any data definitions",
flatbuffers::BinaryMakeRule },
flatbuffers::BinaryMakeRule, nullptr },
{ flatbuffers::GenerateTextFile, "-t", "--json", "text", false, nullptr,
flatbuffers::IDLOptions::kJson,
"Generate text output for any data definitions",
flatbuffers::TextMakeRule },
flatbuffers::TextMakeRule, nullptr },
{ flatbuffers::GenerateCPP, "-c", "--cpp", "C++", true,
flatbuffers::GenerateCppGRPC, flatbuffers::IDLOptions::kCpp,
"Generate C++ headers for tables/structs", flatbuffers::CPPMakeRule },
"Generate C++ headers for tables/structs", flatbuffers::CPPMakeRule,
nullptr },
{ flatbuffers::GenerateGo, "-g", "--go", "Go", true,
flatbuffers::GenerateGoGRPC, flatbuffers::IDLOptions::kGo,
"Generate Go files for tables/structs", nullptr },
"Generate Go files for tables/structs", nullptr, nullptr },
{ flatbuffers::GenerateJava, "-j", "--java", "Java", true,
flatbuffers::GenerateJavaGRPC, flatbuffers::IDLOptions::kJava,
"Generate Java classes for tables/structs", flatbuffers::JavaMakeRule },
"Generate Java classes for tables/structs", flatbuffers::JavaMakeRule,
nullptr },
{ flatbuffers::GenerateDart, "-d", "--dart", "Dart", true, nullptr,
flatbuffers::IDLOptions::kDart,
"Generate Dart classes for tables/structs", flatbuffers::DartMakeRule },
"Generate Dart classes for tables/structs", flatbuffers::DartMakeRule,
nullptr },
{ flatbuffers::GenerateTS, "-T", "--ts", "TypeScript", true,
flatbuffers::GenerateTSGRPC, flatbuffers::IDLOptions::kTs,
"Generate TypeScript code for tables/structs", flatbuffers::TSMakeRule },
"Generate TypeScript code for tables/structs", flatbuffers::TSMakeRule,
nullptr },
{ flatbuffers::GenerateCSharp, "-n", "--csharp", "C#", true, nullptr,
flatbuffers::IDLOptions::kCSharp,
"Generate C# classes for tables/structs", flatbuffers::CSharpMakeRule },
"Generate C# classes for tables/structs", flatbuffers::CSharpMakeRule,
nullptr },
{ flatbuffers::GeneratePython, "-p", "--python", "Python", true,
flatbuffers::GeneratePythonGRPC, flatbuffers::IDLOptions::kPython,
"Generate Python files for tables/structs", nullptr },
"Generate Python files for tables/structs", nullptr, nullptr },
{ flatbuffers::GenerateLobster, nullptr, "--lobster", "Lobster", true,
nullptr, flatbuffers::IDLOptions::kLobster,
"Generate Lobster files for tables/structs", nullptr },
"Generate Lobster files for tables/structs", nullptr, nullptr },
{ flatbuffers::GenerateLua, "-l", "--lua", "Lua", true, nullptr,
flatbuffers::IDLOptions::kLua, "Generate Lua files for tables/structs",
nullptr },
nullptr, bfbs_gen_lua.get() },
{ flatbuffers::GenerateRust, "-r", "--rust", "Rust", true, nullptr,
flatbuffers::IDLOptions::kRust, "Generate Rust files for tables/structs",
flatbuffers::RustMakeRule },
flatbuffers::RustMakeRule, nullptr },
{ flatbuffers::GeneratePhp, nullptr, "--php", "PHP", true, nullptr,
flatbuffers::IDLOptions::kPhp, "Generate PHP files for tables/structs",
nullptr },
nullptr, nullptr },
{ flatbuffers::GenerateKotlin, nullptr, "--kotlin", "Kotlin", true, nullptr,
flatbuffers::IDLOptions::kKotlin,
"Generate Kotlin classes for tables/structs", nullptr },
"Generate Kotlin classes for tables/structs", nullptr, nullptr },
{ flatbuffers::GenerateJsonSchema, nullptr, "--jsonschema", "JsonSchema",
true, nullptr, flatbuffers::IDLOptions::kJsonSchema,
"Generate Json schema", nullptr },
"Generate Json schema", nullptr, nullptr },
{ flatbuffers::GenerateSwift, nullptr, "--swift", "swift", true,
flatbuffers::GenerateSwiftGRPC, flatbuffers::IDLOptions::kSwift,
"Generate Swift files for tables/structs", nullptr },
"Generate Swift files for tables/structs", nullptr, nullptr },
};
flatbuffers::FlatCompiler::InitParams params;

View File

@@ -3669,7 +3669,7 @@ Offset<reflection::Field> FieldDef::Serialize(FlatBufferBuilder *builder,
IsInteger(value.type.base_type) ? StringToInt(value.constant.c_str()) : 0,
// result may be platform-dependent if underlying is float (not double)
IsFloat(value.type.base_type) ? d : 0.0, deprecated, IsRequired(), key,
attr__, docs__, IsOptional());
attr__, docs__, IsOptional(), static_cast<uint16_t>(padding));
// TODO: value.constant is almost always "0", we could save quite a bit of
// space by sharing it. Same for common values of value.type.
}
@@ -3685,6 +3685,7 @@ bool FieldDef::Deserialize(Parser &parser, const reflection::Field *field) {
value.constant = FloatToString(field->default_real(), 16);
}
presence = FieldDef::MakeFieldPresence(field->optional(), field->required());
padding = field->padding();
key = field->key();
if (!DeserializeAttributes(parser, field->attributes())) return false;
// TODO: this should probably be handled by a separate attribute
@@ -3828,7 +3829,8 @@ Offset<reflection::Type> Type::Serialize(FlatBufferBuilder *builder) const {
*builder, static_cast<reflection::BaseType>(base_type),
static_cast<reflection::BaseType>(element),
struct_def ? struct_def->index : (enum_def ? enum_def->index : -1),
fixed_length);
fixed_length, static_cast<uint32_t>(SizeOf(base_type)),
static_cast<uint32_t>(SizeOf(element)));
}
bool Type::Deserialize(const Parser &parser, const reflection::Type *type) {

View File

@@ -23,7 +23,7 @@
namespace flatbuffers {
int64_t GetAnyValueI(reflection::BaseType type, const uint8_t *data) {
// clang-format off
// clang-format off
#define FLATBUFFERS_GET(T) static_cast<int64_t>(ReadScalar<T>(data))
switch (type) {
case reflection::UType:
@@ -121,7 +121,7 @@ std::string GetAnyValueS(reflection::BaseType type, const uint8_t *data,
}
void SetAnyValueI(reflection::BaseType type, uint8_t *data, int64_t val) {
// clang-format off
// clang-format off
#define FLATBUFFERS_SET(T) WriteScalar(data, static_cast<T>(val))
switch (type) {
case reflection::UType: