C++98 (stlport) support for core FlatBuffers and FlexBuffers.

* Added internal - limited - implementation of flatbuffers::unique_ptr
  for STLs that don't ship with std::unique_ptr.  In C++11 and beyond
  this is just an alias for std::unique_ptr.
* Aliased used type traits structs is_scalar is_floating_point is_unsigned
  into flatbuffers namespace so they can be replaced in C++98 implementations.
  Right now these point at stlport's TR1 implementations.
* Wrapped vector::data() in vector_data().
* Wrapped vector::emplace_back() in vector_emplace_back().
* Wrapper string::back() in string_back().
* Added variants of FlatBufferBuilder::CreateVector() and
  FlatBufferBuilder::CreateVectorOfStructs() that allow the use of plain
  function pointers.
  Generated code has also been modified to use plain functions to build objects
  rather than std::function() so all generated code will work in C++98
  applications.
* Added flexbuffers::Builder::Vector(), flexbuffers::Builder::TypedVector()
  and flexbuffers::Builder::Map() methods that allow the use of plain function
  pointers.
* Changed Parser to internally use plain function pointers when parsing table
  and vector delimiters.
* Added specializations of NumToString() for 64-bit types that aren't supported
  by stringstream in stlport.
* Overloaded numeric_limits for 64-bit types not supported by stlport.
* Replaced build_apk.sh (which was broken by deprecation of the
  "android" tool in the Android SDK) with build.gradle and the
  appropriate gradle wrapper to build an APK.
* Switched Android build to build against all STL variants.
* Updated travis configuration to build Android test and sample.

Tested:
* Verified all tests continue to work on Linux, OSX and Android.
* Verified Travis build is green.

Change-Id: I9e634363793f85b9f141d21454b10686020a2065
This commit is contained in:
Stewart Miles
2017-07-13 06:27:39 -07:00
parent 2e2063cbeb
commit a892322203
41 changed files with 1445 additions and 1246 deletions

View File

@@ -16,6 +16,7 @@
#include "flatbuffers/code_generators.h"
#include <assert.h>
#include "flatbuffers/base.h"
#include "flatbuffers/util.h"
#if defined(_MSC_VER)
@@ -58,7 +59,7 @@ void CodeWriter::operator+=(std::string text) {
// Update the text to everything after the }}.
text = text.substr(end + 2);
}
if (!text.empty() && text.back() == '\\') {
if (!text.empty() && string_back(text) == '\\') {
text.pop_back();
stream_ << text;
} else {
@@ -129,8 +130,8 @@ std::string BaseGenerator::GetNameSpace(const Definition &def) const {
std::string qualified_name = qualifying_start_;
for (auto it = ns->components.begin(); it != ns->components.end(); ++it) {
qualified_name += *it;
if (std::next(it) != ns->components.end()) {
qualified_name += qualifying_separator_;
if ((it + 1) != ns->components.end()) {
qualified_name += qualifying_separator_;
}
}
@@ -167,4 +168,3 @@ void GenComment(const std::vector<std::string> &dc, std::string *code_ptr,
#if defined(_MSC_VER)
#pragma warning(pop)
#endif

View File

@@ -722,6 +722,7 @@ class CppGenerator : public BaseGenerator {
code_ += " void Reset();";
code_ += "";
if (!enum_def.uses_type_aliases) {
code_ += "#ifndef FLATBUFFERS_CPP98_STL";
code_ += " template <typename T>";
code_ += " void Set(T&& val) {";
code_ += " Reset();";
@@ -730,6 +731,7 @@ class CppGenerator : public BaseGenerator {
code_ += " value = new T(std::forward<T>(val));";
code_ += " }";
code_ += " }";
code_ += "#endif // FLATBUFFERS_CPP98_STL";
code_ += "";
}
code_ += " " + UnionUnPackSignature(enum_def, true) + ";";
@@ -1802,11 +1804,13 @@ class CppGenerator : public BaseGenerator {
code += "(" + value + ")";
} else {
code += "_fbb.CreateVector<flatbuffers::Offset<";
code += WrapInNameSpace(*vector_type.struct_def) + ">>";
code += "(" + value + ".size(), [&](size_t i) {";
code += " return Create" + vector_type.struct_def->name;
code += "(_fbb, " + value + "[i]" + GenPtrGet(field) + ", ";
code += "_rehasher); })";
code += WrapInNameSpace(*vector_type.struct_def) + ">> ";
code += "(" + value + ".size(), ";
code += "[](size_t i, _VectorArgs *__va) { ";
code += "return Create" + vector_type.struct_def->name;
code += "(*__va->__fbb, __va->_" + value + "[i]" +
GenPtrGet(field) + ", ";
code += "__va->__rehasher); }, &_va )";
}
break;
}
@@ -1815,16 +1819,19 @@ class CppGenerator : public BaseGenerator {
break;
}
case BASE_TYPE_UNION: {
code += "_fbb.CreateVector<flatbuffers::Offset<void>>(" + value +
".size(), [&](size_t i) { return " + value +
"[i].Pack(_fbb, _rehasher); })";
code += "_fbb.CreateVector<flatbuffers::"
"Offset<void>>(" + value +
".size(), [](size_t i, _VectorArgs *__va) { "
"return __va->_" + value +
"[i].Pack(*__va->__fbb, __va->__rehasher); }, &_va)";
break;
}
case BASE_TYPE_UTYPE: {
value = StripUnionType(value);
code += "_fbb.CreateVector<uint8_t>(" + value +
".size(), [&](size_t i) { return static_cast<uint8_t>(" + value +
"[i].type); })";
".size(), [](size_t i, _VectorArgs *__va) { "
"return static_cast<uint8_t>(__va->_" + value +
"[i].type); }, &_va)";
break;
}
default: {
@@ -1935,6 +1942,15 @@ class CppGenerator : public BaseGenerator {
code_ += " (void)_rehasher;";
code_ += " (void)_o;";
code_ +=
" struct _VectorArgs "
"{ flatbuffers::FlatBufferBuilder *__fbb; "
"const " +
NativeName(struct_def.name, &struct_def) +
"* __o; "
"const flatbuffers::rehasher_function_t *__rehasher; } _va = { "
"&_fbb, _o, _rehasher}; (void)_va;";
for (auto it = struct_def.fields.vec.begin();
it != struct_def.fields.vec.end(); ++it) {
auto &field = **it;

View File

@@ -21,6 +21,10 @@
#include "flatbuffers/util.h"
#include "flatbuffers/code_generators.h"
#if defined(FLATBUFFERS_CPP98_STL)
#include <cctype>
#endif // defined(FLATBUFFERS_CPP98_STL)
namespace flatbuffers {
// Convert an underscore_based_indentifier in to camelCase.
@@ -204,7 +208,7 @@ class GeneralGenerator : public BaseGenerator {
std::string code;
if (lang_.language == IDLOptions::kCSharp) {
code = "// <auto-generated>\n"
code = "// <auto-generated>\n"
"// " + std::string(FlatBuffersGeneratedWarning()) + "\n"
"// </auto-generated>\n\n";
} else {

View File

@@ -106,8 +106,8 @@ CheckedError Parser::CheckInRange(int64_t val, int64_t min, int64_t max) {
template<typename T> inline CheckedError atot(const char *s, Parser &parser,
T *val) {
int64_t i = StringToInt(s);
const int64_t min = std::numeric_limits<T>::min();
const int64_t max = std::numeric_limits<T>::max();
const int64_t min = flatbuffers::numeric_limits<T>::min();
const int64_t max = flatbuffers::numeric_limits<T>::max();
ECHECK(parser.CheckInRange(i, min, max));
*val = (T)i;
return NoError();
@@ -870,7 +870,8 @@ void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) {
CheckedError Parser::ParseTableDelimiters(size_t &fieldn,
const StructDef *struct_def,
const std::function<CheckedError(const std::string &name)> &body) {
ParseTableDelimitersBody body,
void *state) {
// We allow tables both as JSON object{ .. } with field names
// or vector[..] with all fields in order
char terminator = '}';
@@ -898,7 +899,7 @@ CheckedError Parser::ParseTableDelimiters(size_t &fieldn,
}
if (!opts.protobuf_ascii_alike || !(Is('{') || Is('['))) EXPECT(':');
}
ECHECK(body(name));
ECHECK(body(name, fieldn, struct_def, state));
if (Is(terminator)) break;
ECHECK(ParseComma());
}
@@ -911,52 +912,56 @@ CheckedError Parser::ParseTableDelimiters(size_t &fieldn,
CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
uoffset_t *ovalue) {
size_t fieldn = 0;
auto err = ParseTableDelimiters(fieldn, &struct_def,
[&](const std::string &name) -> CheckedError {
size_t fieldn_outer = 0;
auto err = ParseTableDelimiters(fieldn_outer, &struct_def,
[](const std::string &name, size_t &fieldn,
const StructDef *struct_def_inner,
void *state) -> CheckedError {
Parser *parser = static_cast<Parser *>(state);
if (name == "$schema") {
EXPECT(kTokenStringConstant);
ECHECK(parser->Expect(kTokenStringConstant));
return NoError();
}
auto field = struct_def.fields.Lookup(name);
auto field = struct_def_inner->fields.Lookup(name);
if (!field) {
if (!opts.skip_unexpected_fields_in_json) {
return Error("unknown field: " + name);
if (!parser->opts.skip_unexpected_fields_in_json) {
return parser->Error("unknown field: " + name);
} else {
ECHECK(SkipAnyJsonValue());
ECHECK(parser->SkipAnyJsonValue());
}
} else {
if (Is(kTokenNull)) {
NEXT(); // Ignore this field.
if (parser->Is(kTokenNull)) {
ECHECK(parser->Next()); // Ignore this field.
} else {
Value val = field->value;
if (field->flexbuffer) {
flexbuffers::Builder builder(1024,
flexbuffers::BUILDER_FLAG_SHARE_ALL);
ECHECK(ParseFlexBufferValue(&builder));
ECHECK(parser->ParseFlexBufferValue(&builder));
builder.Finish();
auto off = builder_.CreateVector(builder.GetBuffer());
auto off = parser->builder_.CreateVector(builder.GetBuffer());
val.constant = NumToString(off.o);
} else {
ECHECK(ParseAnyValue(val, field, fieldn, &struct_def));
ECHECK(parser->ParseAnyValue(val, field, fieldn, struct_def_inner));
}
// Hardcoded insertion-sort with error-check.
// If fields are specified in order, then this loop exits immediately.
auto elem = field_stack_.rbegin();
for (; elem != field_stack_.rbegin() + fieldn; ++elem) {
auto elem = parser->field_stack_.rbegin();
for (; elem != parser->field_stack_.rbegin() + fieldn; ++elem) {
auto existing_field = elem->second;
if (existing_field == field)
return Error("field set more than once: " + field->name);
return parser->Error("field set more than once: " + field->name);
if (existing_field->value.offset < field->value.offset) break;
}
// Note: elem points to before the insertion point, thus .base() points
// to the correct spot.
field_stack_.insert(elem.base(), std::make_pair(val, field));
parser->field_stack_.insert(elem.base(),
std::make_pair(val, field));
fieldn++;
}
}
return NoError();
});
}, this);
ECHECK(err);
// Check if all required fields are parsed.
@@ -968,7 +973,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
continue;
}
bool found = false;
for (auto pf_it = field_stack_.end() - fieldn;
for (auto pf_it = field_stack_.end() - fieldn_outer;
pf_it != field_stack_.end();
++pf_it) {
auto parsed_field = pf_it->second;
@@ -982,7 +987,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
}
}
if (struct_def.fixed && fieldn != struct_def.fields.vec.size())
if (struct_def.fixed && fieldn_outer != struct_def.fields.vec.size())
return Error("struct: wrong number of initializers: " + struct_def.name);
auto start = struct_def.fixed
@@ -993,8 +998,9 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
size;
size /= 2) {
// Go through elements in reverse, since we're building the data backwards.
for (auto it = field_stack_.rbegin();
it != field_stack_.rbegin() + fieldn; ++it) {
for (auto it = field_stack_.rbegin(); it != field_stack_.rbegin() +
fieldn_outer;
++it) {
auto &field_value = it->first;
auto field = it->second;
if (!struct_def.sortbysize ||
@@ -1035,7 +1041,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
}
}
}
for (size_t i = 0; i < fieldn; i++) field_stack_.pop_back();
for (size_t i = 0; i < fieldn_outer; i++) field_stack_.pop_back();
if (struct_def.fixed) {
builder_.ClearOffsets();
@@ -1058,11 +1064,12 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
}
CheckedError Parser::ParseVectorDelimiters(size_t &count,
const std::function<CheckedError()> &body) {
ParseVectorDelimitersBody body,
void *state) {
EXPECT('[');
for (;;) {
if ((!opts.strict_json || !count) && Is(']')) break;
ECHECK(body());
ECHECK(body(count, state));
count++;
if (Is(']')) break;
ECHECK(ParseComma());
@@ -1073,13 +1080,18 @@ CheckedError Parser::ParseVectorDelimiters(size_t &count,
CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) {
size_t count = 0;
auto err = ParseVectorDelimiters(count, [&]() -> CheckedError {
std::pair<Parser *, const Type &> parser_and_type_state(this, type);
auto err = ParseVectorDelimiters(count,
[](size_t &, void *state) -> CheckedError {
auto *parser_and_type =
static_cast<std::pair<Parser *, const Type &> *>(state);
auto *parser = parser_and_type->first;
Value val;
val.type = type;
ECHECK(ParseAnyValue(val, nullptr, 0, nullptr));
field_stack_.push_back(std::make_pair(val, nullptr));
val.type = parser_and_type->second;
ECHECK(parser->ParseAnyValue(val, nullptr, 0, nullptr));
parser->field_stack_.push_back(std::make_pair(val, nullptr));
return NoError();
});
}, &parser_and_type_state);
ECHECK(err);
builder_.StartVector(count * InlineSize(type) / InlineAlignment(type),
@@ -1927,17 +1939,25 @@ CheckedError Parser::ParseTypeFromProtoType(Type *type) {
CheckedError Parser::SkipAnyJsonValue() {
switch (token_) {
case '{': {
size_t fieldn = 0;
return ParseTableDelimiters(fieldn, nullptr,
[&](const std::string &) -> CheckedError {
ECHECK(SkipAnyJsonValue());
fieldn++;
return NoError();
});
size_t fieldn_outer = 0;
return ParseTableDelimiters(fieldn_outer, nullptr,
[](const std::string &,
size_t &fieldn, const StructDef *,
void *state) -> CheckedError {
auto *parser = static_cast<Parser *>(state);
ECHECK(parser->SkipAnyJsonValue());
fieldn++;
return NoError();
},
this);
}
case '[': {
size_t count = 0;
return ParseVectorDelimiters(count, [&]() { return SkipAnyJsonValue(); });
return ParseVectorDelimiters(count, [](size_t &,
void *state) -> CheckedError {
return static_cast<Parser *>(state)->SkipAnyJsonValue();
},
this);
}
case kTokenStringConstant:
EXPECT(kTokenStringConstant);
@@ -1957,15 +1977,25 @@ CheckedError Parser::SkipAnyJsonValue() {
CheckedError Parser::ParseFlexBufferValue(flexbuffers::Builder *builder) {
switch (token_) {
case '{': {
std::pair<Parser *, flexbuffers::Builder *> parser_and_builder_state(
this, builder);
auto start = builder->StartMap();
size_t fieldn = 0;
auto err = ParseTableDelimiters(fieldn, nullptr,
[&](const std::string &name) -> CheckedError {
builder->Key(name);
ECHECK(ParseFlexBufferValue(builder));
fieldn++;
return NoError();
});
size_t fieldn_outer = 0;
auto err = ParseTableDelimiters(fieldn_outer, nullptr,
[](const std::string &name,
size_t &fieldn, const StructDef *,
void *state) -> CheckedError {
auto *parser_and_builder =
static_cast<std::pair<Parser *, flexbuffers::Builder *> *>(
state);
auto *parser = parser_and_builder->first;
auto *current_builder = parser_and_builder->second;
current_builder->Key(name);
ECHECK(parser->ParseFlexBufferValue(current_builder));
fieldn++;
return NoError();
},
&parser_and_builder_state);
ECHECK(err);
builder->EndMap(start);
break;
@@ -1973,9 +2003,17 @@ CheckedError Parser::ParseFlexBufferValue(flexbuffers::Builder *builder) {
case '[':{
auto start = builder->StartVector();
size_t count = 0;
ECHECK(ParseVectorDelimiters(count, [&]() {
return ParseFlexBufferValue(builder);
}));
std::pair<Parser *, flexbuffers::Builder *> parser_and_builder_state(
this, builder);
ECHECK(ParseVectorDelimiters(count, [](size_t &,
void *state) -> CheckedError {
auto *parser_and_builder =
static_cast<std::pair<Parser *, flexbuffers::Builder *> *>(
state);
return parser_and_builder->first->ParseFlexBufferValue(
parser_and_builder->second);
},
&parser_and_builder_state));
builder->EndVector(start, false, false);
break;
}
@@ -2082,7 +2120,7 @@ CheckedError Parser::DoParse(const char *source,
ECHECK(ParseProtoDecl());
} else if (Is(kTokenNativeInclude)) {
NEXT();
native_included_files_.emplace_back(attribute_);
vector_emplace_back(&native_included_files_, attribute_);
EXPECT(kTokenStringConstant);
} else if (Is(kTokenInclude) ||
(opts.proto_mode &&
@@ -2201,7 +2239,10 @@ std::set<std::string> Parser::GetIncludedFilesRecursive(
to_process.pop_front();
included_files.insert(current);
auto new_files = files_included_per_file_.at(current);
// Workaround the lack of const accessor in C++98 maps.
auto &new_files =
(*const_cast<std::map<std::string, std::set<std::string>> *>(
&files_included_per_file_))[current];
for (auto it = new_files.begin(); it != new_files.end(); ++it) {
if (included_files.find(*it) == included_files.end())
to_process.push_back(*it);

View File

@@ -165,15 +165,15 @@ class ResizeContext {
ResizeContext(const reflection::Schema &schema, uoffset_t start, int delta,
std::vector<uint8_t> *flatbuf,
const reflection::Object *root_table = nullptr)
: schema_(schema), startptr_(flatbuf->data() + start),
: schema_(schema), startptr_(vector_data(*flatbuf) + start),
delta_(delta), buf_(*flatbuf),
dag_check_(flatbuf->size() / sizeof(uoffset_t), false) {
auto mask = static_cast<int>(sizeof(largest_scalar_t) - 1);
delta_ = (delta_ + mask) & ~mask;
if (!delta_) return; // We can't shrink by less than largest_scalar_t.
// Now change all the offsets by delta_.
auto root = GetAnyRoot(buf_.data());
Straddle<uoffset_t, 1>(buf_.data(), root, buf_.data());
auto root = GetAnyRoot(vector_data(buf_));
Straddle<uoffset_t, 1>(vector_data(buf_), root, vector_data(buf_));
ResizeTable(root_table ? *root_table : *schema.root_table(), root);
// We can now add or remove bytes at start.
if (delta_ > 0) buf_.insert(buf_.begin() + start, delta_, 0);
@@ -200,7 +200,7 @@ class ResizeContext {
// will straddle and which won't.
uint8_t &DagCheck(const void *offsetloc) {
auto dag_idx = reinterpret_cast<const uoffset_t *>(offsetloc) -
reinterpret_cast<const uoffset_t *>(buf_.data());
reinterpret_cast<const uoffset_t *>(vector_data(buf_));
return dag_check_[dag_idx];
}
@@ -296,19 +296,19 @@ void SetString(const reflection::Schema &schema, const std::string &val,
const reflection::Object *root_table) {
auto delta = static_cast<int>(val.size()) - static_cast<int>(str->Length());
auto str_start = static_cast<uoffset_t>(
reinterpret_cast<const uint8_t *>(str) - flatbuf->data());
reinterpret_cast<const uint8_t *>(str) - vector_data(*flatbuf));
auto start = str_start + static_cast<uoffset_t>(sizeof(uoffset_t));
if (delta) {
// Clear the old string, since we don't want parts of it remaining.
memset(flatbuf->data() + start, 0, str->Length());
memset(vector_data(*flatbuf) + start, 0, str->Length());
// Different size, we must expand (or contract).
ResizeContext(schema, start, delta, flatbuf, root_table);
// Set the new length.
WriteScalar(flatbuf->data() + str_start,
WriteScalar(vector_data(*flatbuf) + str_start,
static_cast<uoffset_t>(val.size()));
}
// Copy new data. Safe because we created the right amount of space.
memcpy(flatbuf->data() + start, val.c_str(), val.size() + 1);
memcpy(vector_data(*flatbuf) + start, val.c_str(), val.size() + 1);
}
uint8_t *ResizeAnyVector(const reflection::Schema &schema, uoffset_t newsize,
@@ -317,7 +317,8 @@ uint8_t *ResizeAnyVector(const reflection::Schema &schema, uoffset_t newsize,
const reflection::Object *root_table) {
auto delta_elem = static_cast<int>(newsize) - static_cast<int>(num_elems);
auto delta_bytes = delta_elem * static_cast<int>(elem_size);
auto vec_start = reinterpret_cast<const uint8_t *>(vec) - flatbuf->data();
auto vec_start = reinterpret_cast<const uint8_t *>(vec) -
vector_data(*flatbuf);
auto start = static_cast<uoffset_t>(vec_start + sizeof(uoffset_t) +
elem_size * num_elems);
if (delta_bytes) {
@@ -325,16 +326,16 @@ uint8_t *ResizeAnyVector(const reflection::Schema &schema, uoffset_t newsize,
// Clear elements we're throwing away, since some might remain in the
// buffer.
auto size_clear = -delta_elem * elem_size;
memset(flatbuf->data() + start - size_clear, 0, size_clear);
memset(vector_data(*flatbuf) + start - size_clear, 0, size_clear);
}
ResizeContext(schema, start, delta_bytes, flatbuf, root_table);
WriteScalar(flatbuf->data() + vec_start, newsize); // Length field.
WriteScalar(vector_data(*flatbuf) + vec_start, newsize); // Length field.
// Set new elements to 0.. this can be overwritten by the caller.
if (delta_elem > 0) {
memset(flatbuf->data() + start, 0, delta_elem * elem_size);
memset(vector_data(*flatbuf) + start, 0, delta_elem * elem_size);
}
}
return flatbuf->data() + start;
return vector_data(*flatbuf) + start;
}
const uint8_t *AddFlatBuffer(std::vector<uint8_t> &flatbuf,
@@ -349,7 +350,7 @@ const uint8_t *AddFlatBuffer(std::vector<uint8_t> &flatbuf,
// Insert the entire FlatBuffer minus the root pointer.
flatbuf.insert(flatbuf.end(), newbuf + sizeof(uoffset_t), newbuf + newlen);
auto root_offset = ReadScalar<uoffset_t>(newbuf) - sizeof(uoffset_t);
return flatbuf.data() + insertion_point + root_offset;
return vector_data(flatbuf) + insertion_point + root_offset;
}
void CopyInline(FlatBufferBuilder &fbb, const reflection::Field &fielddef,