BigfootDev/flatbuffers
1
0
Fork 1
mirror of https://github.com/google/flatbuffers.git synced 2026-07-01 11:51:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed vector of union JSON parsing.

Browse Source 
This for some reason never had a test case, and was broken.

Change-Id: If832f5eb8b6c5ba8a75257464892634b38719c55
This commit is contained in:
Wouter van Oortmerssen
2019-03-04 14:56:07 -08:00
parent cb7b2bf87e
commit 71628dad0d
4 changed files with 122 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
include/flatbuffers/idl.h
View File

@@ -719,7 +719,9 @@ class Parser : public ParserState {
FLATBUFFERS_CHECKED_ERROR ParseComma(); FLATBUFFERS_CHECKED_ERROR ParseComma();
FLATBUFFERS_CHECKED_ERROR ParseAnyValue(Value &val, FieldDef *field, FLATBUFFERS_CHECKED_ERROR ParseAnyValue(Value &val, FieldDef *field,
size_t parent_fieldn, size_t parent_fieldn,
const StructDef *parent_struct_def); const StructDef *parent_struct_def,
uoffset_t count,
bool inside_vector = false);
template<typename F> template<typename F>
FLATBUFFERS_CHECKED_ERROR ParseTableDelimiters(size_t &fieldn, FLATBUFFERS_CHECKED_ERROR ParseTableDelimiters(size_t &fieldn,
const StructDef *struct_def, const StructDef *struct_def,
@@ -728,8 +730,9 @@ class Parser : public ParserState {
std::string *value, uoffset_t *ovalue); std::string *value, uoffset_t *ovalue);
void SerializeStruct(const StructDef &struct_def, const Value &val); void SerializeStruct(const StructDef &struct_def, const Value &val);
template<typename F> template<typename F>
FLATBUFFERS_CHECKED_ERROR ParseVectorDelimiters(size_t &count, F body); FLATBUFFERS_CHECKED_ERROR ParseVectorDelimiters(uoffset_t &count, F body);
FLATBUFFERS_CHECKED_ERROR ParseVector(const Type &type, uoffset_t *ovalue); FLATBUFFERS_CHECKED_ERROR ParseVector(const Type &type, uoffset_t *ovalue,
FieldDef *field, size_t fieldn);
FLATBUFFERS_CHECKED_ERROR ParseNestedFlatbuffer(Value &val, FieldDef *field, FLATBUFFERS_CHECKED_ERROR ParseNestedFlatbuffer(Value &val, FieldDef *field,
size_t fieldn, size_t fieldn,
const StructDef *parent_struct_def); const StructDef *parent_struct_def);
@@ -775,7 +778,7 @@ class Parser : public ParserState {
const char *suffix, const char *suffix,
BaseType baseType); BaseType baseType);
bool SupportsVectorOfUnions() const; bool SupportsAdvancedUnionFeatures() const;
Namespace *UniqueNamespace(Namespace *ns); Namespace *UniqueNamespace(Namespace *ns);
FLATBUFFERS_CHECKED_ERROR RecurseError(); FLATBUFFERS_CHECKED_ERROR RecurseError();

101
src/idl_parser.cpp
View File

@@ -650,7 +650,7 @@ CheckedError Parser::ParseField(StructDef &struct_def) {
} else if (type.base_type == BASE_TYPE_VECTOR && } else if (type.base_type == BASE_TYPE_VECTOR &&
type.element == BASE_TYPE_UNION) { type.element == BASE_TYPE_UNION) {
// Only cpp, js and ts supports the union vector feature so far. // Only cpp, js and ts supports the union vector feature so far.
if (!SupportsVectorOfUnions()) { if (!SupportsAdvancedUnionFeatures()) {
return Error( return Error(
"Vectors of unions are not yet supported in all " "Vectors of unions are not yet supported in all "
"the specified programming languages."); "the specified programming languages.");
@@ -843,25 +843,45 @@ CheckedError Parser::ParseComma() {
CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field, CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
size_t parent_fieldn, size_t parent_fieldn,
const StructDef *parent_struct_def) { const StructDef *parent_struct_def,
uoffset_t count,
bool inside_vector) {
switch (val.type.base_type) { switch (val.type.base_type) {
case BASE_TYPE_UNION: { case BASE_TYPE_UNION: {
FLATBUFFERS_ASSERT(field); FLATBUFFERS_ASSERT(field);
std::string constant; std::string constant;
Vector<uint8_t> *vector_of_union_types = nullptr;
// Find corresponding type field we may have already parsed. // Find corresponding type field we may have already parsed.
for (auto elem = field_stack_.rbegin(); for (auto elem = field_stack_.rbegin() + count;
elem != field_stack_.rbegin() + parent_fieldn; ++elem) { elem != field_stack_.rbegin() + parent_fieldn + count; ++elem) {
auto &type = elem->second->value.type; auto &type = elem->second->value.type;
if (type.base_type == BASE_TYPE_UTYPE && if (type.enum_def == val.type.enum_def) {
type.enum_def == val.type.enum_def) { if (inside_vector) {
constant = elem->first.constant; if (type.base_type == BASE_TYPE_VECTOR &&
break; type.element == BASE_TYPE_UTYPE) {
// Vector of union type field.
uoffset_t offset;
ECHECK(atot(elem->first.constant.c_str(), *this, &offset));
vector_of_union_types = reinterpret_cast<Vector<uint8_t> *>(
builder_.GetCurrentBufferPointer() +
builder_.GetSize() - offset);
break;
}
} else {
if (type.base_type == BASE_TYPE_UTYPE) {
// Union type field.
constant = elem->first.constant;
break;
}
}
} }
} }
if (constant.empty()) { if (constant.empty() && !inside_vector) {
// We haven't seen the type field yet. Sadly a lot of JSON writers // We haven't seen the type field yet. Sadly a lot of JSON writers
// output these in alphabetical order, meaning it comes after this // output these in alphabetical order, meaning it comes after this
// value. So we scan past the value to find it, then come back here. // value. So we scan past the value to find it, then come back here.
// We currently don't do this for vectors of unions because the
// scanning/serialization logic would get very complicated.
auto type_name = field->name + UnionTypeFieldSuffix(); auto type_name = field->name + UnionTypeFieldSuffix();
FLATBUFFERS_ASSERT(parent_struct_def); FLATBUFFERS_ASSERT(parent_struct_def);
auto type_field = parent_struct_def->fields.Lookup(type_name); auto type_field = parent_struct_def->fields.Lookup(type_name);
@@ -876,18 +896,25 @@ CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
} else { } else {
EXPECT(kTokenIdentifier); EXPECT(kTokenIdentifier);
} }
if (next_name != type_name) if (next_name == type_name) {
return Error("missing type field after this union value: " + EXPECT(':');
type_name); Value type_val = type_field->value;
EXPECT(':'); ECHECK(ParseAnyValue(type_val, type_field, 0, nullptr, 0));
Value type_val = type_field->value; constant = type_val.constant;
ECHECK(ParseAnyValue(type_val, type_field, 0, nullptr)); // Got the information we needed, now rewind:
constant = type_val.constant; *static_cast<ParserState *>(this) = backup;
// Got the information we needed, now rewind: }
*static_cast<ParserState *>(this) = backup; }
if (constant.empty() && !vector_of_union_types) {
return Error("missing type field for this union value: " +
field->name);
} }
uint8_t enum_idx; uint8_t enum_idx;
ECHECK(atot(constant.c_str(), *this, &enum_idx)); if (vector_of_union_types) {
enum_idx = vector_of_union_types->Get(count);
} else {
ECHECK(atot(constant.c_str(), *this, &enum_idx));
}
auto enum_val = val.type.enum_def->ReverseLookup(enum_idx); auto enum_val = val.type.enum_def->ReverseLookup(enum_idx);
if (!enum_val) return Error("illegal type id for: " + field->name); if (!enum_val) return Error("illegal type id for: " + field->name);
if (enum_val->union_type.base_type == BASE_TYPE_STRUCT) { if (enum_val->union_type.base_type == BASE_TYPE_STRUCT) {
@@ -915,7 +942,7 @@ CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
} }
case BASE_TYPE_VECTOR: { case BASE_TYPE_VECTOR: {
uoffset_t off; uoffset_t off;
ECHECK(ParseVector(val.type.VectorType(), &off)); ECHECK(ParseVector(val.type.VectorType(), &off, field, parent_fieldn));
val.constant = NumToString(off); val.constant = NumToString(off);
break; break;
} }
@@ -1026,7 +1053,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
ParseNestedFlatbuffer(val, field, fieldn, struct_def_inner)); ParseNestedFlatbuffer(val, field, fieldn, struct_def_inner));
} else { } else {
ECHECK(Recurse([&]() { ECHECK(Recurse([&]() {
return ParseAnyValue(val, field, fieldn, struct_def_inner); return ParseAnyValue(val, field, fieldn, struct_def_inner, 0);
})); }));
} }
// Hardcoded insertion-sort with error-check. // Hardcoded insertion-sort with error-check.
@@ -1144,7 +1171,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
} }
template <typename F> template <typename F>
CheckedError Parser::ParseVectorDelimiters(size_t &count, F body) { CheckedError Parser::ParseVectorDelimiters(uoffset_t &count, F body) {
EXPECT('['); EXPECT('[');
for (;;) { for (;;) {
if ((!opts.strict_json || !count) && Is(']')) break; if ((!opts.strict_json || !count) && Is(']')) break;
@@ -1157,12 +1184,15 @@ CheckedError Parser::ParseVectorDelimiters(size_t &count, F body) {
return NoError(); return NoError();
} }
CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) { CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue,
size_t count = 0; FieldDef *field, size_t fieldn) {
auto err = ParseVectorDelimiters(count, [&](size_t &) -> CheckedError { uoffset_t count = 0;
auto err = ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
Value val; Value val;
val.type = type; val.type = type;
ECHECK(Recurse([&]() { return ParseAnyValue(val, nullptr, 0, nullptr); })); ECHECK(Recurse([&]() {
return ParseAnyValue(val, field, fieldn, nullptr, count, true);
}));
field_stack_.push_back(std::make_pair(val, nullptr)); field_stack_.push_back(std::make_pair(val, nullptr));
return NoError(); return NoError();
}); });
@@ -1170,7 +1200,7 @@ CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) {
builder_.StartVector(count * InlineSize(type) / InlineAlignment(type), builder_.StartVector(count * InlineSize(type) / InlineAlignment(type),
InlineAlignment(type)); InlineAlignment(type));
for (size_t i = 0; i < count; i++) { for (uoffset_t i = 0; i < count; i++) {
// start at the back, since we're building the data backwards. // start at the back, since we're building the data backwards.
auto &val = field_stack_.back().first; auto &val = field_stack_.back().first;
switch (val.type.base_type) { switch (val.type.base_type) {
@@ -1201,7 +1231,7 @@ CheckedError Parser::ParseNestedFlatbuffer(Value &val, FieldDef *field,
size_t fieldn, size_t fieldn,
const StructDef *parent_struct_def) { const StructDef *parent_struct_def) {
if (token_ == '[') { // backwards compat for 'legacy' ubyte buffers if (token_ == '[') { // backwards compat for 'legacy' ubyte buffers
ECHECK(ParseAnyValue(val, field, fieldn, parent_struct_def)); ECHECK(ParseAnyValue(val, field, fieldn, parent_struct_def, 0));
} else { } else {
auto cursor_at_value_begin = cursor_; auto cursor_at_value_begin = cursor_;
ECHECK(SkipAnyJsonValue()); ECHECK(SkipAnyJsonValue());
@@ -1757,11 +1787,12 @@ CheckedError Parser::CheckClash(std::vector<FieldDef *> &fields,
return NoError(); return NoError();
} }
bool Parser::SupportsVectorOfUnions() const { bool Parser::SupportsAdvancedUnionFeatures() const {
return opts.lang_to_generate != 0 && return opts.lang_to_generate != 0 &&
(opts.lang_to_generate & ~(IDLOptions::kCpp | IDLOptions::kJs | (opts.lang_to_generate & ~(IDLOptions::kCpp | IDLOptions::kJs |
IDLOptions::kTs | IDLOptions::kPhp | IDLOptions::kTs | IDLOptions::kPhp |
IDLOptions::kJava | IDLOptions::kCSharp)) == 0; IDLOptions::kJava | IDLOptions::kCSharp |
IDLOptions::kBinary)) == 0;
} }
Namespace *Parser::UniqueNamespace(Namespace *ns) { Namespace *Parser::UniqueNamespace(Namespace *ns) {
@@ -2284,8 +2315,8 @@ CheckedError Parser::SkipAnyJsonValue() {
}); });
} }
case '[': { case '[': {
size_t count = 0; uoffset_t count = 0;
return ParseVectorDelimiters(count, [&](size_t &) -> CheckedError { return ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
return Recurse([&]() { return SkipAnyJsonValue(); }); return Recurse([&]() { return SkipAnyJsonValue(); });
}); });
} }
@@ -2321,8 +2352,8 @@ CheckedError Parser::ParseFlexBufferValue(flexbuffers::Builder *builder) {
} }
case '[': { case '[': {
auto start = builder->StartVector(); auto start = builder->StartVector();
size_t count = 0; uoffset_t count = 0;
ECHECK(ParseVectorDelimiters(count, [&](size_t &) -> CheckedError { ECHECK(ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
return ParseFlexBufferValue(builder); return ParseFlexBufferValue(builder);
})); }));
builder->EndVector(start, false, false); builder->EndVector(start, false, false);
@@ -2454,7 +2485,7 @@ CheckedError Parser::ParseRoot(const char *source, const char **include_paths,
for (auto val_it = enum_def.vals.vec.begin(); for (auto val_it = enum_def.vals.vec.begin();
val_it != enum_def.vals.vec.end(); ++val_it) { val_it != enum_def.vals.vec.end(); ++val_it) {
auto &val = **val_it; auto &val = **val_it;
if (!SupportsVectorOfUnions() && val.union_type.struct_def && if (!SupportsAdvancedUnionFeatures() && val.union_type.struct_def &&
val.union_type.struct_def->fixed) val.union_type.struct_def->fixed)
return Error( return Error(
"only tables can be union elements in the generated language: " + "only tables can be union elements in the generated language: " +

29
tests/test.cpp
View File

@@ -2017,17 +2017,20 @@ void InvalidNestedFlatbufferTest() {
} }
void UnionVectorTest() { void UnionVectorTest() {
// load FlatBuffer fbs schema. // load FlatBuffer fbs schema and json.
// TODO: load a JSON file with such a vector when JSON support is ready. std::string schemafile, jsonfile;
std::string schemafile;
TEST_EQ(flatbuffers::LoadFile( TEST_EQ(flatbuffers::LoadFile(
(test_data_path + "union_vector/union_vector.fbs").c_str(), false, (test_data_path + "union_vector/union_vector.fbs").c_str(),
&schemafile), false, &schemafile),
true);
TEST_EQ(flatbuffers::LoadFile(
(test_data_path + "union_vector/union_vector.json").c_str(),
false, &jsonfile),
true); true);
// parse schema. // parse schema.
flatbuffers::IDLOptions idl_opts; flatbuffers::IDLOptions idl_opts;
idl_opts.lang_to_generate |= flatbuffers::IDLOptions::kCpp; idl_opts.lang_to_generate |= flatbuffers::IDLOptions::kBinary;
flatbuffers::Parser parser(idl_opts); flatbuffers::Parser parser(idl_opts);
TEST_EQ(parser.Parse(schemafile.c_str()), true); TEST_EQ(parser.Parse(schemafile.c_str()), true);
@@ -2093,6 +2096,13 @@ void UnionVectorTest() {
TestMovie(flat_movie); TestMovie(flat_movie);
// Also test the JSON we loaded above.
TEST_EQ(parser.Parse(jsonfile.c_str()), true);
auto jbuf = parser.builder_.GetBufferPointer();
flatbuffers::Verifier jverifier(jbuf, parser.builder_.GetSize());
TEST_EQ(VerifyMovieBuffer(jverifier), true);
TestMovie(GetMovie(jbuf));
auto movie_object = flat_movie->UnPack(); auto movie_object = flat_movie->UnPack();
TEST_EQ(movie_object->main_character.AsRapunzel()->hair_length(), 6); TEST_EQ(movie_object->main_character.AsRapunzel()->hair_length(), 6);
TEST_EQ(movie_object->characters[0].AsBelle()->books_read(), 7); TEST_EQ(movie_object->characters[0].AsBelle()->books_read(), 7);
@@ -2150,6 +2160,13 @@ void UnionVectorTest() {
" \"Unused\"\n" " \"Unused\"\n"
" ]\n" " ]\n"
"}"); "}");
flatbuffers::Parser parser2(idl_opts);
TEST_EQ(parser2.Parse("struct Bool { b:bool; }"
"union Any { Bool }"
"table Root { a:Any; }"
"root_type Root;"), true);
TEST_EQ(parser2.Parse("{a_type:Bool,a:{b:true}}"), true);
} }
void ConformTest() { void ConformTest() {

26
tests/union_vector/union_vector.json Normal file
View File

@@ -0,0 +1,26 @@
{
"main_character_type": "Rapunzel",
"main_character": {
"hair_length": 6
},
"characters_type": [
"Belle",
"MuLan",
"BookFan",
"Other",
"Unused"
],
"characters": [
{
"books_read": 7
},
{
"sword_attack_damage": 5
},
{
"books_read": 2
},
"Other",
"Unused"
]
}