From 4d7810424c8f964dbcb8dd3179d8c46cd896c4dc Mon Sep 17 00:00:00 2001 From: Wouter van Oortmerssen Date: Mon, 28 Sep 2015 15:02:41 -0700 Subject: [PATCH] Allow structs to be parsed in JSON with out of order fields. Also simplified the code and made it faster. Change-Id: I1d83b1165a4a9a4380d1bfb5538769c012d2d367 Tested: on Linux. --- include/flatbuffers/idl.h | 9 ++-- src/idl_parser.cpp | 71 ++++++++++++++---------------- tests/MyGame/Example/Monster.cs | 1 + tests/MyGame/Example/Monster.go | 1 + tests/MyGame/Example/Monster.java | 3 ++ tests/MyGame/Example/Monster.py | 1 + tests/monster_test_generated.h | 1 + tests/monsterdata_test.json | 4 +- tests/monsterdata_test.mon | Bin 352 -> 352 bytes tests/test.cpp | 2 +- 10 files changed, 46 insertions(+), 47 deletions(-) diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 82fb18bed..ffd612a1a 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -221,8 +221,7 @@ struct Definition { }; struct FieldDef : public Definition { - FieldDef() : deprecated(false), required(false), key(false), padding(0), - used(false) {} + FieldDef() : deprecated(false), required(false), key(false), padding(0) {} Offset Serialize(FlatBufferBuilder *builder, uint16_t id) const; @@ -233,7 +232,6 @@ struct FieldDef : public Definition { bool required; // Field must always be present. bool key; // Field functions as a key for creating sorted vectors. size_t padding; // Bytes to always pad after this field. - bool used; // Used during JSON parsing to check for repeated fields. }; struct StructDef : public Definition { @@ -378,8 +376,8 @@ class Parser { const std::string &name, const Type &type); void ParseField(StructDef &struct_def); - void ParseAnyValue(Value &val, FieldDef *field); - uoffset_t ParseTable(const StructDef &struct_def); + void ParseAnyValue(Value &val, FieldDef *field, size_t parent_fieldn); + uoffset_t ParseTable(const StructDef &struct_def, std::string *value); void SerializeStruct(const StructDef &struct_def, const Value &val); void AddVector(bool sortbysize, int count); uoffset_t ParseVector(const Type &type); @@ -428,7 +426,6 @@ class Parser { std::vector doc_comment_; std::vector> field_stack_; - std::vector struct_stack_; std::set known_attributes_; diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index 401b6899a..3d57b6bab 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -531,22 +531,22 @@ void Parser::ParseField(StructDef &struct_def) { Expect(';'); } -void Parser::ParseAnyValue(Value &val, FieldDef *field) { +void Parser::ParseAnyValue(Value &val, FieldDef *field, size_t parent_fieldn) { switch (val.type.base_type) { case BASE_TYPE_UNION: { assert(field); - if (!field_stack_.size() || + if (!parent_fieldn || field_stack_.back().second->value.type.base_type != BASE_TYPE_UTYPE) Error("missing type field before this union value: " + field->name); auto enum_idx = atot( field_stack_.back().first.constant.c_str()); auto enum_val = val.type.enum_def->ReverseLookup(enum_idx); if (!enum_val) Error("illegal type id for: " + field->name); - val.constant = NumToString(ParseTable(*enum_val->struct_def)); + ParseTable(*enum_val->struct_def, &val.constant); break; } case BASE_TYPE_STRUCT: - val.constant = NumToString(ParseTable(*val.type.struct_def)); + ParseTable(*val.type.struct_def, &val.constant); break; case BASE_TYPE_STRING: { auto s = attribute_; @@ -578,15 +578,14 @@ void Parser::ParseAnyValue(Value &val, FieldDef *field) { } void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) { - auto off = atot(val.constant.c_str()); - assert(struct_stack_.size() - off == struct_def.bytesize); + assert(val.constant.length() == struct_def.bytesize); builder_.Align(struct_def.minalign); - builder_.PushBytes(&struct_stack_[off], struct_def.bytesize); - struct_stack_.resize(struct_stack_.size() - struct_def.bytesize); + builder_.PushBytes(reinterpret_cast(val.constant.c_str()), + struct_def.bytesize); builder_.AddStructOffset(val.offset, builder_.GetSize()); } -uoffset_t Parser::ParseTable(const StructDef &struct_def) { +uoffset_t Parser::ParseTable(const StructDef &struct_def, std::string *value) { Expect('{'); size_t fieldn = 0; for (;;) { @@ -596,30 +595,26 @@ uoffset_t Parser::ParseTable(const StructDef &struct_def) { Expect(strict_json_ ? kTokenStringConstant : kTokenIdentifier); auto field = struct_def.fields.Lookup(name); if (!field) Error("unknown field: " + name); - if (struct_def.fixed && (fieldn >= struct_def.fields.vec.size() - || struct_def.fields.vec[fieldn] != field)) { - Error("struct field appearing out of order: " + name); - } Expect(':'); Value val = field->value; - ParseAnyValue(val, field); - field_stack_.push_back(std::make_pair(val, field)); + ParseAnyValue(val, field, fieldn); + size_t i = field_stack_.size(); + // Hardcoded insertion-sort with error-check. + // If fields are specified in order, then this loop exits immediately. + for (; i > field_stack_.size() - fieldn; i--) { + auto existing_field = field_stack_[i - 1].second; + if (existing_field == field) + Error("field set more than once: " + field->name); + if (existing_field->value.offset < field->value.offset) break; + } + field_stack_.insert(field_stack_.begin() + i, std::make_pair(val, field)); fieldn++; if (IsNext('}')) break; Expect(','); } - for (auto it = field_stack_.rbegin(); - it != field_stack_.rbegin() + fieldn; ++it) { - if (it->second->used) - Error("field set more than once: " + it->second->name); - it->second->used = true; - } - for (auto it = field_stack_.rbegin(); - it != field_stack_.rbegin() + fieldn; ++it) { - it->second->used = false; - } if (struct_def.fixed && fieldn != struct_def.fields.vec.size()) - Error("incomplete struct initialization: " + struct_def.name); + Error("struct: wrong number of initializers: " + struct_def.name); + auto start = struct_def.fixed ? builder_.StartStruct(struct_def.minalign) : builder_.StartTable(); @@ -670,19 +665,20 @@ uoffset_t Parser::ParseTable(const StructDef &struct_def) { if (struct_def.fixed) { builder_.ClearOffsets(); builder_.EndStruct(); - // Temporarily store this struct in a side buffer, since this data has to - // be stored in-line later in the parent object. - auto off = struct_stack_.size(); - struct_stack_.insert(struct_stack_.end(), - builder_.GetCurrentBufferPointer(), - builder_.GetCurrentBufferPointer() + - struct_def.bytesize); + assert(value); + // Temporarily store this struct in the value string, since it is to + // be serialized in-place elsewhere. + value->assign( + reinterpret_cast(builder_.GetCurrentBufferPointer()), + struct_def.bytesize); builder_.PopBytes(struct_def.bytesize); - return static_cast(off); + return 0xFFFFFFFF; // Value not used by the caller. } else { - return builder_.EndTable( + auto off = builder_.EndTable( start, static_cast(struct_def.fields.vec.size())); + if (value) *value = NumToString(off); + return off; } } @@ -692,7 +688,7 @@ uoffset_t Parser::ParseVector(const Type &type) { if ((!strict_json_ || !count) && IsNext(']')) break; Value val; val.type = type; - ParseAnyValue(val, nullptr); + ParseAnyValue(val, nullptr, 0); field_stack_.push_back(std::make_pair(val, nullptr)); count++; if (IsNext(']')) break; @@ -1435,7 +1431,7 @@ bool Parser::Parse(const char *source, const char **include_paths, if (builder_.GetSize()) { Error("cannot have more than one json object in a file"); } - builder_.Finish(Offset(ParseTable(*root_struct_def_)), + builder_.Finish(Offset
(ParseTable(*root_struct_def_, nullptr)), file_identifier_.length() ? file_identifier_.c_str() : nullptr); } else if (token_ == kTokenEnum) { ParseEnum(false); @@ -1507,7 +1503,6 @@ bool Parser::Parse(const char *source, const char **include_paths, return false; } if (source_filename) files_being_parsed_.pop(); - assert(!struct_stack_.size()); return true; } diff --git a/tests/MyGame/Example/Monster.cs b/tests/MyGame/Example/Monster.cs index 0076595af..258ae45cd 100644 --- a/tests/MyGame/Example/Monster.cs +++ b/tests/MyGame/Example/Monster.cs @@ -5,6 +5,7 @@ namespace MyGame.Example using FlatBuffers; +/// an example documentation comment: monster object public sealed class Monster : Table { public static Monster GetRootAsMonster(ByteBuffer _bb) { return GetRootAsMonster(_bb, new Monster()); } public static Monster GetRootAsMonster(ByteBuffer _bb, Monster obj) { return (obj.__init(_bb.GetInt(_bb.Position) + _bb.Position, _bb)); } diff --git a/tests/MyGame/Example/Monster.go b/tests/MyGame/Example/Monster.go index 6c0c3659d..ee9d51076 100644 --- a/tests/MyGame/Example/Monster.go +++ b/tests/MyGame/Example/Monster.go @@ -5,6 +5,7 @@ package Example import ( flatbuffers "github.com/google/flatbuffers/go" ) +/// an example documentation comment: monster object type Monster struct { _tab flatbuffers.Table } diff --git a/tests/MyGame/Example/Monster.java b/tests/MyGame/Example/Monster.java index 9f9759010..a098a994b 100644 --- a/tests/MyGame/Example/Monster.java +++ b/tests/MyGame/Example/Monster.java @@ -8,6 +8,9 @@ import java.util.*; import com.google.flatbuffers.*; @SuppressWarnings("unused") +/** + * an example documentation comment: monster object + */ public final class Monster extends Table { public static Monster getRootAsMonster(ByteBuffer _bb) { return getRootAsMonster(_bb, new Monster()); } public static Monster getRootAsMonster(ByteBuffer _bb, Monster obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__init(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } diff --git a/tests/MyGame/Example/Monster.py b/tests/MyGame/Example/Monster.py index 6122de59e..171355c62 100644 --- a/tests/MyGame/Example/Monster.py +++ b/tests/MyGame/Example/Monster.py @@ -4,6 +4,7 @@ import flatbuffers +# /// an example documentation comment: monster object class Monster(object): __slots__ = ['_tab'] diff --git a/tests/monster_test_generated.h b/tests/monster_test_generated.h index 6564a9bd7..ece639808 100644 --- a/tests/monster_test_generated.h +++ b/tests/monster_test_generated.h @@ -167,6 +167,7 @@ inline flatbuffers::Offset CreateStat(flatbuffers::FlatBufferBuilder &_fbb return builder_.Finish(); } +/// an example documentation comment: monster object struct Monster FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { const Vec3 *pos() const { return GetStruct(4); } Vec3 *mutable_pos() { return GetStruct(4); } diff --git a/tests/monsterdata_test.json b/tests/monsterdata_test.json index a718efa0d..7ed39a8de 100755 --- a/tests/monsterdata_test.json +++ b/tests/monsterdata_test.json @@ -29,8 +29,8 @@ b: 20 }, { - a: 30, - b: 40 + b: 40, + a: 30 } ], testarrayofstring: [ diff --git a/tests/monsterdata_test.mon b/tests/monsterdata_test.mon index 0505aeeb4b36920b1995a2adfe60022ef627e6d0..eff1e66ca2a5b60af52b6deeffcf0eaba91dd00d 100644 GIT binary patch delta 51 zcmaFB^nhuCnV<`U2SWry0)r1j07C{t0Yk_{cUKX|la+TtxNu2h9|)^3FfdG9Z~y?C CxDoaM delta 51 zcmaFB^nhuCnV<)Q4?_e)0z&{p2tx)#0fWm#cUKV;1_lPlla+TtxNu2h9|%ueZ~y>% C_!0&H diff --git a/tests/test.cpp b/tests/test.cpp index f068b480f..d25fa1358 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -738,7 +738,7 @@ void ErrorTest() { TestError("table X { Y:int; } root_type X; { \"Y\":1, }", "string constant", true); TestError("struct X { Y:int; Z:int; } table W { V:X; } root_type W; " - "{ V:{ Y:1 } }", "incomplete"); + "{ V:{ Y:1 } }", "wrong number"); TestError("enum E:byte { A } table X { Y:E; } root_type X; { Y:U }", "unknown enum value"); TestError("table X { Y:byte; } root_type X; { Y:; }", "starting");