From 8f864aad7b875d645236be59ab1037051b12ba10 Mon Sep 17 00:00:00 2001 From: Wouter van Oortmerssen Date: Mon, 5 Jun 2017 17:45:44 -0700 Subject: [PATCH] Added (nested) FlexBuffer JSON parsing and output. FlexBuffer parser is just 40 lines of code (on top of existing parser!). Change-Id: Idebebadafb661ca5333f5621139031f6df3c3e1a Tested: on Linux. --- include/flatbuffers/flexbuffers.h | 67 +++++++-- include/flatbuffers/idl.h | 24 +++- include/flatbuffers/util.h | 71 ++++++++- src/idl_gen_cpp.cpp | 2 +- src/idl_gen_text.cpp | 74 ++-------- src/idl_parser.cpp | 232 ++++++++++++++++++------------ src/reflection.cpp | 7 +- tests/monster_test.bfbs | Bin 4320 -> 4336 bytes tests/monsterdata_test.golden | 11 +- tests/test.cpp | 36 +++-- 10 files changed, 332 insertions(+), 192 deletions(-) diff --git a/include/flatbuffers/flexbuffers.h b/include/flatbuffers/flexbuffers.h index e75451570..67f82b94c 100644 --- a/include/flatbuffers/flexbuffers.h +++ b/include/flatbuffers/flexbuffers.h @@ -457,25 +457,61 @@ class Reference { } // Unlike AsString(), this will convert any type to a std::string. - std::string ToString() const { + std::string ToString() { + std::string s; + ToString(false, false, s); + return s; + } + + // Convert any type to a JSON-like string. strings_quoted determines if + // string values at the top level receive "" quotes (inside other values + // they always do). keys_quoted determines if keys are quoted, at any level. + // TODO(wvo): add further options to have indentation/newlines. + void ToString(bool strings_quoted, bool keys_quoted, std::string &s) const { if (type_ == TYPE_STRING) { - return String(Indirect(), byte_width_).c_str(); + String str(Indirect(), byte_width_); + if (strings_quoted) { + flatbuffers::EscapeString(str.c_str(), str.length(), &s, true); + } else { + s.append(str.c_str(), str.length()); + } } else if (IsKey()) { - return AsKey(); + auto str = AsKey(); + if (keys_quoted) { + flatbuffers::EscapeString(str, strlen(str), &s, true); + } else { + s += str; + } } else if (IsInt()) { - return flatbuffers::NumToString(AsInt64()); + s += flatbuffers::NumToString(AsInt64()); } else if (IsUInt()) { - return flatbuffers::NumToString(AsUInt64()); + s += flatbuffers::NumToString(AsUInt64()); } else if (IsFloat()) { - return flatbuffers::NumToString(AsDouble()); + s += flatbuffers::NumToString(AsDouble()); } else if (IsNull()) { - return "null"; + s += "null"; } else if (IsMap()) { - return "{..}"; // TODO: show elements. + s += "{ "; + auto m = AsMap(); + auto keys = m.Keys(); + auto vals = m.Values(); + for (size_t i = 0; i < keys.size(); i++) { + keys[i].ToString(true, keys_quoted, s); + s += ": "; + vals[i].ToString(true, keys_quoted, s); + if (i < keys.size() - 1) s += ", "; + } + s += " }"; } else if (IsVector()) { - return "[..]"; // TODO: show elements. + s += "[ "; + auto v = AsVector(); + for (size_t i = 0; i < v.size(); i++) { + v[i].ToString(true, keys_quoted, s); + if (i < v.size() - 1) s += ", "; + } + s += " ]"; } else { - return "(?)"; + s += "(?)"; } } @@ -746,6 +782,17 @@ class Builder FLATBUFFERS_FINAL_CLASS { return buf_; } + // Reset all state so we can re-use the buffer. + void Clear() { + buf_.clear(); + stack_.clear(); + finished_ = false; + // flags_ remains as-is; + force_min_bit_width_ = BIT_WIDTH_8; + key_pool.clear(); + string_pool.clear(); + } + // All value constructing functions below have two versions: one that // takes a key (for placement inside a map) and one that doesn't (for inside // vectors and elsewhere). diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 4e3a24a67..ab0421fe5 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -25,6 +25,7 @@ #include "flatbuffers/flatbuffers.h" #include "flatbuffers/hash.h" #include "flatbuffers/reflection.h" +#include "flatbuffers/flexbuffers.h" // This file defines the data types representing a parsed IDL (Interface // Definition Language) / schema file. @@ -226,18 +227,20 @@ struct Definition { }; struct FieldDef : public Definition { - FieldDef() : deprecated(false), required(false), key(false), padding(0) {} + FieldDef() : deprecated(false), required(false), key(false), + flexbuffer(false), padding(0) {} Offset Serialize(FlatBufferBuilder *builder, uint16_t id, const Parser &parser) const; Value value; - bool deprecated; // Field is allowed to be present in old data, but can't be + bool deprecated; // Field is allowed to be present in old data, but can't be. // written in new data nor accessed in new code. bool required; // Field must always be present. bool key; // Field functions as a key for creating sorted vectors. bool native_inline; // Field will be defined inline (instead of as a pointer) // for native tables if field is a struct. + bool flexbuffer; // This field contains FlexBuffer data. size_t padding; // Bytes to always pad after this field. }; @@ -536,6 +539,11 @@ class Parser : public ParserState { // of the schema provided. Returns non-empty error on any problems. std::string ConformTo(const Parser &base); + // Similar to Parse(), but now only accepts JSON to be parsed into a + // FlexBuffer. + bool ParseFlexBuffer(const char *source, const char *source_filename, + flexbuffers::Builder *builder); + FLATBUFFERS_CHECKED_ERROR CheckInRange(int64_t val, int64_t min, int64_t max); private: @@ -559,15 +567,21 @@ private: FLATBUFFERS_CHECKED_ERROR ParseAnyValue(Value &val, FieldDef *field, size_t parent_fieldn, const StructDef *parent_struct_def); + FLATBUFFERS_CHECKED_ERROR ParseTableDelimiters(size_t &fieldn, + const StructDef *struct_def, + const std::function &body); FLATBUFFERS_CHECKED_ERROR ParseTable(const StructDef &struct_def, std::string *value, uoffset_t *ovalue); void SerializeStruct(const StructDef &struct_def, const Value &val); void AddVector(bool sortbysize, int count); + FLATBUFFERS_CHECKED_ERROR ParseVectorDelimiters(size_t &count, + const std::function &body); FLATBUFFERS_CHECKED_ERROR ParseVector(const Type &type, uoffset_t *ovalue); FLATBUFFERS_CHECKED_ERROR ParseMetaData(SymbolTable *attributes); FLATBUFFERS_CHECKED_ERROR TryTypedValue(int dtoken, bool check, Value &e, BaseType req, bool *destmatch); FLATBUFFERS_CHECKED_ERROR ParseHash(Value &e, FieldDef* field); + FLATBUFFERS_CHECKED_ERROR TokenError(); FLATBUFFERS_CHECKED_ERROR ParseSingleValue(Value &e); FLATBUFFERS_CHECKED_ERROR ParseEnumFromString(Type &type, int64_t *result); StructDef *LookupCreateStruct(const std::string &name, @@ -587,9 +601,9 @@ private: FLATBUFFERS_CHECKED_ERROR ParseProtoCurliesOrIdent(); FLATBUFFERS_CHECKED_ERROR ParseTypeFromProtoType(Type *type); FLATBUFFERS_CHECKED_ERROR SkipAnyJsonValue(); - FLATBUFFERS_CHECKED_ERROR SkipJsonObject(); - FLATBUFFERS_CHECKED_ERROR SkipJsonArray(); - FLATBUFFERS_CHECKED_ERROR SkipJsonString(); + FLATBUFFERS_CHECKED_ERROR ParseFlexBufferValue(flexbuffers::Builder *builder); + FLATBUFFERS_CHECKED_ERROR StartParseFile(const char *source, + const char *source_filename); FLATBUFFERS_CHECKED_ERROR DoParse(const char *_source, const char **include_paths, const char *source_filename, diff --git a/include/flatbuffers/util.h b/include/flatbuffers/util.h index baab3e574..ee62b86d3 100644 --- a/include/flatbuffers/util.h +++ b/include/flatbuffers/util.h @@ -72,9 +72,8 @@ template<> inline std::string NumToString(double t) { // Sadly, std::fixed turns "1" into "1.00000", so here we undo that. auto p = s.find_last_not_of('0'); if (p != std::string::npos) { - s.resize(p + 1); // Strip trailing zeroes. - if (s[s.size() - 1] == '.') - s.erase(s.size() - 1, 1); // Strip '.' if a whole number. + // Strip trailing zeroes. If it is a whole number, keep one zero. + s.resize(p + (s[p] == '.' ? 2 : 1)); } return s; } @@ -361,6 +360,72 @@ inline std::string WordWrap(const std::string in, size_t max_length, return wrapped; } +inline bool EscapeString(const char *s, size_t length, std::string *_text, + bool allow_non_utf8) { + std::string &text = *_text; + text += "\""; + for (uoffset_t i = 0; i < length; i++) { + char c = s[i]; + switch (c) { + case '\n': text += "\\n"; break; + case '\t': text += "\\t"; break; + case '\r': text += "\\r"; break; + case '\b': text += "\\b"; break; + case '\f': text += "\\f"; break; + case '\"': text += "\\\""; break; + case '\\': text += "\\\\"; break; + default: + if (c >= ' ' && c <= '~') { + text += c; + } else { + // Not printable ASCII data. Let's see if it's valid UTF-8 first: + const char *utf8 = s + i; + int ucc = FromUTF8(&utf8); + if (ucc < 0) { + if (allow_non_utf8) { + text += "\\x"; + text += IntToStringHex(static_cast(c), 2); + } else { + // There are two cases here: + // + // 1) We reached here by parsing an IDL file. In that case, + // we previously checked for non-UTF-8, so we shouldn't reach + // here. + // + // 2) We reached here by someone calling GenerateText() + // on a previously-serialized flatbuffer. The data might have + // non-UTF-8 Strings, or might be corrupt. + // + // In both cases, we have to give up and inform the caller + // they have no JSON. + return false; + } + } else { + if (ucc <= 0xFFFF) { + // Parses as Unicode within JSON's \uXXXX range, so use that. + text += "\\u"; + text += IntToStringHex(ucc, 4); + } else if (ucc <= 0x10FFFF) { + // Encode Unicode SMP values to a surrogate pair using two \u escapes. + uint32_t base = ucc - 0x10000; + auto high_surrogate = (base >> 10) + 0xD800; + auto low_surrogate = (base & 0x03FF) + 0xDC00; + text += "\\u"; + text += IntToStringHex(high_surrogate, 4); + text += "\\u"; + text += IntToStringHex(low_surrogate, 4); + } + // Skip past characters recognized. + i = static_cast(utf8 - s - 1); + } + } + break; + } + } + text += "\""; + return true; +} + } // namespace flatbuffers #endif // FLATBUFFERS_UTIL_H_ diff --git a/src/idl_gen_cpp.cpp b/src/idl_gen_cpp.cpp index d5c8b03f5..242c8d9bd 100644 --- a/src/idl_gen_cpp.cpp +++ b/src/idl_gen_cpp.cpp @@ -1343,7 +1343,7 @@ class CppGenerator : public BaseGenerator { code_ += " }"; } - if (field.attributes.Lookup("flexbuffer")) { + if (field.flexbuffer) { code_ += " flexbuffers::Reference {{FIELD_NAME}}_flexbuffer_root()" " const {"; code_ += " auto v = {{FIELD_NAME}}();"; diff --git a/src/idl_gen_text.cpp b/src/idl_gen_text.cpp index dc445e5c4..9d741e0ee 100644 --- a/src/idl_gen_text.cpp +++ b/src/idl_gen_text.cpp @@ -19,6 +19,7 @@ #include "flatbuffers/flatbuffers.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" +#include "flatbuffers/flexbuffers.h" namespace flatbuffers { @@ -101,71 +102,6 @@ template bool PrintVector(const Vector &v, Type type, return true; } -static bool EscapeString(const String &s, std::string *_text, const IDLOptions& opts) { - std::string &text = *_text; - text += "\""; - for (uoffset_t i = 0; i < s.size(); i++) { - char c = s[i]; - switch (c) { - case '\n': text += "\\n"; break; - case '\t': text += "\\t"; break; - case '\r': text += "\\r"; break; - case '\b': text += "\\b"; break; - case '\f': text += "\\f"; break; - case '\"': text += "\\\""; break; - case '\\': text += "\\\\"; break; - default: - if (c >= ' ' && c <= '~') { - text += c; - } else { - // Not printable ASCII data. Let's see if it's valid UTF-8 first: - const char *utf8 = s.c_str() + i; - int ucc = FromUTF8(&utf8); - if (ucc < 0) { - if (opts.allow_non_utf8) { - text += "\\x"; - text += IntToStringHex(static_cast(c), 2); - } else { - // There are two cases here: - // - // 1) We reached here by parsing an IDL file. In that case, - // we previously checked for non-UTF-8, so we shouldn't reach - // here. - // - // 2) We reached here by someone calling GenerateText() - // on a previously-serialized flatbuffer. The data might have - // non-UTF-8 Strings, or might be corrupt. - // - // In both cases, we have to give up and inform the caller - // they have no JSON. - return false; - } - } else { - if (ucc <= 0xFFFF) { - // Parses as Unicode within JSON's \uXXXX range, so use that. - text += "\\u"; - text += IntToStringHex(ucc, 4); - } else if (ucc <= 0x10FFFF) { - // Encode Unicode SMP values to a surrogate pair using two \u escapes. - uint32_t base = ucc - 0x10000; - auto high_surrogate = (base >> 10) + 0xD800; - auto low_surrogate = (base & 0x03FF) + 0xDC00; - text += "\\u"; - text += IntToStringHex(high_surrogate, 4); - text += "\\u"; - text += IntToStringHex(low_surrogate, 4); - } - // Skip past characters recognized. - i = static_cast(utf8 - s.c_str() - 1); - } - } - break; - } - } - text += "\""; - return true; -} - // Specialization of Print above for pointer types. template<> bool Print(const void *val, Type type, int indent, @@ -189,7 +125,8 @@ template<> bool Print(const void *val, } break; case BASE_TYPE_STRING: { - if (!EscapeString(*reinterpret_cast(val), _text, opts)) { + auto s = reinterpret_cast(val); + if (!EscapeString(s->c_str(), s->Length(), _text, opts.allow_non_utf8)) { return false; } break; @@ -238,6 +175,11 @@ static bool GenFieldOffset(const FieldDef &fd, const Table *table, bool fixed, assert(IsStruct(fd.value.type)); val = reinterpret_cast(table)-> GetStruct(fd.value.offset); + } else if (fd.flexbuffer) { + auto vec = table->GetPointer *>(fd.value.offset); + auto root = flexbuffers::GetRoot(vec->data(), vec->size()); + root.ToString(true, false, *_text); + return true; } else { val = IsStruct(fd.value.type) ? table->GetStruct(fd.value.offset) diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index c238bee0c..5a1b8708a 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -726,6 +726,7 @@ CheckedError Parser::ParseField(StructDef &struct_def) { } if (field->attributes.Lookup("flexbuffer")) { + field->flexbuffer = true; uses_flexbuffers_ = true; if (field->value.type.base_type != BASE_TYPE_VECTOR || field->value.type.element != BASE_TYPE_UCHAR) @@ -830,7 +831,6 @@ CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field, break; } case BASE_TYPE_VECTOR: { - EXPECT('['); uoffset_t off; ECHECK(ParseVector(val.type.VectorType(), &off)); val.constant = NumToString(off); @@ -863,27 +863,27 @@ void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) { builder_.AddStructOffset(val.offset, builder_.GetSize()); } -CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, - uoffset_t *ovalue) { - // We allow tables both as JSON object{ .. } with field names +CheckedError Parser::ParseTableDelimiters(size_t &fieldn, + const StructDef *struct_def, + const std::function &body) { + // We allow tables both as JSON object{ .. } with field names // or vector[..] with all fields in order - const bool is_nested_list = Is('['); - if (is_nested_list) { + char terminator = '}'; + bool is_nested_vector = struct_def && Is('['); + if (is_nested_vector) { NEXT(); + terminator = ']'; } else { EXPECT('{'); } - size_t fieldn = 0; for (;;) { - if ((!opts.strict_json || !fieldn) && Is(is_nested_list ? ']' : '}')) { NEXT(); break; } - FieldDef *field = nullptr; + if ((!opts.strict_json || !fieldn) && Is(terminator)) break; std::string name; - if (is_nested_list) { - if (fieldn > struct_def.fields.vec.size()) { + if (is_nested_vector) { + if (fieldn > struct_def->fields.vec.size()) { return Error("too many unnamed fields in nested array"); } - field = struct_def.fields.vec[fieldn]; - name = field->name; + name = struct_def->fields.vec[fieldn]->name; } else { name = attribute_; if (Is(kTokenStringConstant)) { @@ -891,24 +891,46 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, } else { EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier); } - field = struct_def.fields.Lookup(name); + EXPECT(':'); } + ECHECK(body(name)); + if (Is(terminator)) break; + EXPECT(','); + } + NEXT(); + if (is_nested_vector && fieldn != struct_def->fields.vec.size()) { + return Error("wrong number of unnamed fields in table vector"); + } + return NoError(); +} + +CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, + uoffset_t *ovalue) { + size_t fieldn = 0; + auto err = ParseTableDelimiters(fieldn, &struct_def, + [&](const std::string &name) -> CheckedError { + auto field = struct_def.fields.Lookup(name); if (!field) { if (!opts.skip_unexpected_fields_in_json) { return Error("unknown field: " + name); } else { - EXPECT(':'); ECHECK(SkipAnyJsonValue()); } } else { - if (!is_nested_list) { - EXPECT(':'); - } if (Is(kTokenNull)) { NEXT(); // Ignore this field. } else { Value val = field->value; - ECHECK(ParseAnyValue(val, field, fieldn, &struct_def)); + if (field->flexbuffer) { + flexbuffers::Builder builder(1024, + flexbuffers::BUILDER_FLAG_SHARE_ALL); + ECHECK(ParseFlexBufferValue(&builder)); + builder.Finish(); + auto off = builder_.CreateVector(builder.GetBuffer()); + val.constant = NumToString(off.o); + } else { + ECHECK(ParseAnyValue(val, field, fieldn, &struct_def)); + } // Hardcoded insertion-sort with error-check. // If fields are specified in order, then this loop exits immediately. auto elem = field_stack_.rbegin(); @@ -924,12 +946,9 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, fieldn++; } } - if (Is(is_nested_list ? ']' : '}')) { NEXT(); break; } - EXPECT(','); - } - if (is_nested_list && fieldn != struct_def.fields.vec.size()) { - return Error("wrong number of unnamed fields in table vector"); - } + return NoError(); + }); + ECHECK(err); // Check if all required fields are parsed. for (auto field_it = struct_def.fields.vec.begin(); @@ -1029,22 +1048,34 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, return NoError(); } -CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) { - int count = 0; +CheckedError Parser::ParseVectorDelimiters(size_t &count, + const std::function &body) { + EXPECT('['); for (;;) { - if ((!opts.strict_json || !count) && Is(']')) { NEXT(); break; } + if ((!opts.strict_json || !count) && Is(']')) break; + ECHECK(body()); + count++; + if (Is(']')) break; + EXPECT(','); + } + NEXT(); + return NoError(); +} + +CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) { + size_t count = 0; + auto err = ParseVectorDelimiters(count, [&]() { Value val; val.type = type; ECHECK(ParseAnyValue(val, nullptr, 0, nullptr)); field_stack_.push_back(std::make_pair(val, nullptr)); - count++; - if (Is(']')) { NEXT(); break; } - EXPECT(','); - } + return NoError(); + }); + ECHECK(err); builder_.StartVector(count * InlineSize(type) / InlineAlignment(type), InlineAlignment(type)); - for (int i = 0; i < count; i++) { + for (size_t i = 0; i < count; i++) { // start at the back, since we're building the data backwards. auto &val = field_stack_.back().first; switch (val.type.base_type) { @@ -1186,6 +1217,11 @@ CheckedError Parser::ParseHash(Value &e, FieldDef* field) { return NoError(); } +CheckedError Parser::TokenError() { + return Error("cannot parse value starting with: " + + TokenToStringId(token_)); +} + CheckedError Parser::ParseSingleValue(Value &e) { // First see if this could be a conversion function: if (token_ == kTokenIdentifier && *cursor_ == '(') { @@ -1252,9 +1288,7 @@ CheckedError Parser::ParseSingleValue(Value &e) { e, BASE_TYPE_STRING, &match)); - if (!match) - return Error("cannot parse value starting with: " + - TokenToStringId(token_)); + if (!match) return TokenError(); } return NoError(); } @@ -1881,14 +1915,21 @@ CheckedError Parser::ParseTypeFromProtoType(Type *type) { CheckedError Parser::SkipAnyJsonValue() { switch (token_) { - case '{': - ECHECK(SkipJsonObject()); - break; + case '{': { + size_t fieldn = 0; + return ParseTableDelimiters(fieldn, nullptr, + [&](const std::string &) -> CheckedError { + ECHECK(SkipAnyJsonValue()); + fieldn++; + return NoError(); + }); + } + case '[': { + size_t count = 0; + return ParseVectorDelimiters(count, [&]() { return SkipAnyJsonValue(); }); + } case kTokenStringConstant: - ECHECK(SkipJsonString()); - break; - case '[': - ECHECK(SkipJsonArray()); + EXPECT(kTokenStringConstant); break; case kTokenIntegerConstant: EXPECT(kTokenIntegerConstant); @@ -1897,56 +1938,60 @@ CheckedError Parser::SkipAnyJsonValue() { EXPECT(kTokenFloatConstant); break; default: - return Error(std::string("Unexpected token:") + std::string(1, static_cast(token_))); + return TokenError(); } return NoError(); } -CheckedError Parser::SkipJsonObject() { - EXPECT('{'); - size_t fieldn = 0; - - for (;;) { - if ((!opts.strict_json || !fieldn) && Is('}')) break; - - if (!Is(kTokenStringConstant)) { - EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier); +CheckedError Parser::ParseFlexBufferValue(flexbuffers::Builder *builder) { + switch (token_) { + case '{': { + auto start = builder->StartMap(); + size_t fieldn = 0; + auto err = ParseTableDelimiters(fieldn, nullptr, + [&](const std::string &name) -> CheckedError { + builder->Key(name); + ECHECK(ParseFlexBufferValue(builder)); + fieldn++; + return NoError(); + }); + ECHECK(err); + builder->EndMap(start); + break; } - else { - NEXT(); + case '[':{ + auto start = builder->StartVector(); + size_t count = 0; + ECHECK(ParseVectorDelimiters(count, [&]() { + return ParseFlexBufferValue(builder); + })); + builder->EndVector(start, false, false); + break; } - - EXPECT(':'); - ECHECK(SkipAnyJsonValue()); - fieldn++; - - if (Is('}')) break; - EXPECT(','); + case kTokenStringConstant: + builder->String(attribute_); + EXPECT(kTokenStringConstant); + break; + case kTokenIntegerConstant: + builder->Int(StringToInt(attribute_.c_str())); + EXPECT(kTokenIntegerConstant); + break; + case kTokenFloatConstant: + builder->Double(strtod(attribute_.c_str(), nullptr)); + EXPECT(kTokenFloatConstant); + break; + default: + return TokenError(); } - - NEXT(); return NoError(); } -CheckedError Parser::SkipJsonArray() { - EXPECT('['); - - for (;;) { - if (Is(']')) break; - - ECHECK(SkipAnyJsonValue()); - - if (Is(']')) break; - EXPECT(','); - } - - NEXT(); - return NoError(); -} - -CheckedError Parser::SkipJsonString() { - EXPECT(kTokenStringConstant); - return NoError(); +bool Parser::ParseFlexBuffer(const char *source, const char *source_filename, + flexbuffers::Builder *builder) { + auto ok = !StartParseFile(source, source_filename).Check() && + !ParseFlexBufferValue(builder).Check(); + if (ok) builder->Finish(); + return ok; } bool Parser::Parse(const char *source, const char **include_paths, @@ -1954,10 +1999,21 @@ bool Parser::Parse(const char *source, const char **include_paths, return !DoParse(source, include_paths, source_filename, nullptr).Check(); } +CheckedError Parser::StartParseFile(const char *source, const char *source_filename) { + file_being_parsed_ = source_filename ? source_filename : ""; + source_ = cursor_ = source; + line_ = 1; + error_.clear(); + ECHECK(SkipByteOrderMark()); + NEXT(); + if (Is(kTokenEof)) + return Error("input file is empty"); + return NoError(); +} + CheckedError Parser::DoParse(const char *source, const char **include_paths, const char *source_filename, const char *include_filename) { - file_being_parsed_ = source_filename ? source_filename : ""; if (source_filename && included_files_.find(source_filename) == included_files_.end()) { included_files_[source_filename] = include_filename ? include_filename : ""; @@ -1967,18 +2023,12 @@ CheckedError Parser::DoParse(const char *source, const char **include_paths, static const char *current_directory[] = { "", nullptr }; include_paths = current_directory; } - source_ = cursor_ = source; - line_ = 1; - error_.clear(); field_stack_.clear(); builder_.Clear(); // Start with a blank namespace just in case this file doesn't have one. namespaces_.push_back(new Namespace()); - ECHECK(SkipByteOrderMark()); - NEXT(); - if (Is(kTokenEof)) - return Error("input file is empty"); + ECHECK(StartParseFile(source, source_filename)); // Includes must come before type declarations: for (;;) { diff --git a/src/reflection.cpp b/src/reflection.cpp index 3c66b8bff..96b3e4aa1 100644 --- a/src/reflection.cpp +++ b/src/reflection.cpp @@ -87,8 +87,11 @@ std::string GetAnyValueS(reflection::BaseType type, const uint8_t *data, auto &fielddef = **it; if (!table_field->CheckField(fielddef.offset())) continue; auto val = GetAnyFieldS(*table_field, fielddef, schema); - if (fielddef.type()->base_type() == reflection::String) - val = "\"" + val + "\""; // Doesn't deal with escape codes etc. + if (fielddef.type()->base_type() == reflection::String) { + std::string esc; + flatbuffers::EscapeString(val.c_str(), val.length(), &esc, true); + val = esc; + } s += fielddef.name()->str(); s += ": "; s += val; diff --git a/tests/monster_test.bfbs b/tests/monster_test.bfbs index 516b764ef2ffb9e3aae7c083ed55f57bafe5ea12..619025902f577441e5df0870cbba38cb55ee3ae0 100644 GIT binary patch literal 4336 zcmai2PiPcZ82>hj`J-zRW2$LP?N}s;ghi7eA|f%xn1aS)(zNsvCfQlDVRmNOolTP> z9!kkkq==Lrq=(W|@g$`bky46>QhE|kJ(M0i6cO86TWy=^@AtiVn@Jk!gKyt^^S=N8 z-puxi$mrPUBqoe@>5(RBm8^71H+F(wkzv3HU>uNb6r77hS^Qvqb(_f14(x&4w_T*O zQ{*iGj^q%)&xo995O5Vadu~GI@T?RDkkO;ETCk!d{$gY}?*~xsaX=T~?DA>1JINZ1Q^_wvr!As}J&<@~Y8fkvdMfv%dFcF`7RW)JoF(tLoK3 zKV;R*Yy7D9O=wKMR`C|BtuE6EEzdq%jbr+sJ|eG)a}%d9$wzoT09*O|0Koc=yf^sn z)zA+sc&xs-hyOb*H%^=ls2?KsbZvI!OugTItdeCwAjSf8$V zUV!<8;xoh6@F_QKH@2$Rc`q-4$|rwHH{8jeK>LmYMgaQ&iH+x?d9N~oOqndZb6#$& z5|&;I=KNZo3ezqJ@abu>V@7_bYl z!QVEF@-oh*kWin1$xW>9a*aSQVSa%544?@lhShcP>D;CQB(oP0P}D|tcWM#V62=E4#; z#~hR>OTh_e7reO$Gz*X8c(06TB)YL@+~|*j?=4}H+%x*MU0rkyxG%{G#odpy0RZbG zLVdWEin|;Zs$nJa^3|wP1NYItlonNL#y!yYq52th2S6U|18MrGQVcE(BA!MrDiYFv zkK;XZY@Lp*VbT?bM7{)HY|*tuzD0iZB8zBCAHV^a(K=G60sn_78j)V$GS{1gIHWbB z6DJ}s%8_;af|Wgwp6+WKLHLXgm`^}QOsd0etb35nX*-!yDLZGw(68c=TsA&JL`9x} zZnq4squYV!q?>oE^R*+xGlgLBu#0Cmj69~Zx)&t( zT)NuOU|rr`;DSV3#&xW_5&h@NOVLdmI<`K!hxJZ5xQ@>HC2b#ZFB>|wulW5BVLg2C5#-^aQg8A>_CcK}qJ_+F_T;X0H{e=7eDCBVKH`|Yw%jQ*5 z?-4IVCUOpS2RR+2rLMG{O@r?nPO0WcMZXw$P85ddzkK%py(*e7R-Jc>zVCQJ-l>(H zV&LE^a4taKiVI88*7Od@J&oh7@SxR`e%YWWIuiN{p0DCgg;v^E3ZQF4cPAb7C7rG5 zVcZK@bb}T|P?He0q221Z4!Tx&FzrX$$`}w~PxQm=zaPVX3%s881#_wxEaCzPE6cE^RqZnOoy`r? z1Nn|wVNQ=w8sgnXuuj2V3zXb5#MUA0y5h)a7omn+4v zQq&X|;vNQ=(W{YM(?b$%yAwS!nMRF`;YEMa_W|AyNPox3*Z9&VC()o;B@=_g^z{au z**bObG|!`CFo);Mxcjv~tIP2ueNWI5I}E=JG_~<^aC9v>rhX+iaJem{`0?F|+S5<; zD|KOxmBXqm{>?M#V5~V-Kt~+r0sS=q*a_J1UNVUbUwIJsB48N6aUT3=FJ)0j$})LW z3nHvnjb6O|dBO0|7>#Ow zk_^vj6Me<};Chqyu9LIqBg`7|qJC2VjzRFrB)+sUC7*l|?V=EhG(E+1p8>2CJ? zWzJjOC6l(_>fUekRUWG;-47@w`Woh=fK6gE?ad9rt0pevm}&C^z*m5)0FGzixdpmW zHqTjWj1zmy5a!F;zog!a4r1S=74u(wyynPlgJ16tZ)4vG8!dmE2jtHhY&Q5jmKCgR z&M=>-4|DHbK+fRl`_2ult?$!#YzWgSybrJ@<223^`K53Ita(l!QWocaz=k}Zg1Vc1 zk|_;q#0*>i>>nA+e+74HG&h3$({_a)z6&3hxicFDg--)FJV zd}dCNCv%$XO>%;ED2|WwwBDw!%=;*gl41KygT^oI$5~)}t^Gtx6}w*OWWKU2wVAUx zF||Y6nP2p?tM`KG`R2Z}J+BLvt$nRAdR@S$Y}!Ox>b>^eZNKLSbM=oToSV@F53vWC ge+QURWiYBG8wuqRX?4ETu?4GlnJ)4>$ zc?=>&N+~{+B7G=Q`c!=Cg9H&NrSzc>r4L1X@X3cF_GW7@ZohBlO!jzoX$Swz%s=z} zxB2IvvyO3wO z=R^G1{7KKP1db0oMqtN)vXwSb-o;)m5ALnEzMqwi{frCcopLI{T!FTnvKNZZ8-*Z$ z*sYczk5R76v#|FT{Gn{+Ao??*EY3SYbz4$lOn6H57+rf>t5NUF?95?wvKxNyK_8Ff z4K25oav6(vL2rU~gLuZV7{25cJs<4QW@^-WhcRM2nG>dMhfdlsU939f-Nl?Ti866I z0$w}jM}5a})!HEZsmnF{O&pf6pLV|HWTB6Hsge5!l>VI<#eKZ`i#mrc>ZBaz@HSBB z+iZ|`{264*T-nYz=_9^ZI__qQ)hvxpA?xWw+)t72kk65(5OvxvVh)|(fPVr+os?nv zy%<}zYdJbGt*n=+mK-;*gM#N;8L!02l!YtBbhEsLMJE#g^CD~GG9^PZ(a^o1t7TCvJZ4>gDSGo;g zg{%9-LT~a-uO$z%e;Ahs-9%XbA@6yaRCU4J3wL$vqbJ+C4id>W{hPKCN26^-|0Q_C zGST2qPRFr@_bU>I1=rHZI-bH+;y#^dqarJ+m(EEa$-K`F736!n|xfW4auGc z+&zu!isC8}oD1A@l5W5afe*3CD6Tp_wyy6w)(25p;^#c0ah*|IC4#$$^-Z#?0k^7g z-6k$k|6qp<{a&kWl${N@HI3_v;*5X9%>cJSb~NA~Yg{slGyV~`3furrJ?>u`d@OfH zamGL5*0A0$+Z%8>jcZ|rFLL{m97xul*$2~J=u=6UR02{ zZMXj+rmu~8PaQ7E$>xf7uu#qA@Ytc91D0=IT;Ib`UjloW$ByfisOt*&J-8p2Funld zeWp#sbIk7ut5hupg<`>Vtiba`j+6KQH8IE+D%RVDV$pKktW_;r1=m9Pw$4MhZ^Lqw zPgS=svbe2=#i*xncp)e3b7I2X7sR7Act#F~1= zS0IJz(uumjH-Y+5I#p~K+X;+612%~&6OZMTs&(oKQ}Vc<5`Mt`PJGS8_b=-GbZ`Bv z=Y!?4gSgl098c9WUFSNWViYW{q(ICq)7G?1^)0~j%!H3U>u@_xjkvhKN#(D8|KM*M z^RD17;Q{mfWn~M$9Zfv&K%osn3TrM}f^Yq+7`zOkJtmfPOxwqbom6)6{&1aZdqK3l z>=!)0pi*3}YWvlF$@v9jND=9{1=^xENNDk3XhR~=iqnecziJccSq;60jyJ?`l*S#*3jm+EKDyo684OZnrh$#LD0L z{`2gZcV(|4?*mWyw3TtA95$|#{~qWVXcH**zR%%v7=gYUa-gJ)A%D4<)%jjiX3t5p@RNayUePC4S7xRHV8~1T7 z4{Gx5a%cgyhgmZPIf*9v?rVKX%FGt<-Qs4D19iK>4jXgY*e!{;-Mh4%Nn>}5sx6Y3 z0E%|=Tr7~WoAKb8oCWO!@zB0Q_bI5!8J%`Am*-kflex(); - if (flex) { - // flex is a vector of bytes you can memcpy. However, if you - // actually want to access the nested data, this is a convenient - // accessor that directly gives you the root value: - TEST_EQ(monster->flex_flexbuffer_root().AsInt16(), 1234); - } + // flex is a vector of bytes you can memcpy etc. + TEST_EQ(flex->size(), 4); // Encoded FlexBuffer bytes. + // However, if you actually want to access the nested data, this is a + // convenient accessor that directly gives you the root value: + TEST_EQ(monster->flex_flexbuffer_root().AsInt16(), 1234); // Since Flatbuffers uses explicit mechanisms to override the default // compiler alignment, double check that the compiler indeed obeys them: @@ -919,7 +918,8 @@ void FuzzTest2() { AddToSchemaAndInstances((" " + field_name + ":").c_str(), deprecated ? "" : (field_name + ": ").c_str()); // Pick random type: - int base_type = lcg_rand() % (flatbuffers::BASE_TYPE_UNION + 1); + auto base_type = static_cast( + lcg_rand() % (flatbuffers::BASE_TYPE_UNION + 1)); switch (base_type) { case flatbuffers::BASE_TYPE_STRING: if (is_struct) { @@ -970,7 +970,9 @@ void FuzzTest2() { // We want each instance to use its own random value. for (int inst = 0; inst < instances_per_definition; inst++) definitions[definition].instances[inst] += - flatbuffers::NumToString(lcg_rand() % 128).c_str(); + flatbuffers::IsFloat(base_type) + ? flatbuffers::NumToString(lcg_rand() % 128).c_str() + : flatbuffers::NumToString(lcg_rand() % 128).c_str(); } } AddToSchemaAndInstances( @@ -1548,7 +1550,7 @@ void FlexBuffersTest() { TEST_EQ(vec[2].AsDouble(), 4.0); TEST_EQ(vec[2].AsString().IsTheEmptyString(), true); // Wrong Type. TEST_EQ_STR(vec[2].AsString().c_str(), ""); // This still works though. - TEST_EQ_STR(vec[2].ToString().c_str(), "4"); // Or have it converted. + TEST_EQ_STR(vec[2].ToString().c_str(), "4.0"); // Or have it converted. auto tvec = map["bar"].AsTypedVector(); TEST_EQ(tvec.size(), 3); TEST_EQ(tvec[2].AsInt8(), 3); @@ -1570,6 +1572,22 @@ void FlexBuffersTest() { TEST_EQ(vec[2].MutateFloat(2.0f), true); TEST_EQ(vec[2].AsFloat(), 2.0f); TEST_EQ(vec[2].MutateFloat(3.14159), false); // Double does not fit in float. + + // Parse from JSON: + flatbuffers::Parser parser; + slb.Clear(); + auto jsontest = "{ a: [ 123, 456.0 ], b: \"hello\" }"; + TEST_EQ(parser.ParseFlexBuffer(jsontest, nullptr, &slb), + true); + auto jroot = flexbuffers::GetRoot(slb.GetBuffer()); + auto jmap = jroot.AsMap(); + auto jvec = jmap["a"].AsVector(); + TEST_EQ(jvec[0].AsInt64(), 123); + TEST_EQ(jvec[1].AsDouble(), 456.0); + TEST_EQ_STR(jmap["b"].AsString().c_str(), "hello"); + // And from FlexBuffer back to JSON: + auto jsonback = jroot.ToString(); + TEST_EQ_STR(jsontest, jsonback.c_str()); } int main(int /*argc*/, const char * /*argv*/[]) {