diff --git a/include/flatbuffers/flexbuffers.h b/include/flatbuffers/flexbuffers.h index e75451570..67f82b94c 100644 --- a/include/flatbuffers/flexbuffers.h +++ b/include/flatbuffers/flexbuffers.h @@ -457,25 +457,61 @@ class Reference { } // Unlike AsString(), this will convert any type to a std::string. - std::string ToString() const { + std::string ToString() { + std::string s; + ToString(false, false, s); + return s; + } + + // Convert any type to a JSON-like string. strings_quoted determines if + // string values at the top level receive "" quotes (inside other values + // they always do). keys_quoted determines if keys are quoted, at any level. + // TODO(wvo): add further options to have indentation/newlines. + void ToString(bool strings_quoted, bool keys_quoted, std::string &s) const { if (type_ == TYPE_STRING) { - return String(Indirect(), byte_width_).c_str(); + String str(Indirect(), byte_width_); + if (strings_quoted) { + flatbuffers::EscapeString(str.c_str(), str.length(), &s, true); + } else { + s.append(str.c_str(), str.length()); + } } else if (IsKey()) { - return AsKey(); + auto str = AsKey(); + if (keys_quoted) { + flatbuffers::EscapeString(str, strlen(str), &s, true); + } else { + s += str; + } } else if (IsInt()) { - return flatbuffers::NumToString(AsInt64()); + s += flatbuffers::NumToString(AsInt64()); } else if (IsUInt()) { - return flatbuffers::NumToString(AsUInt64()); + s += flatbuffers::NumToString(AsUInt64()); } else if (IsFloat()) { - return flatbuffers::NumToString(AsDouble()); + s += flatbuffers::NumToString(AsDouble()); } else if (IsNull()) { - return "null"; + s += "null"; } else if (IsMap()) { - return "{..}"; // TODO: show elements. + s += "{ "; + auto m = AsMap(); + auto keys = m.Keys(); + auto vals = m.Values(); + for (size_t i = 0; i < keys.size(); i++) { + keys[i].ToString(true, keys_quoted, s); + s += ": "; + vals[i].ToString(true, keys_quoted, s); + if (i < keys.size() - 1) s += ", "; + } + s += " }"; } else if (IsVector()) { - return "[..]"; // TODO: show elements. + s += "[ "; + auto v = AsVector(); + for (size_t i = 0; i < v.size(); i++) { + v[i].ToString(true, keys_quoted, s); + if (i < v.size() - 1) s += ", "; + } + s += " ]"; } else { - return "(?)"; + s += "(?)"; } } @@ -746,6 +782,17 @@ class Builder FLATBUFFERS_FINAL_CLASS { return buf_; } + // Reset all state so we can re-use the buffer. + void Clear() { + buf_.clear(); + stack_.clear(); + finished_ = false; + // flags_ remains as-is; + force_min_bit_width_ = BIT_WIDTH_8; + key_pool.clear(); + string_pool.clear(); + } + // All value constructing functions below have two versions: one that // takes a key (for placement inside a map) and one that doesn't (for inside // vectors and elsewhere). diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 4e3a24a67..ab0421fe5 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -25,6 +25,7 @@ #include "flatbuffers/flatbuffers.h" #include "flatbuffers/hash.h" #include "flatbuffers/reflection.h" +#include "flatbuffers/flexbuffers.h" // This file defines the data types representing a parsed IDL (Interface // Definition Language) / schema file. @@ -226,18 +227,20 @@ struct Definition { }; struct FieldDef : public Definition { - FieldDef() : deprecated(false), required(false), key(false), padding(0) {} + FieldDef() : deprecated(false), required(false), key(false), + flexbuffer(false), padding(0) {} Offset Serialize(FlatBufferBuilder *builder, uint16_t id, const Parser &parser) const; Value value; - bool deprecated; // Field is allowed to be present in old data, but can't be + bool deprecated; // Field is allowed to be present in old data, but can't be. // written in new data nor accessed in new code. bool required; // Field must always be present. bool key; // Field functions as a key for creating sorted vectors. bool native_inline; // Field will be defined inline (instead of as a pointer) // for native tables if field is a struct. + bool flexbuffer; // This field contains FlexBuffer data. size_t padding; // Bytes to always pad after this field. }; @@ -536,6 +539,11 @@ class Parser : public ParserState { // of the schema provided. Returns non-empty error on any problems. std::string ConformTo(const Parser &base); + // Similar to Parse(), but now only accepts JSON to be parsed into a + // FlexBuffer. + bool ParseFlexBuffer(const char *source, const char *source_filename, + flexbuffers::Builder *builder); + FLATBUFFERS_CHECKED_ERROR CheckInRange(int64_t val, int64_t min, int64_t max); private: @@ -559,15 +567,21 @@ private: FLATBUFFERS_CHECKED_ERROR ParseAnyValue(Value &val, FieldDef *field, size_t parent_fieldn, const StructDef *parent_struct_def); + FLATBUFFERS_CHECKED_ERROR ParseTableDelimiters(size_t &fieldn, + const StructDef *struct_def, + const std::function &body); FLATBUFFERS_CHECKED_ERROR ParseTable(const StructDef &struct_def, std::string *value, uoffset_t *ovalue); void SerializeStruct(const StructDef &struct_def, const Value &val); void AddVector(bool sortbysize, int count); + FLATBUFFERS_CHECKED_ERROR ParseVectorDelimiters(size_t &count, + const std::function &body); FLATBUFFERS_CHECKED_ERROR ParseVector(const Type &type, uoffset_t *ovalue); FLATBUFFERS_CHECKED_ERROR ParseMetaData(SymbolTable *attributes); FLATBUFFERS_CHECKED_ERROR TryTypedValue(int dtoken, bool check, Value &e, BaseType req, bool *destmatch); FLATBUFFERS_CHECKED_ERROR ParseHash(Value &e, FieldDef* field); + FLATBUFFERS_CHECKED_ERROR TokenError(); FLATBUFFERS_CHECKED_ERROR ParseSingleValue(Value &e); FLATBUFFERS_CHECKED_ERROR ParseEnumFromString(Type &type, int64_t *result); StructDef *LookupCreateStruct(const std::string &name, @@ -587,9 +601,9 @@ private: FLATBUFFERS_CHECKED_ERROR ParseProtoCurliesOrIdent(); FLATBUFFERS_CHECKED_ERROR ParseTypeFromProtoType(Type *type); FLATBUFFERS_CHECKED_ERROR SkipAnyJsonValue(); - FLATBUFFERS_CHECKED_ERROR SkipJsonObject(); - FLATBUFFERS_CHECKED_ERROR SkipJsonArray(); - FLATBUFFERS_CHECKED_ERROR SkipJsonString(); + FLATBUFFERS_CHECKED_ERROR ParseFlexBufferValue(flexbuffers::Builder *builder); + FLATBUFFERS_CHECKED_ERROR StartParseFile(const char *source, + const char *source_filename); FLATBUFFERS_CHECKED_ERROR DoParse(const char *_source, const char **include_paths, const char *source_filename, diff --git a/include/flatbuffers/util.h b/include/flatbuffers/util.h index baab3e574..ee62b86d3 100644 --- a/include/flatbuffers/util.h +++ b/include/flatbuffers/util.h @@ -72,9 +72,8 @@ template<> inline std::string NumToString(double t) { // Sadly, std::fixed turns "1" into "1.00000", so here we undo that. auto p = s.find_last_not_of('0'); if (p != std::string::npos) { - s.resize(p + 1); // Strip trailing zeroes. - if (s[s.size() - 1] == '.') - s.erase(s.size() - 1, 1); // Strip '.' if a whole number. + // Strip trailing zeroes. If it is a whole number, keep one zero. + s.resize(p + (s[p] == '.' ? 2 : 1)); } return s; } @@ -361,6 +360,72 @@ inline std::string WordWrap(const std::string in, size_t max_length, return wrapped; } +inline bool EscapeString(const char *s, size_t length, std::string *_text, + bool allow_non_utf8) { + std::string &text = *_text; + text += "\""; + for (uoffset_t i = 0; i < length; i++) { + char c = s[i]; + switch (c) { + case '\n': text += "\\n"; break; + case '\t': text += "\\t"; break; + case '\r': text += "\\r"; break; + case '\b': text += "\\b"; break; + case '\f': text += "\\f"; break; + case '\"': text += "\\\""; break; + case '\\': text += "\\\\"; break; + default: + if (c >= ' ' && c <= '~') { + text += c; + } else { + // Not printable ASCII data. Let's see if it's valid UTF-8 first: + const char *utf8 = s + i; + int ucc = FromUTF8(&utf8); + if (ucc < 0) { + if (allow_non_utf8) { + text += "\\x"; + text += IntToStringHex(static_cast(c), 2); + } else { + // There are two cases here: + // + // 1) We reached here by parsing an IDL file. In that case, + // we previously checked for non-UTF-8, so we shouldn't reach + // here. + // + // 2) We reached here by someone calling GenerateText() + // on a previously-serialized flatbuffer. The data might have + // non-UTF-8 Strings, or might be corrupt. + // + // In both cases, we have to give up and inform the caller + // they have no JSON. + return false; + } + } else { + if (ucc <= 0xFFFF) { + // Parses as Unicode within JSON's \uXXXX range, so use that. + text += "\\u"; + text += IntToStringHex(ucc, 4); + } else if (ucc <= 0x10FFFF) { + // Encode Unicode SMP values to a surrogate pair using two \u escapes. + uint32_t base = ucc - 0x10000; + auto high_surrogate = (base >> 10) + 0xD800; + auto low_surrogate = (base & 0x03FF) + 0xDC00; + text += "\\u"; + text += IntToStringHex(high_surrogate, 4); + text += "\\u"; + text += IntToStringHex(low_surrogate, 4); + } + // Skip past characters recognized. + i = static_cast(utf8 - s - 1); + } + } + break; + } + } + text += "\""; + return true; +} + } // namespace flatbuffers #endif // FLATBUFFERS_UTIL_H_ diff --git a/src/idl_gen_cpp.cpp b/src/idl_gen_cpp.cpp index d5c8b03f5..242c8d9bd 100644 --- a/src/idl_gen_cpp.cpp +++ b/src/idl_gen_cpp.cpp @@ -1343,7 +1343,7 @@ class CppGenerator : public BaseGenerator { code_ += " }"; } - if (field.attributes.Lookup("flexbuffer")) { + if (field.flexbuffer) { code_ += " flexbuffers::Reference {{FIELD_NAME}}_flexbuffer_root()" " const {"; code_ += " auto v = {{FIELD_NAME}}();"; diff --git a/src/idl_gen_text.cpp b/src/idl_gen_text.cpp index dc445e5c4..9d741e0ee 100644 --- a/src/idl_gen_text.cpp +++ b/src/idl_gen_text.cpp @@ -19,6 +19,7 @@ #include "flatbuffers/flatbuffers.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" +#include "flatbuffers/flexbuffers.h" namespace flatbuffers { @@ -101,71 +102,6 @@ template bool PrintVector(const Vector &v, Type type, return true; } -static bool EscapeString(const String &s, std::string *_text, const IDLOptions& opts) { - std::string &text = *_text; - text += "\""; - for (uoffset_t i = 0; i < s.size(); i++) { - char c = s[i]; - switch (c) { - case '\n': text += "\\n"; break; - case '\t': text += "\\t"; break; - case '\r': text += "\\r"; break; - case '\b': text += "\\b"; break; - case '\f': text += "\\f"; break; - case '\"': text += "\\\""; break; - case '\\': text += "\\\\"; break; - default: - if (c >= ' ' && c <= '~') { - text += c; - } else { - // Not printable ASCII data. Let's see if it's valid UTF-8 first: - const char *utf8 = s.c_str() + i; - int ucc = FromUTF8(&utf8); - if (ucc < 0) { - if (opts.allow_non_utf8) { - text += "\\x"; - text += IntToStringHex(static_cast(c), 2); - } else { - // There are two cases here: - // - // 1) We reached here by parsing an IDL file. In that case, - // we previously checked for non-UTF-8, so we shouldn't reach - // here. - // - // 2) We reached here by someone calling GenerateText() - // on a previously-serialized flatbuffer. The data might have - // non-UTF-8 Strings, or might be corrupt. - // - // In both cases, we have to give up and inform the caller - // they have no JSON. - return false; - } - } else { - if (ucc <= 0xFFFF) { - // Parses as Unicode within JSON's \uXXXX range, so use that. - text += "\\u"; - text += IntToStringHex(ucc, 4); - } else if (ucc <= 0x10FFFF) { - // Encode Unicode SMP values to a surrogate pair using two \u escapes. - uint32_t base = ucc - 0x10000; - auto high_surrogate = (base >> 10) + 0xD800; - auto low_surrogate = (base & 0x03FF) + 0xDC00; - text += "\\u"; - text += IntToStringHex(high_surrogate, 4); - text += "\\u"; - text += IntToStringHex(low_surrogate, 4); - } - // Skip past characters recognized. - i = static_cast(utf8 - s.c_str() - 1); - } - } - break; - } - } - text += "\""; - return true; -} - // Specialization of Print above for pointer types. template<> bool Print(const void *val, Type type, int indent, @@ -189,7 +125,8 @@ template<> bool Print(const void *val, } break; case BASE_TYPE_STRING: { - if (!EscapeString(*reinterpret_cast(val), _text, opts)) { + auto s = reinterpret_cast(val); + if (!EscapeString(s->c_str(), s->Length(), _text, opts.allow_non_utf8)) { return false; } break; @@ -238,6 +175,11 @@ static bool GenFieldOffset(const FieldDef &fd, const Table *table, bool fixed, assert(IsStruct(fd.value.type)); val = reinterpret_cast(table)-> GetStruct(fd.value.offset); + } else if (fd.flexbuffer) { + auto vec = table->GetPointer *>(fd.value.offset); + auto root = flexbuffers::GetRoot(vec->data(), vec->size()); + root.ToString(true, false, *_text); + return true; } else { val = IsStruct(fd.value.type) ? table->GetStruct(fd.value.offset) diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index c238bee0c..5a1b8708a 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -726,6 +726,7 @@ CheckedError Parser::ParseField(StructDef &struct_def) { } if (field->attributes.Lookup("flexbuffer")) { + field->flexbuffer = true; uses_flexbuffers_ = true; if (field->value.type.base_type != BASE_TYPE_VECTOR || field->value.type.element != BASE_TYPE_UCHAR) @@ -830,7 +831,6 @@ CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field, break; } case BASE_TYPE_VECTOR: { - EXPECT('['); uoffset_t off; ECHECK(ParseVector(val.type.VectorType(), &off)); val.constant = NumToString(off); @@ -863,27 +863,27 @@ void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) { builder_.AddStructOffset(val.offset, builder_.GetSize()); } -CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, - uoffset_t *ovalue) { - // We allow tables both as JSON object{ .. } with field names +CheckedError Parser::ParseTableDelimiters(size_t &fieldn, + const StructDef *struct_def, + const std::function &body) { + // We allow tables both as JSON object{ .. } with field names // or vector[..] with all fields in order - const bool is_nested_list = Is('['); - if (is_nested_list) { + char terminator = '}'; + bool is_nested_vector = struct_def && Is('['); + if (is_nested_vector) { NEXT(); + terminator = ']'; } else { EXPECT('{'); } - size_t fieldn = 0; for (;;) { - if ((!opts.strict_json || !fieldn) && Is(is_nested_list ? ']' : '}')) { NEXT(); break; } - FieldDef *field = nullptr; + if ((!opts.strict_json || !fieldn) && Is(terminator)) break; std::string name; - if (is_nested_list) { - if (fieldn > struct_def.fields.vec.size()) { + if (is_nested_vector) { + if (fieldn > struct_def->fields.vec.size()) { return Error("too many unnamed fields in nested array"); } - field = struct_def.fields.vec[fieldn]; - name = field->name; + name = struct_def->fields.vec[fieldn]->name; } else { name = attribute_; if (Is(kTokenStringConstant)) { @@ -891,24 +891,46 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, } else { EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier); } - field = struct_def.fields.Lookup(name); + EXPECT(':'); } + ECHECK(body(name)); + if (Is(terminator)) break; + EXPECT(','); + } + NEXT(); + if (is_nested_vector && fieldn != struct_def->fields.vec.size()) { + return Error("wrong number of unnamed fields in table vector"); + } + return NoError(); +} + +CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, + uoffset_t *ovalue) { + size_t fieldn = 0; + auto err = ParseTableDelimiters(fieldn, &struct_def, + [&](const std::string &name) -> CheckedError { + auto field = struct_def.fields.Lookup(name); if (!field) { if (!opts.skip_unexpected_fields_in_json) { return Error("unknown field: " + name); } else { - EXPECT(':'); ECHECK(SkipAnyJsonValue()); } } else { - if (!is_nested_list) { - EXPECT(':'); - } if (Is(kTokenNull)) { NEXT(); // Ignore this field. } else { Value val = field->value; - ECHECK(ParseAnyValue(val, field, fieldn, &struct_def)); + if (field->flexbuffer) { + flexbuffers::Builder builder(1024, + flexbuffers::BUILDER_FLAG_SHARE_ALL); + ECHECK(ParseFlexBufferValue(&builder)); + builder.Finish(); + auto off = builder_.CreateVector(builder.GetBuffer()); + val.constant = NumToString(off.o); + } else { + ECHECK(ParseAnyValue(val, field, fieldn, &struct_def)); + } // Hardcoded insertion-sort with error-check. // If fields are specified in order, then this loop exits immediately. auto elem = field_stack_.rbegin(); @@ -924,12 +946,9 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, fieldn++; } } - if (Is(is_nested_list ? ']' : '}')) { NEXT(); break; } - EXPECT(','); - } - if (is_nested_list && fieldn != struct_def.fields.vec.size()) { - return Error("wrong number of unnamed fields in table vector"); - } + return NoError(); + }); + ECHECK(err); // Check if all required fields are parsed. for (auto field_it = struct_def.fields.vec.begin(); @@ -1029,22 +1048,34 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, return NoError(); } -CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) { - int count = 0; +CheckedError Parser::ParseVectorDelimiters(size_t &count, + const std::function &body) { + EXPECT('['); for (;;) { - if ((!opts.strict_json || !count) && Is(']')) { NEXT(); break; } + if ((!opts.strict_json || !count) && Is(']')) break; + ECHECK(body()); + count++; + if (Is(']')) break; + EXPECT(','); + } + NEXT(); + return NoError(); +} + +CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) { + size_t count = 0; + auto err = ParseVectorDelimiters(count, [&]() { Value val; val.type = type; ECHECK(ParseAnyValue(val, nullptr, 0, nullptr)); field_stack_.push_back(std::make_pair(val, nullptr)); - count++; - if (Is(']')) { NEXT(); break; } - EXPECT(','); - } + return NoError(); + }); + ECHECK(err); builder_.StartVector(count * InlineSize(type) / InlineAlignment(type), InlineAlignment(type)); - for (int i = 0; i < count; i++) { + for (size_t i = 0; i < count; i++) { // start at the back, since we're building the data backwards. auto &val = field_stack_.back().first; switch (val.type.base_type) { @@ -1186,6 +1217,11 @@ CheckedError Parser::ParseHash(Value &e, FieldDef* field) { return NoError(); } +CheckedError Parser::TokenError() { + return Error("cannot parse value starting with: " + + TokenToStringId(token_)); +} + CheckedError Parser::ParseSingleValue(Value &e) { // First see if this could be a conversion function: if (token_ == kTokenIdentifier && *cursor_ == '(') { @@ -1252,9 +1288,7 @@ CheckedError Parser::ParseSingleValue(Value &e) { e, BASE_TYPE_STRING, &match)); - if (!match) - return Error("cannot parse value starting with: " + - TokenToStringId(token_)); + if (!match) return TokenError(); } return NoError(); } @@ -1881,14 +1915,21 @@ CheckedError Parser::ParseTypeFromProtoType(Type *type) { CheckedError Parser::SkipAnyJsonValue() { switch (token_) { - case '{': - ECHECK(SkipJsonObject()); - break; + case '{': { + size_t fieldn = 0; + return ParseTableDelimiters(fieldn, nullptr, + [&](const std::string &) -> CheckedError { + ECHECK(SkipAnyJsonValue()); + fieldn++; + return NoError(); + }); + } + case '[': { + size_t count = 0; + return ParseVectorDelimiters(count, [&]() { return SkipAnyJsonValue(); }); + } case kTokenStringConstant: - ECHECK(SkipJsonString()); - break; - case '[': - ECHECK(SkipJsonArray()); + EXPECT(kTokenStringConstant); break; case kTokenIntegerConstant: EXPECT(kTokenIntegerConstant); @@ -1897,56 +1938,60 @@ CheckedError Parser::SkipAnyJsonValue() { EXPECT(kTokenFloatConstant); break; default: - return Error(std::string("Unexpected token:") + std::string(1, static_cast(token_))); + return TokenError(); } return NoError(); } -CheckedError Parser::SkipJsonObject() { - EXPECT('{'); - size_t fieldn = 0; - - for (;;) { - if ((!opts.strict_json || !fieldn) && Is('}')) break; - - if (!Is(kTokenStringConstant)) { - EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier); +CheckedError Parser::ParseFlexBufferValue(flexbuffers::Builder *builder) { + switch (token_) { + case '{': { + auto start = builder->StartMap(); + size_t fieldn = 0; + auto err = ParseTableDelimiters(fieldn, nullptr, + [&](const std::string &name) -> CheckedError { + builder->Key(name); + ECHECK(ParseFlexBufferValue(builder)); + fieldn++; + return NoError(); + }); + ECHECK(err); + builder->EndMap(start); + break; } - else { - NEXT(); + case '[':{ + auto start = builder->StartVector(); + size_t count = 0; + ECHECK(ParseVectorDelimiters(count, [&]() { + return ParseFlexBufferValue(builder); + })); + builder->EndVector(start, false, false); + break; } - - EXPECT(':'); - ECHECK(SkipAnyJsonValue()); - fieldn++; - - if (Is('}')) break; - EXPECT(','); + case kTokenStringConstant: + builder->String(attribute_); + EXPECT(kTokenStringConstant); + break; + case kTokenIntegerConstant: + builder->Int(StringToInt(attribute_.c_str())); + EXPECT(kTokenIntegerConstant); + break; + case kTokenFloatConstant: + builder->Double(strtod(attribute_.c_str(), nullptr)); + EXPECT(kTokenFloatConstant); + break; + default: + return TokenError(); } - - NEXT(); return NoError(); } -CheckedError Parser::SkipJsonArray() { - EXPECT('['); - - for (;;) { - if (Is(']')) break; - - ECHECK(SkipAnyJsonValue()); - - if (Is(']')) break; - EXPECT(','); - } - - NEXT(); - return NoError(); -} - -CheckedError Parser::SkipJsonString() { - EXPECT(kTokenStringConstant); - return NoError(); +bool Parser::ParseFlexBuffer(const char *source, const char *source_filename, + flexbuffers::Builder *builder) { + auto ok = !StartParseFile(source, source_filename).Check() && + !ParseFlexBufferValue(builder).Check(); + if (ok) builder->Finish(); + return ok; } bool Parser::Parse(const char *source, const char **include_paths, @@ -1954,10 +1999,21 @@ bool Parser::Parse(const char *source, const char **include_paths, return !DoParse(source, include_paths, source_filename, nullptr).Check(); } +CheckedError Parser::StartParseFile(const char *source, const char *source_filename) { + file_being_parsed_ = source_filename ? source_filename : ""; + source_ = cursor_ = source; + line_ = 1; + error_.clear(); + ECHECK(SkipByteOrderMark()); + NEXT(); + if (Is(kTokenEof)) + return Error("input file is empty"); + return NoError(); +} + CheckedError Parser::DoParse(const char *source, const char **include_paths, const char *source_filename, const char *include_filename) { - file_being_parsed_ = source_filename ? source_filename : ""; if (source_filename && included_files_.find(source_filename) == included_files_.end()) { included_files_[source_filename] = include_filename ? include_filename : ""; @@ -1967,18 +2023,12 @@ CheckedError Parser::DoParse(const char *source, const char **include_paths, static const char *current_directory[] = { "", nullptr }; include_paths = current_directory; } - source_ = cursor_ = source; - line_ = 1; - error_.clear(); field_stack_.clear(); builder_.Clear(); // Start with a blank namespace just in case this file doesn't have one. namespaces_.push_back(new Namespace()); - ECHECK(SkipByteOrderMark()); - NEXT(); - if (Is(kTokenEof)) - return Error("input file is empty"); + ECHECK(StartParseFile(source, source_filename)); // Includes must come before type declarations: for (;;) { diff --git a/src/reflection.cpp b/src/reflection.cpp index 3c66b8bff..96b3e4aa1 100644 --- a/src/reflection.cpp +++ b/src/reflection.cpp @@ -87,8 +87,11 @@ std::string GetAnyValueS(reflection::BaseType type, const uint8_t *data, auto &fielddef = **it; if (!table_field->CheckField(fielddef.offset())) continue; auto val = GetAnyFieldS(*table_field, fielddef, schema); - if (fielddef.type()->base_type() == reflection::String) - val = "\"" + val + "\""; // Doesn't deal with escape codes etc. + if (fielddef.type()->base_type() == reflection::String) { + std::string esc; + flatbuffers::EscapeString(val.c_str(), val.length(), &esc, true); + val = esc; + } s += fielddef.name()->str(); s += ": "; s += val; diff --git a/tests/monster_test.bfbs b/tests/monster_test.bfbs index 516b764ef..619025902 100644 Binary files a/tests/monster_test.bfbs and b/tests/monster_test.bfbs differ diff --git a/tests/monsterdata_test.golden b/tests/monsterdata_test.golden index f978ed236..2e32501d6 100644 --- a/tests/monsterdata_test.golden +++ b/tests/monsterdata_test.golden @@ -1,9 +1,9 @@ { pos: { - x: 1, - y: 2, - z: 3, - test1: 3, + x: 1.0, + y: 2.0, + z: 3.0, + test1: 3.0, test2: Green, test3: { a: 10, @@ -63,5 +63,6 @@ testhashs32_fnv1a: -1904106383, testhashu32_fnv1a: 2390860913, testhashs64_fnv1a: 4898026182817603057, - testhashu64_fnv1a: 4898026182817603057 + testhashu64_fnv1a: 4898026182817603057, + flex: 1234 } diff --git a/tests/test.cpp b/tests/test.cpp index 5ebeb4d90..f8f905de5 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -309,12 +309,11 @@ void AccessFlatBufferTest(const uint8_t *flatbuf, size_t length, // Test flexbuffer if available: auto flex = monster->flex(); - if (flex) { - // flex is a vector of bytes you can memcpy. However, if you - // actually want to access the nested data, this is a convenient - // accessor that directly gives you the root value: - TEST_EQ(monster->flex_flexbuffer_root().AsInt16(), 1234); - } + // flex is a vector of bytes you can memcpy etc. + TEST_EQ(flex->size(), 4); // Encoded FlexBuffer bytes. + // However, if you actually want to access the nested data, this is a + // convenient accessor that directly gives you the root value: + TEST_EQ(monster->flex_flexbuffer_root().AsInt16(), 1234); // Since Flatbuffers uses explicit mechanisms to override the default // compiler alignment, double check that the compiler indeed obeys them: @@ -919,7 +918,8 @@ void FuzzTest2() { AddToSchemaAndInstances((" " + field_name + ":").c_str(), deprecated ? "" : (field_name + ": ").c_str()); // Pick random type: - int base_type = lcg_rand() % (flatbuffers::BASE_TYPE_UNION + 1); + auto base_type = static_cast( + lcg_rand() % (flatbuffers::BASE_TYPE_UNION + 1)); switch (base_type) { case flatbuffers::BASE_TYPE_STRING: if (is_struct) { @@ -970,7 +970,9 @@ void FuzzTest2() { // We want each instance to use its own random value. for (int inst = 0; inst < instances_per_definition; inst++) definitions[definition].instances[inst] += - flatbuffers::NumToString(lcg_rand() % 128).c_str(); + flatbuffers::IsFloat(base_type) + ? flatbuffers::NumToString(lcg_rand() % 128).c_str() + : flatbuffers::NumToString(lcg_rand() % 128).c_str(); } } AddToSchemaAndInstances( @@ -1548,7 +1550,7 @@ void FlexBuffersTest() { TEST_EQ(vec[2].AsDouble(), 4.0); TEST_EQ(vec[2].AsString().IsTheEmptyString(), true); // Wrong Type. TEST_EQ_STR(vec[2].AsString().c_str(), ""); // This still works though. - TEST_EQ_STR(vec[2].ToString().c_str(), "4"); // Or have it converted. + TEST_EQ_STR(vec[2].ToString().c_str(), "4.0"); // Or have it converted. auto tvec = map["bar"].AsTypedVector(); TEST_EQ(tvec.size(), 3); TEST_EQ(tvec[2].AsInt8(), 3); @@ -1570,6 +1572,22 @@ void FlexBuffersTest() { TEST_EQ(vec[2].MutateFloat(2.0f), true); TEST_EQ(vec[2].AsFloat(), 2.0f); TEST_EQ(vec[2].MutateFloat(3.14159), false); // Double does not fit in float. + + // Parse from JSON: + flatbuffers::Parser parser; + slb.Clear(); + auto jsontest = "{ a: [ 123, 456.0 ], b: \"hello\" }"; + TEST_EQ(parser.ParseFlexBuffer(jsontest, nullptr, &slb), + true); + auto jroot = flexbuffers::GetRoot(slb.GetBuffer()); + auto jmap = jroot.AsMap(); + auto jvec = jmap["a"].AsVector(); + TEST_EQ(jvec[0].AsInt64(), 123); + TEST_EQ(jvec[1].AsDouble(), 456.0); + TEST_EQ_STR(jmap["b"].AsString().c_str(), "hello"); + // And from FlexBuffer back to JSON: + auto jsonback = jroot.ToString(); + TEST_EQ_STR(jsontest, jsonback.c_str()); } int main(int /*argc*/, const char * /*argv*/[]) {