BigfootDev/flatbuffers
1
0
Fork 1
mirror of https://github.com/google/flatbuffers.git synced 2026-06-16 17:22:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added (nested) FlexBuffer JSON parsing and output.

Browse Source 
FlexBuffer parser is just 40 lines of code (on top of existing parser!).

Change-Id: Idebebadafb661ca5333f5621139031f6df3c3e1a
Tested: on Linux.
This commit is contained in:
Wouter van Oortmerssen
2017-06-05 17:45:44 -07:00
parent dddd0865cb
commit 8f864aad7b
10 changed files with 332 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
include/flatbuffers/flexbuffers.h
View File

@@ -457,25 +457,61 @@ class Reference {
}
// Unlike AsString(), this will convert any type to a std::string.
std::string ToString() const {
std::string ToString() {
std::string s;
ToString(false, false, s);
return s;
}
// Convert any type to a JSON-like string. strings_quoted determines if
// string values at the top level receive "" quotes (inside other values
// they always do). keys_quoted determines if keys are quoted, at any level.
// TODO(wvo): add further options to have indentation/newlines.
void ToString(bool strings_quoted, bool keys_quoted, std::string &s) const {
if (type_ == TYPE_STRING) {
return String(Indirect(), byte_width_).c_str();
String str(Indirect(), byte_width_);
if (strings_quoted) {
flatbuffers::EscapeString(str.c_str(), str.length(), &s, true);
} else {
s.append(str.c_str(), str.length());
}
} else if (IsKey()) {
return AsKey();
auto str = AsKey();
if (keys_quoted) {
flatbuffers::EscapeString(str, strlen(str), &s, true);
} else {
s += str;
}
} else if (IsInt()) {
return flatbuffers::NumToString(AsInt64());
s += flatbuffers::NumToString(AsInt64());
} else if (IsUInt()) {
return flatbuffers::NumToString(AsUInt64());
s += flatbuffers::NumToString(AsUInt64());
} else if (IsFloat()) {
return flatbuffers::NumToString(AsDouble());
s += flatbuffers::NumToString(AsDouble());
} else if (IsNull()) {
return "null";
s += "null";
} else if (IsMap()) {
return "{..}"; // TODO: show elements.
s += "{ ";
auto m = AsMap();
auto keys = m.Keys();
auto vals = m.Values();
for (size_t i = 0; i < keys.size(); i++) {
keys[i].ToString(true, keys_quoted, s);
s += ": ";
vals[i].ToString(true, keys_quoted, s);
if (i < keys.size() - 1) s += ", ";
}
s += " }";
} else if (IsVector()) {
return "[..]"; // TODO: show elements.
s += "[ ";
auto v = AsVector();
for (size_t i = 0; i < v.size(); i++) {
v[i].ToString(true, keys_quoted, s);
if (i < v.size() - 1) s += ", ";
}
s += " ]";
} else {
return "(?)";
s += "(?)";
}
}
@@ -746,6 +782,17 @@ class Builder FLATBUFFERS_FINAL_CLASS {
return buf_;
}
// Reset all state so we can re-use the buffer.
void Clear() {
buf_.clear();
stack_.clear();
finished_ = false;
// flags_ remains as-is;
force_min_bit_width_ = BIT_WIDTH_8;
key_pool.clear();
string_pool.clear();
}
// All value constructing functions below have two versions: one that
// takes a key (for placement inside a map) and one that doesn't (for inside
// vectors and elsewhere).

24
include/flatbuffers/idl.h
View File

@@ -25,6 +25,7 @@
#include "flatbuffers/flatbuffers.h"
#include "flatbuffers/hash.h"
#include "flatbuffers/reflection.h"
#include "flatbuffers/flexbuffers.h"
// This file defines the data types representing a parsed IDL (Interface
// Definition Language) / schema file.
@@ -226,18 +227,20 @@ struct Definition {
};
struct FieldDef : public Definition {
FieldDef() : deprecated(false), required(false), key(false), padding(0) {}
FieldDef() : deprecated(false), required(false), key(false),
flexbuffer(false), padding(0) {}
Offset<reflection::Field> Serialize(FlatBufferBuilder *builder, uint16_t id,
const Parser &parser) const;
Value value;
bool deprecated; // Field is allowed to be present in old data, but can't be
bool deprecated; // Field is allowed to be present in old data, but can't be.
// written in new data nor accessed in new code.
bool required; // Field must always be present.
bool key; // Field functions as a key for creating sorted vectors.
bool native_inline; // Field will be defined inline (instead of as a pointer)
// for native tables if field is a struct.
bool flexbuffer; // This field contains FlexBuffer data.
size_t padding; // Bytes to always pad after this field.
};
@@ -536,6 +539,11 @@ class Parser : public ParserState {
// of the schema provided. Returns non-empty error on any problems.
std::string ConformTo(const Parser &base);
// Similar to Parse(), but now only accepts JSON to be parsed into a
// FlexBuffer.
bool ParseFlexBuffer(const char *source, const char *source_filename,
flexbuffers::Builder *builder);
FLATBUFFERS_CHECKED_ERROR CheckInRange(int64_t val, int64_t min, int64_t max);
private:
@@ -559,15 +567,21 @@ private:
FLATBUFFERS_CHECKED_ERROR ParseAnyValue(Value &val, FieldDef *field,
size_t parent_fieldn,
const StructDef *parent_struct_def);
FLATBUFFERS_CHECKED_ERROR ParseTableDelimiters(size_t &fieldn,
const StructDef *struct_def,
const std::function<CheckedError(const std::string &name)> &body);
FLATBUFFERS_CHECKED_ERROR ParseTable(const StructDef &struct_def,
std::string *value, uoffset_t *ovalue);
void SerializeStruct(const StructDef &struct_def, const Value &val);
void AddVector(bool sortbysize, int count);
FLATBUFFERS_CHECKED_ERROR ParseVectorDelimiters(size_t &count,
const std::function<CheckedError()> &body);
FLATBUFFERS_CHECKED_ERROR ParseVector(const Type &type, uoffset_t *ovalue);
FLATBUFFERS_CHECKED_ERROR ParseMetaData(SymbolTable<Value> *attributes);
FLATBUFFERS_CHECKED_ERROR TryTypedValue(int dtoken, bool check, Value &e,
BaseType req, bool *destmatch);
FLATBUFFERS_CHECKED_ERROR ParseHash(Value &e, FieldDef* field);
FLATBUFFERS_CHECKED_ERROR TokenError();
FLATBUFFERS_CHECKED_ERROR ParseSingleValue(Value &e);
FLATBUFFERS_CHECKED_ERROR ParseEnumFromString(Type &type, int64_t *result);
StructDef *LookupCreateStruct(const std::string &name,
@@ -587,9 +601,9 @@ private:
FLATBUFFERS_CHECKED_ERROR ParseProtoCurliesOrIdent();
FLATBUFFERS_CHECKED_ERROR ParseTypeFromProtoType(Type *type);
FLATBUFFERS_CHECKED_ERROR SkipAnyJsonValue();
FLATBUFFERS_CHECKED_ERROR SkipJsonObject();
FLATBUFFERS_CHECKED_ERROR SkipJsonArray();
FLATBUFFERS_CHECKED_ERROR SkipJsonString();
FLATBUFFERS_CHECKED_ERROR ParseFlexBufferValue(flexbuffers::Builder *builder);
FLATBUFFERS_CHECKED_ERROR StartParseFile(const char *source,
const char *source_filename);
FLATBUFFERS_CHECKED_ERROR DoParse(const char *_source,
const char **include_paths,
const char *source_filename,

71
include/flatbuffers/util.h
View File

@@ -72,9 +72,8 @@ template<> inline std::string NumToString<double>(double t) {
// Sadly, std::fixed turns "1" into "1.00000", so here we undo that.
auto p = s.find_last_not_of('0');
if (p != std::string::npos) {
s.resize(p + 1); // Strip trailing zeroes.
if (s[s.size() - 1] == '.')
s.erase(s.size() - 1, 1); // Strip '.' if a whole number.
// Strip trailing zeroes. If it is a whole number, keep one zero.
s.resize(p + (s[p] == '.' ? 2 : 1));
}
return s;
}
@@ -361,6 +360,72 @@ inline std::string WordWrap(const std::string in, size_t max_length,
return wrapped;
}
inline bool EscapeString(const char *s, size_t length, std::string *_text,
bool allow_non_utf8) {
std::string &text = *_text;
text += "\"";
for (uoffset_t i = 0; i < length; i++) {
char c = s[i];
switch (c) {
case '\n': text += "\\n"; break;
case '\t': text += "\\t"; break;
case '\r': text += "\\r"; break;
case '\b': text += "\\b"; break;
case '\f': text += "\\f"; break;
case '\"': text += "\\\""; break;
case '\\': text += "\\\\"; break;
default:
if (c >= ' ' && c <= '~') {
text += c;
} else {
// Not printable ASCII data. Let's see if it's valid UTF-8 first:
const char *utf8 = s + i;
int ucc = FromUTF8(&utf8);
if (ucc < 0) {
if (allow_non_utf8) {
text += "\\x";
text += IntToStringHex(static_cast<uint8_t>(c), 2);
} else {
// There are two cases here:
//
// 1) We reached here by parsing an IDL file. In that case,
// we previously checked for non-UTF-8, so we shouldn't reach
// here.
//
// 2) We reached here by someone calling GenerateText()
// on a previously-serialized flatbuffer. The data might have
// non-UTF-8 Strings, or might be corrupt.
//
// In both cases, we have to give up and inform the caller
// they have no JSON.
return false;
}
} else {
if (ucc <= 0xFFFF) {
// Parses as Unicode within JSON's \uXXXX range, so use that.
text += "\\u";
text += IntToStringHex(ucc, 4);
} else if (ucc <= 0x10FFFF) {
// Encode Unicode SMP values to a surrogate pair using two \u escapes.
uint32_t base = ucc - 0x10000;
auto high_surrogate = (base >> 10) + 0xD800;
auto low_surrogate = (base & 0x03FF) + 0xDC00;
text += "\\u";
text += IntToStringHex(high_surrogate, 4);
text += "\\u";
text += IntToStringHex(low_surrogate, 4);
}
// Skip past characters recognized.
i = static_cast<uoffset_t>(utf8 - s - 1);
}
}
break;
}
}
text += "\"";
return true;
}
} // namespace flatbuffers
#endif // FLATBUFFERS_UTIL_H_

2
src/idl_gen_cpp.cpp
View File

@@ -1343,7 +1343,7 @@ class CppGenerator : public BaseGenerator {
code_ += " }";
}
if (field.attributes.Lookup("flexbuffer")) {
if (field.flexbuffer) {
code_ += " flexbuffers::Reference {{FIELD_NAME}}_flexbuffer_root()"
" const {";
code_ += " auto v = {{FIELD_NAME}}();";

74
src/idl_gen_text.cpp
View File

@@ -19,6 +19,7 @@
#include "flatbuffers/flatbuffers.h"
#include "flatbuffers/idl.h"
#include "flatbuffers/util.h"
#include "flatbuffers/flexbuffers.h"
namespace flatbuffers {
@@ -101,71 +102,6 @@ template<typename T> bool PrintVector(const Vector<T> &v, Type type,
return true;
}
static bool EscapeString(const String &s, std::string *_text, const IDLOptions& opts) {
std::string &text = *_text;
text += "\"";
for (uoffset_t i = 0; i < s.size(); i++) {
char c = s[i];
switch (c) {
case '\n': text += "\\n"; break;
case '\t': text += "\\t"; break;
case '\r': text += "\\r"; break;
case '\b': text += "\\b"; break;
case '\f': text += "\\f"; break;
case '\"': text += "\\\""; break;
case '\\': text += "\\\\"; break;
default:
if (c >= ' ' && c <= '~') {
text += c;
} else {
// Not printable ASCII data. Let's see if it's valid UTF-8 first:
const char *utf8 = s.c_str() + i;
int ucc = FromUTF8(&utf8);
if (ucc < 0) {
if (opts.allow_non_utf8) {
text += "\\x";
text += IntToStringHex(static_cast<uint8_t>(c), 2);
} else {
// There are two cases here:
//
// 1) We reached here by parsing an IDL file. In that case,
// we previously checked for non-UTF-8, so we shouldn't reach
// here.
//
// 2) We reached here by someone calling GenerateText()
// on a previously-serialized flatbuffer. The data might have
// non-UTF-8 Strings, or might be corrupt.
//
// In both cases, we have to give up and inform the caller
// they have no JSON.
return false;
}
} else {
if (ucc <= 0xFFFF) {
// Parses as Unicode within JSON's \uXXXX range, so use that.
text += "\\u";
text += IntToStringHex(ucc, 4);
} else if (ucc <= 0x10FFFF) {
// Encode Unicode SMP values to a surrogate pair using two \u escapes.
uint32_t base = ucc - 0x10000;
auto high_surrogate = (base >> 10) + 0xD800;
auto low_surrogate = (base & 0x03FF) + 0xDC00;
text += "\\u";
text += IntToStringHex(high_surrogate, 4);
text += "\\u";
text += IntToStringHex(low_surrogate, 4);
}
// Skip past characters recognized.
i = static_cast<uoffset_t>(utf8 - s.c_str() - 1);
}
}
break;
}
}
text += "\"";
return true;
}
// Specialization of Print above for pointer types.
template<> bool Print<const void *>(const void *val,
Type type, int indent,
@@ -189,7 +125,8 @@ template<> bool Print<const void *>(const void *val,
}
break;
case BASE_TYPE_STRING: {
if (!EscapeString(*reinterpret_cast<const String *>(val), _text, opts)) {
auto s = reinterpret_cast<const String *>(val);
if (!EscapeString(s->c_str(), s->Length(), _text, opts.allow_non_utf8)) {
return false;
}
break;
@@ -238,6 +175,11 @@ static bool GenFieldOffset(const FieldDef &fd, const Table *table, bool fixed,
assert(IsStruct(fd.value.type));
val = reinterpret_cast<const Struct *>(table)->
GetStruct<const void *>(fd.value.offset);
} else if (fd.flexbuffer) {
auto vec = table->GetPointer<const Vector<uint8_t> *>(fd.value.offset);
auto root = flexbuffers::GetRoot(vec->data(), vec->size());
root.ToString(true, false, *_text);
return true;
} else {
val = IsStruct(fd.value.type)
? table->GetStruct<const void *>(fd.value.offset)

232
src/idl_parser.cpp
View File

@@ -726,6 +726,7 @@ CheckedError Parser::ParseField(StructDef &struct_def) {
}
if (field->attributes.Lookup("flexbuffer")) {
field->flexbuffer = true;
uses_flexbuffers_ = true;
if (field->value.type.base_type != BASE_TYPE_VECTOR ||
field->value.type.element != BASE_TYPE_UCHAR)
@@ -830,7 +831,6 @@ CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
break;
}
case BASE_TYPE_VECTOR: {
EXPECT('[');
uoffset_t off;
ECHECK(ParseVector(val.type.VectorType(), &off));
val.constant = NumToString(off);
@@ -863,27 +863,27 @@ void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) {
builder_.AddStructOffset(val.offset, builder_.GetSize());
}
CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
uoffset_t *ovalue) {
// We allow tables both as JSON object{ .. } with field names
CheckedError Parser::ParseTableDelimiters(size_t &fieldn,
const StructDef *struct_def,
const std::function<CheckedError(const std::string &name)> &body) {
// We allow tables both as JSON object{ .. } with field names
// or vector[..] with all fields in order
const bool is_nested_list = Is('[');
if (is_nested_list) {
char terminator = '}';
bool is_nested_vector = struct_def && Is('[');
if (is_nested_vector) {
NEXT();
terminator = ']';
} else {
EXPECT('{');
}
size_t fieldn = 0;
for (;;) {
if ((!opts.strict_json || !fieldn) && Is(is_nested_list ? ']' : '}')) { NEXT(); break; }
FieldDef *field = nullptr;
if ((!opts.strict_json || !fieldn) && Is(terminator)) break;
std::string name;
if (is_nested_list) {
if (fieldn > struct_def.fields.vec.size()) {
if (is_nested_vector) {
if (fieldn > struct_def->fields.vec.size()) {
return Error("too many unnamed fields in nested array");
}
field = struct_def.fields.vec[fieldn];
name = field->name;
name = struct_def->fields.vec[fieldn]->name;
} else {
name = attribute_;
if (Is(kTokenStringConstant)) {
@@ -891,24 +891,46 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
} else {
EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier);
}
field = struct_def.fields.Lookup(name);
EXPECT(':');
}
ECHECK(body(name));
if (Is(terminator)) break;
EXPECT(',');
}
NEXT();
if (is_nested_vector && fieldn != struct_def->fields.vec.size()) {
return Error("wrong number of unnamed fields in table vector");
}
return NoError();
}
CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
uoffset_t *ovalue) {
size_t fieldn = 0;
auto err = ParseTableDelimiters(fieldn, &struct_def,
[&](const std::string &name) -> CheckedError {
auto field = struct_def.fields.Lookup(name);
if (!field) {
if (!opts.skip_unexpected_fields_in_json) {
return Error("unknown field: " + name);
} else {
EXPECT(':');
ECHECK(SkipAnyJsonValue());
}
} else {
if (!is_nested_list) {
EXPECT(':');
}
if (Is(kTokenNull)) {
NEXT(); // Ignore this field.
} else {
Value val = field->value;
ECHECK(ParseAnyValue(val, field, fieldn, &struct_def));
if (field->flexbuffer) {
flexbuffers::Builder builder(1024,
flexbuffers::BUILDER_FLAG_SHARE_ALL);
ECHECK(ParseFlexBufferValue(&builder));
builder.Finish();
auto off = builder_.CreateVector(builder.GetBuffer());
val.constant = NumToString(off.o);
} else {
ECHECK(ParseAnyValue(val, field, fieldn, &struct_def));
}
// Hardcoded insertion-sort with error-check.
// If fields are specified in order, then this loop exits immediately.
auto elem = field_stack_.rbegin();
@@ -924,12 +946,9 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
fieldn++;
}
}
if (Is(is_nested_list ? ']' : '}')) { NEXT(); break; }
EXPECT(',');
}
if (is_nested_list && fieldn != struct_def.fields.vec.size()) {
return Error("wrong number of unnamed fields in table vector");
}
return NoError();
});
ECHECK(err);
// Check if all required fields are parsed.
for (auto field_it = struct_def.fields.vec.begin();
@@ -1029,22 +1048,34 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
return NoError();
}
CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) {
int count = 0;
CheckedError Parser::ParseVectorDelimiters(size_t &count,
const std::function<CheckedError()> &body) {
EXPECT('[');
for (;;) {
if ((!opts.strict_json || !count) && Is(']')) { NEXT(); break; }
if ((!opts.strict_json || !count) && Is(']')) break;
ECHECK(body());
count++;
if (Is(']')) break;
EXPECT(',');
}
NEXT();
return NoError();
}
CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) {
size_t count = 0;
auto err = ParseVectorDelimiters(count, [&]() {
Value val;
val.type = type;
ECHECK(ParseAnyValue(val, nullptr, 0, nullptr));
field_stack_.push_back(std::make_pair(val, nullptr));
count++;
if (Is(']')) { NEXT(); break; }
EXPECT(',');
}
return NoError();
});
ECHECK(err);
builder_.StartVector(count * InlineSize(type) / InlineAlignment(type),
InlineAlignment(type));
for (int i = 0; i < count; i++) {
for (size_t i = 0; i < count; i++) {
// start at the back, since we're building the data backwards.
auto &val = field_stack_.back().first;
switch (val.type.base_type) {
@@ -1186,6 +1217,11 @@ CheckedError Parser::ParseHash(Value &e, FieldDef* field) {
return NoError();
}
CheckedError Parser::TokenError() {
return Error("cannot parse value starting with: " +
TokenToStringId(token_));
}
CheckedError Parser::ParseSingleValue(Value &e) {
// First see if this could be a conversion function:
if (token_ == kTokenIdentifier && *cursor_ == '(') {
@@ -1252,9 +1288,7 @@ CheckedError Parser::ParseSingleValue(Value &e) {
e,
BASE_TYPE_STRING,
&match));
if (!match)
return Error("cannot parse value starting with: " +
TokenToStringId(token_));
if (!match) return TokenError();
}
return NoError();
}
@@ -1881,14 +1915,21 @@ CheckedError Parser::ParseTypeFromProtoType(Type *type) {
CheckedError Parser::SkipAnyJsonValue() {
switch (token_) {
case '{':
ECHECK(SkipJsonObject());
break;
case '{': {
size_t fieldn = 0;
return ParseTableDelimiters(fieldn, nullptr,
[&](const std::string &) -> CheckedError {
ECHECK(SkipAnyJsonValue());
fieldn++;
return NoError();
});
}
case '[': {
size_t count = 0;
return ParseVectorDelimiters(count, [&]() { return SkipAnyJsonValue(); });
}
case kTokenStringConstant:
ECHECK(SkipJsonString());
break;
case '[':
ECHECK(SkipJsonArray());
EXPECT(kTokenStringConstant);
break;
case kTokenIntegerConstant:
EXPECT(kTokenIntegerConstant);
@@ -1897,56 +1938,60 @@ CheckedError Parser::SkipAnyJsonValue() {
EXPECT(kTokenFloatConstant);
break;
default:
return Error(std::string("Unexpected token:") + std::string(1, static_cast<char>(token_)));
return TokenError();
}
return NoError();
}
CheckedError Parser::SkipJsonObject() {
EXPECT('{');
size_t fieldn = 0;
for (;;) {
if ((!opts.strict_json || !fieldn) && Is('}')) break;
if (!Is(kTokenStringConstant)) {
EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier);
CheckedError Parser::ParseFlexBufferValue(flexbuffers::Builder *builder) {
switch (token_) {
case '{': {
auto start = builder->StartMap();
size_t fieldn = 0;
auto err = ParseTableDelimiters(fieldn, nullptr,
[&](const std::string &name) -> CheckedError {
builder->Key(name);
ECHECK(ParseFlexBufferValue(builder));
fieldn++;
return NoError();
});
ECHECK(err);
builder->EndMap(start);
break;
}
else {
NEXT();
case '[':{
auto start = builder->StartVector();
size_t count = 0;
ECHECK(ParseVectorDelimiters(count, [&]() {
return ParseFlexBufferValue(builder);
}));
builder->EndVector(start, false, false);
break;
}
EXPECT(':');
ECHECK(SkipAnyJsonValue());
fieldn++;
if (Is('}')) break;
EXPECT(',');
case kTokenStringConstant:
builder->String(attribute_);
EXPECT(kTokenStringConstant);
break;
case kTokenIntegerConstant:
builder->Int(StringToInt(attribute_.c_str()));
EXPECT(kTokenIntegerConstant);
break;
case kTokenFloatConstant:
builder->Double(strtod(attribute_.c_str(), nullptr));
EXPECT(kTokenFloatConstant);
break;
default:
return TokenError();
}
NEXT();
return NoError();
}
CheckedError Parser::SkipJsonArray() {
EXPECT('[');
for (;;) {
if (Is(']')) break;
ECHECK(SkipAnyJsonValue());
if (Is(']')) break;
EXPECT(',');
}
NEXT();
return NoError();
}
CheckedError Parser::SkipJsonString() {
EXPECT(kTokenStringConstant);
return NoError();
bool Parser::ParseFlexBuffer(const char *source, const char *source_filename,
flexbuffers::Builder *builder) {
auto ok = !StartParseFile(source, source_filename).Check() &&
!ParseFlexBufferValue(builder).Check();
if (ok) builder->Finish();
return ok;
}
bool Parser::Parse(const char *source, const char **include_paths,
@@ -1954,10 +1999,21 @@ bool Parser::Parse(const char *source, const char **include_paths,
return !DoParse(source, include_paths, source_filename, nullptr).Check();
}
CheckedError Parser::StartParseFile(const char *source, const char *source_filename) {
file_being_parsed_ = source_filename ? source_filename : "";
source_ = cursor_ = source;
line_ = 1;
error_.clear();
ECHECK(SkipByteOrderMark());
NEXT();
if (Is(kTokenEof))
return Error("input file is empty");
return NoError();
}
CheckedError Parser::DoParse(const char *source, const char **include_paths,
const char *source_filename,
const char *include_filename) {
file_being_parsed_ = source_filename ? source_filename : "";
if (source_filename &&
included_files_.find(source_filename) == included_files_.end()) {
included_files_[source_filename] = include_filename ? include_filename : "";
@@ -1967,18 +2023,12 @@ CheckedError Parser::DoParse(const char *source, const char **include_paths,
static const char *current_directory[] = { "", nullptr };
include_paths = current_directory;
}
source_ = cursor_ = source;
line_ = 1;
error_.clear();
field_stack_.clear();
builder_.Clear();
// Start with a blank namespace just in case this file doesn't have one.
namespaces_.push_back(new Namespace());
ECHECK(SkipByteOrderMark());
NEXT();
if (Is(kTokenEof))
return Error("input file is empty");
ECHECK(StartParseFile(source, source_filename));
// Includes must come before type declarations:
for (;;) {

7
src/reflection.cpp
View File

@@ -87,8 +87,11 @@ std::string GetAnyValueS(reflection::BaseType type, const uint8_t *data,
auto &fielddef = **it;
if (!table_field->CheckField(fielddef.offset())) continue;
auto val = GetAnyFieldS(*table_field, fielddef, schema);
if (fielddef.type()->base_type() == reflection::String)
val = "\"" + val + "\""; // Doesn't deal with escape codes etc.
if (fielddef.type()->base_type() == reflection::String) {
std::string esc;
flatbuffers::EscapeString(val.c_str(), val.length(), &esc, true);
val = esc;
}
s += fielddef.name()->str();
s += ": ";
s += val;

BIN
tests/monster_test.bfbs
View File

Binary file not shown.

11
tests/monsterdata_test.golden
View File

@@ -1,9 +1,9 @@
{
pos: {
x: 1,
y: 2,
z: 3,
test1: 3,
x: 1.0,
y: 2.0,
z: 3.0,
test1: 3.0,
test2: Green,
test3: {
a: 10,
@@ -63,5 +63,6 @@
testhashs32_fnv1a: -1904106383,
testhashu32_fnv1a: 2390860913,
testhashs64_fnv1a: 4898026182817603057,
testhashu64_fnv1a: 4898026182817603057
testhashu64_fnv1a: 4898026182817603057,
flex: 1234
}

36
tests/test.cpp
View File

@@ -309,12 +309,11 @@ void AccessFlatBufferTest(const uint8_t *flatbuf, size_t length,
// Test flexbuffer if available:
auto flex = monster->flex();
if (flex) {
// flex is a vector of bytes you can memcpy. However, if you
// actually want to access the nested data, this is a convenient
// accessor that directly gives you the root value:
TEST_EQ(monster->flex_flexbuffer_root().AsInt16(), 1234);
}
// flex is a vector of bytes you can memcpy etc.
TEST_EQ(flex->size(), 4); // Encoded FlexBuffer bytes.
// However, if you actually want to access the nested data, this is a
// convenient accessor that directly gives you the root value:
TEST_EQ(monster->flex_flexbuffer_root().AsInt16(), 1234);
// Since Flatbuffers uses explicit mechanisms to override the default
// compiler alignment, double check that the compiler indeed obeys them:
@@ -919,7 +918,8 @@ void FuzzTest2() {
AddToSchemaAndInstances((" " + field_name + ":").c_str(),
deprecated ? "" : (field_name + ": ").c_str());
// Pick random type:
int base_type = lcg_rand() % (flatbuffers::BASE_TYPE_UNION + 1);
auto base_type = static_cast<flatbuffers::BaseType>(
lcg_rand() % (flatbuffers::BASE_TYPE_UNION + 1));
switch (base_type) {
case flatbuffers::BASE_TYPE_STRING:
if (is_struct) {
@@ -970,7 +970,9 @@ void FuzzTest2() {
// We want each instance to use its own random value.
for (int inst = 0; inst < instances_per_definition; inst++)
definitions[definition].instances[inst] +=
flatbuffers::NumToString(lcg_rand() % 128).c_str();
flatbuffers::IsFloat(base_type)
? flatbuffers::NumToString<double>(lcg_rand() % 128).c_str()
: flatbuffers::NumToString<int>(lcg_rand() % 128).c_str();
}
}
AddToSchemaAndInstances(
@@ -1548,7 +1550,7 @@ void FlexBuffersTest() {
TEST_EQ(vec[2].AsDouble(), 4.0);
TEST_EQ(vec[2].AsString().IsTheEmptyString(), true); // Wrong Type.
TEST_EQ_STR(vec[2].AsString().c_str(), ""); // This still works though.
TEST_EQ_STR(vec[2].ToString().c_str(), "4"); // Or have it converted.
TEST_EQ_STR(vec[2].ToString().c_str(), "4.0"); // Or have it converted.
auto tvec = map["bar"].AsTypedVector();
TEST_EQ(tvec.size(), 3);
TEST_EQ(tvec[2].AsInt8(), 3);
@@ -1570,6 +1572,22 @@ void FlexBuffersTest() {
TEST_EQ(vec[2].MutateFloat(2.0f), true);
TEST_EQ(vec[2].AsFloat(), 2.0f);
TEST_EQ(vec[2].MutateFloat(3.14159), false); // Double does not fit in float.
// Parse from JSON:
flatbuffers::Parser parser;
slb.Clear();
auto jsontest = "{ a: [ 123, 456.0 ], b: \"hello\" }";
TEST_EQ(parser.ParseFlexBuffer(jsontest, nullptr, &slb),
true);
auto jroot = flexbuffers::GetRoot(slb.GetBuffer());
auto jmap = jroot.AsMap();
auto jvec = jmap["a"].AsVector();
TEST_EQ(jvec[0].AsInt64(), 123);
TEST_EQ(jvec[1].AsDouble(), 456.0);
TEST_EQ_STR(jmap["b"].AsString().c_str(), "hello");
// And from FlexBuffer back to JSON:
auto jsonback = jroot.ToString();
TEST_EQ_STR(jsontest, jsonback.c_str());
}
int main(int /*argc*/, const char * /*argv*/[]) {