Added (nested) FlexBuffer JSON parsing and output.

FlexBuffer parser is just 40 lines of code (on top of existing parser!). Change-Id: Idebebadafb661ca5333f5621139031f6df3c3e1a Tested: on Linux.
2026-06-09 06:30:54 +00:00 · 2017-06-05 17:45:44 -07:00
parent dddd0865cb
commit 8f864aad7b
10 changed files with 332 additions and 192 deletions
--- a/src/idl_parser.cpp
+++ b/src/idl_parser.cpp
@@ -726,6 +726,7 @@ CheckedError Parser::ParseField(StructDef &struct_def) {
  }

  if (field->attributes.Lookup("flexbuffer")) {
+    field->flexbuffer = true;
    uses_flexbuffers_ = true;
    if (field->value.type.base_type != BASE_TYPE_VECTOR ||
        field->value.type.element != BASE_TYPE_UCHAR)
@@ -830,7 +831,6 @@ CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
      break;
    }
    case BASE_TYPE_VECTOR: {
-      EXPECT('[');
      uoffset_t off;
      ECHECK(ParseVector(val.type.VectorType(), &off));
      val.constant = NumToString(off);
@@ -863,27 +863,27 @@ void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) {
  builder_.AddStructOffset(val.offset, builder_.GetSize());
 }

-CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
-                                uoffset_t *ovalue) {
-  // We allow tables both as JSON object{ .. } with field names 
+CheckedError Parser::ParseTableDelimiters(size_t &fieldn,
+                                          const StructDef *struct_def,
+    const std::function<CheckedError(const std::string &name)> &body) {
+  // We allow tables both as JSON object{ .. } with field names
  // or vector[..] with all fields in order
-  const bool is_nested_list = Is('['); 
-  if (is_nested_list) {
+  char terminator = '}';
+  bool is_nested_vector = struct_def && Is('[');
+  if (is_nested_vector) {
    NEXT();
+    terminator = ']';
  } else {
    EXPECT('{');
  }
-  size_t fieldn = 0;
  for (;;) {
-    if ((!opts.strict_json || !fieldn) && Is(is_nested_list ? ']' : '}')) { NEXT(); break; }
-    FieldDef *field = nullptr;
+    if ((!opts.strict_json || !fieldn) && Is(terminator)) break;
    std::string name;
-    if (is_nested_list) {
-      if (fieldn > struct_def.fields.vec.size()) {
+    if (is_nested_vector) {
+      if (fieldn > struct_def->fields.vec.size()) {
        return Error("too many unnamed fields in nested array");
      }
-      field = struct_def.fields.vec[fieldn];
-      name = field->name;
+      name = struct_def->fields.vec[fieldn]->name;
    } else {
      name = attribute_;
      if (Is(kTokenStringConstant)) {
@@ -891,24 +891,46 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
      } else {
        EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier);
      }
-      field = struct_def.fields.Lookup(name);
+      EXPECT(':');
    }
+    ECHECK(body(name));
+    if (Is(terminator)) break;
+    EXPECT(',');
+  }
+  NEXT();
+  if (is_nested_vector && fieldn != struct_def->fields.vec.size()) {
+    return Error("wrong number of unnamed fields in table vector");
+  }
+  return NoError();
+}
+
+CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
+                                uoffset_t *ovalue) {
+  size_t fieldn = 0;
+  auto err = ParseTableDelimiters(fieldn, &struct_def,
+                                  [&](const std::string &name) -> CheckedError {
+    auto field = struct_def.fields.Lookup(name);
    if (!field) {
      if (!opts.skip_unexpected_fields_in_json) {
        return Error("unknown field: " + name);
      } else {
-        EXPECT(':');
        ECHECK(SkipAnyJsonValue());
      }
    } else {
-      if (!is_nested_list) {
-        EXPECT(':');
-      }
      if (Is(kTokenNull)) {
        NEXT(); // Ignore this field.
      } else {
        Value val = field->value;
-        ECHECK(ParseAnyValue(val, field, fieldn, &struct_def));
+        if (field->flexbuffer) {
+          flexbuffers::Builder builder(1024,
+                                       flexbuffers::BUILDER_FLAG_SHARE_ALL);
+          ECHECK(ParseFlexBufferValue(&builder));
+          builder.Finish();
+          auto off = builder_.CreateVector(builder.GetBuffer());
+          val.constant = NumToString(off.o);
+        } else {
+          ECHECK(ParseAnyValue(val, field, fieldn, &struct_def));
+        }
        // Hardcoded insertion-sort with error-check.
        // If fields are specified in order, then this loop exits immediately.
        auto elem = field_stack_.rbegin();
@@ -924,12 +946,9 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
        fieldn++;
      }
    }
-    if (Is(is_nested_list ? ']' : '}')) { NEXT(); break; }
-    EXPECT(',');
-  }
-  if (is_nested_list && fieldn != struct_def.fields.vec.size()) {
-    return Error("wrong number of unnamed fields in table vector");
-  }
+    return NoError();
+  });
+  ECHECK(err);

  // Check if all required fields are parsed.
  for (auto field_it = struct_def.fields.vec.begin();
@@ -1029,22 +1048,34 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
  return NoError();
 }

-CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) {
-  int count = 0;
+CheckedError Parser::ParseVectorDelimiters(size_t &count,
+                                    const std::function<CheckedError()> &body) {
+  EXPECT('[');
  for (;;) {
-    if ((!opts.strict_json || !count) && Is(']')) { NEXT(); break; }
+    if ((!opts.strict_json || !count) && Is(']')) break;
+    ECHECK(body());
+    count++;
+    if (Is(']')) break;
+    EXPECT(',');
+  }
+  NEXT();
+  return NoError();
+}
+
+CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) {
+  size_t count = 0;
+  auto err = ParseVectorDelimiters(count, [&]() {
    Value val;
    val.type = type;
    ECHECK(ParseAnyValue(val, nullptr, 0, nullptr));
    field_stack_.push_back(std::make_pair(val, nullptr));
-    count++;
-    if (Is(']')) { NEXT(); break; }
-    EXPECT(',');
-  }
+    return NoError();
+  });
+  ECHECK(err);

  builder_.StartVector(count * InlineSize(type) / InlineAlignment(type),
                       InlineAlignment(type));
-  for (int i = 0; i < count; i++) {
+  for (size_t i = 0; i < count; i++) {
    // start at the back, since we're building the data backwards.
    auto &val = field_stack_.back().first;
    switch (val.type.base_type) {
@@ -1186,6 +1217,11 @@ CheckedError Parser::ParseHash(Value &e, FieldDef* field) {
  return NoError();
 }

+CheckedError Parser::TokenError() {
+  return Error("cannot parse value starting with: " +
+               TokenToStringId(token_));
+}
+
 CheckedError Parser::ParseSingleValue(Value &e) {
  // First see if this could be a conversion function:
  if (token_ == kTokenIdentifier && *cursor_ == '(') {
@@ -1252,9 +1288,7 @@ CheckedError Parser::ParseSingleValue(Value &e) {
                         e,
                         BASE_TYPE_STRING,
                         &match));
-    if (!match)
-      return Error("cannot parse value starting with: " +
-                   TokenToStringId(token_));
+    if (!match) return TokenError();
  }
  return NoError();
 }
@@ -1881,14 +1915,21 @@ CheckedError Parser::ParseTypeFromProtoType(Type *type) {

 CheckedError Parser::SkipAnyJsonValue() {
  switch (token_) {
-    case '{':
-      ECHECK(SkipJsonObject());
-      break;
+    case '{': {
+      size_t fieldn = 0;
+      return ParseTableDelimiters(fieldn, nullptr,
+                                  [&](const std::string &) -> CheckedError {
+        ECHECK(SkipAnyJsonValue());
+        fieldn++;
+        return NoError();
+      });
+    }
+    case '[': {
+      size_t count = 0;
+      return ParseVectorDelimiters(count, [&]() { return SkipAnyJsonValue(); });
+    }
    case kTokenStringConstant:
-      ECHECK(SkipJsonString());
-      break;
-    case '[':
-      ECHECK(SkipJsonArray());
+      EXPECT(kTokenStringConstant);
      break;
    case kTokenIntegerConstant:
      EXPECT(kTokenIntegerConstant);
@@ -1897,56 +1938,60 @@ CheckedError Parser::SkipAnyJsonValue() {
      EXPECT(kTokenFloatConstant);
      break;
    default:
-      return Error(std::string("Unexpected token:") + std::string(1, static_cast<char>(token_)));
+      return TokenError();
  }
  return NoError();
 }

-CheckedError Parser::SkipJsonObject() {
-  EXPECT('{');
-  size_t fieldn = 0;
-
-  for (;;) {
-    if ((!opts.strict_json || !fieldn) && Is('}')) break;
-
-    if (!Is(kTokenStringConstant)) {
-      EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier);
+CheckedError Parser::ParseFlexBufferValue(flexbuffers::Builder *builder) {
+  switch (token_) {
+    case '{': {
+      auto start = builder->StartMap();
+      size_t fieldn = 0;
+      auto err = ParseTableDelimiters(fieldn, nullptr,
+                                  [&](const std::string &name) -> CheckedError {
+        builder->Key(name);
+        ECHECK(ParseFlexBufferValue(builder));
+        fieldn++;
+        return NoError();
+      });
+      ECHECK(err);
+      builder->EndMap(start);
+      break;
    }
-    else {
-      NEXT();
+    case '[':{
+      auto start = builder->StartVector();
+      size_t count = 0;
+      ECHECK(ParseVectorDelimiters(count, [&]() {
+        return ParseFlexBufferValue(builder);
+      }));
+      builder->EndVector(start, false, false);
+      break;
    }
-
-    EXPECT(':');
-    ECHECK(SkipAnyJsonValue());
-    fieldn++;
-
-    if (Is('}')) break;
-    EXPECT(',');
+    case kTokenStringConstant:
+      builder->String(attribute_);
+      EXPECT(kTokenStringConstant);
+      break;
+    case kTokenIntegerConstant:
+      builder->Int(StringToInt(attribute_.c_str()));
+      EXPECT(kTokenIntegerConstant);
+      break;
+    case kTokenFloatConstant:
+      builder->Double(strtod(attribute_.c_str(), nullptr));
+      EXPECT(kTokenFloatConstant);
+      break;
+    default:
+      return TokenError();
  }
-
-  NEXT();
  return NoError();
 }

-CheckedError Parser::SkipJsonArray() {
-  EXPECT('[');
-
-  for (;;) {
-    if (Is(']')) break;
-
-    ECHECK(SkipAnyJsonValue());
-
-    if (Is(']')) break;
-    EXPECT(',');
-  }
-
-  NEXT();
-  return NoError();
-}
-
-CheckedError Parser::SkipJsonString() {
-  EXPECT(kTokenStringConstant);
-  return NoError();
+bool Parser::ParseFlexBuffer(const char *source, const char *source_filename,
+                             flexbuffers::Builder *builder) {
+  auto ok = !StartParseFile(source, source_filename).Check() &&
+            !ParseFlexBufferValue(builder).Check();
+  if (ok) builder->Finish();
+  return ok;
 }

 bool Parser::Parse(const char *source, const char **include_paths,
@@ -1954,10 +1999,21 @@ bool Parser::Parse(const char *source, const char **include_paths,
  return !DoParse(source, include_paths, source_filename, nullptr).Check();
 }

+CheckedError Parser::StartParseFile(const char *source, const char *source_filename) {
+  file_being_parsed_ = source_filename ? source_filename : "";
+  source_ = cursor_ = source;
+  line_ = 1;
+  error_.clear();
+  ECHECK(SkipByteOrderMark());
+  NEXT();
+  if (Is(kTokenEof))
+      return Error("input file is empty");
+  return NoError();
+}
+
 CheckedError Parser::DoParse(const char *source, const char **include_paths,
                             const char *source_filename,
                             const char *include_filename) {
-  file_being_parsed_ = source_filename ? source_filename : "";
  if (source_filename &&
      included_files_.find(source_filename) == included_files_.end()) {
    included_files_[source_filename] = include_filename ? include_filename : "";
@@ -1967,18 +2023,12 @@ CheckedError Parser::DoParse(const char *source, const char **include_paths,
    static const char *current_directory[] = { "", nullptr };
    include_paths = current_directory;
  }
-  source_ = cursor_ = source;
-  line_ = 1;
-  error_.clear();
  field_stack_.clear();
  builder_.Clear();
  // Start with a blank namespace just in case this file doesn't have one.
  namespaces_.push_back(new Namespace());
-  ECHECK(SkipByteOrderMark());
-  NEXT();

-  if (Is(kTokenEof))
-      return Error("input file is empty");
+  ECHECK(StartParseFile(source, source_filename));

  // Includes must come before type declarations:
  for (;;) {