Merge pull request #2 from google/master

Catchup

include/flatbuffers/idl.h

@@ -320,6 +320,7 @@ struct IDLOptions {
   bool one_file;
   bool proto_mode;
   bool generate_all;
+  bool skip_unexpected_fields_in_json;
 
   // Possible options for the more general generator below.
   enum Language { kJava, kCSharp, kGo, kMAX };
@@ -337,6 +338,7 @@ struct IDLOptions {
       one_file(false),
       proto_mode(false),
       generate_all(false),
+      skip_unexpected_fields_in_json(false),
       lang(IDLOptions::kJava) {}
 };
 
@@ -477,7 +479,10 @@ private:
   CHECKED_ERROR ParseProtoDecl();
   CHECKED_ERROR ParseProtoCurliesOrIdent();
   CHECKED_ERROR ParseTypeFromProtoType(Type *type);
-
+  CHECKED_ERROR SkipAnyJsonValue();
+  CHECKED_ERROR SkipJsonObject();
+  CHECKED_ERROR SkipJsonArray();
+  CHECKED_ERROR SkipJsonString();
   CHECKED_ERROR DoParse(const char *_source, const char **include_paths,
                         const char *source_filename);
 

php/ByteBuffer.php

@@ -399,8 +399,13 @@ class ByteBuffer
         $sign = $index + (ByteBuffer::isLittleEndian() ? 3 : 0);
         $issigned = isset($this->_buffer[$sign]) && ord($this->_buffer[$sign]) & 0x80;
 
-        // 4294967296 = 1 << 32 = Maximum unsigned 32-bit int
+        if (PHP_INT_SIZE > 4) {
-        return $issigned ? $result - 4294967296 : $result;
+            // 4294967296 = 1 << 32 = Maximum unsigned 32-bit int
+            return $issigned ? $result - 4294967296 : $result;
+        } else {
+            // 32bit / Windows treated number as signed integer.
+            return $result;
+        }
     }
 
     /**

src/flatc.cpp

@@ -100,12 +100,15 @@ static void Error(const std::string &err, bool usage, bool show_exe_name) {
       "                  no trailing commas in tables/vectors.\n"
       "  --defaults-json Output fields whose value is the default when\n"
       "                  writing JSON\n"
+      "  --unknown-json  Allow fields in JSON that are not defined in the\n"
+      "                  schema. These fields will be discared when generating\n"
+      "                  binaries.\n"
       "  --no-prefix     Don\'t prefix enum values with the enum type in C++.\n"
       "  --scoped-enums  Use C++11 style scoped and strongly typed enums.\n"
       "                  also implies --no-prefix.\n"
       "  --gen-includes  (deprecated), this is the default behavior.\n"
       "                  If the original behavior is required (no include\n"
-    "                  statements) use --no-includes.\n"
+      "                  statements) use --no-includes.\n"
       "  --no-includes   Don\'t generate include statements for included\n"
       "                  schemas the generated file depends on (C++).\n"
       "  --gen-mutable   Generate accessors that can mutate buffers in-place.\n"
@@ -155,6 +158,8 @@ int main(int argc, const char *argv[]) {
         opts.skip_js_exports = true;
       } else if(arg == "--defaults-json") {
         opts.output_default_scalars_in_json = true;
+      } else if (arg == "--unknown-json") {
+        opts.skip_unexpected_fields_in_json = true;
       } else if(arg == "--no-prefix") {
         opts.prefixed_enums = false;
       } else if(arg == "--scoped-enums") {

src/idl_gen_php.cpp

@@ -249,7 +249,13 @@ namespace php {
         NumToString(field.value.offset) +
         ");\n";
       code += Indent + Indent;
-      code += "return $o != 0 ? $obj->init($o + $this->bb_pos, $this->bb) : ";
+      code += "return $o != 0 ? $obj->init(";
+	  if (field.value.type.struct_def->fixed)
+      {
+        code += "$o + $this->bb_pos, $this->bb) : ";
+      } else {
+        code += "$this->__indirect($o + $this->bb_pos), $this->bb) : ";
+      }
       code += GenDefaultValue(field.value) + ";\n";
       code += Indent + "}\n\n";
     }

src/idl_parser.cpp

@@ -673,24 +673,33 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
       EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier);
     }
     auto field = struct_def.fields.Lookup(name);
-    if (!field) return Error("unknown field: " + name);
+    if (!field) {
-    EXPECT(':');
+      if (!opts.skip_unexpected_fields_in_json) {
-    Value val = field->value;
+        return Error("unknown field: " + name);
-    ECHECK(ParseAnyValue(val, field, fieldn));
+      } else {
-    size_t i = field_stack_.size();
+        EXPECT(':');
-    // Hardcoded insertion-sort with error-check.
+        ECHECK(SkipAnyJsonValue());
-    // If fields are specified in order, then this loop exits immediately.
+      }
-    for (; i > field_stack_.size() - fieldn; i--) {
+    } else {
-      auto existing_field = field_stack_[i - 1].second;
+      EXPECT(':');
-      if (existing_field == field)
+      Value val = field->value;
-        return Error("field set more than once: " + field->name);
+      ECHECK(ParseAnyValue(val, field, fieldn));
-      if (existing_field->value.offset < field->value.offset) break;
+      size_t i = field_stack_.size();
+      // Hardcoded insertion-sort with error-check.
+      // If fields are specified in order, then this loop exits immediately.
+      for (; i > field_stack_.size() - fieldn; i--) {
+        auto existing_field = field_stack_[i - 1].second;
+        if (existing_field == field)
+          return Error("field set more than once: " + field->name);
+        if (existing_field->value.offset < field->value.offset) break;
+      }
+      field_stack_.insert(field_stack_.begin() + i, std::make_pair(val, field));
+      fieldn++;
     }
-    field_stack_.insert(field_stack_.begin() + i, std::make_pair(val, field));
-    fieldn++;
     if (Is('}')) { NEXT(); break; }
     EXPECT(',');
   }
+
   if (struct_def.fixed && fieldn != struct_def.fields.vec.size())
     return Error("struct: wrong number of initializers: " + struct_def.name);
 
@@ -1480,6 +1489,72 @@ CheckedError Parser::ParseTypeFromProtoType(Type *type) {
   return NoError();
 }
 
+CheckedError Parser::SkipAnyJsonValue() {
+  switch (token_) {
+    case '{':
+      ECHECK(SkipJsonObject());
+      break;
+    case kTokenStringConstant:
+      ECHECK(SkipJsonString());
+      break;
+    case '[':
+      ECHECK(SkipJsonArray());
+      break;
+    case kTokenIntegerConstant:
+      EXPECT(kTokenIntegerConstant);
+      break;
+    case kTokenFloatConstant:
+      EXPECT(kTokenFloatConstant);
+      break;
+    default:
+      return Error(std::string("Unexpected token:") + std::string(1, token_));
+  }
+  return NoError();
+}
+
+CheckedError Parser::SkipJsonObject() {
+  EXPECT('{');
+  size_t fieldn = 0;
+
+  while (true) {
+    if ((!opts.strict_json || !fieldn) && Is('}')) break;
+
+    if (!Is(kTokenStringConstant))
+      EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier);
+
+    EXPECT(':');
+    ECHECK(SkipAnyJsonValue());
+    fieldn++;
+
+    if (Is('}')) break;
+    EXPECT(',');
+  }
+
+  NEXT();
+  return NoError();
+}
+
+CheckedError Parser::SkipJsonArray() {
+  EXPECT('[');
+  
+  while (true) {
+    if (Is(']')) break;
+        
+    ECHECK(SkipAnyJsonValue());
+
+    if (Is(']')) break;
+    EXPECT(',');
+  }
+
+  NEXT();
+  return NoError();
+}
+
+CheckedError Parser::SkipJsonString() {
+  EXPECT(kTokenStringConstant);
+  return NoError();
+}
+
 bool Parser::Parse(const char *source, const char **include_paths,
                    const char *source_filename) {
   return !DoParse(source, include_paths, source_filename).Check();

tests/MyGame/Example/Monster.php

@@ -188,7 +188,7 @@ class Monster extends Table
     {
         $obj = new Monster();
         $o = $this->__offset(28);
-        return $o != 0 ? $obj->init($o + $this->bb_pos, $this->bb) : 0;
+        return $o != 0 ? $obj->init($this->__indirect($o + $this->bb_pos), $this->bb) : 0;
     }
 
     /**
@@ -214,7 +214,7 @@ class Monster extends Table
     {
         $obj = new Stat();
         $o = $this->__offset(32);
-        return $o != 0 ? $obj->init($o + $this->bb_pos, $this->bb) : 0;
+        return $o != 0 ? $obj->init($this->__indirect($o + $this->bb_pos), $this->bb) : 0;
     }
 
     /**

tests/monsterdata_test.json

@@ -37,6 +37,9 @@
     "test1",
     "test2"
   ],
+  enemy: {
+    name: "Fred"
+  },
   testarrayofbools:[
     true, false, true
   ],

tests/phpTest.php

@@ -30,6 +30,10 @@ function main()
 
     // We set up the same values as monsterdata.json:
     $str = $fbb->createString("MyMonster");
+    $name = $fbb->createString('Fred');
+    \MyGame\Example\Monster::startMonster($fbb);
+    \MyGame\Example\Monster::addName($fbb, $name);
+    $enemy = \MyGame\Example\Monster::endMonster($fbb);
 
     $inv = \MyGame\Example\Monster::CreateInventoryVector($fbb, array(0, 1, 2, 3, 4));
 
@@ -62,6 +66,7 @@ function main()
     \MyGame\Example\Monster::AddTest($fbb, $mon2);
     \MyGame\Example\Monster::AddTest4($fbb, $test4);
     \MyGame\Example\Monster::AddTestarrayofstring($fbb, $testArrayOfString);
+    \MyGame\Example\Monster::AddEnemy($fbb, $enemy);
     \MyGame\Example\Monster::AddTestbool($fbb, false);
     $mon = \MyGame\Example\Monster::EndMonster($fbb);
 
@@ -132,6 +137,10 @@ function test_buffer(Assert $assert, Google\FlatBuffers\ByteBuffer $bb) {
     $assert->strictEqual($monster->GetTestarrayofstringLength(), 2);
     $assert->strictEqual($monster->GetTestarrayofstring(0), 'test1');
     $assert->strictEqual($monster->GetTestarrayofstring(1), 'test2');
+
+    $fred = $monster->getEnemy();
+    $assert->Equal('Fred', $fred->getName());
+
     $assert->strictEqual($monster->GetTestbool(), false);
 }
 

tests/test.cpp

@@ -813,6 +813,26 @@ void UnicodeTest() {
                      "\\u5225\\u30B5\\u30A4\\u30C8\\x01\\x80\"}", true);
 }
 
+void UnknownFieldsTest() {
+  flatbuffers::IDLOptions opts;
+  opts.skip_unexpected_fields_in_json = true;
+  flatbuffers::Parser parser(opts);
+
+  TEST_EQ(parser.Parse("table T { str:string; i:int;}"
+                       "root_type T;"
+                       "{ str:\"test\","
+                       "unknown_int:10,"
+                       "unknown_float:1.0,"
+                       "unknown_array: [ 1, 2, 3, 4],"
+                       "unknown_object: { i: 10 },"
+                       "i:10}"), true);
+
+  std::string jsongen;
+  parser.opts.indent_step = -1;
+  GenerateText(parser, parser.builder_.GetBufferPointer(), &jsongen);
+  TEST_EQ(jsongen == "{str: \"test\",i: 10}", true);
+}
+
 int main(int /*argc*/, const char * /*argv*/[]) {
   // Run our various test suites:
 
@@ -837,6 +857,7 @@ int main(int /*argc*/, const char * /*argv*/[]) {
   ScientificTest();
   EnumStringsTest();
   UnicodeTest();
+  UnknownFieldsTest();
 
   if (!testing_fails) {
     TEST_OUTPUT_LINE("ALL TESTS PASSED");