flatc can now convert flatbuffer binary files into json.

Also removed it appending _wire to filenames, renamed the json golden
file to .golden to not clash with generated files.

Bug: 15781201
Change-Id: I8322861e50d1e5b6a5ab5e4b5e5d8ae13c356eb2
Tested: on Windows and Linux

build/VS2010/flatc.vcxproj.user

@@ -3,14 +3,14 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <LocalDebuggerWorkingDirectory>..\..\tests</LocalDebuggerWorkingDirectory>
     <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>
-    <LocalDebuggerCommandArguments>-j -c -b -t monster_test.fbs monsterdata_test.json</LocalDebuggerCommandArguments>
+    <LocalDebuggerCommandArguments>-j -c -b -t monster_test.fbs monsterdata_test.golden</LocalDebuggerCommandArguments>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <LocalDebuggerWorkingDirectory>..\..</LocalDebuggerWorkingDirectory>
     <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
-    <LocalDebuggerCommandArguments>-j -c -b -t monster_test.fbs monsterdata_test.json</LocalDebuggerCommandArguments>
+    <LocalDebuggerCommandArguments>-j -c -b -t monster_test.fbs monsterdata_test.golden</LocalDebuggerCommandArguments>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <LocalDebuggerWorkingDirectory>..\..\tests</LocalDebuggerWorkingDirectory>

docs/html/md__compiler.html

@@ -53,13 +53,16 @@ $(document).ready(function(){initNavTree('md__compiler.html','');});
 <div class="title">Using the schema compiler </div>  </div>
 </div><!--header-->
 <div class="contents">
-<div class="textblock"><p>Usage: </p><pre class="fragment">flatc [ -c ] [ -j ] [ -b ] [ -t ] [ -o PATH ] [ -S ] file1 file2 ..
-</pre><p>The files are read and parsed in order, and can contain either schemas or data (see below). Later files can make use of definitions in earlier files. Depending on the flags passed, additional files may be generated for each file processed:</p>
+<div class="textblock"><p>Usage: </p><pre class="fragment">flatc [ -c ] [ -j ] [ -b ] [ -t ] [ -o PATH ] [ -S ] FILES...
+      [ -- FILES...]
+</pre><p>The files are read and parsed in order, and can contain either schemas or data (see below). Later files can make use of definitions in earlier files.</p>
+<p><code>--</code> indicates that the following files are binary files in FlatBuffer format conforming to the schema(s) indicated before it. Incompatible binary files currently will give unpredictable results (!)</p>
+<p>Depending on the flags passed, additional files may be generated for each file processed:</p>
 <ul>
-<li><code>-c</code> : Generate a C++ header for all definitions in this file (as <code>filename_generated.h</code>). Skips data.</li>
-<li><code>-j</code> : Generate Java classes.</li>
-<li><code>-b</code> : If data is contained in this file, generate a <code>filename_wire.bin</code> containing the binary flatbuffer.</li>
-<li><code>-t</code> : If data is contained in this file, generate a <code>filename_wire.txt</code> (for debugging).</li>
+<li><code>-c</code> : Generate a C++ header for all definitions in this file (as <code>filename_generated.h</code>). Skipped for data.</li>
+<li><code>-j</code> : Generate Java classes. Skipped for data.</li>
+<li><code>-b</code> : If data is contained in this file, generate a <code>filename.bin</code> containing the binary flatbuffer.</li>
+<li><code>-t</code> : If data is contained in this file, generate a <code>filename.json</code> representing the data in the flatbuffer.</li>
 <li><code>-o PATH</code> : Output all generated files to PATH (either absolute, or relative to the current directory). If omitted, PATH will be the current directory. PATH should end in your systems path separator, e.g. <code>/</code> or <code>\</code>.</li>
 <li><code>-S</code> : Generate strict JSON (field names are enclosed in quotes). By default, no quotes are generated. </li>
 </ul>

docs/source/Compiler.md

@@ -2,23 +2,30 @@
 
 Usage:
 
-    flatc [ -c ] [ -j ] [ -b ] [ -t ] [ -o PATH ] [ -S ] file1 file2 ..
+    flatc [ -c ] [ -j ] [ -b ] [ -t ] [ -o PATH ] [ -S ] FILES...
+          [ -- FILES...]
 
 The files are read and parsed in order, and can contain either schemas
 or data (see below). Later files can make use of definitions in earlier
-files. Depending on the flags passed, additional files may
+files.
+
+`--` indicates that the following files are binary files in
+FlatBuffer format conforming to the schema(s) indicated before it.
+Incompatible binary files currently will give unpredictable results (!)
+
+Depending on the flags passed, additional files may
 be generated for each file processed:
 
 -   `-c` : Generate a C++ header for all definitions in this file (as
-    `filename_generated.h`). Skips data.
+    `filename_generated.h`). Skipped for data.
 
--   `-j` : Generate Java classes.
+-   `-j` : Generate Java classes. Skipped for data.
 
 -   `-b` : If data is contained in this file, generate a
-    `filename_wire.bin` containing the binary flatbuffer.
+    `filename.bin` containing the binary flatbuffer.
 
 -   `-t` : If data is contained in this file, generate a
-    `filename_wire.txt` (for debugging).
+    `filename.json` representing the data in the flatbuffer.
 
 -   `-o PATH` : Output all generated files to PATH (either absolute, or
     relative to the current directory). If omitted, PATH will be the

src/flatc.cpp

@@ -29,7 +29,7 @@ bool GenerateBinary(const Parser &parser,
                     const GeneratorOptions & /*opts*/) {
   return !parser.builder_.GetSize() ||
          flatbuffers::SaveFile(
-           (path + file_name + "_wire.bin").c_str(),
+           (path + file_name + ".bin").c_str(),
            reinterpret_cast<char *>(parser.builder_.GetBufferPointer()),
            parser.builder_.GetSize(),
            true);
@@ -44,7 +44,7 @@ bool GenerateTextFile(const Parser &parser,
   std::string text;
   GenerateText(parser, parser.builder_.GetBufferPointer(), opts,
                &text);
-  return flatbuffers::SaveFile((path + file_name + "_wire.txt").c_str(),
+  return flatbuffers::SaveFile((path + file_name + ".json").c_str(),
                                text,
                                false);
 
@@ -78,16 +78,17 @@ const Generator generators[] = {
 const char *program_name = NULL;
 
 static void Error(const char *err, const char *obj, bool usage) {
-  printf("%s: %s\n", program_name, err);
+  printf("%s: %s", program_name, err);
   if (obj) printf(": %s", obj);
   printf("\n");
   if (usage) {
-    printf("usage: %s [OPTION]... FILE...\n", program_name);
+    printf("usage: %s [OPTION]... FILE... [-- FILE...]\n", program_name);
     for (size_t i = 0; i < sizeof(generators) / sizeof(generators[0]); ++i)
       printf("  -%s      %s.\n", generators[i].extension, generators[i].help);
     printf("  -o PATH Prefix PATH to all generated files.\n"
            "  -S      Strict JSON: add quotes to field names.\n"
            "FILEs may depend on declarations in earlier files.\n"
+           "FILEs after the -- must be binary flatbuffer format files.\n"
            "Output files are named using the base file name of the input,"
            "and written to the current directory or the path given by -o.\n"
            "example: %s -c -b schema1.fbs schema2.fbs data.json\n",
@@ -121,10 +122,11 @@ int main(int argc, const char *argv[]) {
   bool generator_enabled[num_generators] = { false };
   bool any_generator = false;
   std::vector<std::string> filenames;
+  size_t binary_files_from = std::numeric_limits<size_t>::max();
   for (int i = 1; i < argc; i++) {
     const char *arg = argv[i];
     if (arg[0] == '-') {
-      if (filenames.size())
+      if (filenames.size() && arg[1] != '-')
         Error("invalid option location", arg, true);
       if (strlen(arg) != 2)
         Error("invalid commandline argument", arg, true);
@@ -136,6 +138,9 @@ int main(int argc, const char *argv[]) {
         case 'S':
           opts.strict_json = true;
           break;
+        case '-':  // Separator between text and binary input files.
+          binary_files_from = filenames.size();
+          break;
         default:
           for (size_t i = 0; i < num_generators; ++i) {
             if(!strcmp(arg+1, generators[i].extension)) {
@@ -167,8 +172,17 @@ int main(int argc, const char *argv[]) {
       if (!flatbuffers::LoadFile(file_it->c_str(), true, &contents))
         Error("unable to load file", file_it->c_str());
 
-      if (!parser.Parse(contents.c_str()))
-        Error(parser.error_.c_str());
+      bool is_binary = static_cast<size_t>(file_it - filenames.begin()) >=
+                       binary_files_from;
+      if (is_binary) {
+        parser.builder_.Clear();
+        parser.builder_.PushBytes(
+          reinterpret_cast<const uint8_t *>(contents.c_str()),
+          contents.length());
+      } else {
+        if (!parser.Parse(contents.c_str()))
+          Error(parser.error_.c_str());
+      }
 
       std::string filebase = StripPath(StripExtension(*file_it));
 

tests/JavaTest.java

@@ -26,7 +26,7 @@ class JavaTest {
         // This file was generated from monsterdata_test.json
 
         byte[] data = null;
-        File file = new File("monsterdata_test_wire.bin");
+        File file = new File("monsterdata_test.bin");
         RandomAccessFile f = null;
         try {
             f = new RandomAccessFile(file, "r");

tests/monsterdata_test.golden

@@ -0,0 +1,36 @@
+{
+  pos: {
+    x: 1,
+    y: 2,
+    z: 3,
+    test1: 3,
+    test2: 4,
+    test3: {
+      a: 5,
+      b: 6
+    }
+  },
+  hp: 80,
+  name: "MyMonster",
+  inventory: [
+    0,
+    1,
+    2,
+    3,
+    4
+  ],
+  test_type: 1,
+  test: {
+    hp: 20
+  },
+  test4: [
+    {
+      a: 10,
+      b: 20
+    },
+    {
+      a: 30,
+      b: 40
+    }
+  ]
+}

tests/test.cpp

@@ -181,7 +181,7 @@ void ParseAndGenerateTextTest() {
   TEST_EQ(flatbuffers::LoadFile(
     "tests/monster_test.fbs", false, &schemafile), true);
   TEST_EQ(flatbuffers::LoadFile(
-    "tests/monsterdata_test.json", false, &jsonfile), true);
+    "tests/monsterdata_test.golden", false, &jsonfile), true);
 
   // parse schema first, so we can use it to parse the data after
   flatbuffers::Parser parser;