Enable --gen-onefile in Python (#6953)

* Enable --gen-onefile in Python

Made it possible to generate all python code in one file. Modified
py_test.py so that it can switch between the multi-file code and
the one-file code.

Updated PythonTest.sh and py_test.py so that the multi-file code
and the one-file code can be tested based on the same test code.

* Sync with google/flatbuffers

* Add --gen-onefile to generate_code.py

docs/source/Compiler.md

@@ -101,7 +101,7 @@ Additional options:
 -   `--gen-mutable` : Generate additional non-const accessors for mutating
     FlatBuffers in-place.
 
--   `--gen-onefile` : Generate single output file for C# and Go.
+-   `--gen-onefile` : Generate single output file for C#, Go, and Python.
 
 -   `--gen-name-strings` : Generate type name functions for C++.
 

scripts/generate_code.py

@@ -162,6 +162,13 @@ flatc(
     data="monsterdata_test.json",
 )
 
+flatc(
+    options=BASE_OPTS + ["--python", "--gen-onefile"],
+    schema="monster_test.fbs",
+    include="include_test",
+    data="monsterdata_test.json",
+)
+
 # For Rust we currently generate two independent schemas, with namespace_test2
 # duplicating the types in namespace_test1
 flatc(

src/flatc.cpp

@@ -102,7 +102,7 @@ std::string FlatCompiler::GetUsageString(const char *program_name) const {
     "  --no-includes          Don\'t generate include statements for included\n"
     "                         schemas the generated file depends on (C++ / Python).\n"
     "  --gen-mutable          Generate accessors that can mutate buffers in-place.\n"
-    "  --gen-onefile          Generate single output file for C# and Go.\n"
+    "  --gen-onefile          Generate single output file for C#, Go, and Python.\n"
     "  --gen-name-strings     Generate type name functions for C++ and Rust.\n"
     "  --gen-object-api       Generate an additional object-based API.\n"
     "  --gen-compare          Generate operator== for object-based API types.\n"
@@ -331,6 +331,7 @@ int FlatCompiler::Compile(int argc, const char **argv) {
         opts.include_dependence_headers = false;
       } else if (arg == "--gen-onefile") {
         opts.one_file = true;
+        opts.include_dependence_headers = false;
       } else if (arg == "--raw-binary") {
         raw_binary = true;
       } else if (arg == "--size-prefixed") {

src/idl_gen_python.cpp

@@ -113,12 +113,6 @@ class PythonGenerator : public BaseGenerator {
     code += enum_def.ToString(ev) + "\n";
   }
 
-  // End enum code.
-  void EndEnum(std::string *code_ptr) {
-    auto &code = *code_ptr;
-    code += "\n";
-  }
-
   // Initialize a new struct or table from existing data.
   void NewRootTypeFromBuffer(const StructDef &struct_def,
                              std::string *code_ptr) {
@@ -562,10 +556,12 @@ class PythonGenerator : public BaseGenerator {
     code += NumToString(struct_def.fields.vec.size());
     code += ")\n";
 
-    // Generate method without struct name.
-    code += "def Start(builder):\n";
-    code +=
-        Indent + "return " + NormalizedName(struct_def) + "Start(builder)\n";
+    if (!parser_.opts.one_file) {
+      // Generate method without struct name.
+      code += "def Start(builder):\n";
+      code +=
+          Indent + "return " + NormalizedName(struct_def) + "Start(builder)\n";
+    }
   }
 
   // Set the value of a table's field.
@@ -595,16 +591,18 @@ class PythonGenerator : public BaseGenerator {
                 : field.value.constant;
     code += ")\n";
 
-    // Generate method without struct name.
-    code += "def Add" + MakeCamel(NormalizedName(field));
-    code += "(builder, ";
-    code += MakeCamel(NormalizedName(field), false);
-    code += "):\n";
-    code += Indent + "return " + NormalizedName(struct_def) + "Add" +
-            MakeCamel(NormalizedName(field));
-    code += "(builder, ";
-    code += MakeCamel(NormalizedName(field), false);
-    code += ")\n";
+    if (!parser_.opts.one_file) {
+      // Generate method without struct name.
+      code += "def Add" + MakeCamel(NormalizedName(field));
+      code += "(builder, ";
+      code += MakeCamel(NormalizedName(field), false);
+      code += "):\n";
+      code += Indent + "return " + NormalizedName(struct_def) + "Add" +
+              MakeCamel(NormalizedName(field));
+      code += "(builder, ";
+      code += MakeCamel(NormalizedName(field), false);
+      code += ")\n";
+    }
   }
 
   // Set the value of one of the members of a table's vector.
@@ -623,13 +621,15 @@ class PythonGenerator : public BaseGenerator {
     code += ", numElems, " + NumToString(alignment);
     code += ")\n";
 
-    // Generate method without struct name.
-    code += "def Start";
-    code += MakeCamel(NormalizedName(field));
-    code += "Vector(builder, numElems):\n";
-    code += Indent + "return " + NormalizedName(struct_def) + "Start";
-    code += MakeCamel(NormalizedName(field));
-    code += "Vector(builder, numElems)\n";
+    if (!parser_.opts.one_file) {
+      // Generate method without struct name.
+      code += "def Start";
+      code += MakeCamel(NormalizedName(field));
+      code += "Vector(builder, numElems):\n";
+      code += Indent + "return " + NormalizedName(struct_def) + "Start";
+      code += MakeCamel(NormalizedName(field));
+      code += "Vector(builder, numElems)\n";
+    }
   }
 
   // Set the value of one of the members of a table's vector and fills in the
@@ -670,12 +670,14 @@ class PythonGenerator : public BaseGenerator {
     code += " = bytes\n";
     code += Indent + "return builder.EndVector()\n";
 
-    // Generate method without struct and field name.
-    code += "def Make" + MakeCamel(NormalizedName(field)) +
-            "VectorFromBytes(builder, bytes):\n";
-    code += Indent + "return " + NormalizedName(struct_def) + "Make" +
-            MakeCamel(NormalizedName(field)) +
-            "VectorFromBytes(builder, bytes)\n";
+    if (!parser_.opts.one_file) {
+      // Generate method without struct and field name.
+      code += "def Make" + MakeCamel(NormalizedName(field)) +
+              "VectorFromBytes(builder, bytes):\n";
+      code += Indent + "return " + NormalizedName(struct_def) + "Make" +
+              MakeCamel(NormalizedName(field)) +
+              "VectorFromBytes(builder, bytes)\n";
+    }
   }
 
   // Get the offset of the end of a table.
@@ -687,9 +689,11 @@ class PythonGenerator : public BaseGenerator {
     code += "(builder): ";
     code += "return builder.EndObject()\n";
 
-    // Generate method without struct name.
-    code += "def End(builder):\n";
-    code += Indent + "return " + NormalizedName(struct_def) + "End(builder)";
+    if (!parser_.opts.one_file) {
+      // Generate method without struct name.
+      code += "def End(builder):\n";
+      code += Indent + "return " + NormalizedName(struct_def) + "End(builder)";
+    }
   }
 
   // Generate the receiver for function signatures.
@@ -734,8 +738,11 @@ class PythonGenerator : public BaseGenerator {
           }
           break;
         }
-        case BASE_TYPE_UNION: GetUnionField(struct_def, field, code_ptr); break;
-        default: FLATBUFFERS_ASSERT(0);
+        case BASE_TYPE_UNION:
+          GetUnionField(struct_def, field, code_ptr);
+          break;
+        default:
+          FLATBUFFERS_ASSERT(0);
       }
     }
     if (IsVector(field.value.type) || IsArray(field.value.type)) {
@@ -906,9 +913,14 @@ class PythonGenerator : public BaseGenerator {
             import_list->insert("import " + package_reference);
           }
           break;
-        case BASE_TYPE_STRING: field_type += "str"; break;
-        case BASE_TYPE_NONE: field_type += "None"; break;
-        default: break;
+        case BASE_TYPE_STRING:
+          field_type += "str";
+          break;
+        case BASE_TYPE_NONE:
+          field_type += "None";
+          break;
+        default:
+          break;
       }
       field_types += field_type + separator_string;
     }
@@ -1241,7 +1253,8 @@ class PythonGenerator : public BaseGenerator {
           GenUnPackForScalarVector(struct_def, field, &code);
           break;
         }
-        default: GenUnPackForScalar(struct_def, field, &code);
+        default:
+          GenUnPackForScalar(struct_def, field, &code);
       }
     }
 
@@ -1639,7 +1652,6 @@ class PythonGenerator : public BaseGenerator {
       GenComment(ev.doc_comment, code_ptr, &def_comment, Indent.c_str());
       EnumMember(enum_def, ev, code_ptr);
     }
-    EndEnum(code_ptr);
   }
 
   // Returns the function name that is able to read a value of the given type.
@@ -1709,13 +1721,20 @@ class PythonGenerator : public BaseGenerator {
   }
 
   bool generate() {
-    if (!generateEnums()) return false;
-    if (!generateStructs()) return false;
+    std::string one_file_code;
+    if (!generateEnums(&one_file_code)) return false;
+    if (!generateStructs(&one_file_code)) return false;
+
+    if (parser_.opts.one_file) {
+      return SaveType(file_name_ + "_generated", *parser_.current_namespace_,
+                      one_file_code, true);
+    }
+
     return true;
   }
 
  private:
-  bool generateEnums() {
+  bool generateEnums(std::string *one_file_code) {
     for (auto it = parser_.enums_.vec.begin(); it != parser_.enums_.vec.end();
          ++it) {
       auto &enum_def = **it;
@@ -1724,12 +1743,19 @@ class PythonGenerator : public BaseGenerator {
       if (parser_.opts.generate_object_based_api & enum_def.is_union) {
         GenUnionCreator(enum_def, &enumcode);
       }
-      if (!SaveType(enum_def, enumcode, false)) return false;
+
+      if (parser_.opts.one_file && !enumcode.empty()) {
+        *one_file_code += enumcode + "\n\n";
+      } else {
+        if (!SaveType(enum_def.name, *enum_def.defined_namespace, enumcode,
+                      false))
+          return false;
+      }
     }
     return true;
   }
 
-  bool generateStructs() {
+  bool generateStructs(std::string *one_file_code) {
     for (auto it = parser_.structs_.vec.begin();
          it != parser_.structs_.vec.end(); ++it) {
       auto &struct_def = **it;
@@ -1738,7 +1764,14 @@ class PythonGenerator : public BaseGenerator {
       if (parser_.opts.generate_object_based_api) {
         GenStructForObjectAPI(struct_def, &declcode);
       }
-      if (!SaveType(struct_def, declcode, true)) return false;
+
+      if (parser_.opts.one_file && !declcode.empty()) {
+        *one_file_code += declcode + "\n\n";
+      } else {
+        if (!SaveType(struct_def.name, *struct_def.defined_namespace, declcode,
+                      true))
+          return false;
+      }
     }
     return true;
   }
@@ -1757,12 +1790,12 @@ class PythonGenerator : public BaseGenerator {
   }
 
   // Save out the generated code for a Python Table type.
-  bool SaveType(const Definition &def, const std::string &classcode,
-                bool needs_imports) {
+  bool SaveType(const std::string &defname, const Namespace &ns,
+                const std::string &classcode, bool needs_imports) {
     if (!classcode.length()) return true;
 
     std::string namespace_dir = path_;
-    auto &namespaces = def.defined_namespace->components;
+    auto &namespaces = ns.components;
     for (auto it = namespaces.begin(); it != namespaces.end(); ++it) {
       if (it != namespaces.begin()) namespace_dir += kPathSeparator;
       namespace_dir += *it;
@@ -1771,10 +1804,9 @@ class PythonGenerator : public BaseGenerator {
     }
 
     std::string code = "";
-    BeginFile(LastNamespacePart(*def.defined_namespace), needs_imports, &code);
+    BeginFile(LastNamespacePart(ns), needs_imports, &code);
     code += classcode;
-    std::string filename =
-        NamespaceDir(*def.defined_namespace) + NormalizedName(def) + ".py";
+    std::string filename = NamespaceDir(ns) + defname + ".py";
     return SaveFile(filename.c_str(), code, false);
   }
 

tests/MyGame/Example/Any.py

@@ -8,7 +8,6 @@ class Any(object):
     TestSimpleTableWithEnum = 2
     MyGame_Example2_Monster = 3
 
-
 def AnyCreator(unionType, table):
     from flatbuffers.table import Table
     if not isinstance(table, Table):

tests/MyGame/Example/AnyAmbiguousAliases.py

@@ -8,7 +8,6 @@ class AnyAmbiguousAliases(object):
     M2 = 2
     M3 = 3
 
-
 def AnyAmbiguousAliasesCreator(unionType, table):
     from flatbuffers.table import Table
     if not isinstance(table, Table):

tests/MyGame/Example/AnyUniqueAliases.py

@@ -8,7 +8,6 @@ class AnyUniqueAliases(object):
     TS = 2
     M2 = 3
 
-
 def AnyUniqueAliasesCreator(unionType, table):
     from flatbuffers.table import Table
     if not isinstance(table, Table):

tests/MyGame/Example/Color.py

@@ -10,4 +10,3 @@ class Color(object):
     Green = 2
     # \brief color Blue (1u << 3)
     Blue = 8
-

tests/MyGame/Example/Race.py

@@ -7,4 +7,3 @@ class Race(object):
     Human = 0
     Dwarf = 1
     Elf = 2
-

tests/MyGame/Example/TestEnum.py

@@ -6,4 +6,3 @@ class TestEnum(object):
     A = 0
     B = 1
     C = 2
-

tests/PythonTest.sh

@@ -21,6 +21,7 @@ runtime_library_dir=${test_dir}/../python
 
 # Emit Python code for the example schema in the test dir:
 ${test_dir}/../flatc -p -o ${gen_code_path} -I include_test monster_test.fbs --gen-object-api
+${test_dir}/../flatc -p -o ${gen_code_path} -I include_test monster_test.fbs --gen-object-api --gen-onefile
 ${test_dir}/../flatc -p -o ${gen_code_path} -I include_test monster_extra.fbs --gen-object-api
 
 # Syntax: run_tests <interpreter> <benchmark vtable dedupes>
@@ -35,7 +36,7 @@ function run_tests() {
     JYTHONPATH=${runtime_library_dir}:${gen_code_path} \
     COMPARE_GENERATED_TO_GO=0 \
     COMPARE_GENERATED_TO_JAVA=0 \
-    $1 py_test.py $2 $3 $4
+    $1 py_test.py $2 $3 $4 $5
     if [ $1 = python3 ]; then
       PYTHONDONTWRITEBYTECODE=1 \
       PYTHONPATH=${runtime_library_dir}:${gen_code_path} \
@@ -47,10 +48,12 @@ function run_tests() {
 }
 
 # Run test suite with these interpreters. The arguments are benchmark counts.
-run_tests python2.6 100 100 100
-run_tests python2.7 100 100 100
-run_tests python3 100 100 100
-run_tests pypy 100 100 100
+run_tests python2.6 100 100 100 false
+run_tests python2.7 100 100 100 false
+run_tests python2.7 100 100 100 true
+run_tests python3 100 100 100 false
+run_tests python3 100 100 100 true
+run_tests pypy 100 100 100 false
 
 # NOTE: We'd like to support python2.5 in the future.
 

tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.py

@@ -6,4 +6,3 @@ class EnumInNestedNS(object):
     A = 0
     B = 1
     C = 2
-

tests/namespace_test/NamespaceA/NamespaceB/UnionInNestedNS.py

@@ -6,7 +6,6 @@ class UnionInNestedNS(object):
     NONE = 0
     TableInNestedNS = 1
 
-
 def UnionInNestedNSCreator(unionType, table):
     from flatbuffers.table import Table
     if not isinstance(table, Table):