From 413115858c1e89b340b0e4465a005d0f0ac45478 Mon Sep 17 00:00:00 2001 From: Joshua Smith <102520999+joshua-smith8@users.noreply.github.com> Date: Thu, 22 Sep 2022 19:08:09 +0100 Subject: [PATCH] [Python] Python fixed size array (#7529) * feat: Added support for fixed sized arrays to python Problem: We encountered that using fixed arrays from C++ to python that python would not read those arrays correctly due to no size information being encoded in the byte array itself. Fix: Encode the sizes within the generated python file during code generation. Specfically we add GetArrayAsNumpy to the python version of table, which takes as input the length of the vector. When generating the python message files we include this length from the VectorType().fixed_length. * fix: added digit support for camel case to snake case conversion Problem: When including a number in the message name we would encounter cases where SnakeCase would not add the appropirate breaks. e.g. Int32Stamped -> int_32stamped rather than int_32_stamped. Fix: To fix this we can add the condition that we check if the current character is not lower and not a digit, that we check if the previous character was a lower or digit. If it was a lower or digit then we add the break. * fix: Array support for structures Problem: The python generated code for handling non-struct and struct vectors and arrays was inconsistent. The calls to populate the obj api was creating incorrect code. Solution: To fix this the VectorOfStruct and VectorOfNonStruct was rewritten to handle array cases and bring the two methods in line which each other. Testing: PythonTesting.sh now correctly runs and generates the code for array_test.fbs. Minor modifications were done on the test to use the new index accessor for struct arrays and the script correctly sources the location of the python code. * chore: clang format changes * Added code generated by scripts/generate_code. Modified GetArrayOfNonStruct slightly to allow for function overloading allowing the user to get a single element of an array or the whole array. * Added new_line parameter to OffsetPrefix to allow optional new lines to be added. This allows us to use the GenIndents method that automatically adds new lines instead. * Reupload of generated code from the scripts/generate_code.py * Removed new line in GetVectorAsNumpy. * Updated Array lengths to use Length methods where possible. Added fallthrough for GenTypePointer. Added digit check to CamelToSnake method. Added and modified tests for ToSnakeCase and CamelToSnake. * Added range check on the getter methods for vector and array types. Renamed == as is for python --- python/flatbuffers/table.py | 11 +- src/idl_gen_python.cpp | 165 +++++++++++++----- src/util.cpp | 4 +- tests/MyGame/Example/Ability.py | 3 +- tests/MyGame/Example/ArrayStruct.py | 63 ++++--- tests/MyGame/Example/ArrayTable.py | 3 +- tests/MyGame/Example/Monster.py | 3 +- tests/MyGame/Example/NestedStruct.py | 66 ++++--- tests/MyGame/Example/Referrable.py | 3 +- tests/MyGame/Example/Stat.py | 3 +- tests/MyGame/Example/StructOfStructs.py | 3 +- .../Example/StructOfStructsOfStructs.py | 3 +- tests/MyGame/Example/Test.py | 3 +- .../MyGame/Example/TestSimpleTableWithEnum.py | 3 +- tests/MyGame/Example/TypeAliases.py | 3 +- tests/MyGame/Example/Vec3.py | 3 +- tests/MyGame/Example2/Monster.py | 3 +- tests/MyGame/InParentNamespace.py | 3 +- tests/MyGame/MonsterExtra.py | 3 +- tests/PythonTest.sh | 5 +- tests/monster_test_generated.py | 36 ++-- tests/optional_scalars/ScalarStuff.py | 3 +- tests/py_test.py | 21 ++- tests/util_test.cpp | 10 +- 24 files changed, 293 insertions(+), 133 deletions(-) diff --git a/python/flatbuffers/table.py b/python/flatbuffers/table.py index adc76ca8d..d5336ca6b 100644 --- a/python/flatbuffers/table.py +++ b/python/flatbuffers/table.py @@ -113,6 +113,15 @@ class Table(object): numpy_dtype = N.to_numpy_type(flags) return encode.GetVectorAsNumpy(numpy_dtype, self.Bytes, length, offset) + def GetArrayAsNumpy(self, flags, off, length): + """ + GetArrayAsNumpy returns the array with fixed width that starts at `Vector(offset)` + with length `length` as a numpy array with the type specified by `flags`. The + array is a `view` into Bytes so modifying the returned will modify Bytes in place. + """ + numpy_dtype = N.to_numpy_type(flags) + return encode.GetVectorAsNumpy(numpy_dtype, self.Bytes, length, off) + def GetVOffsetTSlot(self, slot, d): """ GetVOffsetTSlot retrieves the VOffsetT that the given vtable location @@ -125,5 +134,5 @@ class Table(object): off = self.Offset(slot) if off == 0: - return d + return d return off diff --git a/src/idl_gen_python.cpp b/src/idl_gen_python.cpp index 0b8ffa821..289b18691 100644 --- a/src/idl_gen_python.cpp +++ b/src/idl_gen_python.cpp @@ -83,11 +83,11 @@ class PythonGenerator : public BaseGenerator { // Most field accessors need to retrieve and test the field offset first, // this is the prefix code for that. - std::string OffsetPrefix(const FieldDef &field) const { + std::string OffsetPrefix(const FieldDef &field, bool new_line = true) const { return "\n" + Indent + Indent + "o = flatbuffers.number_types.UOffsetTFlags.py_type" + "(self._tab.Offset(" + NumToString(field.value.offset) + "))\n" + - Indent + Indent + "if o != 0:\n"; + Indent + Indent + "if o != 0:" + (new_line ? "\n" : ""); } // Begin a class declaration. @@ -164,9 +164,14 @@ class PythonGenerator : public BaseGenerator { GenReceiver(struct_def, code_ptr); code += namer_.Method(field) + "Length(self"; - code += "):" + OffsetPrefix(field); - code += Indent + Indent + Indent + "return self._tab.VectorLen(o)\n"; - code += Indent + Indent + "return 0\n\n"; + code += "):"; + if(!IsArray(field.value.type)){ + code += OffsetPrefix(field,false); + code += GenIndents(3) + "return self._tab.VectorLen(o)"; + code += GenIndents(2) + "return 0\n\n"; + }else{ + code += GenIndents(2) + "return "+NumToString(field.value.type.fixed_length)+"\n\n"; + } } // Determines whether a vector is none or not. @@ -177,10 +182,15 @@ class PythonGenerator : public BaseGenerator { GenReceiver(struct_def, code_ptr); code += namer_.Method(field) + "IsNone(self"; code += "):"; - code += GenIndents(2) + - "o = flatbuffers.number_types.UOffsetTFlags.py_type" + - "(self._tab.Offset(" + NumToString(field.value.offset) + "))"; - code += GenIndents(2) + "return o == 0"; + if(!IsArray(field.value.type)){ + code += GenIndents(2) + + "o = flatbuffers.number_types.UOffsetTFlags.py_type" + + "(self._tab.Offset(" + NumToString(field.value.offset) + "))"; + code += GenIndents(2) + "return o == 0"; + } else { + //assume that we always have an array as memory is preassigned + code += GenIndents(2) + "return False"; + } code += "\n\n"; } @@ -244,21 +254,42 @@ class PythonGenerator : public BaseGenerator { const auto vec_type = field.value.type.VectorType(); GenReceiver(struct_def, code_ptr); code += namer_.Method(field); - if (IsStruct(vec_type)) { - code += "(self, obj, i):\n"; - code += Indent + Indent + "obj.Init(self._tab.Bytes, self._tab.Pos + "; - code += NumToString(field.value.offset) + " + i * "; - code += NumToString(InlineSize(vec_type)); - code += ")\n" + Indent + Indent + "return obj\n\n"; - } else { - auto getter = GenGetter(vec_type); - code += "(self): return [" + getter; - code += "self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type("; - code += NumToString(field.value.offset) + " + i * "; - code += NumToString(InlineSize(vec_type)); - code += ")) for i in range("; - code += NumToString(field.value.type.fixed_length) + ")]\n"; + code += "(self, i: int):"; + if (parser_.opts.include_dependence_headers) { + code += GenIndents(2); + code += "from " + GenPackageReference(field.value.type) + " import " + + TypeName(field); } + code += GenIndents(2) + "obj = " + TypeName(field) + "()"; + code += GenIndents(2) + "obj.Init(self._tab.Bytes, self._tab.Pos + "; + code += NumToString(field.value.offset) + " + i * "; + code += NumToString(InlineSize(vec_type)); + code += ")" + GenIndents(2) + "return obj\n\n"; + } + + // Get the value of a vector's non-struct member. Uses a named return + // argument to conveniently set the zero value for the result. + void GetArrayOfNonStruct(const StructDef &struct_def, const FieldDef &field, + std::string *code_ptr) const { + auto &code = *code_ptr; + GenReceiver(struct_def, code_ptr); + code += namer_.Method(field); + code += "(self, j = None):"; + code += GenIndents(2) + "if j is None:"; + code += GenIndents(3) + "return [" + GenGetter(field.value.type); + code += "self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type("; + code += NumToString(field.value.offset) + " + i * "; + code += NumToString(InlineSize(field.value.type.VectorType())); + code += ")) for i in range("; + code += "self."+namer_.Method(field)+"Length()" + ")]"; + code += GenIndents(2) +"elif j >= 0 and j < self."+namer_.Method(field)+"Length():"; + code += GenIndents(3) + "return " + GenGetter(field.value.type); + code += "self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type("; + code += NumToString(field.value.offset) + " + j * "; + code += NumToString(InlineSize(field.value.type.VectorType())); + code += "))"; + code += GenIndents(2) + "else:"; + code += GenIndents(3) + "return None\n\n"; } // Get a struct by initializing an existing struct. @@ -403,18 +434,25 @@ class PythonGenerator : public BaseGenerator { GenReceiver(struct_def, code_ptr); code += namer_.Method(field) + "AsNumpy(self):"; - code += OffsetPrefix(field); + if(!IsArray(field.value.type)){ + code += OffsetPrefix(field, false); - code += Indent + Indent + Indent; - code += "return "; - code += "self._tab.GetVectorAsNumpy(flatbuffers.number_types."; - code += namer_.Method(GenTypeGet(field.value.type)); - code += "Flags, o)\n"; + code += GenIndents(3); + code += "return "; + code += "self._tab.GetVectorAsNumpy(flatbuffers.number_types."; + code += namer_.Method(GenTypeGet(field.value.type)); + code += "Flags, o)"; - if (IsString(vectortype)) { - code += Indent + Indent + "return \"\"\n"; - } else { - code += Indent + Indent + "return 0\n"; + if (IsString(vectortype)) { + code += GenIndents(2) + "return \"\"\n"; + } else { + code += GenIndents(2) + "return 0\n"; + } + }else{ + code += GenIndents(2) + "return "; + code += "self._tab.GetArrayAsNumpy(flatbuffers.number_types."; + code += namer_.Method(GenTypeGet(field.value.type.VectorType())); + code += "Flags, self._tab.Pos + "+NumToString(field.value.offset)+", "+NumToString("self."+namer_.Method(field)+"Length()")+")\n"; } code += "\n"; } @@ -714,8 +752,6 @@ class PythonGenerator : public BaseGenerator { } else { GetScalarFieldOfTable(struct_def, field, code_ptr); } - } else if (IsArray(field.value.type)) { - GetArrayOfStruct(struct_def, field, code_ptr); } else { switch (field.value.type.base_type) { case BASE_TYPE_STRUCT: @@ -739,6 +775,17 @@ class PythonGenerator : public BaseGenerator { } break; } + case BASE_TYPE_ARRAY: { + auto vectortype = field.value.type.VectorType(); + if (vectortype.base_type == BASE_TYPE_STRUCT) { + GetArrayOfStruct(struct_def, field, code_ptr); + } else { + GetArrayOfNonStruct(struct_def, field, code_ptr); + GetVectorOfNonStructAsNumpy(struct_def, field, code_ptr); + GetVectorAsNestedFlatbuffer(struct_def, field, code_ptr); + } + break; + } case BASE_TYPE_UNION: GetUnionField(struct_def, field, code_ptr); break; default: FLATBUFFERS_ASSERT(0); } @@ -1061,8 +1108,9 @@ class PythonGenerator : public BaseGenerator { code += GenIndents(1) + "@classmethod"; code += GenIndents(1) + "def InitFromBuf(cls, buf, pos):"; + code += GenIndents(2) + "n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0)"; code += GenIndents(2) + struct_var + " = " + struct_type + "()"; - code += GenIndents(2) + struct_var + ".Init(buf, pos)"; + code += GenIndents(2) + struct_var + ".Init(buf, pos+n)"; code += GenIndents(2) + "return cls.InitFromObj(" + struct_var + ")"; code += "\n"; } @@ -1143,12 +1191,41 @@ class PythonGenerator : public BaseGenerator { auto field_type = TypeName(field); auto one_instance = field_type + "_"; one_instance[0] = CharToLower(one_instance[0]); - if (parser_.opts.include_dependence_headers) { auto package_reference = GenPackageReference(field.value.type); field_type = package_reference + "." + TypeName(field); } + code += GenIndents(4) + "if " + struct_var + "." + field_method + + "(i) is None:"; + code += GenIndents(5) + "self." + field_field + ".append(None)"; + code += GenIndents(4) + "else:"; + code += GenIndents(5) + one_instance + " = " + field_type + + "T.InitFromObj(" + struct_var + "." + field_method + "(i))"; + code += + GenIndents(5) + "self." + field_field + ".append(" + one_instance + ")"; + } + void GenUnpackForTableVector(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) const { + auto &code = *code_ptr; + const auto field_field = namer_.Field(field); + const auto field_method = namer_.Method(field); + const auto struct_var = namer_.Variable(struct_def); + + code += GenIndents(2) + "if not " + struct_var + "." + field_method + + "IsNone():"; + code += GenIndents(3) + "self." + field_field + " = []"; + code += GenIndents(3) + "for i in range(" + struct_var + "." + + field_method + "Length()):"; + + auto field_type = TypeName(field); + auto one_instance = field_type + "_"; + one_instance[0] = CharToLower(one_instance[0]); + if (parser_.opts.include_dependence_headers) { + auto package_reference = GenPackageReference(field.value.type); + field_type = package_reference + "." + TypeName(field); + } code += GenIndents(4) + "if " + struct_var + "." + field_method + "(i) is None:"; code += GenIndents(5) + "self." + field_field + ".append(None)"; @@ -1233,6 +1310,7 @@ class PythonGenerator : public BaseGenerator { GenUnPackForUnion(struct_def, field, &code); break; } + case BASE_TYPE_ARRAY: case BASE_TYPE_VECTOR: { auto vectortype = field.value.type.VectorType(); if (vectortype.base_type == BASE_TYPE_STRUCT) { @@ -1242,10 +1320,6 @@ class PythonGenerator : public BaseGenerator { } break; } - case BASE_TYPE_ARRAY: { - GenUnPackForScalarVector(struct_def, field, &code); - break; - } default: GenUnPackForScalar(struct_def, field, &code); } } @@ -1482,6 +1556,7 @@ class PythonGenerator : public BaseGenerator { GenPackForUnionField(struct_def, field, &code_prefix, &code); break; } + case BASE_TYPE_ARRAY: case BASE_TYPE_VECTOR: { auto vectortype = field.value.type.VectorType(); if (vectortype.base_type == BASE_TYPE_STRUCT) { @@ -1491,10 +1566,6 @@ class PythonGenerator : public BaseGenerator { } break; } - case BASE_TYPE_ARRAY: { - GenPackForScalarVectorField(struct_def, field, &code_prefix, &code); - break; - } case BASE_TYPE_STRING: { code_prefix += GenIndents(2) + "if self." + field_field + " is not None:"; @@ -1666,7 +1737,9 @@ class PythonGenerator : public BaseGenerator { std::string GenTypePointer(const Type &type) const { switch (type.base_type) { case BASE_TYPE_STRING: return "string"; - case BASE_TYPE_VECTOR: return GenTypeGet(type.VectorType()); + case BASE_TYPE_VECTOR: + // fall through + case BASE_TYPE_ARRAY: return GenTypeGet(type.VectorType()); case BASE_TYPE_STRUCT: return type.struct_def->name; case BASE_TYPE_UNION: // fall through diff --git a/src/util.cpp b/src/util.cpp index df324e837..aabc23aa4 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -109,7 +109,7 @@ static std::string ToSnakeCase(const std::string &input, bool screaming) { } else if (!islower(input[i])) { // Prevent duplicate underscores for Upper_Snake_Case strings // and UPPERCASE strings. - if (islower(input[i - 1])) { s += '_'; } + if (islower(input[i - 1]) || (isdigit(input[i-1]) && !isdigit(input[i]))) { s += '_'; } s += screaming ? CharToUpper(input[i]) : CharToLower(input[i]); } else { s += screaming ? CharToUpper(input[i]) : input[i]; @@ -135,7 +135,7 @@ std::string CamelToSnake(const std::string &input) { } else if (!islower(input[i])) { // Prevent duplicate underscores for Upper_Snake_Case strings // and UPPERCASE strings. - if (islower(input[i - 1])) { s += '_'; } + if (islower(input[i - 1]) || (isdigit(input[i-1]) && !isdigit(input[i]))) { s += '_'; } s += CharToLower(input[i]); } else { s += input[i]; diff --git a/tests/MyGame/Example/Ability.py b/tests/MyGame/Example/Ability.py index e57dfd74f..7d3b9552b 100644 --- a/tests/MyGame/Example/Ability.py +++ b/tests/MyGame/Example/Ability.py @@ -38,8 +38,9 @@ class AbilityT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) ability = Ability() - ability.Init(buf, pos) + ability.Init(buf, pos+n) return cls.InitFromObj(ability) @classmethod diff --git a/tests/MyGame/Example/ArrayStruct.py b/tests/MyGame/Example/ArrayStruct.py index c80bf6882..3b54cb237 100644 --- a/tests/MyGame/Example/ArrayStruct.py +++ b/tests/MyGame/Example/ArrayStruct.py @@ -20,53 +20,65 @@ class ArrayStruct(object): # ArrayStruct def A(self): return self._tab.Get(flatbuffers.number_types.Float32Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(0)) # ArrayStruct - def B(self): return [self._tab.Get(flatbuffers.number_types.Int32Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(4 + i * 4)) for i in range(15)] + def B(self, j = None): + if j is None: + return [self._tab.Get(flatbuffers.number_types.Int32Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(4 + i * 4)) for i in range(self.BLength())] + elif j >= 0 and j < self.BLength(): + return self._tab.Get(flatbuffers.number_types.Int32Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(4 + j * 4)) + else: + return None + + # ArrayStruct + def BAsNumpy(self): + return self._tab.GetArrayAsNumpy(flatbuffers.number_types.Int32Flags, self._tab.Pos + 4, self.BLength()) + # ArrayStruct def BLength(self): - o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) - if o != 0: - return self._tab.VectorLen(o) - return 0 + return 15 # ArrayStruct def BIsNone(self): - o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) - return o == 0 + return False # ArrayStruct def C(self): return self._tab.Get(flatbuffers.number_types.Int8Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(64)) # ArrayStruct - def D(self, obj, i): + def D(self, i: int): + from MyGame.Example.NestedStruct import NestedStruct + obj = NestedStruct() obj.Init(self._tab.Bytes, self._tab.Pos + 72 + i * 32) return obj # ArrayStruct def DLength(self): - o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(72)) - if o != 0: - return self._tab.VectorLen(o) - return 0 + return 2 # ArrayStruct def DIsNone(self): - o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(72)) - return o == 0 + return False # ArrayStruct def E(self): return self._tab.Get(flatbuffers.number_types.Int32Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(136)) # ArrayStruct - def F(self): return [self._tab.Get(flatbuffers.number_types.Int64Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(144 + i * 8)) for i in range(2)] + def F(self, j = None): + if j is None: + return [self._tab.Get(flatbuffers.number_types.Int64Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(144 + i * 8)) for i in range(self.FLength())] + elif j >= 0 and j < self.FLength(): + return self._tab.Get(flatbuffers.number_types.Int64Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(144 + j * 8)) + else: + return None + + # ArrayStruct + def FAsNumpy(self): + return self._tab.GetArrayAsNumpy(flatbuffers.number_types.Int64Flags, self._tab.Pos + 144, self.FLength()) + # ArrayStruct def FLength(self): - o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(144)) - if o != 0: - return self._tab.VectorLen(o) - return 0 + return 2 # ArrayStruct def FIsNone(self): - o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(144)) - return o == 0 + return False def CreateArrayStruct(builder, a, b, c, d_a, d_b, d_c, d_d, e, f): @@ -111,8 +123,9 @@ class ArrayStructT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) arrayStruct = ArrayStruct() - arrayStruct.Init(buf, pos) + arrayStruct.Init(buf, pos+n) return cls.InitFromObj(arrayStruct) @classmethod @@ -137,7 +150,11 @@ class ArrayStructT(object): if not arrayStruct.DIsNone(): self.d = [] for i in range(arrayStruct.DLength()): - self.d.append(arrayStruct.D(i)) + if arrayStruct.D(i) is None: + self.d.append(None) + else: + nestedStruct_ = MyGame.Example.NestedStruct.NestedStructT.InitFromObj(arrayStruct.D(i)) + self.d.append(nestedStruct_) self.e = arrayStruct.E() if not arrayStruct.FIsNone(): if np is None: diff --git a/tests/MyGame/Example/ArrayTable.py b/tests/MyGame/Example/ArrayTable.py index 62d71afa4..0eacdb99d 100644 --- a/tests/MyGame/Example/ArrayTable.py +++ b/tests/MyGame/Example/ArrayTable.py @@ -62,8 +62,9 @@ class ArrayTableT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) arrayTable = ArrayTable() - arrayTable.Init(buf, pos) + arrayTable.Init(buf, pos+n) return cls.InitFromObj(arrayTable) @classmethod diff --git a/tests/MyGame/Example/Monster.py b/tests/MyGame/Example/Monster.py index 0044f8d18..654fdcbe3 100644 --- a/tests/MyGame/Example/Monster.py +++ b/tests/MyGame/Example/Monster.py @@ -1131,8 +1131,9 @@ class MonsterT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) monster = Monster() - monster.Init(buf, pos) + monster.Init(buf, pos+n) return cls.InitFromObj(monster) @classmethod diff --git a/tests/MyGame/Example/NestedStruct.py b/tests/MyGame/Example/NestedStruct.py index a9db014f9..21065efae 100644 --- a/tests/MyGame/Example/NestedStruct.py +++ b/tests/MyGame/Example/NestedStruct.py @@ -18,48 +18,69 @@ class NestedStruct(object): self._tab = flatbuffers.table.Table(buf, pos) # NestedStruct - def A(self): return [self._tab.Get(flatbuffers.number_types.Int32Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(0 + i * 4)) for i in range(2)] + def A(self, j = None): + if j is None: + return [self._tab.Get(flatbuffers.number_types.Int32Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(0 + i * 4)) for i in range(self.ALength())] + elif j >= 0 and j < self.ALength(): + return self._tab.Get(flatbuffers.number_types.Int32Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(0 + j * 4)) + else: + return None + + # NestedStruct + def AAsNumpy(self): + return self._tab.GetArrayAsNumpy(flatbuffers.number_types.Int32Flags, self._tab.Pos + 0, self.ALength()) + # NestedStruct def ALength(self): - o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(0)) - if o != 0: - return self._tab.VectorLen(o) - return 0 + return 2 # NestedStruct def AIsNone(self): - o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(0)) - return o == 0 + return False # NestedStruct def B(self): return self._tab.Get(flatbuffers.number_types.Int8Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(8)) # NestedStruct - def C(self): return [self._tab.Get(flatbuffers.number_types.Int8Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(9 + i * 1)) for i in range(2)] + def C(self, j = None): + if j is None: + return [self._tab.Get(flatbuffers.number_types.Int8Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(9 + i * 1)) for i in range(self.CLength())] + elif j >= 0 and j < self.CLength(): + return self._tab.Get(flatbuffers.number_types.Int8Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(9 + j * 1)) + else: + return None + + # NestedStruct + def CAsNumpy(self): + return self._tab.GetArrayAsNumpy(flatbuffers.number_types.Int8Flags, self._tab.Pos + 9, self.CLength()) + # NestedStruct def CLength(self): - o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(9)) - if o != 0: - return self._tab.VectorLen(o) - return 0 + return 2 # NestedStruct def CIsNone(self): - o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(9)) - return o == 0 + return False # NestedStruct - def D(self): return [self._tab.Get(flatbuffers.number_types.Int64Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(16 + i * 8)) for i in range(2)] + def D(self, j = None): + if j is None: + return [self._tab.Get(flatbuffers.number_types.Int64Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(16 + i * 8)) for i in range(self.DLength())] + elif j >= 0 and j < self.DLength(): + return self._tab.Get(flatbuffers.number_types.Int64Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(16 + j * 8)) + else: + return None + + # NestedStruct + def DAsNumpy(self): + return self._tab.GetArrayAsNumpy(flatbuffers.number_types.Int64Flags, self._tab.Pos + 16, self.DLength()) + # NestedStruct def DLength(self): - o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) - if o != 0: - return self._tab.VectorLen(o) - return 0 + return 2 # NestedStruct def DIsNone(self): - o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) - return o == 0 + return False def CreateNestedStruct(builder, a, b, c, d): @@ -90,8 +111,9 @@ class NestedStructT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) nestedStruct = NestedStruct() - nestedStruct.Init(buf, pos) + nestedStruct.Init(buf, pos+n) return cls.InitFromObj(nestedStruct) @classmethod diff --git a/tests/MyGame/Example/Referrable.py b/tests/MyGame/Example/Referrable.py index 4b1da6ae4..8bf6ceba3 100644 --- a/tests/MyGame/Example/Referrable.py +++ b/tests/MyGame/Example/Referrable.py @@ -53,8 +53,9 @@ class ReferrableT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) referrable = Referrable() - referrable.Init(buf, pos) + referrable.Init(buf, pos+n) return cls.InitFromObj(referrable) @classmethod diff --git a/tests/MyGame/Example/Stat.py b/tests/MyGame/Example/Stat.py index 3b19a3b84..7845eb3df 100644 --- a/tests/MyGame/Example/Stat.py +++ b/tests/MyGame/Example/Stat.py @@ -75,8 +75,9 @@ class StatT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) stat = Stat() - stat.Init(buf, pos) + stat.Init(buf, pos+n) return cls.InitFromObj(stat) @classmethod diff --git a/tests/MyGame/Example/StructOfStructs.py b/tests/MyGame/Example/StructOfStructs.py index 0dcf3d558..f12ca5e61 100644 --- a/tests/MyGame/Example/StructOfStructs.py +++ b/tests/MyGame/Example/StructOfStructs.py @@ -64,8 +64,9 @@ class StructOfStructsT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) structOfStructs = StructOfStructs() - structOfStructs.Init(buf, pos) + structOfStructs.Init(buf, pos+n) return cls.InitFromObj(structOfStructs) @classmethod diff --git a/tests/MyGame/Example/StructOfStructsOfStructs.py b/tests/MyGame/Example/StructOfStructsOfStructs.py index edba98215..afef10ea7 100644 --- a/tests/MyGame/Example/StructOfStructsOfStructs.py +++ b/tests/MyGame/Example/StructOfStructsOfStructs.py @@ -52,8 +52,9 @@ class StructOfStructsOfStructsT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) structOfStructsOfStructs = StructOfStructsOfStructs() - structOfStructsOfStructs.Init(buf, pos) + structOfStructsOfStructs.Init(buf, pos+n) return cls.InitFromObj(structOfStructsOfStructs) @classmethod diff --git a/tests/MyGame/Example/Test.py b/tests/MyGame/Example/Test.py index 8357ec209..17a71b82a 100644 --- a/tests/MyGame/Example/Test.py +++ b/tests/MyGame/Example/Test.py @@ -39,8 +39,9 @@ class TestT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) test = Test() - test.Init(buf, pos) + test.Init(buf, pos+n) return cls.InitFromObj(test) @classmethod diff --git a/tests/MyGame/Example/TestSimpleTableWithEnum.py b/tests/MyGame/Example/TestSimpleTableWithEnum.py index 7f18ed24e..7caf78f3c 100644 --- a/tests/MyGame/Example/TestSimpleTableWithEnum.py +++ b/tests/MyGame/Example/TestSimpleTableWithEnum.py @@ -53,8 +53,9 @@ class TestSimpleTableWithEnumT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) testSimpleTableWithEnum = TestSimpleTableWithEnum() - testSimpleTableWithEnum.Init(buf, pos) + testSimpleTableWithEnum.Init(buf, pos+n) return cls.InitFromObj(testSimpleTableWithEnum) @classmethod diff --git a/tests/MyGame/Example/TypeAliases.py b/tests/MyGame/Example/TypeAliases.py index 8b0ea19c5..6f08bd8cb 100644 --- a/tests/MyGame/Example/TypeAliases.py +++ b/tests/MyGame/Example/TypeAliases.py @@ -224,8 +224,9 @@ class TypeAliasesT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) typeAliases = TypeAliases() - typeAliases.Init(buf, pos) + typeAliases.Init(buf, pos+n) return cls.InitFromObj(typeAliases) @classmethod diff --git a/tests/MyGame/Example/Vec3.py b/tests/MyGame/Example/Vec3.py index 69cd511f4..c0a44a81d 100644 --- a/tests/MyGame/Example/Vec3.py +++ b/tests/MyGame/Example/Vec3.py @@ -68,8 +68,9 @@ class Vec3T(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) vec3 = Vec3() - vec3.Init(buf, pos) + vec3.Init(buf, pos+n) return cls.InitFromObj(vec3) @classmethod diff --git a/tests/MyGame/Example2/Monster.py b/tests/MyGame/Example2/Monster.py index 893ea9bbb..fda0a0417 100644 --- a/tests/MyGame/Example2/Monster.py +++ b/tests/MyGame/Example2/Monster.py @@ -43,8 +43,9 @@ class MonsterT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) monster = Monster() - monster.Init(buf, pos) + monster.Init(buf, pos+n) return cls.InitFromObj(monster) @classmethod diff --git a/tests/MyGame/InParentNamespace.py b/tests/MyGame/InParentNamespace.py index 197072b84..807133d6a 100644 --- a/tests/MyGame/InParentNamespace.py +++ b/tests/MyGame/InParentNamespace.py @@ -43,8 +43,9 @@ class InParentNamespaceT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) inParentNamespace = InParentNamespace() - inParentNamespace.Init(buf, pos) + inParentNamespace.Init(buf, pos+n) return cls.InitFromObj(inParentNamespace) @classmethod diff --git a/tests/MyGame/MonsterExtra.py b/tests/MyGame/MonsterExtra.py index f7288b6c9..4569b513f 100644 --- a/tests/MyGame/MonsterExtra.py +++ b/tests/MyGame/MonsterExtra.py @@ -202,8 +202,9 @@ class MonsterExtraT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) monsterExtra = MonsterExtra() - monsterExtra.Init(buf, pos) + monsterExtra.Init(buf, pos+n) return cls.InitFromObj(monsterExtra) @classmethod diff --git a/tests/PythonTest.sh b/tests/PythonTest.sh index 00772eae3..0d0ee695b 100755 --- a/tests/PythonTest.sh +++ b/tests/PythonTest.sh @@ -17,12 +17,13 @@ pushd "$(dirname $0)" >/dev/null test_dir="$(pwd)" gen_code_path=${test_dir} -runtime_library_dir=${test_dir}/../python +runtime_library_dir=${test_dir}/../../python # Emit Python code for the example schema in the test dir: ${test_dir}/../flatc -p -o ${gen_code_path} -I include_test monster_test.fbs --gen-object-api ${test_dir}/../flatc -p -o ${gen_code_path} -I include_test monster_test.fbs --gen-object-api --gen-onefile ${test_dir}/../flatc -p -o ${gen_code_path} -I include_test monster_extra.fbs --gen-object-api +${test_dir}/../flatc -p -o ${gen_code_path} -I include_test arrays_test.fbs --gen-object-api # Syntax: run_tests # @@ -73,7 +74,7 @@ if $(which coverage >/dev/null); then PYTHONDONTWRITEBYTECODE=1 \ PYTHONPATH=${runtime_library_dir}:${gen_code_path} \ - coverage run --source=flatbuffers,MyGame py_test.py 0 0 0 > /dev/null + coverage run --source=flatbuffers,MyGame py_test.py 0 0 0 false > /dev/null echo cov_result=`coverage report --omit="*flatbuffers/vendor*,*py_test*" \ diff --git a/tests/monster_test_generated.py b/tests/monster_test_generated.py index 0fa214056..27aff92da 100644 --- a/tests/monster_test_generated.py +++ b/tests/monster_test_generated.py @@ -120,8 +120,9 @@ class InParentNamespaceT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) inParentNamespace = InParentNamespace() - inParentNamespace.Init(buf, pos) + inParentNamespace.Init(buf, pos+n) return cls.InitFromObj(inParentNamespace) @classmethod @@ -176,8 +177,9 @@ class MonsterT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) monster = Monster() - monster.Init(buf, pos) + monster.Init(buf, pos+n) return cls.InitFromObj(monster) @classmethod @@ -231,8 +233,9 @@ class TestT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) test = Test() - test.Init(buf, pos) + test.Init(buf, pos+n) return cls.InitFromObj(test) @classmethod @@ -295,8 +298,9 @@ class TestSimpleTableWithEnumT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) testSimpleTableWithEnum = TestSimpleTableWithEnum() - testSimpleTableWithEnum.Init(buf, pos) + testSimpleTableWithEnum.Init(buf, pos+n) return cls.InitFromObj(testSimpleTableWithEnum) @classmethod @@ -380,8 +384,9 @@ class Vec3T(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) vec3 = Vec3() - vec3.Init(buf, pos) + vec3.Init(buf, pos+n) return cls.InitFromObj(vec3) @classmethod @@ -439,8 +444,9 @@ class AbilityT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) ability = Ability() - ability.Init(buf, pos) + ability.Init(buf, pos+n) return cls.InitFromObj(ability) @classmethod @@ -517,8 +523,9 @@ class StructOfStructsT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) structOfStructs = StructOfStructs() - structOfStructs.Init(buf, pos) + structOfStructs.Init(buf, pos+n) return cls.InitFromObj(structOfStructs) @classmethod @@ -588,8 +595,9 @@ class StructOfStructsOfStructsT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) structOfStructsOfStructs = StructOfStructsOfStructs() - structOfStructsOfStructs.Init(buf, pos) + structOfStructsOfStructs.Init(buf, pos+n) return cls.InitFromObj(structOfStructsOfStructs) @classmethod @@ -670,8 +678,9 @@ class StatT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) stat = Stat() - stat.Init(buf, pos) + stat.Init(buf, pos+n) return cls.InitFromObj(stat) @classmethod @@ -743,8 +752,9 @@ class ReferrableT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) referrable = Referrable() - referrable.Init(buf, pos) + referrable.Init(buf, pos+n) return cls.InitFromObj(referrable) @classmethod @@ -1716,8 +1726,9 @@ class MonsterT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) monster = Monster() - monster.Init(buf, pos) + monster.Init(buf, pos+n) return cls.InitFromObj(monster) @classmethod @@ -2353,8 +2364,9 @@ class TypeAliasesT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) typeAliases = TypeAliases() - typeAliases.Init(buf, pos) + typeAliases.Init(buf, pos+n) return cls.InitFromObj(typeAliases) @classmethod diff --git a/tests/optional_scalars/ScalarStuff.py b/tests/optional_scalars/ScalarStuff.py index 4dea123be..3e7d2a221 100644 --- a/tests/optional_scalars/ScalarStuff.py +++ b/tests/optional_scalars/ScalarStuff.py @@ -438,8 +438,9 @@ class ScalarStuffT(object): @classmethod def InitFromBuf(cls, buf, pos): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) scalarStuff = ScalarStuff() - scalarStuff.Init(buf, pos) + scalarStuff.Init(buf, pos+n) return cls.InitFromObj(scalarStuff) @classmethod diff --git a/tests/py_test.py b/tests/py_test.py index 5e3c54735..1332257fd 100644 --- a/tests/py_test.py +++ b/tests/py_test.py @@ -2642,23 +2642,26 @@ class TestFixedLengthArrays(unittest.TestCase): self.assertEqual(table.A().B(), \ [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]) self.assertEqual(table.A().C(), 1) - self.assertEqual(table.A().D(nested, 0).A(), [1, 2]) - self.assertEqual(table.A().D(nested, 1).A(), [3, 4]) - self.assertEqual(table.A().D(nested, 0).B(), \ + self.assertEqual(table.A().D(0).A(), [1, 2]) + self.assertEqual(table.A().D(1).A(), [3, 4]) + self.assertEqual(table.A().D(0).B(), \ MyGame.Example.TestEnum.TestEnum.B) - self.assertEqual(table.A().D(nested, 1).B(), \ + self.assertEqual(table.A().D(1).B(), \ MyGame.Example.TestEnum.TestEnum.C) - self.assertEqual(table.A().D(nested, 0).C(), \ + self.assertEqual(table.A().D(0).C(), \ [MyGame.Example.TestEnum.TestEnum.A, \ MyGame.Example.TestEnum.TestEnum.B]) - self.assertEqual(table.A().D(nested, 1).C(), \ + self.assertEqual(table.A().D(1).C(), \ [MyGame.Example.TestEnum.TestEnum.C, \ MyGame.Example.TestEnum.TestEnum.B]) - self.assertEqual(table.A().D(nested, 0).D(), [-1, 1]) - self.assertEqual(table.A().D(nested, 1).D(), [-2, 2]) + self.assertEqual(table.A().D(0).D(), [-1, 1]) + self.assertEqual(table.A().D(1).D(), [-2, 2]) self.assertEqual(table.A().E(), 2) self.assertEqual(table.A().F(), [-1, 1]) - + self.assertEqual(table.A().D(0).D(0), -1) + self.assertEqual(table.A().D(0).D(1), 1) + self.assertEqual(table.A().D(1).D(0), -2) + self.assertEqual(table.A().D(1).D(1), 2) def CheckAgainstGoldDataGo(): try: diff --git a/tests/util_test.cpp b/tests/util_test.cpp index d6db91673..f2821a40c 100644 --- a/tests/util_test.cpp +++ b/tests/util_test.cpp @@ -149,11 +149,19 @@ void UtilConvertCase() { cases.push_back({ flatbuffers::Case::kUpperCamel, "tHe_qUiCk_BrOwN_fOx", flatbuffers::Case::kKeep, "tHe_qUiCk_BrOwN_fOx" }); cases.push_back({ flatbuffers::Case::kLowerCamel, "theQuick12345Fox", - flatbuffers::Case::kSnake, "the_quick_12345fox" }); + flatbuffers::Case::kSnake, "the_quick_12345_fox" }); cases.push_back({ flatbuffers::Case::kLowerCamel, "a12b34c45", flatbuffers::Case::kSnake, "a_12b_34c_45" }); cases.push_back({ flatbuffers::Case::kLowerCamel, "a12b34c45", flatbuffers::Case::kSnake2, "a12_b34_c45" }); + cases.push_back({ flatbuffers::Case::kUpperCamel, "Int32Stamped", + flatbuffers::Case::kSnake, "int_32_stamped" }); + cases.push_back({ flatbuffers::Case::kUpperCamel, "101DogsTest", + flatbuffers::Case::kSnake, "101_dogs_test" }); + cases.push_back({ flatbuffers::Case::kUpperCamel, "Int32Stamped", + flatbuffers::Case::kScreamingSnake, "INT_32_STAMPED" }); + cases.push_back({ flatbuffers::Case::kUpperCamel, "101DogsTest", + flatbuffers::Case::kScreamingSnake, "101_DOGS_TEST" }); for (auto &test_case : cases) { TEST_EQ(test_case.expected_output,