BigfootDev/flatbuffers
1
0
Fork 1
mirror of https://github.com/google/flatbuffers.git synced 2026-07-03 17:24:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Optimize Pack method using numpy (#5662)

Browse Source 
Add the support to pack using numpy for scalar vectors when numpy is available.
This commit is contained in:
lu-wang-g
2019-12-26 18:42:11 -08:00
committed by Robert Winslow
parent 901b89e733
commit a2c12900aa
4 changed files with 157 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
src/idl_gen_python.cpp
View File

@@ -1201,7 +1201,7 @@ class PythonGenerator : public BaseGenerator {
code_prefix += GenIndents(4) + "builder.PrependUOffsetTRelative" + "(" + code_prefix += GenIndents(4) + "builder.PrependUOffsetTRelative" + "(" +
field_instance_name + "list[i])"; field_instance_name + "list[i])";
code_prefix += GenIndents(3) + field_instance_name + code_prefix += GenIndents(3) + field_instance_name +
" = builder.EndVector(len(self." + field_instance_name + " = builder.EndVector(len(self." + field_instance_name +
"))"; "))";
} }
@@ -1211,38 +1211,20 @@ class PythonGenerator : public BaseGenerator {
"(builder, " + field_instance_name + ")"; "(builder, " + field_instance_name + ")";
} }
void GenPackForScalarVectorField(const StructDef &struct_def, void GenPackForScalarVectorFieldHelper(const StructDef &struct_def,
const FieldDef &field, const FieldDef &field,
std::string *code_prefix_ptr, std::string *code_ptr, int indents) {
std::string *code_ptr) {
auto &code_prefix = *code_prefix_ptr;
auto &code = *code_ptr; auto &code = *code_ptr;
auto field_instance_name = MakeLowerCamel(field); auto field_instance_name = MakeLowerCamel(field);
auto field_accessor_name = MakeUpperCamel(field); auto field_accessor_name = MakeUpperCamel(field);
auto struct_name = NormalizedName(struct_def); auto struct_name = NormalizedName(struct_def);
// Creates the field.
code_prefix +=
GenIndents(2) + "if self." + field_instance_name + " is not None:";
// If the vector is a string vector, we need to first build accessor for
// each string element. And this generated code, needs to be
// placed ahead of code+prefix.
auto vectortype = field.value.type.VectorType(); auto vectortype = field.value.type.VectorType();
if (vectortype.base_type == BASE_TYPE_STRING) {
code_prefix += GenIndents(3) + MakeLowerCamel(field) + "list = []";
code_prefix += GenIndents(3);
code_prefix += "for i in range(len(self." + field_instance_name + ")):";
code_prefix += GenIndents(4) + MakeLowerCamel(field) +
"list.append(builder.CreateString(self." +
field_instance_name + "[i]))";
}
code_prefix += GenIndents(3) + struct_name + "Start" + field_accessor_name + code += GenIndents(indents) + struct_name + "Start" + field_accessor_name +
"Vector(builder, len(self." + field_instance_name + "))"; "Vector(builder, len(self." + field_instance_name + "))";
code_prefix += GenIndents(3) + "for i in reversed(range(len(self." + code += GenIndents(indents) + "for i in reversed(range(len(self." +
field_instance_name + "))):"; field_instance_name + "))):";
code_prefix += GenIndents(4) + "builder.Prepend"; code += GenIndents(indents + 1) + "builder.Prepend";
std::string type_name; std::string type_name;
switch (vectortype.base_type) { switch (vectortype.base_type) {
@@ -1286,20 +1268,57 @@ class PythonGenerator : public BaseGenerator {
type_name = "VOffsetT"; type_name = "VOffsetT";
break; break;
} }
code += type_name;
}
if (vectortype.base_type == BASE_TYPE_STRING) { void GenPackForScalarVectorField(const StructDef &struct_def,
code_prefix += type_name + "(" + MakeLowerCamel(field) + "list[i])"; const FieldDef &field,
} else { std::string *code_prefix_ptr,
code_prefix += type_name + "(self." + field_instance_name + "[i])"; std::string *code_ptr) {
} auto &code = *code_ptr;
code_prefix += GenIndents(3) + field_instance_name + auto &code_prefix = *code_prefix_ptr;
" = builder.EndVector(len(self." + field_instance_name + auto field_instance_name = MakeLowerCamel(field);
"))"; auto field_accessor_name = MakeUpperCamel(field);
auto struct_name = NormalizedName(struct_def);
// Adds the field into the struct. // Adds the field into the struct.
code += GenIndents(2) + "if self." + field_instance_name + " is not None:"; code += GenIndents(2) + "if self." + field_instance_name + " is not None:";
code += GenIndents(3) + struct_name + "Add" + field_accessor_name + code += GenIndents(3) + struct_name + "Add" + field_accessor_name +
"(builder, " + field_instance_name + ")"; "(builder, " + field_instance_name + ")";
// Creates the field.
code_prefix +=
GenIndents(2) + "if self." + field_instance_name + " is not None:";
// If the vector is a string vector, we need to first build accessor for
// each string element. And this generated code, needs to be
// placed ahead of code_prefix.
auto vectortype = field.value.type.VectorType();
if (vectortype.base_type == BASE_TYPE_STRING) {
code_prefix += GenIndents(3) + MakeLowerCamel(field) + "list = []";
code_prefix += GenIndents(3) + "for i in range(len(self." +
field_instance_name + ")):";
code_prefix += GenIndents(4) + MakeLowerCamel(field) +
"list.append(builder.CreateString(self." +
field_instance_name + "[i]))";
GenPackForScalarVectorFieldHelper(struct_def, field, code_prefix_ptr, 3);
code_prefix += "(" + MakeLowerCamel(field) + "list[i])";
code_prefix += GenIndents(3) + field_instance_name +
" = builder.EndVector(len(self." + field_instance_name +
"))";
return;
}
code_prefix += GenIndents(3) + "if np is not None and type(self." +
field_instance_name + ") is np.ndarray:";
code_prefix += GenIndents(4) + field_instance_name +
" = builder.CreateNumpyVector(self." + field_instance_name +
")";
code_prefix += GenIndents(3) + "else:";
GenPackForScalarVectorFieldHelper(struct_def, field, code_prefix_ptr, 4);
code_prefix += "(self." + field_instance_name + "[i])";
code_prefix += GenIndents(4) + field_instance_name +
" = builder.EndVector(len(self." + field_instance_name +
"))";
} }
void GenPackForStructField(const StructDef &struct_def, const FieldDef &field, void GenPackForStructField(const StructDef &struct_def, const FieldDef &field,

120
tests/MyGame/Example/Monster.py
View File

@@ -1039,10 +1039,13 @@ class MonsterT(object):
if self.name is not None: if self.name is not None:
name = builder.CreateString(self.name) name = builder.CreateString(self.name)
if self.inventory is not None: if self.inventory is not None:
MonsterStartInventoryVector(builder, len(self.inventory)) if np is not None and type(self.inventory) is np.ndarray:
for i in reversed(range(len(self.inventory))): inventory = builder.CreateNumpyVector(self.inventory)
builder.PrependUint8(self.inventory[i]) else:
inventory = builder.EndVector(len(self.inventory)) MonsterStartInventoryVector(builder, len(self.inventory))
for i in reversed(range(len(self.inventory))):
builder.PrependUint8(self.inventory[i])
inventory = builder.EndVector(len(self.inventory))
if self.test is not None: if self.test is not None:
test = self.test.Pack(builder) test = self.test.Pack(builder)
if self.test4 is not None: if self.test4 is not None:
@@ -1057,7 +1060,7 @@ class MonsterT(object):
MonsterStartTestarrayofstringVector(builder, len(self.testarrayofstring)) MonsterStartTestarrayofstringVector(builder, len(self.testarrayofstring))
for i in reversed(range(len(self.testarrayofstring))): for i in reversed(range(len(self.testarrayofstring))):
builder.PrependUOffsetTRelative(testarrayofstringlist[i]) builder.PrependUOffsetTRelative(testarrayofstringlist[i])
testarrayofstring = builder.EndVector(len(self.testarrayofstring)) testarrayofstring = builder.EndVector(len(self.testarrayofstring))
if self.testarrayoftables is not None: if self.testarrayoftables is not None:
testarrayoftableslist = [] testarrayoftableslist = []
for i in range(len(self.testarrayoftables)): for i in range(len(self.testarrayoftables)):
@@ -1065,21 +1068,27 @@ class MonsterT(object):
MonsterStartTestarrayoftablesVector(builder, len(self.testarrayoftables)) MonsterStartTestarrayoftablesVector(builder, len(self.testarrayoftables))
for i in reversed(range(len(self.testarrayoftables))): for i in reversed(range(len(self.testarrayoftables))):
builder.PrependUOffsetTRelative(testarrayoftableslist[i]) builder.PrependUOffsetTRelative(testarrayoftableslist[i])
testarrayoftables = builder.EndVector(len(self.testarrayoftables)) testarrayoftables = builder.EndVector(len(self.testarrayoftables))
if self.enemy is not None: if self.enemy is not None:
enemy = self.enemy.Pack(builder) enemy = self.enemy.Pack(builder)
if self.testnestedflatbuffer is not None: if self.testnestedflatbuffer is not None:
MonsterStartTestnestedflatbufferVector(builder, len(self.testnestedflatbuffer)) if np is not None and type(self.testnestedflatbuffer) is np.ndarray:
for i in reversed(range(len(self.testnestedflatbuffer))): testnestedflatbuffer = builder.CreateNumpyVector(self.testnestedflatbuffer)
builder.PrependUint8(self.testnestedflatbuffer[i]) else:
testnestedflatbuffer = builder.EndVector(len(self.testnestedflatbuffer)) MonsterStartTestnestedflatbufferVector(builder, len(self.testnestedflatbuffer))
for i in reversed(range(len(self.testnestedflatbuffer))):
builder.PrependUint8(self.testnestedflatbuffer[i])
testnestedflatbuffer = builder.EndVector(len(self.testnestedflatbuffer))
if self.testempty is not None: if self.testempty is not None:
testempty = self.testempty.Pack(builder) testempty = self.testempty.Pack(builder)
if self.testarrayofbools is not None: if self.testarrayofbools is not None:
MonsterStartTestarrayofboolsVector(builder, len(self.testarrayofbools)) if np is not None and type(self.testarrayofbools) is np.ndarray:
for i in reversed(range(len(self.testarrayofbools))): testarrayofbools = builder.CreateNumpyVector(self.testarrayofbools)
builder.PrependBool(self.testarrayofbools[i]) else:
testarrayofbools = builder.EndVector(len(self.testarrayofbools)) MonsterStartTestarrayofboolsVector(builder, len(self.testarrayofbools))
for i in reversed(range(len(self.testarrayofbools))):
builder.PrependBool(self.testarrayofbools[i])
testarrayofbools = builder.EndVector(len(self.testarrayofbools))
if self.testarrayofstring2 is not None: if self.testarrayofstring2 is not None:
testarrayofstring2list = [] testarrayofstring2list = []
for i in range(len(self.testarrayofstring2)): for i in range(len(self.testarrayofstring2)):
@@ -1087,32 +1096,41 @@ class MonsterT(object):
MonsterStartTestarrayofstring2Vector(builder, len(self.testarrayofstring2)) MonsterStartTestarrayofstring2Vector(builder, len(self.testarrayofstring2))
for i in reversed(range(len(self.testarrayofstring2))): for i in reversed(range(len(self.testarrayofstring2))):
builder.PrependUOffsetTRelative(testarrayofstring2list[i]) builder.PrependUOffsetTRelative(testarrayofstring2list[i])
testarrayofstring2 = builder.EndVector(len(self.testarrayofstring2)) testarrayofstring2 = builder.EndVector(len(self.testarrayofstring2))
if self.testarrayofsortedstruct is not None: if self.testarrayofsortedstruct is not None:
MonsterStartTestarrayofsortedstructVector(builder, len(self.testarrayofsortedstruct)) MonsterStartTestarrayofsortedstructVector(builder, len(self.testarrayofsortedstruct))
for i in reversed(range(len(self.testarrayofsortedstruct))): for i in reversed(range(len(self.testarrayofsortedstruct))):
self.testarrayofsortedstruct[i].Pack(builder) self.testarrayofsortedstruct[i].Pack(builder)
testarrayofsortedstruct = builder.EndVector(len(self.testarrayofsortedstruct)) testarrayofsortedstruct = builder.EndVector(len(self.testarrayofsortedstruct))
if self.flex is not None: if self.flex is not None:
MonsterStartFlexVector(builder, len(self.flex)) if np is not None and type(self.flex) is np.ndarray:
for i in reversed(range(len(self.flex))): flex = builder.CreateNumpyVector(self.flex)
builder.PrependUint8(self.flex[i]) else:
flex = builder.EndVector(len(self.flex)) MonsterStartFlexVector(builder, len(self.flex))
for i in reversed(range(len(self.flex))):
builder.PrependUint8(self.flex[i])
flex = builder.EndVector(len(self.flex))
if self.test5 is not None: if self.test5 is not None:
MonsterStartTest5Vector(builder, len(self.test5)) MonsterStartTest5Vector(builder, len(self.test5))
for i in reversed(range(len(self.test5))): for i in reversed(range(len(self.test5))):
self.test5[i].Pack(builder) self.test5[i].Pack(builder)
test5 = builder.EndVector(len(self.test5)) test5 = builder.EndVector(len(self.test5))
if self.vectorOfLongs is not None: if self.vectorOfLongs is not None:
MonsterStartVectorOfLongsVector(builder, len(self.vectorOfLongs)) if np is not None and type(self.vectorOfLongs) is np.ndarray:
for i in reversed(range(len(self.vectorOfLongs))): vectorOfLongs = builder.CreateNumpyVector(self.vectorOfLongs)
builder.PrependInt64(self.vectorOfLongs[i]) else:
vectorOfLongs = builder.EndVector(len(self.vectorOfLongs)) MonsterStartVectorOfLongsVector(builder, len(self.vectorOfLongs))
for i in reversed(range(len(self.vectorOfLongs))):
builder.PrependInt64(self.vectorOfLongs[i])
vectorOfLongs = builder.EndVector(len(self.vectorOfLongs))
if self.vectorOfDoubles is not None: if self.vectorOfDoubles is not None:
MonsterStartVectorOfDoublesVector(builder, len(self.vectorOfDoubles)) if np is not None and type(self.vectorOfDoubles) is np.ndarray:
for i in reversed(range(len(self.vectorOfDoubles))): vectorOfDoubles = builder.CreateNumpyVector(self.vectorOfDoubles)
builder.PrependFloat64(self.vectorOfDoubles[i]) else:
vectorOfDoubles = builder.EndVector(len(self.vectorOfDoubles)) MonsterStartVectorOfDoublesVector(builder, len(self.vectorOfDoubles))
for i in reversed(range(len(self.vectorOfDoubles))):
builder.PrependFloat64(self.vectorOfDoubles[i])
vectorOfDoubles = builder.EndVector(len(self.vectorOfDoubles))
if self.parentNamespaceTest is not None: if self.parentNamespaceTest is not None:
parentNamespaceTest = self.parentNamespaceTest.Pack(builder) parentNamespaceTest = self.parentNamespaceTest.Pack(builder)
if self.vectorOfReferrables is not None: if self.vectorOfReferrables is not None:
@@ -1122,12 +1140,15 @@ class MonsterT(object):
MonsterStartVectorOfReferrablesVector(builder, len(self.vectorOfReferrables)) MonsterStartVectorOfReferrablesVector(builder, len(self.vectorOfReferrables))
for i in reversed(range(len(self.vectorOfReferrables))): for i in reversed(range(len(self.vectorOfReferrables))):
builder.PrependUOffsetTRelative(vectorOfReferrableslist[i]) builder.PrependUOffsetTRelative(vectorOfReferrableslist[i])
vectorOfReferrables = builder.EndVector(len(self.vectorOfReferrables)) vectorOfReferrables = builder.EndVector(len(self.vectorOfReferrables))
if self.vectorOfWeakReferences is not None: if self.vectorOfWeakReferences is not None:
MonsterStartVectorOfWeakReferencesVector(builder, len(self.vectorOfWeakReferences)) if np is not None and type(self.vectorOfWeakReferences) is np.ndarray:
for i in reversed(range(len(self.vectorOfWeakReferences))): vectorOfWeakReferences = builder.CreateNumpyVector(self.vectorOfWeakReferences)
builder.PrependUint64(self.vectorOfWeakReferences[i]) else:
vectorOfWeakReferences = builder.EndVector(len(self.vectorOfWeakReferences)) MonsterStartVectorOfWeakReferencesVector(builder, len(self.vectorOfWeakReferences))
for i in reversed(range(len(self.vectorOfWeakReferences))):
builder.PrependUint64(self.vectorOfWeakReferences[i])
vectorOfWeakReferences = builder.EndVector(len(self.vectorOfWeakReferences))
if self.vectorOfStrongReferrables is not None: if self.vectorOfStrongReferrables is not None:
vectorOfStrongReferrableslist = [] vectorOfStrongReferrableslist = []
for i in range(len(self.vectorOfStrongReferrables)): for i in range(len(self.vectorOfStrongReferrables)):
@@ -1135,26 +1156,35 @@ class MonsterT(object):
MonsterStartVectorOfStrongReferrablesVector(builder, len(self.vectorOfStrongReferrables)) MonsterStartVectorOfStrongReferrablesVector(builder, len(self.vectorOfStrongReferrables))
for i in reversed(range(len(self.vectorOfStrongReferrables))): for i in reversed(range(len(self.vectorOfStrongReferrables))):
builder.PrependUOffsetTRelative(vectorOfStrongReferrableslist[i]) builder.PrependUOffsetTRelative(vectorOfStrongReferrableslist[i])
vectorOfStrongReferrables = builder.EndVector(len(self.vectorOfStrongReferrables)) vectorOfStrongReferrables = builder.EndVector(len(self.vectorOfStrongReferrables))
if self.vectorOfCoOwningReferences is not None: if self.vectorOfCoOwningReferences is not None:
MonsterStartVectorOfCoOwningReferencesVector(builder, len(self.vectorOfCoOwningReferences)) if np is not None and type(self.vectorOfCoOwningReferences) is np.ndarray:
for i in reversed(range(len(self.vectorOfCoOwningReferences))): vectorOfCoOwningReferences = builder.CreateNumpyVector(self.vectorOfCoOwningReferences)
builder.PrependUint64(self.vectorOfCoOwningReferences[i]) else:
vectorOfCoOwningReferences = builder.EndVector(len(self.vectorOfCoOwningReferences)) MonsterStartVectorOfCoOwningReferencesVector(builder, len(self.vectorOfCoOwningReferences))
for i in reversed(range(len(self.vectorOfCoOwningReferences))):
builder.PrependUint64(self.vectorOfCoOwningReferences[i])
vectorOfCoOwningReferences = builder.EndVector(len(self.vectorOfCoOwningReferences))
if self.vectorOfNonOwningReferences is not None: if self.vectorOfNonOwningReferences is not None:
MonsterStartVectorOfNonOwningReferencesVector(builder, len(self.vectorOfNonOwningReferences)) if np is not None and type(self.vectorOfNonOwningReferences) is np.ndarray:
for i in reversed(range(len(self.vectorOfNonOwningReferences))): vectorOfNonOwningReferences = builder.CreateNumpyVector(self.vectorOfNonOwningReferences)
builder.PrependUint64(self.vectorOfNonOwningReferences[i]) else:
vectorOfNonOwningReferences = builder.EndVector(len(self.vectorOfNonOwningReferences)) MonsterStartVectorOfNonOwningReferencesVector(builder, len(self.vectorOfNonOwningReferences))
for i in reversed(range(len(self.vectorOfNonOwningReferences))):
builder.PrependUint64(self.vectorOfNonOwningReferences[i])
vectorOfNonOwningReferences = builder.EndVector(len(self.vectorOfNonOwningReferences))
if self.anyUnique is not None: if self.anyUnique is not None:
anyUnique = self.anyUnique.Pack(builder) anyUnique = self.anyUnique.Pack(builder)
if self.anyAmbiguous is not None: if self.anyAmbiguous is not None:
anyAmbiguous = self.anyAmbiguous.Pack(builder) anyAmbiguous = self.anyAmbiguous.Pack(builder)
if self.vectorOfEnums is not None: if self.vectorOfEnums is not None:
MonsterStartVectorOfEnumsVector(builder, len(self.vectorOfEnums)) if np is not None and type(self.vectorOfEnums) is np.ndarray:
for i in reversed(range(len(self.vectorOfEnums))): vectorOfEnums = builder.CreateNumpyVector(self.vectorOfEnums)
builder.PrependUint8(self.vectorOfEnums[i]) else:
vectorOfEnums = builder.EndVector(len(self.vectorOfEnums)) MonsterStartVectorOfEnumsVector(builder, len(self.vectorOfEnums))
for i in reversed(range(len(self.vectorOfEnums))):
builder.PrependUint8(self.vectorOfEnums[i])
vectorOfEnums = builder.EndVector(len(self.vectorOfEnums))
MonsterStart(builder) MonsterStart(builder)
if self.pos is not None: if self.pos is not None:
pos = self.pos.Pack(builder) pos = self.pos.Pack(builder)

22
tests/MyGame/Example/TypeAliases.py
View File

@@ -231,15 +231,21 @@ class TypeAliasesT(object):
# TypeAliasesT # TypeAliasesT
def Pack(self, builder): def Pack(self, builder):
if self.v8 is not None: if self.v8 is not None:
TypeAliasesStartV8Vector(builder, len(self.v8)) if np is not None and type(self.v8) is np.ndarray:
for i in reversed(range(len(self.v8))): v8 = builder.CreateNumpyVector(self.v8)
builder.PrependByte(self.v8[i]) else:
v8 = builder.EndVector(len(self.v8)) TypeAliasesStartV8Vector(builder, len(self.v8))
for i in reversed(range(len(self.v8))):
builder.PrependByte(self.v8[i])
v8 = builder.EndVector(len(self.v8))
if self.vf64 is not None: if self.vf64 is not None:
TypeAliasesStartVf64Vector(builder, len(self.vf64)) if np is not None and type(self.vf64) is np.ndarray:
for i in reversed(range(len(self.vf64))): vf64 = builder.CreateNumpyVector(self.vf64)
builder.PrependFloat64(self.vf64[i]) else:
vf64 = builder.EndVector(len(self.vf64)) TypeAliasesStartVf64Vector(builder, len(self.vf64))
for i in reversed(range(len(self.vf64))):
builder.PrependFloat64(self.vf64[i])
vf64 = builder.EndVector(len(self.vf64))
TypeAliasesStart(builder) TypeAliasesStart(builder)
TypeAliasesAddI8(builder, self.i8) TypeAliasesAddI8(builder, self.i8)
TypeAliasesAddU8(builder, self.u8) TypeAliasesAddU8(builder, self.u8)

22
tests/MyGame/MonsterExtra.py
View File

@@ -211,15 +211,21 @@ class MonsterExtraT(object):
# MonsterExtraT # MonsterExtraT
def Pack(self, builder): def Pack(self, builder):
if self.dvec is not None: if self.dvec is not None:
MonsterExtraStartDvecVector(builder, len(self.dvec)) if np is not None and type(self.dvec) is np.ndarray:
for i in reversed(range(len(self.dvec))): dvec = builder.CreateNumpyVector(self.dvec)
builder.PrependFloat64(self.dvec[i]) else:
dvec = builder.EndVector(len(self.dvec)) MonsterExtraStartDvecVector(builder, len(self.dvec))
for i in reversed(range(len(self.dvec))):
builder.PrependFloat64(self.dvec[i])
dvec = builder.EndVector(len(self.dvec))
if self.fvec is not None: if self.fvec is not None:
MonsterExtraStartFvecVector(builder, len(self.fvec)) if np is not None and type(self.fvec) is np.ndarray:
for i in reversed(range(len(self.fvec))): fvec = builder.CreateNumpyVector(self.fvec)
builder.PrependFloat32(self.fvec[i]) else:
fvec = builder.EndVector(len(self.fvec)) MonsterExtraStartFvecVector(builder, len(self.fvec))
for i in reversed(range(len(self.fvec))):
builder.PrependFloat32(self.fvec[i])
fvec = builder.EndVector(len(self.fvec))
MonsterExtraStart(builder) MonsterExtraStart(builder)
MonsterExtraAddD0(builder, self.d0) MonsterExtraAddD0(builder, self.d0)
MonsterExtraAddD1(builder, self.d1) MonsterExtraAddD1(builder, self.d1)