Python: Added support for file_identifiers (#5123)

* Python: Added support for file_identifiers

* Added tests. Fixed file_identifier code.

* Python: Fixed excessive padding of file_identifier. Repaired tests.

* Python: Made code compatible with python2.7

* Python: Typo fix in @endcond

* whitespace normaalization

* Stylistic change from if(not X is None) to if(X is not None). Added comment to type string.

* Python: Added support for automatic code generation of file_identifiers. Added tests for said code generation.

* converted sprintf to snprintf

* Bugfix, added snprint deffinition for MSVC

* changed snprint deffinition for MSVC to sprint_s

* changed scanf to IntToStringHex. Renamed HasFileIdentifier to GenHasFileIdentifier.

* Added updated genereated code to commit

* Python bugix: flatc no longer produces HasFileIdentfier for shcemas with no file identifier

* Added tests to verify `MonsterBufferHasIdentifier` returns false on no Identifier

* Python: added tests for GetBufferIdentifier and BufferHasIdentifier
Python: removed unessasary parenethesis in if statements
Minor format changes.

* Python : correceted instances of keyword arguments being called as positional arguments

* fixed typos and grammer in comments

* Minor style fixes

* Indentation fix

* Equals style changes

* Python: Fixed Alignment Issues. Changed test code to test against atual output

* Ran make(forgot to run make last commit)

* Python: Style changes

* Style changes

* indentation and style

* readded CONTRIBUTING.md

* Formatting tweak

Mostly to make CI run again

* More formatting fixes

* More formatting fixes

* More formatting fixes

* More formatting fixes

* Formatting fix

* More formatting fixes

* Formatting

* ran generate_code.sh

tests/MyGame/Example/ArrayTable.py

@@ -14,6 +14,10 @@ class ArrayTable(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def ArrayTableBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x41\x52\x52\x54", size_prefixed=size_prefixed)
+
     # ArrayTable
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/MyGame/Example/Monster.py

@@ -15,6 +15,10 @@ class Monster(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def MonsterBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
+
     # Monster
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/MyGame/Example/Referrable.py

@@ -14,6 +14,10 @@ class Referrable(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def ReferrableBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
+
     # Referrable
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/MyGame/Example/Stat.py

@@ -14,6 +14,10 @@ class Stat(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def StatBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
+
     # Stat
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/MyGame/Example/TestSimpleTableWithEnum.py

@@ -14,6 +14,10 @@ class TestSimpleTableWithEnum(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def TestSimpleTableWithEnumBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
+
     # TestSimpleTableWithEnum
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/MyGame/Example/TypeAliases.py

@@ -14,6 +14,10 @@ class TypeAliases(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def TypeAliasesBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
+
     # TypeAliases
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/MyGame/Example2/Monster.py

@@ -14,6 +14,10 @@ class Monster(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def MonsterBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
+
     # Monster
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/MyGame/InParentNamespace.py

@@ -14,6 +14,10 @@ class InParentNamespace(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def InParentNamespaceBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x53", size_prefixed=size_prefixed)
+
     # InParentNamespace
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/MyGame/MonsterExtra.py

@@ -14,6 +14,10 @@ class MonsterExtra(object):
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def MonsterExtraBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x4D\x4F\x4E\x45", size_prefixed=size_prefixed)
+
     # MonsterExtra
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)

tests/py_test.py

@@ -65,8 +65,9 @@ class TestWireFormat(unittest.TestCase):
         # returning errors, and is interpreted correctly, for size prefixed
         # representation and regular:
         for sizePrefix in [True, False]:
-            gen_buf, gen_off = make_monster_from_generated_code(sizePrefix = sizePrefix)
-            CheckReadBuffer(gen_buf, gen_off, sizePrefix = sizePrefix)
+            for file_identifier in [None, b"MONS"]:
+                gen_buf, gen_off = make_monster_from_generated_code(sizePrefix=sizePrefix, file_identifier=file_identifier)
+                CheckReadBuffer(gen_buf, gen_off, sizePrefix=sizePrefix, file_identifier=file_identifier)
 
         # Verify that the canonical flatbuffer file is readable by the
         # generated Python code. Note that context managers are not part of
@@ -74,7 +75,7 @@ class TestWireFormat(unittest.TestCase):
         f = open('monsterdata_test.mon', 'rb')
         canonicalWireData = f.read()
         f.close()
-        CheckReadBuffer(bytearray(canonicalWireData), 0)
+        CheckReadBuffer(bytearray(canonicalWireData), 0, file_identifier=b'MONS')
 
         # Write the generated buffer out to a file:
         f = open('monsterdata_python_wire.mon', 'wb')
@@ -82,7 +83,7 @@ class TestWireFormat(unittest.TestCase):
         f.close()
 
 
-def CheckReadBuffer(buf, offset, sizePrefix = False):
+def CheckReadBuffer(buf, offset, sizePrefix=False, file_identifier=None):
     ''' CheckReadBuffer checks that the given buffer is evaluated correctly
         as the example Monster. '''
 
@@ -90,12 +91,18 @@ def CheckReadBuffer(buf, offset, sizePrefix = False):
         ''' An assertion helper that is separated from TestCase classes. '''
         if not stmt:
             raise AssertionError('CheckReadBuffer case failed')
-
+    if file_identifier:
+        # test prior to removal of size_prefix
+        asserter(util.GetBufferIdentifier(buf, offset, size_prefixed=sizePrefix) == file_identifier)
+        asserter(util.BufferHasIdentifier(buf, offset, file_identifier=file_identifier, size_prefixed=sizePrefix))
     if sizePrefix:
         size = util.GetSizePrefix(buf, offset)
-        # taken from the size of monsterdata_python_wire.mon, minus 4
-        asserter(size == 340)
+        asserter(size == len(buf[offset:])-4)
         buf, offset = util.RemoveSizePrefix(buf, offset)
+    if file_identifier:
+        asserter(MyGame.Example.Monster.Monster.MonsterBufferHasIdentifier(buf, offset))
+    else:
+        asserter(not MyGame.Example.Monster.Monster.MonsterBufferHasIdentifier(buf, offset))
     monster = MyGame.Example.Monster.Monster.GetRootAsMonster(buf, offset)
 
     asserter(monster.Hp() == 80)
@@ -1083,7 +1090,7 @@ class TestByteLayout(unittest.TestCase):
         ])
 
 
-def make_monster_from_generated_code(sizePrefix = False):
+def make_monster_from_generated_code(sizePrefix = False, file_identifier=None):
     ''' Use generated code to build the example Monster. '''
 
     b = flatbuffers.Builder(0)
@@ -1145,9 +1152,9 @@ def make_monster_from_generated_code(sizePrefix = False):
     mon = MyGame.Example.Monster.MonsterEnd(b)
 
     if sizePrefix:
-        b.FinishSizePrefixed(mon)
+        b.FinishSizePrefixed(mon, file_identifier)
     else:
-        b.Finish(mon)
+        b.Finish(mon, file_identifier)
 
     return b.Bytes, b.Head()