Java/C#/Python prefixed size support (#4445)

* initial changes to support size prefixed buffers in Java

* add slice equivalent to CSharp ByteBuffer

* resolve TODO for slicing in CSharp code generation

* add newly generated Java and CSharp test sources

* fix typo in comment

* add FinishSizePrefixed methods to CSharp FlatBufferBuilder as well

* add option to allow writing the prefix as well

* generate size-prefixed monster binary as well

* extend JavaTest to test the size prefixed binary as well

* use constants for size prefix length

* fuse common code for getRootAs and getSizePrefixedRootAs

* pulled file identifier out of if

* add FinishSizePrefixed, GetSizePrefixedRootAs support for Python

* Revert "extend JavaTest to test the size prefixed binary as well"

This reverts commit 68be4420dd.

* Revert "generate size-prefixed monster binary as well"

This reverts commit 2939516fdf.

* fix ByteBuffer.cs Slice() method; add proper CSharp and Java tests

* fix unused parameter

* increment version number

* pulled out generated methods into separate utility class

* pulled out generated methods into separate utility class for Python

* fix indentation

* remove unnecessary comment

* fix newline and copyright

* add ByteBufferUtil to csproj compilation

* hide ByteBuffer's internal data; track offset into parent's array

* test unsafe versions as well; compile and run in debug mode

* clarify help text for size prefix

* move ByteBuffer slicing behavior to subclass

* fix protection levels

* add size prefix support for text generation

* add ByteBufferSlice to csproj compilation

* revert size prefix handling for nested buffers

* use duplicate instead of slice for removing size prefix

* remove slice subclass and use duplicate for removing size prefix

* remove slice specific tests

* remove superfluous command line option

tests/py_test.py

@@ -25,6 +25,7 @@ import unittest
 
 
 from flatbuffers import compat
+from flatbuffers import util
 from flatbuffers.compat import range_func as compat_range
 from flatbuffers.compat import NumpyRequiredForThisFeature
 
@@ -56,9 +57,11 @@ def assertRaises(test_case, fn, exception_class):
 class TestWireFormat(unittest.TestCase):
     def test_wire_format(self):
         # Verify that using the generated Python code builds a buffer without
-        # returning errors, and is interpreted correctly:
-        gen_buf, gen_off = make_monster_from_generated_code()
-        CheckReadBuffer(gen_buf, gen_off)
+        # returning errors, and is interpreted correctly, for size prefixed
+        # representation and regular:
+        for sizePrefix in [True, False]:
+            gen_buf, gen_off = make_monster_from_generated_code(sizePrefix = sizePrefix)
+            CheckReadBuffer(gen_buf, gen_off, sizePrefix = sizePrefix)
 
         # Verify that the canonical flatbuffer file is readable by the
         # generated Python code. Note that context managers are not part of
@@ -74,7 +77,7 @@ class TestWireFormat(unittest.TestCase):
         f.close()
 
 
-def CheckReadBuffer(buf, offset):
+def CheckReadBuffer(buf, offset, sizePrefix = False):
     ''' CheckReadBuffer checks that the given buffer is evaluated correctly
         as the example Monster. '''
 
@@ -83,6 +86,11 @@ def CheckReadBuffer(buf, offset):
         if not stmt:
             raise AssertionError('CheckReadBuffer case failed')
 
+    if sizePrefix:
+        size = util.GetSizePrefix(buf, offset)
+        # taken from the size of monsterdata_python_wire.mon, minus 4
+        asserter(size == 348)
+        buf, offset = util.RemoveSizePrefix(buf, offset)
     monster = MyGame.Example.Monster.Monster.GetRootAsMonster(buf, offset)
 
     asserter(monster.Hp() == 80)
@@ -810,7 +818,7 @@ class TestByteLayout(unittest.TestCase):
         ])
 
 
-def make_monster_from_generated_code():
+def make_monster_from_generated_code(sizePrefix = False):
     ''' Use generated code to build the example Monster. '''
 
     b = flatbuffers.Builder(0)
@@ -871,7 +879,10 @@ def make_monster_from_generated_code():
     MyGame.Example.Monster.MonsterAddVectorOfDoubles(b, VectorOfDoubles)
     mon = MyGame.Example.Monster.MonsterEnd(b)
 
-    b.Finish(mon)
+    if sizePrefix:
+        b.FinishSizePrefixed(mon)
+    else:
+        b.Finish(mon)
 
     return b.Bytes, b.Head()