Java/C#/Python prefixed size support (#4445)

* initial changes to support size prefixed buffers in Java

* add slice equivalent to CSharp ByteBuffer

* resolve TODO for slicing in CSharp code generation

* add newly generated Java and CSharp test sources

* fix typo in comment

* add FinishSizePrefixed methods to CSharp FlatBufferBuilder as well

* add option to allow writing the prefix as well

* generate size-prefixed monster binary as well

* extend JavaTest to test the size prefixed binary as well

* use constants for size prefix length

* fuse common code for getRootAs and getSizePrefixedRootAs

* pulled file identifier out of if

* add FinishSizePrefixed, GetSizePrefixedRootAs support for Python

* Revert "extend JavaTest to test the size prefixed binary as well"

This reverts commit 68be4420dd.

* Revert "generate size-prefixed monster binary as well"

This reverts commit 2939516fdf.

* fix ByteBuffer.cs Slice() method; add proper CSharp and Java tests

* fix unused parameter

* increment version number

* pulled out generated methods into separate utility class

* pulled out generated methods into separate utility class for Python

* fix indentation

* remove unnecessary comment

* fix newline and copyright

* add ByteBufferUtil to csproj compilation

* hide ByteBuffer's internal data; track offset into parent's array

* test unsafe versions as well; compile and run in debug mode

* clarify help text for size prefix

* move ByteBuffer slicing behavior to subclass

* fix protection levels

* add size prefix support for text generation

* add ByteBufferSlice to csproj compilation

* revert size prefix handling for nested buffers

* use duplicate instead of slice for removing size prefix

* remove slice subclass and use duplicate for removing size prefix

* remove slice specific tests

* remove superfluous command line option

python/flatbuffers/builder.py

@@ -483,14 +483,32 @@ class Builder(object):
         self.current_vtable[slotnum] = self.Offset()
     ## @endcond
 
-    def Finish(self, rootTable):
+    def __Finish(self, rootTable, sizePrefix):
         """Finish finalizes a buffer, pointing to the given `rootTable`."""
         N.enforce_number(rootTable, N.UOffsetTFlags)
-        self.Prep(self.minalign, N.UOffsetTFlags.bytewidth)
+        prepSize = N.UOffsetTFlags.bytewidth
+        if sizePrefix:
+            prepSize += N.Int32Flags.bytewidth
+        self.Prep(self.minalign, prepSize)
         self.PrependUOffsetTRelative(rootTable)
+        if sizePrefix:
+            size = len(self.Bytes) - self.Head()
+            N.enforce_number(size, N.Int32Flags)
+            self.PrependInt32(size)
         self.finished = True
         return self.Head()
 
+    def Finish(self, rootTable):
+        """Finish finalizes a buffer, pointing to the given `rootTable`."""
+        return self.__Finish(rootTable, False)
+
+    def FinishSizePrefixed(self, rootTable):
+        """
+        Finish finalizes a buffer, pointing to the given `rootTable`,
+        with the size prefixed.
+        """
+        return self.__Finish(rootTable, True)
+
     ## @cond FLATBUFFERS_INTERNAL
     def Prepend(self, flags, off):
         self.Prep(flags.bytewidth, 0)

python/flatbuffers/util.py

@@ -0,0 +1,28 @@
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import encode
+from . import number_types
+from . import packer
+
+def GetSizePrefix(buf, offset):
+	"""Extract the size prefix from a buffer."""
+	return encode.Get(packer.int32, buf, offset)
+
+def RemoveSizePrefix(buf, offset):
+	"""
+	Create a slice of a size-prefixed buffer that has
+	its position advanced just past the size prefix.
+	"""
+	return buf, offset + number_types.Int32Flags.bytewidth