Java/C#/Python prefixed size support (#4445)

* initial changes to support size prefixed buffers in Java

* add slice equivalent to CSharp ByteBuffer

* resolve TODO for slicing in CSharp code generation

* add newly generated Java and CSharp test sources

* fix typo in comment

* add FinishSizePrefixed methods to CSharp FlatBufferBuilder as well

* add option to allow writing the prefix as well

* generate size-prefixed monster binary as well

* extend JavaTest to test the size prefixed binary as well

* use constants for size prefix length

* fuse common code for getRootAs and getSizePrefixedRootAs

* pulled file identifier out of if

* add FinishSizePrefixed, GetSizePrefixedRootAs support for Python

* Revert "extend JavaTest to test the size prefixed binary as well"

This reverts commit 68be4420dd.

* Revert "generate size-prefixed monster binary as well"

This reverts commit 2939516fdf.

* fix ByteBuffer.cs Slice() method; add proper CSharp and Java tests

* fix unused parameter

* increment version number

* pulled out generated methods into separate utility class

* pulled out generated methods into separate utility class for Python

* fix indentation

* remove unnecessary comment

* fix newline and copyright

* add ByteBufferUtil to csproj compilation

* hide ByteBuffer's internal data; track offset into parent's array

* test unsafe versions as well; compile and run in debug mode

* clarify help text for size prefix

* move ByteBuffer slicing behavior to subclass

* fix protection levels

* add size prefix support for text generation

* add ByteBufferSlice to csproj compilation

* revert size prefix handling for nested buffers

* use duplicate instead of slice for removing size prefix

* remove slice subclass and use duplicate for removing size prefix

* remove slice specific tests

* remove superfluous command line option

net/FlatBuffers/ByteBuffer.cs

@@ -30,6 +30,8 @@
 //
 
 using System;
+using System.IO;
+using System.Text;
 
 namespace FlatBuffers
 {
@@ -38,12 +40,12 @@ namespace FlatBuffers
     /// </summary>
     public class ByteBuffer
     {
-        private readonly byte[] _buffer;
+        protected byte[] _buffer;
         private int _pos;  // Must track start of the buffer.
 
         public int Length { get { return _buffer.Length; } }
 
-        public byte[] Data { get { return _buffer; } }
+        public ByteBuffer(int size) : this(new byte[size]) { }
 
         public ByteBuffer(byte[] buffer) : this(buffer, 0) { }
 
@@ -63,11 +65,64 @@ namespace FlatBuffers
             _pos = 0;
         }
 
+        // Create a new ByteBuffer on the same underlying data.
+        // The new ByteBuffer's position will be same as this buffer's.
+        public ByteBuffer Duplicate()
+        {
+            return new ByteBuffer(_buffer, Position);
+        }
+
+        // Increases the size of the ByteBuffer, and copies the old data towards
+        // the end of the new buffer.
+        public void GrowFront(int newSize)
+        {
+            if ((Length & 0xC0000000) != 0)
+                throw new Exception(
+                    "ByteBuffer: cannot grow buffer beyond 2 gigabytes.");
+
+            if (newSize < Length)
+                throw new Exception("ByteBuffer: cannot truncate buffer.");
+
+            byte[] newBuffer = new byte[newSize];
+            Buffer.BlockCopy(_buffer, 0, newBuffer, newSize - Length,
+                             Length);
+            _buffer = newBuffer;
+        }
+
+        public byte[] ToArray(int pos, int len)
+        {
+            byte[] arr = new byte[len];
+            Buffer.BlockCopy(_buffer, pos, arr, 0, len);
+            return arr;
+        }
+
+        public byte[] ToSizedArray()
+        {
+            return ToArray(Position, Length - Position);
+        }
+
+        public byte[] ToFullArray()
+        {
+            return ToArray(0, Length);
+        }
+
+        public ArraySegment<byte> ToArraySegment(int pos, int len)
+        {
+            return new ArraySegment<byte>(_buffer, pos, len);
+        }
+
+        public MemoryStream ToMemoryStream(int pos, int len)
+        {
+            return new MemoryStream(_buffer, pos, len);
+        }
+
+#if !UNSAFE_BYTEBUFFER
         // Pre-allocated helper arrays for convertion.
         private float[] floathelper = new[] { 0.0f };
         private int[] inthelper = new[] { 0 };
         private double[] doublehelper = new[] { 0.0 };
         private ulong[] ulonghelper = new[] { 0UL };
+#endif // !UNSAFE_BYTEBUFFER
 
         // Helper functions for the unsafe version.
         static public ushort ReverseBytes(ushort input)
@@ -136,7 +191,6 @@ namespace FlatBuffers
         }
 #endif // !UNSAFE_BYTEBUFFER
 
-
         private void AssertOffsetAndLength(int offset, int length)
         {
             #if !BYTEBUFFER_NO_BOUNDS_CHECK
@@ -171,6 +225,13 @@ namespace FlatBuffers
             PutByte(offset, value);
         }
 
+        public void PutStringUTF8(int offset, string value)
+        {
+            AssertOffsetAndLength(offset, value.Length);
+            Encoding.UTF8.GetBytes(value, 0, value.Length,
+                _buffer, offset);
+        }
+
 #if UNSAFE_BYTEBUFFER
         // Unsafe but more efficient versions of Put*.
         public void PutShort(int offset, short value)
@@ -321,6 +382,11 @@ namespace FlatBuffers
             return _buffer[index];
         }
 
+        public string GetStringUTF8(int startPos, int len)
+        {
+            return Encoding.UTF8.GetString(_buffer, startPos, len);
+        }
+
 #if UNSAFE_BYTEBUFFER
         // Unsafe but more efficient versions of Get*.
         public short GetShort(int offset)

net/FlatBuffers/ByteBufferUtil.cs

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+namespace FlatBuffers
+{
+	/// <summary>
+	/// Class that collects utility functions around `ByteBuffer`.
+	/// </summary>
+	public class ByteBufferUtil
+	{
+		// Extract the size prefix from a `ByteBuffer`.
+		public static int GetSizePrefix(ByteBuffer bb) {
+			return bb.GetInt(bb.Position);
+		}
+
+		// Create a duplicate of a size-prefixed `ByteBuffer` that has its position
+		// advanced just past the size prefix.
+		public static ByteBuffer RemoveSizePrefix(ByteBuffer bb) {
+			ByteBuffer s = bb.Duplicate();
+			s.Position += FlatBufferConstants.SizePrefixLength;
+			return s;
+		}
+	}
+}

net/FlatBuffers/FlatBufferBuilder.cs

@@ -59,7 +59,7 @@ namespace FlatBuffers
                 throw new ArgumentOutOfRangeException("initialSize",
                     initialSize, "Must be greater than zero");
             _space = initialSize;
-            _bb = new ByteBuffer(new byte[initialSize]);
+            _bb = new ByteBuffer(initialSize);
         }
 
         /// <summary>
@@ -99,18 +99,7 @@ namespace FlatBuffers
         // the end of the new buffer (since we build the buffer backwards).
         void GrowBuffer()
         {
-            var oldBuf = _bb.Data;
-            var oldBufSize = oldBuf.Length;
-            if ((oldBufSize & 0xC0000000) != 0)
-                throw new Exception(
-                    "FlatBuffers: cannot grow buffer beyond 2 gigabytes.");
-
-            var newBufSize = oldBufSize << 1;
-            var newBuf = new byte[newBufSize];
-
-            Buffer.BlockCopy(oldBuf, 0, newBuf, newBufSize - oldBufSize,
-                             oldBufSize);
-            _bb = new ByteBuffer(newBuf, newBufSize);
+            _bb.GrowFront(_bb.Length << 1);
         }
 
         // Prepare to write an element of `size` after `additional_bytes`
@@ -475,7 +464,7 @@ namespace FlatBuffers
             AddByte(0);
             var utf8StringLen = Encoding.UTF8.GetByteCount(s);
             StartVector(1, utf8StringLen, 1);
-            Encoding.UTF8.GetBytes(s, 0, s.Length, _bb.Data, _space -= utf8StringLen);
+            _bb.PutStringUTF8(_space -= utf8StringLen, s);
             return new StringOffset(EndVector().Value);
         }
 
@@ -580,6 +569,25 @@ namespace FlatBuffers
         }
         /// @endcond
 
+        /// <summary>
+        /// Finalize a buffer, pointing to the given `root_table`.
+        /// </summary>
+        /// <param name="rootTable">
+        /// An offset to be added to the buffer.
+        /// </param>
+        /// <param name="sizePrefix">
+        /// Whether to prefix the size to the buffer.
+        /// </param>
+        protected void Finish(int rootTable, bool sizePrefix)
+        {
+            Prep(_minAlign, sizeof(int) + (sizePrefix ? sizeof(int) : 0));
+            AddOffset(rootTable);
+            if (sizePrefix) {
+                AddInt(_bb.Length - _space);
+            }
+            _bb.Position = _space;
+        }
+
         /// <summary>
         /// Finalize a buffer, pointing to the given `root_table`.
         /// </summary>
@@ -588,9 +596,18 @@ namespace FlatBuffers
         /// </param>
         public void Finish(int rootTable)
         {
-            Prep(_minAlign, sizeof(int));
-            AddOffset(rootTable);
-            _bb.Position = _space;
+            Finish(rootTable, false);
+        }
+
+        /// <summary>
+        /// Finalize a buffer, pointing to the given `root_table`, with the size prefixed.
+        /// </summary>
+        /// <param name="rootTable">
+        /// An offset to be added to the buffer.
+        /// </param>
+        public void FinishSizePrefixed(int rootTable)
+        {
+            Finish(rootTable, true);
         }
 
         /// <summary>
@@ -615,41 +632,69 @@ namespace FlatBuffers
         /// </returns>
         public byte[] SizedByteArray()
         {
-            var newArray = new byte[_bb.Data.Length - _bb.Position];
-            Buffer.BlockCopy(_bb.Data, _bb.Position, newArray, 0,
-                             _bb.Data.Length - _bb.Position);
-            return newArray;
+            return _bb.ToSizedArray();
         }
 
-         /// <summary>
-         /// Finalize a buffer, pointing to the given `rootTable`.
-         /// </summary>
-         /// <param name="rootTable">
-         /// An offset to be added to the buffer.
-         /// </param>
-         /// <param name="fileIdentifier">
-         /// A FlatBuffer file identifier to be added to the buffer before
-         /// `root_table`.
-         /// </param>
-         public void Finish(int rootTable, string fileIdentifier)
-         {
-             Prep(_minAlign, sizeof(int) +
-                             FlatBufferConstants.FileIdentifierLength);
-             if (fileIdentifier.Length !=
-                 FlatBufferConstants.FileIdentifierLength)
-                 throw new ArgumentException(
-                     "FlatBuffers: file identifier must be length " +
-                     FlatBufferConstants.FileIdentifierLength,
-                     "fileIdentifier");
-             for (int i = FlatBufferConstants.FileIdentifierLength - 1; i >= 0;
-                  i--)
-             {
-                AddByte((byte)fileIdentifier[i]);
-             }
-             Finish(rootTable);
+        /// <summary>
+        /// Finalize a buffer, pointing to the given `rootTable`.
+        /// </summary>
+        /// <param name="rootTable">
+        /// An offset to be added to the buffer.
+        /// </param>
+        /// <param name="fileIdentifier">
+        /// A FlatBuffer file identifier to be added to the buffer before
+        /// `root_table`.
+        /// </param>
+        /// <param name="sizePrefix">
+        /// Whether to prefix the size to the buffer.
+        /// </param>
+        protected void Finish(int rootTable, string fileIdentifier, bool sizePrefix)
+        {
+            Prep(_minAlign, sizeof(int) + (sizePrefix ? sizeof(int) : 0) +
+                            FlatBufferConstants.FileIdentifierLength);
+            if (fileIdentifier.Length !=
+                FlatBufferConstants.FileIdentifierLength)
+                throw new ArgumentException(
+                    "FlatBuffers: file identifier must be length " +
+                    FlatBufferConstants.FileIdentifierLength,
+                    "fileIdentifier");
+            for (int i = FlatBufferConstants.FileIdentifierLength - 1; i >= 0;
+                 i--)
+            {
+               AddByte((byte)fileIdentifier[i]);
+            }
+            Finish(rootTable, sizePrefix);
         }
 
+        /// <summary>
+        /// Finalize a buffer, pointing to the given `rootTable`.
+        /// </summary>
+        /// <param name="rootTable">
+        /// An offset to be added to the buffer.
+        /// </param>
+        /// <param name="fileIdentifier">
+        /// A FlatBuffer file identifier to be added to the buffer before
+        /// `root_table`.
+        /// </param>
+        public void Finish(int rootTable, string fileIdentifier)
+        {
+            Finish(rootTable, fileIdentifier, false);
+        }
 
+        /// <summary>
+        /// Finalize a buffer, pointing to the given `rootTable`, with the size prefixed.
+        /// </summary>
+        /// <param name="rootTable">
+        /// An offset to be added to the buffer.
+        /// </param>
+        /// <param name="fileIdentifier">
+        /// A FlatBuffer file identifier to be added to the buffer before
+        /// `root_table`.
+        /// </param>
+        public void FinishSizePrefixed(int rootTable, string fileIdentifier)
+        {
+            Finish(rootTable, fileIdentifier, true);
+        }
     }
 }
 

net/FlatBuffers/FlatBufferConstants.cs

@@ -24,5 +24,6 @@ namespace FlatBuffers
     public static class FlatBufferConstants
     {
         public const int FileIdentifierLength = 4;
+        public const int SizePrefixLength = 4;
     }
 }

net/FlatBuffers/Table.cs

@@ -60,7 +60,7 @@ namespace FlatBuffers
             offset += bb.GetInt(offset);
             var len = bb.GetInt(offset);
             var startPos = offset + sizeof(int);
-            return Encoding.UTF8.GetString(bb.Data, startPos , len);
+            return bb.GetStringUTF8(startPos, len);
         }
 
         // Get the length of a vector whose offset is stored at "offset" in this object.
@@ -91,7 +91,7 @@ namespace FlatBuffers
 
             var pos = this.__vector(o);
             var len = this.__vector_len(o);
-            return new ArraySegment<byte>(this.bb.Data, pos, len);
+            return bb.ToArraySegment(pos, len);
         }
 
         // Initialize any Table-derived type to point to the union at the given offset.
@@ -126,10 +126,11 @@ namespace FlatBuffers
             var startPos_1 = offset_1 + sizeof(int);
             var startPos_2 = offset_2 + sizeof(int);
             var len = Math.Min(len_1, len_2);
-            byte[] bbArray = bb.Data;
             for(int i = 0; i < len; i++) {
-                if (bbArray[i + startPos_1] != bbArray[i + startPos_2])
-                    return bbArray[i + startPos_1] - bbArray[i + startPos_2];
+                byte b1 = bb.Get(i + startPos_1);
+                byte b2 = bb.Get(i + startPos_2);
+                if (b1 != b2)
+                    return b1 - b2;
             }
             return len_1 - len_2;
         }
@@ -142,10 +143,10 @@ namespace FlatBuffers
             var len_2 = key.Length;
             var startPos_1 = offset_1 + sizeof(int);
             var len = Math.Min(len_1, len_2);
-            byte[] bbArray = bb.Data;
             for (int i = 0; i < len; i++) {
-                if (bbArray[i + startPos_1] != key[i])
-                    return bbArray[i + startPos_1] - key[i];
+                byte b = bb.Get(i + startPos_1);
+                if (b != key[i])
+                    return b - key[i];
             }
             return len_1 - len_2;
         }