Java/C#/Python prefixed size support (#4445)

* initial changes to support size prefixed buffers in Java

* add slice equivalent to CSharp ByteBuffer

* resolve TODO for slicing in CSharp code generation

* add newly generated Java and CSharp test sources

* fix typo in comment

* add FinishSizePrefixed methods to CSharp FlatBufferBuilder as well

* add option to allow writing the prefix as well

* generate size-prefixed monster binary as well

* extend JavaTest to test the size prefixed binary as well

* use constants for size prefix length

* fuse common code for getRootAs and getSizePrefixedRootAs

* pulled file identifier out of if

* add FinishSizePrefixed, GetSizePrefixedRootAs support for Python

* Revert "extend JavaTest to test the size prefixed binary as well"

This reverts commit 68be4420dd.

* Revert "generate size-prefixed monster binary as well"

This reverts commit 2939516fdf.

* fix ByteBuffer.cs Slice() method; add proper CSharp and Java tests

* fix unused parameter

* increment version number

* pulled out generated methods into separate utility class

* pulled out generated methods into separate utility class for Python

* fix indentation

* remove unnecessary comment

* fix newline and copyright

* add ByteBufferUtil to csproj compilation

* hide ByteBuffer's internal data; track offset into parent's array

* test unsafe versions as well; compile and run in debug mode

* clarify help text for size prefix

* move ByteBuffer slicing behavior to subclass

* fix protection levels

* add size prefix support for text generation

* add ByteBufferSlice to csproj compilation

* revert size prefix handling for nested buffers

* use duplicate instead of slice for removing size prefix

* remove slice subclass and use duplicate for removing size prefix

* remove slice specific tests

* remove superfluous command line option

tests/JavaTest.java

@@ -21,6 +21,8 @@ import java.nio.channels.FileChannel;
 import MyGame.Example.*;
 import NamespaceA.*;
 import NamespaceA.NamespaceB.*;
+import com.google.flatbuffers.ByteBufferUtil;
+import static com.google.flatbuffers.Constants.*;
 import com.google.flatbuffers.FlatBufferBuilder;
 
 class JavaTest {
@@ -53,7 +55,8 @@ class JavaTest {
         // better for performance.
         FlatBufferBuilder fbb = new FlatBufferBuilder(1);
 
-        TestBuilderBasics(fbb);
+        TestBuilderBasics(fbb, true);
+        TestBuilderBasics(fbb, false);
 
         TestExtendedBuffer(fbb.dataBuffer().asReadOnlyBuffer());
 
@@ -244,14 +247,14 @@ class JavaTest {
 
         FlatBufferBuilder fbb = new FlatBufferBuilder(1, new MappedByteBufferFactory());
 
-        TestBuilderBasics(fbb);
+        TestBuilderBasics(fbb, false);
     }
 
     static void TestSizedInputStream() {
         // Test on default FlatBufferBuilder that uses HeapByteBuffer
         FlatBufferBuilder fbb = new FlatBufferBuilder(1);
 
-        TestBuilderBasics(fbb);
+        TestBuilderBasics(fbb, false);
 
         InputStream in = fbb.sizedInputStream();
         byte[] array = fbb.sizedByteArray();
@@ -271,7 +274,7 @@ class JavaTest {
         TestEq(count, array.length);
     }
 
-    static void TestBuilderBasics(FlatBufferBuilder fbb) {
+    static void TestBuilderBasics(FlatBufferBuilder fbb, boolean sizePrefix) {
         int[] names = {fbb.createString("Frodo"), fbb.createString("Barney"), fbb.createString("Wilma")};
         int[] off = new int[3];
         Monster.startMonster(fbb);
@@ -321,7 +324,11 @@ class JavaTest {
         Monster.addTestarrayoftables(fbb, sortMons);
         int mon = Monster.endMonster(fbb);
 
-        Monster.finishMonsterBuffer(fbb, mon);
+        if (sizePrefix) {
+            Monster.finishSizePrefixedMonsterBuffer(fbb, mon);
+        } else {
+            Monster.finishMonsterBuffer(fbb, mon);
+        }
 
         // Write the result to a file for debugging purposes:
         // Note that the binaries are not necessarily identical, since the JSON
@@ -329,7 +336,8 @@ class JavaTest {
         // Java code. They are functionally equivalent though.
 
         try {
-            FileChannel fc = new FileOutputStream("monsterdata_java_wire.mon").getChannel();
+            String filename = "monsterdata_java_wire" + (sizePrefix ? "_sp" : "") + ".mon";
+            FileChannel fc = new FileOutputStream(filename).getChannel();
             fc.write(fbb.dataBuffer().duplicate());
             fc.close();
         } catch(java.io.IOException e) {
@@ -338,18 +346,24 @@ class JavaTest {
         }
 
         // Test it:
-        TestExtendedBuffer(fbb.dataBuffer());
+        ByteBuffer dataBuffer = fbb.dataBuffer();
+        if (sizePrefix) {
+            TestEq(ByteBufferUtil.getSizePrefix(dataBuffer) + SIZE_PREFIX_LENGTH,
+                   dataBuffer.remaining());
+            dataBuffer = ByteBufferUtil.removeSizePrefix(dataBuffer);
+        }
+        TestExtendedBuffer(dataBuffer);
 
         // Make sure it also works with read only ByteBuffers. This is slower,
         // since creating strings incurs an additional copy
         // (see Table.__string).
-        TestExtendedBuffer(fbb.dataBuffer().asReadOnlyBuffer());
+        TestExtendedBuffer(dataBuffer.asReadOnlyBuffer());
 
         TestEnums();
 
         //Attempt to mutate Monster fields and check whether the buffer has been mutated properly
         // revert to original values after testing
-        Monster monster = Monster.getRootAsMonster(fbb.dataBuffer());
+        Monster monster = Monster.getRootAsMonster(dataBuffer);
 
         // mana is optional and does not exist in the buffer so the mutation should fail
         // the mana field should retain its default value