Add overloads for C# ByteBuffer/FlatBufferBuilder to allow adding vector blocks from ArraySegments or IntPtr (#7193)

* Add overloads to Add/Put for ArraySegment and IntPtr

In order to allow using code to reduce memory allocations, add overloads to ByteBuffer's and FlatBuffersBuilder's Put/Add methods that take ArraySegment<T> or IntPtr respectively.
Also, adaptions to the c# code generator in flatc to emit corresponding CreateVectorBlock() overloads

* Add missing files generated with generate_code.py

The previous commit changed the C# code generate, but didn't contain the updated generated test files.

* Incorporate review findings

(1) Adhere to 80 characters limit.
(2) In FlatBufferBuilder.Add(IntPtr,int), move zero length check topmost and add sanity check against negative input

tests/FlatBuffers.Test/ByteBufferTests.cs

@@ -15,6 +15,7 @@
  */
 
 using System;
+using System.Runtime.InteropServices;
 
 namespace FlatBuffers.Test
 {
@@ -408,6 +409,60 @@ namespace FlatBuffers.Test
             Assert.ArrayEqual(data, bbArray);
         }
 
+        public void ByteBuffer_Put_ArraySegment_Helper<T>(ArraySegment<T> data, int typeSize)
+            where T : struct
+        {
+            // Create the Byte Buffer
+            var uut = new ByteBuffer(1024);
+
+            // Put the data into the buffer and make sure the offset is
+            // calculated correctly
+            int nOffset = uut.Put(1024, data);
+            Assert.AreEqual(1024 - typeSize * data.Count, nOffset);
+      
+            // Get the full array back out and ensure they are equivalent
+            var bbArray = uut.ToArray<T>(nOffset, data.Count);
+            Assert.ArrayEqual(data.ToArray(), bbArray);
+        }
+    
+        public unsafe void ByteBuffer_Put_IntPtr_Helper<T>(T[] data, int typeSize)
+            where T : struct
+        {
+            GCHandle handle = GCHandle.Alloc(data, GCHandleType.Pinned);
+            try
+            {
+                var dataPtr = handle.AddrOfPinnedObject();
+                var sizeInBytes = data.Length * typeSize;
+
+                // Create the Byte Buffer
+                var uut = new ByteBuffer(1024);
+
+                // Put the data into the buffer and make sure the offset is
+                // calculated correctly
+                int nOffset = uut.Put<T>(1024, dataPtr, sizeInBytes);
+                Assert.AreEqual(1024 - sizeInBytes, nOffset);
+
+                // Get the full array back out and ensure they are equivalent
+                var bbArray = uut.ToArray<T>(nOffset, data.Length);
+                Assert.ArrayEqual(data, bbArray);
+            }
+            finally
+            {
+                handle.Free();
+            }
+        }
+
+        public void ByteBuffer_Put_ArrayTypes_Helper<T>(T[] data, int typeSize)
+            where T : struct
+        {
+            ByteBuffer_Put_Array_Helper(data, typeSize);
+
+            var arraySegment = CreateArraySegment(data);
+            ByteBuffer_Put_ArraySegment_Helper(arraySegment, typeSize);
+
+            ByteBuffer_Put_IntPtr_Helper(data, typeSize);
+        }
+
         [FlatBuffersTestMethod]
         public void ByteBuffer_Put_Array_Float()
         {
@@ -425,7 +480,7 @@ namespace FlatBuffers.Test
             data[7] = 15.9994F;
             data[8] = 18.9984F;
 
-            ByteBuffer_Put_Array_Helper(data, sizeof(float));
+            ByteBuffer_Put_ArrayTypes_Helper(data, sizeof(float));
         }
 
         [FlatBuffersTestMethod]
@@ -445,7 +500,7 @@ namespace FlatBuffers.Test
             data[7] = 15.9994;
             data[8] = 18.9984;
 
-            ByteBuffer_Put_Array_Helper(data, sizeof(double));
+            ByteBuffer_Put_ArrayTypes_Helper(data, sizeof(double));
         }
 
         [FlatBuffersTestMethod]
@@ -465,7 +520,7 @@ namespace FlatBuffers.Test
             data[7] = 15;
             data[8] = 18;
 
-            ByteBuffer_Put_Array_Helper(data, sizeof(int));
+            ByteBuffer_Put_ArrayTypes_Helper(data, sizeof(int));
         }
 
 
@@ -486,7 +541,7 @@ namespace FlatBuffers.Test
             data[7] = 15;
             data[8] = 18;
 
-            ByteBuffer_Put_Array_Helper(data, sizeof(uint));
+            ByteBuffer_Put_ArrayTypes_Helper(data, sizeof(uint));
         }
 
         [FlatBuffersTestMethod]
@@ -506,7 +561,7 @@ namespace FlatBuffers.Test
             data[7] = true;
             data[8] = false;
 
-            ByteBuffer_Put_Array_Helper(data, sizeof(bool));
+            ByteBuffer_Put_ArrayTypes_Helper(data, sizeof(bool));
         }
 
         [FlatBuffersTestMethod]
@@ -526,7 +581,7 @@ namespace FlatBuffers.Test
             data[7] = 15;
             data[8] = 18;
 
-            ByteBuffer_Put_Array_Helper(data, sizeof(long));
+            ByteBuffer_Put_ArrayTypes_Helper(data, sizeof(long));
         }
 
         [FlatBuffersTestMethod]
@@ -546,7 +601,7 @@ namespace FlatBuffers.Test
             data[7] = 15;
             data[8] = 18;
 
-            ByteBuffer_Put_Array_Helper(data, sizeof(byte));
+            ByteBuffer_Put_ArrayTypes_Helper(data, sizeof(byte));
         }
 
         [FlatBuffersTestMethod]
@@ -566,7 +621,16 @@ namespace FlatBuffers.Test
             data[7] = 15;
             data[8] = 18;
 
-            ByteBuffer_Put_Array_Helper(data, sizeof(sbyte));
+            ByteBuffer_Put_ArrayTypes_Helper(data, sizeof(sbyte));
+        }
+
+        private static ArraySegment<T> CreateArraySegment<T>(T[] data)
+            where T : struct
+        {
+            const int arraySegmentPadding = 7;
+            var newData = new T[data.Length + 2 * arraySegmentPadding];
+            Array.Copy(data, 0, newData, arraySegmentPadding, data.Length);
+            return new ArraySegment<T>(newData, arraySegmentPadding, data.Length);
         }
 
         [FlatBuffersTestMethod]
@@ -578,10 +642,17 @@ namespace FlatBuffers.Test
             // create a null array and try to put it into the buffer
             float[] data = null;
             Assert.Throws<ArgumentNullException>(() => uut.Put(1024, data));
+
+            ArraySegment<float> dataArraySegment = default;
+            Assert.Throws<ArgumentNullException>(() => uut.Put(1024, dataArraySegment));
+
+            IntPtr dataPtr = IntPtr.Zero;
+            int dataPtrLength = 100;
+            Assert.Throws<ArgumentNullException>(() => uut.Put<float>(1024, dataPtr, dataPtrLength));
         }
 
         [FlatBuffersTestMethod]
-        public void ByteBuffer_Put_Array_Empty_Throws()
+        public unsafe void ByteBuffer_Put_Array_Empty_Throws()
         {
             // Create the Byte Buffer
             var uut = new ByteBuffer(1024);
@@ -589,6 +660,32 @@ namespace FlatBuffers.Test
             // create an array of length == 0, and try to put it into the buffer
             float[] data = new float[0];
             Assert.Throws<ArgumentException>(() => uut.Put(1024, data));
+
+            var dataArraySegment = new ArraySegment<float>(new float[10], 5, 0);
+            Assert.Throws<ArgumentException>(() => uut.Put(1024, dataArraySegment));
+
+            fixed(float* floatPtr = data)
+            {
+                var dataPtr = (IntPtr)floatPtr;
+                var dataPtrLength = 0;
+                Assert.Throws<ArgumentException>(() => uut.Put<float>(1024, dataPtr, dataPtrLength));
+            }
+        }
+
+        [FlatBuffersTestMethod]
+        public unsafe void ByteBuffer_Put_IntPtr_NegativeSize_Throws()
+        {
+            // Create the Byte Buffer
+            var uut = new ByteBuffer(1024);
+
+            // create an array of length == 0, and try to put it into the buffer
+            float[] data = new float[10];
+            fixed(float* floatPtr = data)
+            {
+                var dataPtr = (IntPtr)floatPtr;
+                var dataPtrLength = -1;
+                Assert.Throws<ArgumentException>(() => uut.Put<float>(1024, dataPtr, dataPtrLength));
+            }
         }
 
         #pragma warning disable 0169
@@ -601,7 +698,7 @@ namespace FlatBuffers.Test
         #pragma warning restore 0169
 
         [FlatBuffersTestMethod]
-        public void ByteBuffer_Put_Array_IncorrectType_Throws()
+        public unsafe void ByteBuffer_Put_Array_IncorrectType_Throws()
         {
             // Create the Byte Buffer
             var uut = new ByteBuffer(1024);
@@ -610,6 +707,16 @@ namespace FlatBuffers.Test
             // able to be put into the buffer
             var data = new dummyStruct[10];
             Assert.Throws<ArgumentException>(() => uut.Put(1024, data));
+
+            var dataArraySegment = new ArraySegment<dummyStruct>(data);
+            Assert.Throws<ArgumentException>(() => uut.Put(1024, dataArraySegment));
+
+            fixed(dummyStruct* floatPtr = data)
+            {
+                var dataPtr = (IntPtr)floatPtr;
+                var dataPtrLength = data.Length * sizeof(dummyStruct);
+                Assert.Throws<ArgumentException>(() => uut.Put<dummyStruct>(1024, dataPtr, dataPtrLength));
+            }
         }
 
         [FlatBuffersTestMethod]

tests/FlatBuffers.Test/FlatBufferBuilderTests.cs

@@ -553,6 +553,18 @@ namespace FlatBuffers.Test
 
             Assert.Throws<ArgumentNullException>(() => fbb.Add(data));
         }
+        
+        [FlatBuffersTestMethod]
+        public unsafe void FlatBufferBuilder_Add_Array_UnsupportedType_Throws()
+        {
+            var fbb = CreateBuffer(false);
+                  
+            var storedOffset = fbb.Offset;
+
+            // Construct the data array
+            var data = new DummyStruct[10];
+            Assert.Throws<ArgumentException>(() => fbb.Add(data));
+        }
 
         [FlatBuffersTestMethod]
         public void FlatBufferBuilder_Add_Array_Empty_Noop()
@@ -570,6 +582,159 @@ namespace FlatBuffers.Test
             var endOffset = fbb.Offset;
             Assert.AreEqual(endOffset, storedOffset);
         }
+    
+        [FlatBuffersTestMethod]
+        public void FlatBufferBuilder_Add_ArraySegment_Default_Throws()
+        {
+            var fbb = CreateBuffer(false);
+
+            // Construct the data array
+            ArraySegment<float> data = default;
+
+            Assert.Throws<ArgumentNullException>(() => fbb.Add(data));
+        }
+            
+        [FlatBuffersTestMethod]
+        public unsafe void FlatBufferBuilder_Add_ArraySegment_UnsupportedType_Throws()
+        {
+            var fbb = CreateBuffer(false);
+                  
+            var storedOffset = fbb.Offset;
+
+            // Construct the data array
+            var array = new DummyStruct[10];
+            var data = new ArraySegment<DummyStruct>(array);
+            Assert.Throws<ArgumentException>(() => fbb.Add(data));
+        }
+
+        [FlatBuffersTestMethod]
+        public void FlatBufferBuilder_Add_ArraySegment_Empty_Noop()
+        {
+            var fbb = CreateBuffer(false);
+                  
+            var storedOffset = fbb.Offset;
+
+            // Construct the data array
+            var array = new float[10];
+            var data = new ArraySegment<float>(array, 5, 0);
+            fbb.Add(data);
+
+            // Make sure the offset didn't change since nothing
+            // was really added
+            var endOffset = fbb.Offset;
+            Assert.AreEqual(endOffset, storedOffset);
+        }
+    
+        [FlatBuffersTestMethod]
+        public void FlatBufferBuilder_Add_IntPtr_Zero_Throws()
+        {
+            var fbb = CreateBuffer(false);
+
+            // Construct the data array
+            var data = IntPtr.Zero;
+            var length = 100;
+
+            Assert.Throws<ArgumentNullException>(() => fbb.Add<float>(data, length));
+        }
+            
+        [FlatBuffersTestMethod]
+        public unsafe void FlatBufferBuilder_Add_IntPtr_SizeNegative_Throws()
+        {
+            var fbb = CreateBuffer(false);
+
+            // Construct the data array
+            var array = new float[10];
+            fixed(float* ptr = array)
+            {
+                var data = (IntPtr)ptr;
+                var length = -1;
+                Assert.Throws<ArgumentOutOfRangeException>(() => fbb.Add<float>(data, length));
+            }
+        }
+
+        [FlatBuffersTestMethod]
+        public void FlatBufferBuilder_Add_IntPtr_Zero_Empty_Noop()
+        {
+            var fbb = CreateBuffer(false);
+
+            var storedOffset = fbb.Offset;
+
+            // Construct the data array
+            var data = IntPtr.Zero;
+            var length = 0;
+
+            fbb.Add<float>(data, length);
+
+            // make sure that a length of 0 doesn't throw also if ptr is Zero as well
+            // and that nothing was really added
+            var endOffset = fbb.Offset;
+            Assert.AreEqual(endOffset, storedOffset);
+        }
+
+        [FlatBuffersTestMethod]
+        public unsafe void FlatBufferBuilder_Add_IntPtr_Empty_Noop()
+        {
+            var fbb = CreateBuffer(false);
+                  
+            var storedOffset = fbb.Offset;
+
+            // Construct the data array
+            var array = new float[10];
+            fixed(float* ptr = array)
+            {
+                var data = (IntPtr)ptr;
+                var length = 0;
+                fbb.Add<float>(data, length);
+            }
+
+            // Make sure the offset didn't change since nothing
+            // was really added
+            var endOffset = fbb.Offset;
+            Assert.AreEqual(endOffset, storedOffset);
+        }
+    
+        [FlatBuffersTestMethod]
+        public unsafe void FlatBufferBuilder_Add_IntPtr_SizeInBytesNotMatchingDataType_Throws()
+        {
+            var fbb = CreateBuffer(false);
+                  
+            var storedOffset = fbb.Offset;
+
+            // Construct the data array
+            var array = new float[10];
+            fixed(float* ptr = array)
+            {
+                const int invalidBytes = 1;
+                var data = (IntPtr)ptr;
+                // add some invalid bytes to the length
+                var length = 2 * sizeof(float) + invalidBytes;
+            
+                Assert.Throws<ArgumentException>(() => fbb.Add<float>(data, length));
+            }
+        }
+    
+        [FlatBuffersTestMethod]
+        public unsafe void FlatBufferBuilder_Add_IntPtr_UnsupportedType_Throws()
+        {
+            var fbb = CreateBuffer(false);
+                  
+            var storedOffset = fbb.Offset;
+
+            // Construct the data array
+            var array = new DummyStruct[10];
+            fixed(DummyStruct* ptr = array)
+            {
+                var data = (IntPtr)ptr;
+                var length = 10 * sizeof(DummyStruct);
+            
+                Assert.Throws<ArgumentException>(() => fbb.Add<DummyStruct>(data, length));
+            }
+        }
+
+        private struct DummyStruct
+        {
+            int value;
+        }
 
         [FlatBuffersTestMethod]
         public void FlatBufferBuilder_Add_null_String()

tests/FlatBuffers.Test/FlatBuffers.Core.Test.csproj

@@ -19,6 +19,14 @@
     <DefineConstants>$(DefineConstants);ENABLE_SPAN_T</DefineConstants>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
   <ItemGroup>
     <Compile Include="..\..\net\FlatBuffers\ByteBuffer.cs">
       <Link>FlatBuffers\ByteBuffer.cs</Link>

tests/FlatBuffers.Test/FlatBuffers.Test.csproj

@@ -21,6 +21,7 @@
     <DefineConstants>TRACE;DEBUG</DefineConstants>
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
     <DebugType>pdbonly</DebugType>
@@ -29,6 +30,7 @@
     <DefineConstants>TRACE</DefineConstants>
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
   <PropertyGroup>
     <StartupObject />