Dart: binary lists (typed_data) (#6839)

* Dart - add eager mode to Uint8ListReader

* Dart - add Int8ListReader

* Dart - use binary reader where useful

* Dart - test binary list reader laziness

* Dart - update generated code

dart/lib/flat_buffers.dart

@@ -155,7 +155,8 @@ class Builder {
     }
   }
 
-  /// Calculate the finished buffer size (aligned).@pragma('vm:prefer-inline')
+  /// Calculate the finished buffer size (aligned).
+  @pragma('vm:prefer-inline')
   int size() => _tail + ((-_tail) & (_maxAlign - 1));
 
   /// Add the [field] with the given boolean [value].  The field is not added if
@@ -1170,11 +1171,17 @@ class Uint16Reader extends Reader<int> {
   int read(BufferContext bc, int offset) => bc._getUint16(offset);
 }
 
-/// Reader of lists of unsigned 8-bit integer values.
+/// Reader of unmodifiable binary data (a list of unsigned 8-bit integers).
-///
-/// The returned unmodifiable lists lazily read values on access.
 class Uint8ListReader extends Reader<List<int>> {
-  const Uint8ListReader();
+  /// Enables lazy reading of the list
+  ///
+  /// If true, the returned unmodifiable list lazily reads bytes on access.
+  /// Therefore, the underlying buffer must not change while accessing the list.
+  ///
+  /// If false, reads the whole list immediately as an Uint8List.
+  final bool lazy;
+
+  const Uint8ListReader({this.lazy = true});
 
   @override
   @pragma('vm:prefer-inline')
@@ -1182,8 +1189,18 @@ class Uint8ListReader extends Reader<List<int>> {
 
   @override
   @pragma('vm:prefer-inline')
-  List<int> read(BufferContext bc, int offset) =>
+  List<int> read(BufferContext bc, int offset) {
-      _FbUint8List(bc, bc.derefObject(offset));
+    final listOffset = bc.derefObject(offset);
+    if (lazy) return _FbUint8List(bc, listOffset);
+
+    final length = bc._getUint32(listOffset);
+    final result = Uint8List(length);
+    var pos = listOffset + _sizeofUint32;
+    for (var i = 0; i < length; i++, pos++) {
+      result[i] = bc._getUint8(pos);
+    }
+    return result;
+  }
 }
 
 /// The reader of unsigned 8-bit integers.
@@ -1199,6 +1216,38 @@ class Uint8Reader extends Reader<int> {
   int read(BufferContext bc, int offset) => bc._getUint8(offset);
 }
 
+/// Reader of unmodifiable binary data (a list of signed 8-bit integers).
+class Int8ListReader extends Reader<List<int>> {
+  /// Enables lazy reading of the list
+  ///
+  /// If true, the returned unmodifiable list lazily reads bytes on access.
+  /// Therefore, the underlying buffer must not change while accessing the list.
+  ///
+  /// If false, reads the whole list immediately as an Uint8List.
+  final bool lazy;
+
+  const Int8ListReader({this.lazy = true});
+
+  @override
+  @pragma('vm:prefer-inline')
+  int get size => _sizeofUint32;
+
+  @override
+  @pragma('vm:prefer-inline')
+  List<int> read(BufferContext bc, int offset) {
+    final listOffset = bc.derefObject(offset);
+    if (lazy) return _FbUint8List(bc, listOffset);
+
+    final length = bc._getUint32(listOffset);
+    final result = Int8List(length);
+    var pos = listOffset + _sizeofUint32;
+    for (var i = 0; i < length; i++, pos++) {
+      result[i] = bc._getInt8(pos);
+    }
+    return result;
+  }
+}
+
 /// The list backed by 64-bit values - Uint64 length and Float64.
 class _FbFloat64List extends _FbList<double> {
   _FbFloat64List(BufferContext bc, int offset) : super(bc, offset);
@@ -1286,6 +1335,15 @@ class _FbUint8List extends _FbList<int> {
   int operator [](int i) => bc._getUint8(offset + 4 + i);
 }
 
+/// List backed by 8-bit signed integers.
+class _FbInt8List extends _FbList<int> {
+  _FbInt8List(BufferContext bc, int offset) : super(bc, offset);
+
+  @override
+  @pragma('vm:prefer-inline')
+  int operator [](int i) => bc._getInt8(offset + 4 + i);
+}
+
 /// List backed by 8-bit unsigned integers.
 class _FbBoolList extends _FbList<bool> {
   _FbBoolList(BufferContext bc, int offset) : super(bc, offset);

dart/lib/src/reference.dart

@@ -373,8 +373,7 @@ class Reference {
     return null;
   }
 
-  int _diffKeys(
+  int _diffKeys(List<int> input, int index, int indirectOffset, int byteWidth) {
-      List<int> input, int index, int indirectOffset, int byteWidth) {
     final keyOffset = indirectOffset + index * byteWidth;
     final keyIndirectOffset =
         keyOffset - _readUInt(keyOffset, BitWidthUtil.fromByteWidth(byteWidth));

dart/test/flat_buffers_test.dart

@@ -658,15 +658,25 @@ class BuilderTest {
     List<int> byteList;
     {
       Builder builder = Builder(initialSize: 0);
-      int offset = builder.writeListUint8(<int>[1, 2, 3, 4, 0x9A]);
+      int offset = builder.writeListUint8(<int>[1, 2, 3, 4, 0x9A, 0xFA]);
       builder.finish(offset);
       byteList = builder.buffer;
     }
     // read and verify
     BufferContext buf = BufferContext.fromBytes(byteList);
-    List<int> items = const Uint8ListReader().read(buf, 0);
+    const buffOffset = 8; // 32-bit offset to the list, + 32-bit length
-    expect(items, hasLength(5));
+    for (final lazy in [true, false]) {
-    expect(items, orderedEquals(<int>[1, 2, 3, 4, 0x9A]));
+      List<int> items = Uint8ListReader(lazy: lazy).read(buf, 0);
+      expect(items, hasLength(6));
+      expect(items, orderedEquals(<int>[1, 2, 3, 4, 0x9A, 0xFA]));
+
+      // overwrite the buffer to verify the laziness
+      buf.buffer.setUint8(buffOffset + 1, 99);
+      expect(items, orderedEquals(<int>[1, lazy ? 99 : 2, 3, 4, 0x9A, 0xFA]));
+
+      // restore the previous value for the next loop
+      buf.buffer.setUint8(buffOffset + 1, 2);
+    }
   }
 
   void test_reset() {

dart/test/monster_test_my_game.example_generated.dart

@@ -1039,7 +1039,7 @@ class Monster {
   int get mana => const fb.Int16Reader().vTableGet(_bc, _bcOffset, 6, 150);
   int get hp => const fb.Int16Reader().vTableGet(_bc, _bcOffset, 8, 100);
   String? get name => const fb.StringReader().vTableGetNullable(_bc, _bcOffset, 10);
-  List<int>? get inventory => const fb.ListReader<int>(fb.Uint8Reader()).vTableGetNullable(_bc, _bcOffset, 14);
+  List<int>? get inventory => const fb.Uint8ListReader().vTableGetNullable(_bc, _bcOffset, 14);
   Color get color => Color.fromValue(const fb.Uint8Reader().vTableGet(_bc, _bcOffset, 16, 8));
   AnyTypeId? get testType => AnyTypeId._createOrNull(const fb.Uint8Reader().vTableGetNullable(_bc, _bcOffset, 18));
   dynamic get test {
@@ -1056,7 +1056,7 @@ class Monster {
   ///  multiline too
   List<Monster>? get testarrayoftables => const fb.ListReader<Monster>(Monster.reader).vTableGetNullable(_bc, _bcOffset, 26);
   Monster? get enemy => Monster.reader.vTableGetNullable(_bc, _bcOffset, 28);
-  List<int>? get testnestedflatbuffer => const fb.ListReader<int>(fb.Uint8Reader()).vTableGetNullable(_bc, _bcOffset, 30);
+  List<int>? get testnestedflatbuffer => const fb.Uint8ListReader().vTableGetNullable(_bc, _bcOffset, 30);
   Stat? get testempty => Stat.reader.vTableGetNullable(_bc, _bcOffset, 32);
   bool get testbool => const fb.BoolReader().vTableGet(_bc, _bcOffset, 34, false);
   int get testhashs32Fnv1 => const fb.Int32Reader().vTableGet(_bc, _bcOffset, 36, 0);
@@ -1073,7 +1073,7 @@ class Monster {
   double get testf3 => const fb.Float32Reader().vTableGet(_bc, _bcOffset, 58, 0.0);
   List<String>? get testarrayofstring2 => const fb.ListReader<String>(fb.StringReader()).vTableGetNullable(_bc, _bcOffset, 60);
   List<Ability>? get testarrayofsortedstruct => const fb.ListReader<Ability>(Ability.reader).vTableGetNullable(_bc, _bcOffset, 62);
-  List<int>? get flex => const fb.ListReader<int>(fb.Uint8Reader()).vTableGetNullable(_bc, _bcOffset, 64);
+  List<int>? get flex => const fb.Uint8ListReader().vTableGetNullable(_bc, _bcOffset, 64);
   List<Test>? get test5 => const fb.ListReader<Test>(Test.reader).vTableGetNullable(_bc, _bcOffset, 66);
   List<int>? get vectorOfLongs => const fb.ListReader<int>(fb.Int64Reader()).vTableGetNullable(_bc, _bcOffset, 68);
   List<double>? get vectorOfDoubles => const fb.ListReader<double>(fb.Float64Reader()).vTableGetNullable(_bc, _bcOffset, 70);
@@ -1106,7 +1106,7 @@ class Monster {
   }
   List<Color>? get vectorOfEnums => const fb.ListReader<Color>(Color.reader).vTableGetNullable(_bc, _bcOffset, 98);
   Race get signedEnum => Race.fromValue(const fb.Int8Reader().vTableGet(_bc, _bcOffset, 100, -1));
-  List<int>? get testrequirednestedflatbuffer => const fb.ListReader<int>(fb.Uint8Reader()).vTableGetNullable(_bc, _bcOffset, 102);
+  List<int>? get testrequirednestedflatbuffer => const fb.Uint8ListReader().vTableGetNullable(_bc, _bcOffset, 102);
   List<Stat>? get scalarKeySortedTables => const fb.ListReader<Stat>(Stat.reader).vTableGetNullable(_bc, _bcOffset, 104);
 
   @override
@@ -1119,7 +1119,7 @@ class Monster {
       mana: mana,
       hp: hp,
       name: name,
-      inventory: const fb.ListReader<int>(fb.Uint8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 14),
+      inventory: const fb.Uint8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 14),
       color: color,
       testType: testType,
       test: test,
@@ -1127,7 +1127,7 @@ class Monster {
       testarrayofstring: const fb.ListReader<String>(fb.StringReader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 24),
       testarrayoftables: testarrayoftables?.map((e) => e.unpack()).toList(),
       enemy: enemy?.unpack(),
-      testnestedflatbuffer: const fb.ListReader<int>(fb.Uint8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 30),
+      testnestedflatbuffer: const fb.Uint8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 30),
       testempty: testempty?.unpack(),
       testbool: testbool,
       testhashs32Fnv1: testhashs32Fnv1,
@@ -1144,7 +1144,7 @@ class Monster {
       testf3: testf3,
       testarrayofstring2: const fb.ListReader<String>(fb.StringReader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 60),
       testarrayofsortedstruct: testarrayofsortedstruct?.map((e) => e.unpack()).toList(),
-      flex: const fb.ListReader<int>(fb.Uint8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 64),
+      flex: const fb.Uint8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 64),
       test5: test5?.map((e) => e.unpack()).toList(),
       vectorOfLongs: const fb.ListReader<int>(fb.Int64Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 68),
       vectorOfDoubles: const fb.ListReader<double>(fb.Float64Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 70),
@@ -1163,7 +1163,7 @@ class Monster {
       anyAmbiguous: anyAmbiguous,
       vectorOfEnums: const fb.ListReader<Color>(Color.reader, lazy: false).vTableGetNullable(_bc, _bcOffset, 98),
       signedEnum: signedEnum,
-      testrequirednestedflatbuffer: const fb.ListReader<int>(fb.Uint8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 102),
+      testrequirednestedflatbuffer: const fb.Uint8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 102),
       scalarKeySortedTables: scalarKeySortedTables?.map((e) => e.unpack()).toList());
 
   static int pack(fb.Builder fbBuilder, MonsterT? object) {
@@ -1914,7 +1914,7 @@ class TypeAliases {
   int get u64 => const fb.Uint64Reader().vTableGet(_bc, _bcOffset, 18, 0);
   double get f32 => const fb.Float32Reader().vTableGet(_bc, _bcOffset, 20, 0.0);
   double get f64 => const fb.Float64Reader().vTableGet(_bc, _bcOffset, 22, 0.0);
-  List<int>? get v8 => const fb.ListReader<int>(fb.Int8Reader()).vTableGetNullable(_bc, _bcOffset, 24);
+  List<int>? get v8 => const fb.Int8ListReader().vTableGetNullable(_bc, _bcOffset, 24);
   List<double>? get vf64 => const fb.ListReader<double>(fb.Float64Reader()).vTableGetNullable(_bc, _bcOffset, 26);
 
   @override
@@ -1933,7 +1933,7 @@ class TypeAliases {
       u64: u64,
       f32: f32,
       f64: f64,
-      v8: const fb.ListReader<int>(fb.Int8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 24),
+      v8: const fb.Int8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 24),
       vf64: const fb.ListReader<double>(fb.Float64Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 26));
 
   static int pack(fb.Builder fbBuilder, TypeAliasesT? object) {

src/idl_gen_dart.cpp

@@ -335,6 +335,14 @@ class DartGenerator : public BaseGenerator {
     if (type.base_type == BASE_TYPE_BOOL) {
       return prefix + ".BoolReader()";
     } else if (IsVector(type)) {
+      if (!type.VectorType().enum_def) {
+        if (type.VectorType().base_type == BASE_TYPE_CHAR) {
+          return prefix + ".Int8ListReader(" + (lazy ? ")" : "lazy: false)");
+        }
+        if (type.VectorType().base_type == BASE_TYPE_UCHAR) {
+          return prefix + ".Uint8ListReader(" + (lazy ? ")" : "lazy: false)");
+        }
+      }
       return prefix + ".ListReader<" +
              GenDartTypeName(type.VectorType(), current_namespace, def) + ">(" +
              GenReaderTypeName(type.VectorType(), current_namespace, def, true,

tests/monster_test_my_game.example_generated.dart

@@ -1039,7 +1039,7 @@ class Monster {
   int get mana => const fb.Int16Reader().vTableGet(_bc, _bcOffset, 6, 150);
   int get hp => const fb.Int16Reader().vTableGet(_bc, _bcOffset, 8, 100);
   String? get name => const fb.StringReader().vTableGetNullable(_bc, _bcOffset, 10);
-  List<int>? get inventory => const fb.ListReader<int>(fb.Uint8Reader()).vTableGetNullable(_bc, _bcOffset, 14);
+  List<int>? get inventory => const fb.Uint8ListReader().vTableGetNullable(_bc, _bcOffset, 14);
   Color get color => Color.fromValue(const fb.Uint8Reader().vTableGet(_bc, _bcOffset, 16, 8));
   AnyTypeId? get testType => AnyTypeId._createOrNull(const fb.Uint8Reader().vTableGetNullable(_bc, _bcOffset, 18));
   dynamic get test {
@@ -1056,7 +1056,7 @@ class Monster {
   ///  multiline too
   List<Monster>? get testarrayoftables => const fb.ListReader<Monster>(Monster.reader).vTableGetNullable(_bc, _bcOffset, 26);
   Monster? get enemy => Monster.reader.vTableGetNullable(_bc, _bcOffset, 28);
-  List<int>? get testnestedflatbuffer => const fb.ListReader<int>(fb.Uint8Reader()).vTableGetNullable(_bc, _bcOffset, 30);
+  List<int>? get testnestedflatbuffer => const fb.Uint8ListReader().vTableGetNullable(_bc, _bcOffset, 30);
   Stat? get testempty => Stat.reader.vTableGetNullable(_bc, _bcOffset, 32);
   bool get testbool => const fb.BoolReader().vTableGet(_bc, _bcOffset, 34, false);
   int get testhashs32Fnv1 => const fb.Int32Reader().vTableGet(_bc, _bcOffset, 36, 0);
@@ -1073,7 +1073,7 @@ class Monster {
   double get testf3 => const fb.Float32Reader().vTableGet(_bc, _bcOffset, 58, 0.0);
   List<String>? get testarrayofstring2 => const fb.ListReader<String>(fb.StringReader()).vTableGetNullable(_bc, _bcOffset, 60);
   List<Ability>? get testarrayofsortedstruct => const fb.ListReader<Ability>(Ability.reader).vTableGetNullable(_bc, _bcOffset, 62);
-  List<int>? get flex => const fb.ListReader<int>(fb.Uint8Reader()).vTableGetNullable(_bc, _bcOffset, 64);
+  List<int>? get flex => const fb.Uint8ListReader().vTableGetNullable(_bc, _bcOffset, 64);
   List<Test>? get test5 => const fb.ListReader<Test>(Test.reader).vTableGetNullable(_bc, _bcOffset, 66);
   List<int>? get vectorOfLongs => const fb.ListReader<int>(fb.Int64Reader()).vTableGetNullable(_bc, _bcOffset, 68);
   List<double>? get vectorOfDoubles => const fb.ListReader<double>(fb.Float64Reader()).vTableGetNullable(_bc, _bcOffset, 70);
@@ -1106,7 +1106,7 @@ class Monster {
   }
   List<Color>? get vectorOfEnums => const fb.ListReader<Color>(Color.reader).vTableGetNullable(_bc, _bcOffset, 98);
   Race get signedEnum => Race.fromValue(const fb.Int8Reader().vTableGet(_bc, _bcOffset, 100, -1));
-  List<int>? get testrequirednestedflatbuffer => const fb.ListReader<int>(fb.Uint8Reader()).vTableGetNullable(_bc, _bcOffset, 102);
+  List<int>? get testrequirednestedflatbuffer => const fb.Uint8ListReader().vTableGetNullable(_bc, _bcOffset, 102);
   List<Stat>? get scalarKeySortedTables => const fb.ListReader<Stat>(Stat.reader).vTableGetNullable(_bc, _bcOffset, 104);
 
   @override
@@ -1119,7 +1119,7 @@ class Monster {
       mana: mana,
       hp: hp,
       name: name,
-      inventory: const fb.ListReader<int>(fb.Uint8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 14),
+      inventory: const fb.Uint8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 14),
       color: color,
       testType: testType,
       test: test,
@@ -1127,7 +1127,7 @@ class Monster {
       testarrayofstring: const fb.ListReader<String>(fb.StringReader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 24),
       testarrayoftables: testarrayoftables?.map((e) => e.unpack()).toList(),
       enemy: enemy?.unpack(),
-      testnestedflatbuffer: const fb.ListReader<int>(fb.Uint8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 30),
+      testnestedflatbuffer: const fb.Uint8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 30),
       testempty: testempty?.unpack(),
       testbool: testbool,
       testhashs32Fnv1: testhashs32Fnv1,
@@ -1144,7 +1144,7 @@ class Monster {
       testf3: testf3,
       testarrayofstring2: const fb.ListReader<String>(fb.StringReader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 60),
       testarrayofsortedstruct: testarrayofsortedstruct?.map((e) => e.unpack()).toList(),
-      flex: const fb.ListReader<int>(fb.Uint8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 64),
+      flex: const fb.Uint8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 64),
       test5: test5?.map((e) => e.unpack()).toList(),
       vectorOfLongs: const fb.ListReader<int>(fb.Int64Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 68),
       vectorOfDoubles: const fb.ListReader<double>(fb.Float64Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 70),
@@ -1163,7 +1163,7 @@ class Monster {
       anyAmbiguous: anyAmbiguous,
       vectorOfEnums: const fb.ListReader<Color>(Color.reader, lazy: false).vTableGetNullable(_bc, _bcOffset, 98),
       signedEnum: signedEnum,
-      testrequirednestedflatbuffer: const fb.ListReader<int>(fb.Uint8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 102),
+      testrequirednestedflatbuffer: const fb.Uint8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 102),
       scalarKeySortedTables: scalarKeySortedTables?.map((e) => e.unpack()).toList());
 
   static int pack(fb.Builder fbBuilder, MonsterT? object) {
@@ -1914,7 +1914,7 @@ class TypeAliases {
   int get u64 => const fb.Uint64Reader().vTableGet(_bc, _bcOffset, 18, 0);
   double get f32 => const fb.Float32Reader().vTableGet(_bc, _bcOffset, 20, 0.0);
   double get f64 => const fb.Float64Reader().vTableGet(_bc, _bcOffset, 22, 0.0);
-  List<int>? get v8 => const fb.ListReader<int>(fb.Int8Reader()).vTableGetNullable(_bc, _bcOffset, 24);
+  List<int>? get v8 => const fb.Int8ListReader().vTableGetNullable(_bc, _bcOffset, 24);
   List<double>? get vf64 => const fb.ListReader<double>(fb.Float64Reader()).vTableGetNullable(_bc, _bcOffset, 26);
 
   @override
@@ -1933,7 +1933,7 @@ class TypeAliases {
       u64: u64,
       f32: f32,
       f64: f64,
-      v8: const fb.ListReader<int>(fb.Int8Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 24),
+      v8: const fb.Int8ListReader(lazy: false).vTableGetNullable(_bc, _bcOffset, 24),
       vf64: const fb.ListReader<double>(fb.Float64Reader(), lazy: false).vTableGetNullable(_bc, _bcOffset, 26));
 
   static int pack(fb.Builder fbBuilder, TypeAliasesT? object) {