BigfootDev/flatbuffers-bigfoot
1
0
Fork 0
forked from BigfootDev/flatbuffers
Code Pull Requests Activity

[Dart] Adding FlexBuffers support (#5853)

Browse Source 
* Adding FlexBuffers support for Dart language
This commit is contained in:
Maxim Zaks
2020-05-18 16:50:39 +02:00
committed by GitHub
parent 0fa087657e
commit 5aa443d98c
7 changed files with 2020 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
dart/lib/flex_buffers.dart Normal file
View File

@@ -0,0 +1,2 @@
export 'src/builder.dart';
export 'src/reference.dart';

648
dart/lib/src/builder.dart Normal file
View File

@@ -0,0 +1,648 @@
import 'dart:convert';
import 'dart:typed_data';
import 'types.dart';
/// The main builder class for creation of a FlexBuffer.
class Builder {
ByteData _buffer;
List<_StackValue> _stack;
List<_StackPointer> _stackPointers;
int _offset;
bool _finished;
Map<String, _StackValue> _stringCache;
Map<String, _StackValue> _keyCache;
Map<_KeysHash, _StackValue> _keyVectorCache;
Map<int, _StackValue> _indirectIntCache;
Map<double, _StackValue> _indirectDoubleCache;
/// Instantiate the builder if you intent to gradually build up the buffer by calling
/// add... methods and calling [finish] to receive the the resulting byte array.
///
/// The default size of internal buffer is set to 2048. Provide a different value in order to avoid buffer copies.
Builder({int size = 2048}) {
_buffer = ByteData(size);
_stack = [];
_stackPointers = [];
_offset = 0;
_finished = false;
_stringCache = {};
_keyCache = {};
_keyVectorCache = {};
_indirectIntCache = {};
_indirectDoubleCache = {};
}
/// Use this method in order to turn an object into a FlexBuffer directly.
///
/// Use the manual instantiation of the [Builder] and gradual addition of values, if performance is more important than convenience.
static ByteBuffer buildFromObject(Object value) {
final builder = Builder();
builder._add(value);
final buffer = builder.finish();
final byteData = ByteData(buffer.lengthInBytes);
byteData.buffer.asUint8List().setAll(0, buffer);
return byteData.buffer;
}
void _add(Object value) {
if (value == null) {
addNull();
} else if (value is bool) {
addBool(value);
} else if (value is int) {
addInt(value);
} else if (value is double) {
addDouble(value);
} else if (value is ByteBuffer) {
addBlob(value);
} else if (value is String) {
addString(value);
} else if (value is List<dynamic>) {
startVector();
for (var i = 0; i < value.length; i++) {
_add(value[i]);
}
end();
} else if (value is Map<String, dynamic>) {
startMap();
value.forEach((key, value) {
addKey(key);
_add(value);
});
end();
} else {
throw UnsupportedError('Value of unexpected type: $value');
}
}
/// Use this method if you want to store a null value.
///
/// Specifically useful when building up a vector where values can be null.
void addNull() {
_integrityCheckOnValueAddition();
_stack.add(_StackValue.WithNull());
}
/// Adds a string value.
void addInt(int value) {
_integrityCheckOnValueAddition();
_stack.add(_StackValue.WithInt(value));
}
/// Adds a bool value.
void addBool(bool value) {
_integrityCheckOnValueAddition();
_stack.add(_StackValue.WithBool(value));
}
/// Adds a double value.
void addDouble(double value) {
_integrityCheckOnValueAddition();
_stack.add(_StackValue.WithDouble(value));
}
/// Adds a string value.
void addString(String value) {
_integrityCheckOnValueAddition();
if (_stringCache.containsKey(value)) {
_stack.add(_stringCache[value]);
return;
}
final utf8String = utf8.encode(value);
final length = utf8String.length;
final bitWidth = BitWidthUtil.width(length);
final byteWidth = _align(bitWidth);
_writeInt(length, byteWidth);
final stringOffset = _offset;
final newOffset = _newOffset(length + 1);
_pushBuffer(utf8String);
_offset = newOffset;
final stackValue = _StackValue.WithOffset(stringOffset, ValueType.String, bitWidth);
_stack.add(stackValue);
_stringCache[value] = stackValue;
}
/// This methods adds a key to a map and should be followed by an add... value call.
///
/// It also implies that you call this method only after you called [startMap].
void addKey(String value) {
_integrityCheckOnKeyAddition();
if (_keyCache.containsKey(value)) {
_stack.add(_keyCache[value]);
return;
}
final utf8String = utf8.encode(value);
final length = utf8String.length;
final keyOffset = _offset;
final newOffset = _newOffset(length + 1);
_pushBuffer(utf8String);
_offset = newOffset;
final stackValue = _StackValue.WithOffset(keyOffset, ValueType.Key, BitWidth.width8);
_stack.add(stackValue);
_keyCache[value] = stackValue;
}
/// Adds a byte array.
///
/// This method can be used to store any generic BLOB.
void addBlob(ByteBuffer value) {
_integrityCheckOnValueAddition();
final length = value.lengthInBytes;
final bitWidth = BitWidthUtil.width(length);
final byteWidth = _align(bitWidth);
_writeInt(length, byteWidth);
final blobOffset = _offset;
final newOffset = _newOffset(length);
_pushBuffer(value.asUint8List());
_offset = newOffset;
final stackValue = _StackValue.WithOffset(blobOffset, ValueType.Blob, bitWidth);
_stack.add(stackValue);
}
/// Stores int value indirectly in the buffer.
///
/// Adding large integer values indirectly might be beneficial if those values suppose to be store in a vector together with small integer values.
/// This is due to the fact that FlexBuffers will add padding to small integer values, if they are stored together with large integer values.
/// When we add integer indirectly the vector of ints will contain not the value itself, but only the relative offset to the value.
/// By setting the [cache] parameter to true, you make sure that the builder tracks added int value and performs deduplication.
void addIntIndirectly(int value, {bool cache = false}) {
_integrityCheckOnValueAddition();
if (_indirectIntCache.containsKey(value)) {
_stack.add(_indirectIntCache[value]);
return;
}
final stackValue = _StackValue.WithInt(value);
final byteWidth = _align(stackValue.width);
final newOffset = _newOffset(byteWidth);
final valueOffset = _offset;
_pushBuffer(stackValue.asU8List(stackValue.width));
final stackOffset = _StackValue.WithOffset(valueOffset, ValueType.IndirectInt, stackValue.width);
_stack.add(stackOffset);
_offset = newOffset;
if (cache) {
_indirectIntCache[value] = stackOffset;
}
}
/// Stores double value indirectly in the buffer.
///
/// Double are stored as 8 or 4 byte values in FlexBuffers. If they are stored in a mixed vector, values which are smaller than 4 / 8 bytes will be padded.
/// When we add double indirectly, the vector will contain not the value itself, but only the relative offset to the value. Which could occupy only 1 or 2 bytes, reducing the odds for unnecessary padding.
/// By setting the [cache] parameter to true, you make sure that the builder tracks already added double value and performs deduplication.
void addDoubleIndirectly(double value, {bool cache = false}) {
_integrityCheckOnValueAddition();
if (cache && _indirectDoubleCache.containsKey(value)) {
_stack.add(_indirectDoubleCache[value]);
return;
}
final stackValue = _StackValue.WithDouble(value);
final byteWidth = _align(stackValue.width);
final newOffset = _newOffset(byteWidth);
final valueOffset = _offset;
_pushBuffer(stackValue.asU8List(stackValue.width));
final stackOffset = _StackValue.WithOffset(valueOffset, ValueType.IndirectFloat, stackValue.width);
_stack.add(stackOffset);
_offset = newOffset;
if (cache) {
_indirectDoubleCache[value] = stackOffset;
}
}
/// This method starts a vector definition and needs to be followed by 0 to n add... value calls.
///
/// The vector definition needs to be finished with an [end] call.
/// It is also possible to add nested vector or map by calling [startVector] / [startMap].
void startVector() {
_integrityCheckOnValueAddition();
_stackPointers.add(_StackPointer(_stack.length, true));
}
/// This method starts a map definition.
///
/// This method call needs to be followed by 0 to n [addKey] + add... value calls.
/// The map definition needs to be finished with an [end] call.
/// It is also possible to add nested vector or map by calling [startVector] / [startMap] after calling [addKey].
void startMap() {
_integrityCheckOnValueAddition();
_stackPointers.add(_StackPointer(_stack.length, false));
}
/// Marks that the addition of values to the last vector, or map have ended.
void end() {
final pointer = _stackPointers.removeLast();
if (pointer.isVector) {
_endVector(pointer);
} else {
_sortKeysAndEndMap(pointer);
}
}
/// Finish building the FlatBuffer and return array of bytes.
///
/// Can be called multiple times, to get the array of bytes.
/// After the first call, adding values, or starting vectors / maps will result in an exception.
Uint8List finish() {
if (_finished == false) {
_finish();
}
return _buffer.buffer.asUint8List(0, _offset);
}
/// Builds a FlatBuffer with current state without finishing the builder.
///
/// Creates an internal temporary copy of current builder and finishes the copy.
/// Use this method, when the state of a long lasting builder need to be persisted periodically.
ByteBuffer snapshot() {
final tmp = Builder(size: _offset + 200);
tmp._offset = _offset;
tmp._stack = List.from(_stack);
tmp._stackPointers = List.from(_stackPointers);
tmp._buffer.buffer.asUint8List().setAll(0, _buffer.buffer.asUint8List(0, _offset));
for (var i = 0; i < tmp._stackPointers.length; i++){
tmp.end();
}
final buffer = tmp.finish();
final bd = ByteData(buffer.lengthInBytes);
bd.buffer.asUint8List().setAll(0, buffer);
return bd.buffer;
}
void _integrityCheckOnValueAddition() {
if (_finished) {
throw StateError('Adding values after finish is prohibited');
}
if (_stackPointers.isNotEmpty && _stackPointers.last.isVector == false) {
if (_stack.last.type != ValueType.Key) {
throw StateError('Adding value to a map before adding a key is prohibited');
}
}
}
void _integrityCheckOnKeyAddition() {
if (_finished) {
throw StateError('Adding values after finish is prohibited');
}
if (_stackPointers.isEmpty || _stackPointers.last.isVector) {
throw StateError('Adding key before staring a map is prohibited');
}
}
void _finish() {
if (_stack.length != 1) {
throw StateError('Stack has to be exactly 1, but is ${_stack.length}. You have to end all started vectors and maps, before calling [finish]');
}
final value = _stack[0];
final byteWidth = _align(value.elementWidth(_offset, 0));
_writeStackValue(value, byteWidth);
_writeInt(value.storedPackedType(), 1);
_writeInt(byteWidth, 1);
_finished = true;
}
_StackValue _createVector(int start, int vecLength, int step, [_StackValue keys]) {
var bitWidth = BitWidthUtil.width(vecLength);
var prefixElements = 1;
if (keys != null) {
var elemWidth = keys.elementWidth(_offset, 0);
if (elemWidth.index > bitWidth.index) {
bitWidth = elemWidth;
}
prefixElements += 2;
}
var vectorType = ValueType.Key;
var typed = keys == null;
for (var i = start; i < _stack.length; i += step) {
final elemWidth = _stack[i].elementWidth(_offset, i + prefixElements);
if (elemWidth.index > bitWidth.index) {
bitWidth = elemWidth;
}
if (i == start) {
vectorType = _stack[i].type;
typed &= ValueTypeUtils.isTypedVectorElement(vectorType);
} else {
if (vectorType != _stack[i].type) {
typed = false;
}
}
}
final byteWidth = _align(bitWidth);
final fix = typed & ValueTypeUtils.isNumber(vectorType) && vecLength >= 2 && vecLength <= 4;
if (keys != null) {
_writeStackValue(keys, byteWidth);
_writeInt(1 << keys.width.index, byteWidth);
}
if (fix == false) {
_writeInt(vecLength, byteWidth);
}
final vecOffset = _offset;
for (var i = start; i < _stack.length; i += step) {
_writeStackValue(_stack[i], byteWidth);
}
if (typed == false) {
for (var i = start; i < _stack.length; i += step) {
_writeInt(_stack[i].storedPackedType(), 1);
}
}
if (keys != null) {
return _StackValue.WithOffset(vecOffset, ValueType.Map, bitWidth);
}
if (typed) {
final vType = ValueTypeUtils.toTypedVector(vectorType, fix ? vecLength : 0);
return _StackValue.WithOffset(vecOffset, vType, bitWidth);
}
return _StackValue.WithOffset(vecOffset, ValueType.Vector, bitWidth);
}
void _endVector(_StackPointer pointer) {
final vecLength = _stack.length - pointer.stackPosition;
final vec = _createVector(pointer.stackPosition, vecLength, 1);
_stack.removeRange(pointer.stackPosition, _stack.length);
_stack.add(vec);
}
void _sortKeysAndEndMap(_StackPointer pointer) {
if (((_stack.length - pointer.stackPosition) & 1) == 1) {
throw StateError('The stack needs to hold key value pairs (even number of elements). Check if you combined [addKey] with add... method calls properly.');
}
var sorted = true;
for (var i = pointer.stackPosition; i < _stack.length - 2; i += 2) {
if (_shouldFlip(_stack[i], _stack[i+2])) {
sorted = false;
break;
}
}
if (sorted == false) {
for (var i = pointer.stackPosition; i < _stack.length; i += 2) {
var flipIndex = i;
for (var j = i + 2; j < _stack.length; j += 2) {
if (_shouldFlip(_stack[flipIndex], _stack[j])) {
flipIndex = j;
}
}
if (flipIndex != i) {
var k = _stack[flipIndex];
var v = _stack[flipIndex + 1];
_stack[flipIndex] = _stack[i];
_stack[flipIndex + 1] = _stack[i + 1];
_stack[i] = k;
_stack[i + 1] = v;
}
}
}
_endMap(pointer);
}
void _endMap(_StackPointer pointer) {
final vecLength = (_stack.length - pointer.stackPosition) >> 1;
final offsets = <int>[];
for (var i = pointer.stackPosition; i < _stack.length; i += 2) {
offsets.add(_stack[i].offset);
}
final keysHash = _KeysHash(offsets);
var keysStackValue;
if (_keyVectorCache.containsKey(keysHash)) {
keysStackValue = _keyVectorCache[keysHash];
} else {
keysStackValue = _createVector(pointer.stackPosition, vecLength, 2);
_keyVectorCache[keysHash] = keysStackValue;
}
final vec = _createVector(pointer.stackPosition + 1, vecLength, 2, keysStackValue);
_stack.removeRange(pointer.stackPosition, _stack.length);
_stack.add(vec);
}
bool _shouldFlip(_StackValue v1, _StackValue v2) {
if (v1.type != ValueType.Key || v2.type != ValueType.Key) {
throw StateError('Stack values are not keys $v1 | $v2. Check if you combined [addKey] with add... method calls properly.');
}
var c1, c2;
var index = 0;
do {
c1 = _buffer.getUint8(v1.offset + index);
c2 = _buffer.getUint8(v2.offset + index);
if (c2 < c1) return true;
if (c1 < c2) return false;
index += 1;
} while (c1 != 0 && c2 != 0);
return false;
}
int _align(BitWidth width) {
final byteWidth = BitWidthUtil.toByteWidth(width);
_offset += BitWidthUtil.paddingSize(_offset, byteWidth);
return byteWidth;
}
void _writeStackValue(_StackValue value, int byteWidth) {
final newOffset = _newOffset(byteWidth);
if (value.isOffset) {
final relativeOffset = _offset - value.offset;
if (byteWidth == 8 || relativeOffset < (1 << (byteWidth * 8))) {
_writeInt(relativeOffset, byteWidth);
} else {
throw StateError('Unexpected size $byteWidth. This might be a bug. Please create an issue https://github.com/google/flatbuffers/issues/new');
}
} else {
_pushBuffer(value.asU8List(BitWidthUtil.fromByteWidth(byteWidth)));
}
_offset = newOffset;
}
void _writeInt(int value, int byteWidth) {
final newOffset = _newOffset(byteWidth);
_pushInt(value, BitWidthUtil.fromByteWidth(byteWidth));
_offset = newOffset;
}
int _newOffset(int newValueSize) {
final newOffset = _offset + newValueSize;
var size = _buffer.lengthInBytes;
final prevSize = size;
while (size < newOffset) {
size <<= 1;
}
if (prevSize < size) {
final newBuf = ByteData(size);
newBuf.buffer
.asUint8List()
.setAll(0, _buffer.buffer.asUint8List());
}
return newOffset;
}
void _pushInt(int value, BitWidth width) {
switch (width) {
case BitWidth.width8:
_buffer.setInt8(_offset, value);
break;
case BitWidth.width16:
_buffer.setInt16(_offset, value, Endian.little);
break;
case BitWidth.width32:
_buffer.setInt32(_offset, value, Endian.little);
break;
case BitWidth.width64:
_buffer.setInt64(_offset, value, Endian.little);
break;
}
}
void _pushBuffer(List<int> value) {
_buffer.buffer.asUint8List().setAll(_offset, value);
}
}
class _StackValue {
Object _value;
int _offset;
ValueType _type;
BitWidth _width;
_StackValue.WithNull() {
_type = ValueType.Null;
_width = BitWidth.width8;
}
_StackValue.WithInt(int value) {
_type = value != null ? ValueType.Int : ValueType.Null;
_width = BitWidthUtil.width(value);
_value = value;
}
_StackValue.WithBool(bool value) {
_type = value != null ? ValueType.Bool : ValueType.Null;
_width = BitWidth.width8;
_value = value;
}
_StackValue.WithDouble(double value) {
_type = value != null ? ValueType.Float : ValueType.Null;
_width = BitWidthUtil.width(value);
_value = value;
}
_StackValue.WithOffset(int value, ValueType type, BitWidth width) {
_offset = value;
_type = type;
_width = width;
}
BitWidth storedWidth({BitWidth width = BitWidth.width8}) {
return ValueTypeUtils.isInline(_type) ? BitWidthUtil.max(_width, width) : _width;
}
int storedPackedType({BitWidth width = BitWidth.width8}) {
return ValueTypeUtils.packedType(_type, storedWidth(width: width));
}
BitWidth elementWidth(int size, int index) {
if (ValueTypeUtils.isInline(_type)) return _width;
for(var i = 0; i < 4; i++) {
final width = 1 << i;
final offsetLoc = size + BitWidthUtil.paddingSize(size, width) + index * width;
final offset = offsetLoc - _offset;
final bitWidth = BitWidthUtil.width(offset);
if (1 << bitWidth.index == width) {
return bitWidth;
}
}
throw StateError('Element is of unknown. Size: $size at index: $index. This might be a bug. Please create an issue https://github.com/google/flatbuffers/issues/new');
}
List<int> asU8List(BitWidth width) {
if (ValueTypeUtils.isNumber(_type)) {
if (_type == ValueType.Float) {
if (width == BitWidth.width32) {
final result = ByteData(4);
result.setFloat32(0, _value, Endian.little);
return result.buffer.asUint8List();
} else {
final result = ByteData(8);
result.setFloat64(0, _value, Endian.little);
return result.buffer.asUint8List();
}
} else {
switch(width) {
case BitWidth.width8:
final result = ByteData(1);
result.setInt8(0, _value);
return result.buffer.asUint8List();
case BitWidth.width16:
final result = ByteData(2);
result.setInt16(0, _value, Endian.little);
return result.buffer.asUint8List();
case BitWidth.width32:
final result = ByteData(4);
result.setInt32(0, _value, Endian.little);
return result.buffer.asUint8List();
case BitWidth.width64:
final result = ByteData(8);
result.setInt64(0, _value, Endian.little);
return result.buffer.asUint8List();
}
}
}
if (_type == ValueType.Null) {
final result = ByteData(1);
result.setInt8(0, 0);
return result.buffer.asUint8List();
}
if (_type == ValueType.Bool) {
final result = ByteData(1);
result.setInt8(0, _value ? 1 : 0);
return result.buffer.asUint8List();
}
throw StateError('Unexpected type: $_type. This might be a bug. Please create an issue https://github.com/google/flatbuffers/issues/new');
}
ValueType get type {
return _type;
}
BitWidth get width {
return _width;
}
bool get isOffset {
return !ValueTypeUtils.isInline(_type);
}
int get offset => _offset;
bool get isFloat32 {
return _type == ValueType.Float && _width == BitWidth.width32;
}
}
class _StackPointer {
int stackPosition;
bool isVector;
_StackPointer(this.stackPosition, this.isVector);
}
class _KeysHash {
final List<int> keys;
const _KeysHash(this.keys);
@override
bool operator ==(Object other) {
if (other is _KeysHash) {
if (keys.length != other.keys.length) return false;
for (var i = 0; i < keys.length; i++) {
if (keys[i] != other.keys[i]) return false;
}
return true;
}
return false;
}
@override
int get hashCode {
var result = 17;
for (var i = 0; i < keys.length; i++) {
result = result * 23 + keys[i];
}
return result;
}
}

446
dart/lib/src/reference.dart Normal file
View File

@@ -0,0 +1,446 @@
import 'dart:collection';
import 'dart:convert';
import 'dart:typed_data';
import 'types.dart';
/// Main class to read a value out of a FlexBuffer.
///
/// This class let you access values stored in the buffer in a lazy fashion.
class Reference {
final ByteData _buffer;
final int _offset;
final BitWidth _parentWidth;
final String _path;
int _byteWidth;
ValueType _valueType;
int _length;
Reference._(this._buffer, this._offset, this._parentWidth, int packedType, this._path) {
_byteWidth = 1 << (packedType & 3);
_valueType = ValueTypeUtils.fromInt(packedType >> 2);
}
/// Use this method to access the root value of a FlexBuffer.
static Reference fromBuffer(ByteBuffer buffer) {
final len = buffer.lengthInBytes;
if (len < 3) {
throw UnsupportedError('Buffer needs to be bigger than 3');
}
final byteData = ByteData.view(buffer);
final byteWidth = byteData.getUint8(len - 1);
final packedType = byteData.getUint8(len - 2);
final offset = len - byteWidth - 2;
return Reference._(ByteData.view(buffer), offset, BitWidthUtil.fromByteWidth(byteWidth), packedType, "/");
}
/// Returns true if the underlying value is null.
bool get isNull => _valueType == ValueType.Null;
/// Returns true if the underlying value can be represented as [num].
bool get isNum => ValueTypeUtils.isNumber(_valueType) || ValueTypeUtils.isIndirectNumber(_valueType);
/// Returns true if the underlying value was encoded as a float (direct or indirect).
bool get isDouble => _valueType == ValueType.Float || _valueType == ValueType.IndirectFloat;
/// Returns true if the underlying value was encoded as an int or uint (direct or indirect).
bool get isInt => isNum && !isDouble;
/// Returns true if the underlying value was encoded as a string or a key.
bool get isString => _valueType == ValueType.String || _valueType == ValueType.Key;
/// Returns true if the underlying value was encoded as a bool.
bool get isBool => _valueType == ValueType.Bool;
/// Returns true if the underlying value was encoded as a blob.
bool get isBlob => _valueType == ValueType.Blob;
/// Returns true if the underlying value points to a vector.
bool get isVector => ValueTypeUtils.isAVector(_valueType);
/// Returns true if the underlying value points to a map.
bool get isMap => _valueType == ValueType.Map;
/// If this [isBool], returns the bool value. Otherwise, returns null.
bool get boolValue {
if(_valueType == ValueType.Bool) {
return _readInt(_offset, _parentWidth) != 0;
}
return null;
}
/// Returns an [int], if the underlying value can be represented as an int.
///
/// Otherwise returns [null].
int get intValue {
if (_valueType == ValueType.Int) {
return _readInt(_offset, _parentWidth);
}
if (_valueType == ValueType.UInt) {
return _readUInt(_offset, _parentWidth);
}
if (_valueType == ValueType.IndirectInt) {
return _readInt(_indirect, BitWidthUtil.fromByteWidth(_byteWidth));
}
if (_valueType == ValueType.IndirectUInt) {
return _readUInt(_indirect, BitWidthUtil.fromByteWidth(_byteWidth));
}
return null;
}
/// Returns [double], if the underlying value [isDouble].
///
/// Otherwise returns [null].
double get doubleValue {
if (_valueType == ValueType.Float) {
return _readFloat(_offset, _parentWidth);
}
if (_valueType == ValueType.IndirectFloat) {
return _readFloat(_indirect, BitWidthUtil.fromByteWidth(_byteWidth));
}
return null;
}
/// Returns [num], if the underlying value is numeric, be it int uint, or float (direct or indirect).
///
/// Otherwise returns [null].
num get numValue => doubleValue ?? intValue;
/// Returns [String] value or null otherwise.
///
/// This method performers a utf8 decoding, as FlexBuffers format stores strings in utf8 encoding.
String get stringValue {
if (_valueType == ValueType.String || _valueType == ValueType.Key) {
return utf8.decode(_buffer.buffer.asUint8List(_indirect, length));
}
return null;
}
/// Returns [Uint8List] value or null otherwise.
Uint8List get blobValue {
if (_valueType == ValueType.Blob) {
return _buffer.buffer.asUint8List(_indirect, length);
}
return null;
}
/// Can be used with an [int] or a [String] value for key.
/// If the underlying value in FlexBuffer is a vector, then use [int] for access.
/// If the underlying value in FlexBuffer is a map, then use [String] for access.
/// Returns [Reference] value. Throws an exception when [key] is not applicable.
Reference operator [](Object key) {
if (key is int && ValueTypeUtils.isAVector(_valueType)) {
final index = key;
if(index >= length || index < 0) {
throw ArgumentError('Key: [$key] is not applicable on: $_path of: $_valueType length: $length');
}
final elementOffset = _indirect + index * _byteWidth;
final reference = Reference._(_buffer, elementOffset, BitWidthUtil.fromByteWidth(_byteWidth), 0, "$_path[$index]");
reference._byteWidth = 1;
if (ValueTypeUtils.isTypedVector(_valueType)) {
reference._valueType = ValueTypeUtils.typedVectorElementType(_valueType);
return reference;
}
if(ValueTypeUtils.isFixedTypedVector(_valueType)) {
reference._valueType = ValueTypeUtils.fixedTypedVectorElementType(_valueType);
return reference;
}
final packedType = _buffer.getUint8(_indirect + length * _byteWidth + index);
return Reference._(_buffer, elementOffset, BitWidthUtil.fromByteWidth(_byteWidth), packedType, "$_path[$index]");
}
if (key is String && _valueType == ValueType.Map) {
final index = _keyIndex(key);
if (index != null) {
return _valueForIndexWithKey(index, key);
}
}
throw ArgumentError('Key: [$key] is not applicable on: $_path of: $_valueType');
}
/// Get an iterable if the underlying flexBuffer value is a vector.
/// Otherwise throws an exception.
Iterable<Reference> get vectorIterable {
if(isVector == false) {
throw UnsupportedError('Value is not a vector. It is: $_valueType');
}
return _VectorIterator(this);
}
/// Get an iterable for keys if the underlying flexBuffer value is a map.
/// Otherwise throws an exception.
Iterable<String> get mapKeyIterable {
if(isMap == false) {
throw UnsupportedError('Value is not a map. It is: $_valueType');
}
return _MapKeyIterator(this);
}
/// Get an iterable for values if the underlying flexBuffer value is a map.
/// Otherwise throws an exception.
Iterable<Reference> get mapValueIterable {
if(isMap == false) {
throw UnsupportedError('Value is not a map. It is: $_valueType');
}
return _MapValueIterator(this);
}
/// Returns the length of the the underlying FlexBuffer value.
/// If the underlying value is [null] the length is 0.
/// If the underlying value is a number, or a bool, the length is 1.
/// If the underlying value is a vector, or map, the length reflects number of elements / element pairs.
/// If the values is a string or a blob, the length reflects a number of bytes the value occupies (strings are encoded in utf8 format).
int get length {
if (_length != null) {
return _length;
}
// needs to be checked before more generic isAVector
if(ValueTypeUtils.isFixedTypedVector(_valueType)) {
_length = ValueTypeUtils.fixedTypedVectorElementSize(_valueType);
} else if(_valueType == ValueType.Blob || ValueTypeUtils.isAVector(_valueType) || _valueType == ValueType.Map){
_length = _readInt(_indirect - _byteWidth, BitWidthUtil.fromByteWidth(_byteWidth));
} else if (_valueType == ValueType.Null) {
_length = 0;
} else if (_valueType == ValueType.String) {
final indirect = _indirect;
var size_byte_width = _byteWidth;
var size = _readInt(indirect - size_byte_width, BitWidthUtil.fromByteWidth(size_byte_width));
while (_buffer.getInt8(indirect + size) != 0) {
size_byte_width <<= 1;
size = _readInt(indirect - size_byte_width, BitWidthUtil.fromByteWidth(size_byte_width));
}
_length = size;
} else if (_valueType == ValueType.Key) {
final indirect = _indirect;
var size = 1;
while (_buffer.getInt8(indirect + size) != 0) {
size += 1;
}
_length = size;
} else {
_length = 1;
}
return _length;
}
/// Returns a minified JSON representation of the underlying FlexBuffer value.
///
/// This method involves materializing the entire object tree, which may be
/// expensive. It is more efficient to work with [Reference] and access only the needed data.
/// Blob values are represented as base64 encoded string.
String get json {
if(_valueType == ValueType.Bool) {
return boolValue ? 'true' : 'false';
}
if (_valueType == ValueType.Null) {
return 'null';
}
if(ValueTypeUtils.isNumber(_valueType)) {
return jsonEncode(numValue);
}
if (_valueType == ValueType.String) {
return jsonEncode(stringValue);
}
if (_valueType == ValueType.Blob) {
return jsonEncode(base64Encode(blobValue));
}
if (ValueTypeUtils.isAVector(_valueType)) {
final result = StringBuffer();
result.write('[');
for (var i = 0; i < length; i++) {
result.write(this[i].json);
if (i < length - 1) {
result.write(',');
}
}
result.write(']');
return result.toString();
}
if (_valueType == ValueType.Map) {
final result = StringBuffer();
result.write('{');
for (var i = 0; i < length; i++) {
result.write(jsonEncode(_keyForIndex(i)));
result.write(':');
result.write(_valueForIndex(i).json);
if (i < length - 1) {
result.write(',');
}
}
result.write('}');
return result.toString();
}
throw UnsupportedError('Type: $_valueType is not supported for JSON conversion');
}
/// Computes the indirect offset of the value.
///
/// To optimize for the more common case of being called only once, this
/// value is not cached. Callers that need to use it more than once should
/// cache the return value in a local variable.
int get _indirect {
final step = _readInt(_offset, _parentWidth);
return _offset - step;
}
int _readInt(int offset, BitWidth width) {
_validateOffset(offset, width);
if (width == BitWidth.width8) {
return _buffer.getInt8(offset);
}
if (width == BitWidth.width16) {
return _buffer.getInt16(offset, Endian.little);
}
if (width == BitWidth.width32) {
return _buffer.getInt32(offset, Endian.little);
}
return _buffer.getInt64(offset, Endian.little);
}
int _readUInt(int offset, BitWidth width) {
_validateOffset(offset, width);
if (width == BitWidth.width8) {
return _buffer.getUint8(offset);
}
if (width == BitWidth.width16) {
return _buffer.getUint16(offset, Endian.little);
}
if (width == BitWidth.width32) {
return _buffer.getUint32(offset, Endian.little);
}
return _buffer.getUint64(offset, Endian.little);
}
double _readFloat(int offset, BitWidth width) {
_validateOffset(offset, width);
if (width.index < BitWidth.width32.index) {
throw StateError('Bad width: $width');
}
if (width == BitWidth.width32) {
return _buffer.getFloat32(offset, Endian.little);
}
return _buffer.getFloat64(offset, Endian.little);
}
void _validateOffset(int offset, BitWidth width) {
if (_offset < 0 || _buffer.lengthInBytes <= offset + width.index || offset & (BitWidthUtil.toByteWidth(width) - 1) != 0) {
throw StateError('Bad offset: $offset, width: $width');
}
}
int _keyIndex(String key) {
final input = utf8.encode(key);
final keysVectorOffset = _indirect - _byteWidth * 3;
final indirectOffset = keysVectorOffset - _readInt(keysVectorOffset, BitWidthUtil.fromByteWidth(_byteWidth));
final byteWidth = _readInt(keysVectorOffset + _byteWidth, BitWidthUtil.fromByteWidth(_byteWidth));
var low = 0;
var high = length - 1;
while (low <= high) {
final mid = (high + low) >> 1;
final dif = _diffKeys(input, mid, indirectOffset, byteWidth);
if (dif == 0) return mid;
if (dif < 0) {
high = mid - 1;
} else {
low = mid + 1;
}
}
return null;
}
int _diffKeys(List<int> input, int index, int indirect_offset, int byteWidth) {
final keyOffset = indirect_offset + index * byteWidth;
final keyIndirectOffset = keyOffset - _readInt(keyOffset, BitWidthUtil.fromByteWidth(byteWidth));
for (var i = 0; i < input.length; i++) {
final dif = input[i] - _buffer.getUint8(keyIndirectOffset + i);
if (dif != 0) {
return dif;
}
}
return (_buffer.getUint8(keyIndirectOffset + input.length) == 0) ? 0 : -1;
}
Reference _valueForIndexWithKey(int index, String key) {
final indirect = _indirect;
final elementOffset = indirect + index * _byteWidth;
final packedType = _buffer.getUint8(indirect + length * _byteWidth + index);
return Reference._(_buffer, elementOffset, BitWidthUtil.fromByteWidth(_byteWidth), packedType, "$_path/$key");
}
Reference _valueForIndex(int index) {
final indirect = _indirect;
final elementOffset = indirect + index * _byteWidth;
final packedType = _buffer.getUint8(indirect + length * _byteWidth + index);
return Reference._(_buffer, elementOffset, BitWidthUtil.fromByteWidth(_byteWidth), packedType, "$_path/[$index]");
}
String _keyForIndex(int index) {
final keysVectorOffset = _indirect - _byteWidth * 3;
final indirectOffset = keysVectorOffset - _readInt(keysVectorOffset, BitWidthUtil.fromByteWidth(_byteWidth));
final byteWidth = _readInt(keysVectorOffset + _byteWidth, BitWidthUtil.fromByteWidth(_byteWidth));
final keyOffset = indirectOffset + index * byteWidth;
final keyIndirectOffset = keyOffset - _readInt(keyOffset, BitWidthUtil.fromByteWidth(byteWidth));
var length = 0;
while (_buffer.getUint8(keyIndirectOffset + length) != 0) {
length += 1;
}
return utf8.decode(_buffer.buffer.asUint8List(keyIndirectOffset, length));
}
}
class _VectorIterator with IterableMixin<Reference> implements Iterator<Reference> {
final Reference _vector;
int index;
_VectorIterator(this._vector) {
index = -1;
}
@override
Reference get current => _vector[index];
@override
bool moveNext() {
index++;
return index < _vector.length;
}
@override
Iterator<Reference> get iterator => this;
}
class _MapKeyIterator with IterableMixin<String> implements Iterator<String> {
final Reference _map;
int index;
_MapKeyIterator(this._map) {
index = -1;
}
@override
String get current => _map._keyForIndex(index);
@override
bool moveNext() {
index++;
return index < _map.length;
}
@override
Iterator<String> get iterator => this;
}
class _MapValueIterator with IterableMixin<Reference> implements Iterator<Reference> {
final Reference _map;
int index;
_MapValueIterator(this._map) {
index = -1;
}
@override
Reference get current => _map._valueForIndex(index);
@override
bool moveNext() {
index++;
return index < _map.length;
}
@override
Iterator<Reference> get iterator => this;
}

156
dart/lib/src/types.dart Normal file
View File

@@ -0,0 +1,156 @@
import 'dart:typed_data';
/// Represents the number of bits a value occupies.
enum BitWidth {
width8,
width16,
width32,
width64
}
class BitWidthUtil {
static int toByteWidth(BitWidth self) {
return 1 << self.index;
}
static BitWidth width(num value) {
if (value.toInt() == value) {
var v = value.toInt().abs();
if (v >> 7 == 0) return BitWidth.width8;
if (v >> 15 == 0) return BitWidth.width16;
if (v >> 31 == 0) return BitWidth.width32;
return BitWidth.width64;
}
return value == _toF32(value) ? BitWidth.width32 : BitWidth.width64;
}
static BitWidth fromByteWidth(int value) {
if (value == 1) {
return BitWidth.width8;
}
if (value == 2) {
return BitWidth.width16;
}
if (value == 4) {
return BitWidth.width32;
}
if (value == 8) {
return BitWidth.width64;
}
throw Exception('Unexpected value ${value}');
}
static int paddingSize(int bufSize, int scalarSize) {
return (~bufSize + 1) & (scalarSize - 1);
}
static double _toF32(double value) {
var bdata = ByteData(4);
bdata.setFloat32(0, value);
return bdata.getFloat32(0);
}
static BitWidth max(BitWidth self, BitWidth other) {
if (self.index < other.index) {
return other;
}
return self;
}
}
/// Represents all internal FlexBuffer types.
enum ValueType {
Null, Int, UInt, Float,
Key, String, IndirectInt, IndirectUInt, IndirectFloat,
Map, Vector, VectorInt, VectorUInt, VectorFloat, VectorKey,
@Deprecated('VectorString is deprecated due to a flaw in the binary format (https://github.com/google/flatbuffers/issues/5627)')
VectorString,
VectorInt2, VectorUInt2, VectorFloat2,
VectorInt3, VectorUInt3, VectorFloat3,
VectorInt4, VectorUInt4, VectorFloat4,
Blob, Bool, VectorBool
}
class ValueTypeUtils {
static int toInt(ValueType self) {
if (self == ValueType.VectorBool) return 36;
return self.index;
}
static ValueType fromInt(int value) {
if (value == 36) return ValueType.VectorBool;
return ValueType.values[value];
}
static bool isInline(ValueType self) {
return self == ValueType.Bool
|| toInt(self) <= toInt(ValueType.Float);
}
static bool isNumber(ValueType self) {
return toInt(self) >= toInt(ValueType.Int)
&& toInt(self) <= toInt(ValueType.Float);
}
static bool isIndirectNumber(ValueType self) {
return toInt(self) >= toInt(ValueType.IndirectInt)
&& toInt(self) <= toInt(ValueType.IndirectFloat);
}
static bool isTypedVectorElement(ValueType self) {
return self == ValueType.Bool ||
(
toInt(self) >= toInt(ValueType.Int)
&& toInt(self) <= toInt(ValueType.String)
);
}
static bool isTypedVector(ValueType self) {
return self == ValueType.VectorBool ||
(
toInt(self) >= toInt(ValueType.VectorInt)
&& toInt(self) <= toInt(ValueType.VectorString)
);
}
static bool isFixedTypedVector(ValueType self) {
return (
toInt(self) >= toInt(ValueType.VectorInt2)
&& toInt(self) <= toInt(ValueType.VectorFloat4)
);
}
static bool isAVector(ValueType self) {
return (
isTypedVector(self) || isFixedTypedVector(self) || self == ValueType.Vector
);
}
static ValueType toTypedVector(ValueType self, int length) {
if (length == 0) {
return ValueTypeUtils.fromInt(toInt(self) - toInt(ValueType.Int) + toInt(ValueType.VectorInt));
}
if (length == 2) {
return ValueTypeUtils.fromInt(toInt(self) - toInt(ValueType.Int) + toInt(ValueType.VectorInt2));
}
if (length == 3) {
return ValueTypeUtils.fromInt(toInt(self) - toInt(ValueType.Int) + toInt(ValueType.VectorInt3));
}
if (length == 4) {
return ValueTypeUtils.fromInt(toInt(self) - toInt(ValueType.Int) + toInt(ValueType.VectorInt4));
}
throw Exception('unexpected length ' + length.toString());
}
static ValueType typedVectorElementType(ValueType self) {
return ValueTypeUtils.fromInt(toInt(self) - toInt(ValueType.VectorInt) + toInt(ValueType.Int));
}
static ValueType fixedTypedVectorElementType(ValueType self) {
return ValueTypeUtils.fromInt((toInt(self) - toInt(ValueType.VectorInt2)) % 3 + toInt(ValueType.Int));
}
static int fixedTypedVectorElementSize(ValueType self) {
return (toInt(self) - toInt(ValueType.VectorInt2)) ~/ 3 + 2;
}
static int packedType(ValueType self, BitWidth bitWidth) {
return bitWidth.index | (toInt(self) << 2);
}
}