diff --git a/dart/lib/src/builder.dart b/dart/lib/src/builder.dart index afdf7232a..5ce46dcb2 100644 --- a/dart/lib/src/builder.dart +++ b/dart/lib/src/builder.dart @@ -111,9 +111,9 @@ class Builder { } final utf8String = utf8.encode(value); final length = utf8String.length; - final bitWidth = BitWidthUtil.width(length); + final bitWidth = BitWidthUtil.uwidth(length); final byteWidth = _align(bitWidth); - _writeInt(length, byteWidth); + _writeUInt(length, byteWidth); final stringOffset = _offset; final newOffset = _newOffset(length + 1); _pushBuffer(utf8String); @@ -149,9 +149,9 @@ class Builder { void addBlob(ByteBuffer value) { _integrityCheckOnValueAddition(); final length = value.lengthInBytes; - final bitWidth = BitWidthUtil.width(length); + final bitWidth = BitWidthUtil.uwidth(length); final byteWidth = _align(bitWidth); - _writeInt(length, byteWidth); + _writeUInt(length, byteWidth); final blobOffset = _offset; final newOffset = _newOffset(length); _pushBuffer(value.asUint8List()); @@ -295,13 +295,13 @@ class Builder { final value = _stack[0]; final byteWidth = _align(value.elementWidth(_offset, 0)); _writeStackValue(value, byteWidth); - _writeInt(value.storedPackedType(), 1); - _writeInt(byteWidth, 1); + _writeUInt(value.storedPackedType(), 1); + _writeUInt(byteWidth, 1); _finished = true; } _StackValue _createVector(int start, int vecLength, int step, [_StackValue keys]) { - var bitWidth = BitWidthUtil.width(vecLength); + var bitWidth = BitWidthUtil.uwidth(vecLength); var prefixElements = 1; if (keys != null) { var elemWidth = keys.elementWidth(_offset, 0); @@ -330,10 +330,10 @@ class Builder { final fix = typed & ValueTypeUtils.isNumber(vectorType) && vecLength >= 2 && vecLength <= 4; if (keys != null) { _writeStackValue(keys, byteWidth); - _writeInt(1 << keys.width.index, byteWidth); + _writeUInt(1 << keys.width.index, byteWidth); } if (fix == false) { - _writeInt(vecLength, byteWidth); + _writeUInt(vecLength, byteWidth); } final vecOffset = _offset; for (var i = start; i < _stack.length; i += step) { @@ -341,7 +341,7 @@ class Builder { } if (typed == false) { for (var i = start; i < _stack.length; i += step) { - _writeInt(_stack[i].storedPackedType(), 1); + _writeUInt(_stack[i].storedPackedType(), 1); } } if (keys != null) { @@ -442,7 +442,7 @@ class Builder { if (value.isOffset) { final relativeOffset = _offset - value.offset; if (byteWidth == 8 || relativeOffset < (1 << (byteWidth * 8))) { - _writeInt(relativeOffset, byteWidth); + _writeUInt(relativeOffset, byteWidth); } else { throw StateError('Unexpected size $byteWidth. This might be a bug. Please create an issue https://github.com/google/flatbuffers/issues/new'); } @@ -452,9 +452,9 @@ class Builder { _offset = newOffset; } - void _writeInt(int value, int byteWidth) { + void _writeUInt(int value, int byteWidth) { final newOffset = _newOffset(byteWidth); - _pushInt(value, BitWidthUtil.fromByteWidth(byteWidth)); + _pushUInt(value, BitWidthUtil.fromByteWidth(byteWidth)); _offset = newOffset; } @@ -492,6 +492,24 @@ class Builder { } } + void _pushUInt(int value, BitWidth width) { + switch (width) { + + case BitWidth.width8: + _buffer.setUint8(_offset, value); + break; + case BitWidth.width16: + _buffer.setUint16(_offset, value, Endian.little); + break; + case BitWidth.width32: + _buffer.setUint32(_offset, value, Endian.little); + break; + case BitWidth.width64: + _buffer.setUint64(_offset, value, Endian.little); + break; + } + } + void _pushBuffer(List value) { _buffer.buffer.asUint8List().setAll(_offset, value); } @@ -541,7 +559,7 @@ class _StackValue { final width = 1 << i; final offsetLoc = size + BitWidthUtil.paddingSize(size, width) + index * width; final offset = offsetLoc - _offset; - final bitWidth = BitWidthUtil.width(offset); + final bitWidth = BitWidthUtil.uwidth(offset); if (1 << bitWidth.index == width) { return bitWidth; } diff --git a/dart/lib/src/reference.dart b/dart/lib/src/reference.dart index ad3a2a0a6..3954f0687 100644 --- a/dart/lib/src/reference.dart +++ b/dart/lib/src/reference.dart @@ -188,16 +188,16 @@ class Reference { if(ValueTypeUtils.isFixedTypedVector(_valueType)) { _length = ValueTypeUtils.fixedTypedVectorElementSize(_valueType); } else if(_valueType == ValueType.Blob || ValueTypeUtils.isAVector(_valueType) || _valueType == ValueType.Map){ - _length = _readInt(_indirect - _byteWidth, BitWidthUtil.fromByteWidth(_byteWidth)); + _length = _readUInt(_indirect - _byteWidth, BitWidthUtil.fromByteWidth(_byteWidth)); } else if (_valueType == ValueType.Null) { _length = 0; } else if (_valueType == ValueType.String) { final indirect = _indirect; var size_byte_width = _byteWidth; - var size = _readInt(indirect - size_byte_width, BitWidthUtil.fromByteWidth(size_byte_width)); + var size = _readUInt(indirect - size_byte_width, BitWidthUtil.fromByteWidth(size_byte_width)); while (_buffer.getInt8(indirect + size) != 0) { size_byte_width <<= 1; - size = _readInt(indirect - size_byte_width, BitWidthUtil.fromByteWidth(size_byte_width)); + size = _readUInt(indirect - size_byte_width, BitWidthUtil.fromByteWidth(size_byte_width)); } _length = size; } else if (_valueType == ValueType.Key) { @@ -270,7 +270,7 @@ class Reference { /// value is not cached. Callers that need to use it more than once should /// cache the return value in a local variable. int get _indirect { - final step = _readInt(_offset, _parentWidth); + final step = _readUInt(_offset, _parentWidth); return _offset - step; } @@ -324,8 +324,8 @@ class Reference { int _keyIndex(String key) { final input = utf8.encode(key); final keysVectorOffset = _indirect - _byteWidth * 3; - final indirectOffset = keysVectorOffset - _readInt(keysVectorOffset, BitWidthUtil.fromByteWidth(_byteWidth)); - final byteWidth = _readInt(keysVectorOffset + _byteWidth, BitWidthUtil.fromByteWidth(_byteWidth)); + final indirectOffset = keysVectorOffset - _readUInt(keysVectorOffset, BitWidthUtil.fromByteWidth(_byteWidth)); + final byteWidth = _readUInt(keysVectorOffset + _byteWidth, BitWidthUtil.fromByteWidth(_byteWidth)); var low = 0; var high = length - 1; while (low <= high) { @@ -343,7 +343,7 @@ class Reference { int _diffKeys(List input, int index, int indirect_offset, int byteWidth) { final keyOffset = indirect_offset + index * byteWidth; - final keyIndirectOffset = keyOffset - _readInt(keyOffset, BitWidthUtil.fromByteWidth(byteWidth)); + final keyIndirectOffset = keyOffset - _readUInt(keyOffset, BitWidthUtil.fromByteWidth(byteWidth)); for (var i = 0; i < input.length; i++) { final dif = input[i] - _buffer.getUint8(keyIndirectOffset + i); if (dif != 0) { @@ -369,10 +369,10 @@ class Reference { String _keyForIndex(int index) { final keysVectorOffset = _indirect - _byteWidth * 3; - final indirectOffset = keysVectorOffset - _readInt(keysVectorOffset, BitWidthUtil.fromByteWidth(_byteWidth)); - final byteWidth = _readInt(keysVectorOffset + _byteWidth, BitWidthUtil.fromByteWidth(_byteWidth)); + final indirectOffset = keysVectorOffset - _readUInt(keysVectorOffset, BitWidthUtil.fromByteWidth(_byteWidth)); + final byteWidth = _readUInt(keysVectorOffset + _byteWidth, BitWidthUtil.fromByteWidth(_byteWidth)); final keyOffset = indirectOffset + index * byteWidth; - final keyIndirectOffset = keyOffset - _readInt(keyOffset, BitWidthUtil.fromByteWidth(byteWidth)); + final keyIndirectOffset = keyOffset - _readUInt(keyOffset, BitWidthUtil.fromByteWidth(byteWidth)); var length = 0; while (_buffer.getUint8(keyIndirectOffset + length) != 0) { length += 1; diff --git a/dart/lib/src/types.dart b/dart/lib/src/types.dart index 09e552067..8aed27251 100644 --- a/dart/lib/src/types.dart +++ b/dart/lib/src/types.dart @@ -22,6 +22,16 @@ class BitWidthUtil { } return value == _toF32(value) ? BitWidth.width32 : BitWidth.width64; } + static BitWidth uwidth(num value) { + if (value.toInt() == value) { + var v = value.toInt().abs(); + if (v >> 8 == 0) return BitWidth.width8; + if (v >> 16 == 0) return BitWidth.width16; + if (v >> 32 == 0) return BitWidth.width32; + return BitWidth.width64; + } + return value == _toF32(value) ? BitWidth.width32 : BitWidth.width64; + } static BitWidth fromByteWidth(int value) { if (value == 1) { return BitWidth.width8; diff --git a/dart/test/flex_reader_test.dart b/dart/test/flex_reader_test.dart index 47e7a3f98..ec30367b8 100644 --- a/dart/test/flex_reader_test.dart +++ b/dart/test/flex_reader_test.dart @@ -1,6 +1,6 @@ import 'dart:typed_data'; -import 'package:flat_buffers/flex_buffers.dart' show Reference; +import 'package:flat_buffers/flex_buffers.dart' show Reference, Builder; import 'package:test/test.dart'; void main() { @@ -261,6 +261,24 @@ void main() { expect(flx.mapValueIterable.map((e) => e.json).toList(), [flx['address'].json, flx['age'].json, flx['flags'].json, flx['name'].json, flx['weight'].json]); expect(flx['flags'].vectorIterable.map((e) => e.boolValue).toList(), [true, false, true, true]); }); + + test('bug where offest were stored as int instead of uint', (){ + const data = [99, 104, 97, 110, 110, 101, 108, 115, 95, 105, 110, 0, + 100, 105, 108, 97, 116, 105, 111, 110, 95, 104, 101, 105, 103, 104, 116, 95, 102, 97, 99, 116, 111, 114, 0, + 100, 105, 108, 97, 116, 105, 111, 110, 95, 119, 105, 100, 116, 104, 95, 102, 97, 99, 116, 111, 114, 0, + 102, 117, 115, 101, 100, 95, 97, 99, 116, 105, 118, 97, 116, 105, 111, 110, 95, 102, 117, 110, 99, 116, 105, 111, 110, 0, + 112, 97, 100, 95, 118, 97, 108, 117, 101, 115, 0, 112, 97, 100, 100, 105, 110, 103, 0, + 115, 116, 114, 105, 100, 101, 95, 104, 101, 105, 103, 104, 116, 0, + 115, 116, 114, 105, 100, 101, 95, 119, 105, 100, 116, 104, 0, + 8, 130, 119, 97, 76, 51, 41, 34, 21, 8, 1, 8, 64, 1, 1, 1, 1, 0, 1, 1, 4, 4, 4, 4, 4, 4, 4, 4, 16, 36, 1]; + var flx = Reference.fromBuffer(b(data)); + expect(flx.json, '{"channels_in":64,"dilation_height_factor":1,"dilation_width_factor":1,"fused_activation_function":1,"pad_values":1,"padding":0,"stride_height":1,"stride_width":1}'); + const object = {"channels_in":64,"dilation_height_factor":1,"dilation_width_factor":1,"fused_activation_function":1,"pad_values":1,"padding":0,"stride_height":1,"stride_width":1}; + var data1 = Builder.buildFromObject(object).asUint8List(); + expect(data1.length, data.length); + var flx1 = Reference.fromBuffer(b(data1)); + expect(flx1.json, '{"channels_in":64,"dilation_height_factor":1,"dilation_width_factor":1,"fused_activation_function":1,"pad_values":1,"padding":0,"stride_height":1,"stride_width":1}'); + }); } ByteBuffer b(List values) { diff --git a/js/flexbuffers.js b/js/flexbuffers.js index eff06f2f8..59737237a 100644 --- a/js/flexbuffers.js +++ b/js/flexbuffers.js @@ -202,7 +202,7 @@ flexbuffers.toReference = (buffer) => { } function indirect(dataView, offset, width) { - const step = readInt(dataView, offset, width); + const step = readUInt(dataView, offset, width); return offset - step; } @@ -210,8 +210,8 @@ flexbuffers.toReference = (buffer) => { const input = toUTF8Array(key); const keysVectorOffset = indirect(dataView, offset, parentWidth) - byteWidth * 3; const bitWidth = flexbuffers.BitWidthUtil.fromByteWidth(byteWidth); - const indirectOffset = keysVectorOffset - readInt(dataView, keysVectorOffset, bitWidth); - const _byteWidth = readInt(dataView, keysVectorOffset + byteWidth, bitWidth); + const indirectOffset = keysVectorOffset - readUInt(dataView, keysVectorOffset, bitWidth); + const _byteWidth = readUInt(dataView, keysVectorOffset + byteWidth, bitWidth); let low = 0; let high = length - 1; while (low <= high) { @@ -229,7 +229,7 @@ flexbuffers.toReference = (buffer) => { function diffKeys(input, index, dataView, offset, width) { const keyOffset = offset + index * width; - const keyIndirectOffset = keyOffset - readInt(dataView, keyOffset, flexbuffers.BitWidthUtil.fromByteWidth(width)); + const keyIndirectOffset = keyOffset - readUInt(dataView, keyOffset, flexbuffers.BitWidthUtil.fromByteWidth(width)); for (let i = 0; i < input.length; i++) { const dif = input[i] - dataView.getUint8(keyIndirectOffset + i); if (dif !== 0) { @@ -248,12 +248,12 @@ flexbuffers.toReference = (buffer) => { function keyForIndex(index, dataView, offset, parentWidth, byteWidth) { const keysVectorOffset = indirect(dataView, offset, parentWidth) - byteWidth * 3; - const bitWidth = flexbuffers.BitWidthUtil.fromByteWidth(byteWidth) - const indirectOffset = keysVectorOffset - readInt(dataView, keysVectorOffset, bitWidth); - const _byteWidth = readInt(dataView, keysVectorOffset + byteWidth, bitWidth); + const bitWidth = flexbuffers.BitWidthUtil.fromByteWidth(byteWidth); + const indirectOffset = keysVectorOffset - readUInt(dataView, keysVectorOffset, bitWidth); + const _byteWidth = readUInt(dataView, keysVectorOffset + byteWidth, bitWidth); const keyOffset = indirectOffset + index * _byteWidth; - const keyIndirectOffset = keyOffset - readInt(dataView, keyOffset, flexbuffers.BitWidthUtil.fromByteWidth(_byteWidth)); - var length = 0; + const keyIndirectOffset = keyOffset - readUInt(dataView, keyOffset, flexbuffers.BitWidthUtil.fromByteWidth(_byteWidth)); + let length = 0; while (dataView.getUint8(keyIndirectOffset + length) !== 0) { length++; } @@ -363,16 +363,16 @@ flexbuffers.toReference = (buffer) => { } else if (valueType === flexbuffers.ValueType.BLOB || valueType === flexbuffers.ValueType.MAP || flexbuffers.ValueTypeUtil.isAVector(valueType)) { - length = readInt(dataView, indirect(dataView, offset, parentWidth) - byteWidth, flexbuffers.BitWidthUtil.fromByteWidth(byteWidth)) + length = readUInt(dataView, indirect(dataView, offset, parentWidth) - byteWidth, flexbuffers.BitWidthUtil.fromByteWidth(byteWidth)) } else if (valueType === flexbuffers.ValueType.NULL) { length = 0; } else if (valueType === flexbuffers.ValueType.STRING) { const _indirect = indirect(dataView, offset, parentWidth); let sizeByteWidth = byteWidth; - size = readInt(dataView, _indirect - sizeByteWidth, flexbuffers.BitWidthUtil.fromByteWidth(byteWidth)); + size = readUInt(dataView, _indirect - sizeByteWidth, flexbuffers.BitWidthUtil.fromByteWidth(byteWidth)); while (dataView.getInt8(_indirect + size) !== 0) { sizeByteWidth <<= 1; - size = readInt(dataView, _indirect - sizeByteWidth, flexbuffers.BitWidthUtil.fromByteWidth(byteWidth)); + size = readUInt(dataView, _indirect - sizeByteWidth, flexbuffers.BitWidthUtil.fromByteWidth(byteWidth)); } length = size; } else if (valueType === flexbuffers.ValueType.KEY) { @@ -411,7 +411,10 @@ flexbuffers.toReference = (buffer) => { if (this.isBool()) { return this.boolValue(); } - return this.numericValue() || this.blobValue() || this.stringValue(); + if (this.isNumber()) { + return this.numericValue(); + } + return this.blobValue() || this.stringValue(); } } } @@ -509,11 +512,17 @@ flexbuffers.builder = (size = 2048, deduplicateString = true, deduplicateKeys = offset = newOffset; } + function writeUInt(value, byteWidth) { + const newOffset = computeOffset(byteWidth); + pushUInt(value, flexbuffers.BitWidthUtil.fromByteWidth(byteWidth)); + offset = newOffset; + } + function writeBlob(arrayBuffer) { const length = arrayBuffer.byteLength; - const bitWidth = flexbuffers.BitWidthUtil.iwidth(length); + const bitWidth = flexbuffers.BitWidthUtil.uwidth(length); const byteWidth = align(bitWidth); - writeInt(length, byteWidth); + writeUInt(length, byteWidth); const blobOffset = offset; const newOffset = computeOffset(length); new Uint8Array(buffer).set(new Uint8Array(arrayBuffer), blobOffset); @@ -528,9 +537,9 @@ flexbuffers.builder = (size = 2048, deduplicateString = true, deduplicateKeys = } const utf8 = toUTF8Array(str); const length = utf8.length; - const bitWidth = flexbuffers.BitWidthUtil.iwidth(length); + const bitWidth = flexbuffers.BitWidthUtil.uwidth(length); const byteWidth = align(bitWidth); - writeInt(length, byteWidth); + writeUInt(length, byteWidth); const stringOffset = offset; const newOffset = computeOffset(length + 1); new Uint8Array(buffer).set(utf8, stringOffset); @@ -564,7 +573,7 @@ flexbuffers.builder = (size = 2048, deduplicateString = true, deduplicateKeys = if (value.isOffset) { const relativeOffset = offset - value.offset; if (byteWidth === 8 || BigInt(relativeOffset) < (1n << BigInt(byteWidth * 8))) { - writeInt(relativeOffset, byteWidth); + writeUInt(relativeOffset, byteWidth); } else { throw `Unexpected size ${byteWidth}. This might be a bug. Please create an issue https://github.com/google/flatbuffers/issues/new` } @@ -744,7 +753,7 @@ flexbuffers.builder = (size = 2048, deduplicateString = true, deduplicateKeys = } function createVector(start, vecLength, step, keys = null) { - let bitWidth = flexbuffers.BitWidthUtil.iwidth(vecLength); + let bitWidth = flexbuffers.BitWidthUtil.uwidth(vecLength); let prefixElements = 1; if (keys !== null) { const elementWidth = keys.elementWidth(offset, 0); @@ -773,10 +782,10 @@ flexbuffers.builder = (size = 2048, deduplicateString = true, deduplicateKeys = const fix = typed && flexbuffers.ValueTypeUtil.isNumber(vectorType) && vecLength >= 2 && vecLength <= 4; if (keys !== null) { writeStackValue(keys, byteWidth); - writeInt(1 << keys.width, byteWidth); + writeUInt(1 << keys.width, byteWidth); } if (!fix) { - writeInt(vecLength, byteWidth); + writeUInt(vecLength, byteWidth); } const vecOffset = offset; for (let i = start; i < stack.length; i += step) { @@ -784,7 +793,7 @@ flexbuffers.builder = (size = 2048, deduplicateString = true, deduplicateKeys = } if (!typed) { for (let i = start; i < stack.length; i += step) { - writeInt(stack[i].storedPackedType(), 1); + writeUInt(stack[i].storedPackedType(), 1); } } if (keys !== null) { @@ -809,7 +818,7 @@ flexbuffers.builder = (size = 2048, deduplicateString = true, deduplicateKeys = const width = 1 << i; const offsetLoc = size + flexbuffers.BitWidthUtil.paddingSize(size, width) + index * width; const offset = offsetLoc - this.offset; - const bitWidth = flexbuffers.BitWidthUtil.iwidth(offset); + const bitWidth = flexbuffers.BitWidthUtil.uwidth(offset); if (1 << bitWidth === width) { return bitWidth; } @@ -880,8 +889,8 @@ flexbuffers.builder = (size = 2048, deduplicateString = true, deduplicateKeys = const value = stack[0]; const byteWidth = align(value.elementWidth(offset, 0)); writeStackValue(value, byteWidth); - writeInt(value.storedPackedType(), 1); - writeInt(byteWidth, 1); + writeUInt(value.storedPackedType(), 1); + writeUInt(byteWidth, 1); finished = true; } diff --git a/tests/JavaScriptFlexBuffersTest.js b/tests/JavaScriptFlexBuffersTest.js index 417095663..11c46c974 100644 --- a/tests/JavaScriptFlexBuffersTest.js +++ b/tests/JavaScriptFlexBuffersTest.js @@ -15,6 +15,7 @@ function main() { testRoundTrip(); testRoundTripWithBuilder(); testDeduplicationOff(); + testBugWhereOffestWereStoredAsIntInsteadOfUInt(); } function testSingleValueBuffers() { @@ -360,5 +361,29 @@ function testGoldBuffer() { }); } +function testBugWhereOffestWereStoredAsIntInsteadOfUInt() { + // Reported in https://github.com/google/flatbuffers/issues/5949#issuecomment-688421193 + const object = {'channels_in': 64, 'dilation_height_factor': 1, 'dilation_width_factor': 1, 'fused_activation_function': 1, 'pad_values': 1, 'padding': 0, 'stride_height': 1, 'stride_width': 1}; + let data1 = flexbuffers.encode(object); + const data = [99, 104, 97, 110, 110, 101, 108, 115, 95, 105, 110, 0, + 100, 105, 108, 97, 116, 105, 111, 110, 95, 104, 101, 105, 103, 104, 116, 95, 102, 97, 99, 116, 111, 114, 0, + 100, 105, 108, 97, 116, 105, 111, 110, 95, 119, 105, 100, 116, 104, 95, 102, 97, 99, 116, 111, 114, 0, + 102, 117, 115, 101, 100, 95, 97, 99, 116, 105, 118, 97, 116, 105, 111, 110, 95, 102, 117, 110, 99, 116, 105, 111, 110, 0, + 112, 97, 100, 95, 118, 97, 108, 117, 101, 115, 0, 112, 97, 100, 100, 105, 110, 103, 0, + 115, 116, 114, 105, 100, 101, 95, 104, 101, 105, 103, 104, 116, 0, + 115, 116, 114, 105, 100, 101, 95, 119, 105, 100, 116, 104, 0, + 8, 130, 119, 97, 76, 51, 41, 34, 21, 8, 1, 8, 64, 1, 1, 1, 1, 0, 1, 1, 4, 4, 4, 4, 4, 4, 4, 4, 16, 36, 1]; + let object2 = flexbuffers.toObject(new Uint8Array(data).buffer); + let object1 = flexbuffers.toObject(new Uint8Array(data1).buffer); + assert.deepStrictEqual(object, object2); + assert.deepStrictEqual(object, object1); + assert.strictEqual(data.length, data1.length); + let ref = flexbuffers.toReference(new Uint8Array(data).buffer); + assert.strictEqual(ref.isMap(), true); + assert.strictEqual(ref.length(), 8); + assert.strictEqual(ref.get("channels_in").numericValue(), 64); + assert.strictEqual(ref.get("padding").isNumber(), true); +} + main();