BigfootDev/flatbuffers
1
0
Fork 1
mirror of https://github.com/google/flatbuffers.git synced 2026-07-01 19:23:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

[JS] FlexBuffers Fix for wrong type of offset and length values (#6107)

Browse Source 
* Adding FlexBuffers support for Dart language

* Introduce snapshot method.

* Fix docu

* Replacing extension methods with static methods in order to support older Dart version

* Improving code based on PR feedback. Mainly rename refactoring.

* Addressing all PR feedback which does not need clarification

* exchange dynamic type with Object

* Adds better API documentation.
[] operator throws a very descriptive exception in case of a bad key.

* Implementation of JavaScript FlexBuffers decoder

* implements JS FlexBuffers builder

* replacing _toF32 with Math.fround

* Introducing test for BigInt number

* Moving functions from BitWitdth & ValueType object into BitWidthUtil and ValueTypeUtil accordingly.
Removing defensive checks.
Introducing fix for large int numbers by converting them to BigInt type.
Introducing handling for BigInt type in `add` method.
Using TextEncoder and Decoder to handle string / utf8 conversion.

* rename variable

* Lets user turn deduplication strategies for strings, keys and vector of keys off while building FlexBuffer.
Implements quick sort and choses quick sort if the number of keys is bigger then 20.
Removes unnecessary dict lookups in BitWidthUtil helper functions

* make iwidth and uwidth computation simpler and faster

* Making redInt and readUint a bit faster and shim the BigInt / BigUint usage

* Fixing a bug in FlexBuffers JS, where offsets and lengths are stored and read as int and not as uint values.

* Fixing a bug in FlexBuffers Dart, where offset and length values are stored and read as int values instead of uint values
This commit is contained in:
Maxim Zaks
2020-09-10 21:36:37 +02:00
committed by GitHub
parent 6cea45dcd3
commit 92a8c1a0f2
6 changed files with 130 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
dart/lib/src/builder.dart
View File

@@ -111,9 +111,9 @@ class Builder {
} }
final utf8String = utf8.encode(value); final utf8String = utf8.encode(value);
final length = utf8String.length; final length = utf8String.length;
final bitWidth = BitWidthUtil.width(length); final bitWidth = BitWidthUtil.uwidth(length);
final byteWidth = _align(bitWidth); final byteWidth = _align(bitWidth);
_writeInt(length, byteWidth); _writeUInt(length, byteWidth);
final stringOffset = _offset; final stringOffset = _offset;
final newOffset = _newOffset(length + 1); final newOffset = _newOffset(length + 1);
_pushBuffer(utf8String); _pushBuffer(utf8String);
@@ -149,9 +149,9 @@ class Builder {
void addBlob(ByteBuffer value) { void addBlob(ByteBuffer value) {
_integrityCheckOnValueAddition(); _integrityCheckOnValueAddition();
final length = value.lengthInBytes; final length = value.lengthInBytes;
final bitWidth = BitWidthUtil.width(length); final bitWidth = BitWidthUtil.uwidth(length);
final byteWidth = _align(bitWidth); final byteWidth = _align(bitWidth);
_writeInt(length, byteWidth); _writeUInt(length, byteWidth);
final blobOffset = _offset; final blobOffset = _offset;
final newOffset = _newOffset(length); final newOffset = _newOffset(length);
_pushBuffer(value.asUint8List()); _pushBuffer(value.asUint8List());
@@ -295,13 +295,13 @@ class Builder {
final value = _stack[0]; final value = _stack[0];
final byteWidth = _align(value.elementWidth(_offset, 0)); final byteWidth = _align(value.elementWidth(_offset, 0));
_writeStackValue(value, byteWidth); _writeStackValue(value, byteWidth);
_writeInt(value.storedPackedType(), 1); _writeUInt(value.storedPackedType(), 1);
_writeInt(byteWidth, 1); _writeUInt(byteWidth, 1);
_finished = true; _finished = true;
} }
_StackValue _createVector(int start, int vecLength, int step, [_StackValue keys]) { _StackValue _createVector(int start, int vecLength, int step, [_StackValue keys]) {
var bitWidth = BitWidthUtil.width(vecLength); var bitWidth = BitWidthUtil.uwidth(vecLength);
var prefixElements = 1; var prefixElements = 1;
if (keys != null) { if (keys != null) {
var elemWidth = keys.elementWidth(_offset, 0); var elemWidth = keys.elementWidth(_offset, 0);
@@ -330,10 +330,10 @@ class Builder {
final fix = typed & ValueTypeUtils.isNumber(vectorType) && vecLength >= 2 && vecLength <= 4; final fix = typed & ValueTypeUtils.isNumber(vectorType) && vecLength >= 2 && vecLength <= 4;
if (keys != null) { if (keys != null) {
_writeStackValue(keys, byteWidth); _writeStackValue(keys, byteWidth);
_writeInt(1 << keys.width.index, byteWidth); _writeUInt(1 << keys.width.index, byteWidth);
} }
if (fix == false) { if (fix == false) {
_writeInt(vecLength, byteWidth); _writeUInt(vecLength, byteWidth);
} }
final vecOffset = _offset; final vecOffset = _offset;
for (var i = start; i < _stack.length; i += step) { for (var i = start; i < _stack.length; i += step) {
@@ -341,7 +341,7 @@ class Builder {
} }
if (typed == false) { if (typed == false) {
for (var i = start; i < _stack.length; i += step) { for (var i = start; i < _stack.length; i += step) {
_writeInt(_stack[i].storedPackedType(), 1); _writeUInt(_stack[i].storedPackedType(), 1);
} }
} }
if (keys != null) { if (keys != null) {
@@ -442,7 +442,7 @@ class Builder {
if (value.isOffset) { if (value.isOffset) {
final relativeOffset = _offset - value.offset; final relativeOffset = _offset - value.offset;
if (byteWidth == 8 || relativeOffset < (1 << (byteWidth * 8))) { if (byteWidth == 8 || relativeOffset < (1 << (byteWidth * 8))) {
_writeInt(relativeOffset, byteWidth); _writeUInt(relativeOffset, byteWidth);
} else { } else {
throw StateError('Unexpected size $byteWidth. This might be a bug. Please create an issue https://github.com/google/flatbuffers/issues/new'); throw StateError('Unexpected size $byteWidth. This might be a bug. Please create an issue https://github.com/google/flatbuffers/issues/new');
} }
@@ -452,9 +452,9 @@ class Builder {
_offset = newOffset; _offset = newOffset;
} }
void _writeInt(int value, int byteWidth) { void _writeUInt(int value, int byteWidth) {
final newOffset = _newOffset(byteWidth); final newOffset = _newOffset(byteWidth);
_pushInt(value, BitWidthUtil.fromByteWidth(byteWidth)); _pushUInt(value, BitWidthUtil.fromByteWidth(byteWidth));
_offset = newOffset; _offset = newOffset;
} }
@@ -492,6 +492,24 @@ class Builder {
} }
} }
void _pushUInt(int value, BitWidth width) {
switch (width) {
case BitWidth.width8:
_buffer.setUint8(_offset, value);
break;
case BitWidth.width16:
_buffer.setUint16(_offset, value, Endian.little);
break;
case BitWidth.width32:
_buffer.setUint32(_offset, value, Endian.little);
break;
case BitWidth.width64:
_buffer.setUint64(_offset, value, Endian.little);
break;
}
}
void _pushBuffer(List<int> value) { void _pushBuffer(List<int> value) {
_buffer.buffer.asUint8List().setAll(_offset, value); _buffer.buffer.asUint8List().setAll(_offset, value);
} }
@@ -541,7 +559,7 @@ class _StackValue {
final width = 1 << i; final width = 1 << i;
final offsetLoc = size + BitWidthUtil.paddingSize(size, width) + index * width; final offsetLoc = size + BitWidthUtil.paddingSize(size, width) + index * width;
final offset = offsetLoc - _offset; final offset = offsetLoc - _offset;
final bitWidth = BitWidthUtil.width(offset); final bitWidth = BitWidthUtil.uwidth(offset);
if (1 << bitWidth.index == width) { if (1 << bitWidth.index == width) {
return bitWidth; return bitWidth;
} }

20
dart/lib/src/reference.dart
View File

@@ -188,16 +188,16 @@ class Reference {
if(ValueTypeUtils.isFixedTypedVector(_valueType)) { if(ValueTypeUtils.isFixedTypedVector(_valueType)) {
_length = ValueTypeUtils.fixedTypedVectorElementSize(_valueType); _length = ValueTypeUtils.fixedTypedVectorElementSize(_valueType);
} else if(_valueType == ValueType.Blob || ValueTypeUtils.isAVector(_valueType) || _valueType == ValueType.Map){ } else if(_valueType == ValueType.Blob || ValueTypeUtils.isAVector(_valueType) || _valueType == ValueType.Map){
_length = _readInt(_indirect - _byteWidth, BitWidthUtil.fromByteWidth(_byteWidth)); _length = _readUInt(_indirect - _byteWidth, BitWidthUtil.fromByteWidth(_byteWidth));
} else if (_valueType == ValueType.Null) { } else if (_valueType == ValueType.Null) {
_length = 0; _length = 0;
} else if (_valueType == ValueType.String) { } else if (_valueType == ValueType.String) {
final indirect = _indirect; final indirect = _indirect;
var size_byte_width = _byteWidth; var size_byte_width = _byteWidth;
var size = _readInt(indirect - size_byte_width, BitWidthUtil.fromByteWidth(size_byte_width)); var size = _readUInt(indirect - size_byte_width, BitWidthUtil.fromByteWidth(size_byte_width));
while (_buffer.getInt8(indirect + size) != 0) { while (_buffer.getInt8(indirect + size) != 0) {
size_byte_width <<= 1; size_byte_width <<= 1;
size = _readInt(indirect - size_byte_width, BitWidthUtil.fromByteWidth(size_byte_width)); size = _readUInt(indirect - size_byte_width, BitWidthUtil.fromByteWidth(size_byte_width));
} }
_length = size; _length = size;
} else if (_valueType == ValueType.Key) { } else if (_valueType == ValueType.Key) {
@@ -270,7 +270,7 @@ class Reference {
/// value is not cached. Callers that need to use it more than once should /// value is not cached. Callers that need to use it more than once should
/// cache the return value in a local variable. /// cache the return value in a local variable.
int get _indirect { int get _indirect {
final step = _readInt(_offset, _parentWidth); final step = _readUInt(_offset, _parentWidth);
return _offset - step; return _offset - step;
} }
@@ -324,8 +324,8 @@ class Reference {
int _keyIndex(String key) { int _keyIndex(String key) {
final input = utf8.encode(key); final input = utf8.encode(key);
final keysVectorOffset = _indirect - _byteWidth * 3; final keysVectorOffset = _indirect - _byteWidth * 3;
final indirectOffset = keysVectorOffset - _readInt(keysVectorOffset, BitWidthUtil.fromByteWidth(_byteWidth)); final indirectOffset = keysVectorOffset - _readUInt(keysVectorOffset, BitWidthUtil.fromByteWidth(_byteWidth));
final byteWidth = _readInt(keysVectorOffset + _byteWidth, BitWidthUtil.fromByteWidth(_byteWidth)); final byteWidth = _readUInt(keysVectorOffset + _byteWidth, BitWidthUtil.fromByteWidth(_byteWidth));
var low = 0; var low = 0;
var high = length - 1; var high = length - 1;
while (low <= high) { while (low <= high) {
@@ -343,7 +343,7 @@ class Reference {
int _diffKeys(List<int> input, int index, int indirect_offset, int byteWidth) { int _diffKeys(List<int> input, int index, int indirect_offset, int byteWidth) {
final keyOffset = indirect_offset + index * byteWidth; final keyOffset = indirect_offset + index * byteWidth;
final keyIndirectOffset = keyOffset - _readInt(keyOffset, BitWidthUtil.fromByteWidth(byteWidth)); final keyIndirectOffset = keyOffset - _readUInt(keyOffset, BitWidthUtil.fromByteWidth(byteWidth));
for (var i = 0; i < input.length; i++) { for (var i = 0; i < input.length; i++) {
final dif = input[i] - _buffer.getUint8(keyIndirectOffset + i); final dif = input[i] - _buffer.getUint8(keyIndirectOffset + i);
if (dif != 0) { if (dif != 0) {
@@ -369,10 +369,10 @@ class Reference {
String _keyForIndex(int index) { String _keyForIndex(int index) {
final keysVectorOffset = _indirect - _byteWidth * 3; final keysVectorOffset = _indirect - _byteWidth * 3;
final indirectOffset = keysVectorOffset - _readInt(keysVectorOffset, BitWidthUtil.fromByteWidth(_byteWidth)); final indirectOffset = keysVectorOffset - _readUInt(keysVectorOffset, BitWidthUtil.fromByteWidth(_byteWidth));
final byteWidth = _readInt(keysVectorOffset + _byteWidth, BitWidthUtil.fromByteWidth(_byteWidth)); final byteWidth = _readUInt(keysVectorOffset + _byteWidth, BitWidthUtil.fromByteWidth(_byteWidth));
final keyOffset = indirectOffset + index * byteWidth; final keyOffset = indirectOffset + index * byteWidth;
final keyIndirectOffset = keyOffset - _readInt(keyOffset, BitWidthUtil.fromByteWidth(byteWidth)); final keyIndirectOffset = keyOffset - _readUInt(keyOffset, BitWidthUtil.fromByteWidth(byteWidth));
var length = 0; var length = 0;
while (_buffer.getUint8(keyIndirectOffset + length) != 0) { while (_buffer.getUint8(keyIndirectOffset + length) != 0) {
length += 1; length += 1;

10
dart/lib/src/types.dart
View File

@@ -22,6 +22,16 @@ class BitWidthUtil {
} }
return value == _toF32(value) ? BitWidth.width32 : BitWidth.width64; return value == _toF32(value) ? BitWidth.width32 : BitWidth.width64;
} }
static BitWidth uwidth(num value) {
if (value.toInt() == value) {
var v = value.toInt().abs();
if (v >> 8 == 0) return BitWidth.width8;
if (v >> 16 == 0) return BitWidth.width16;
if (v >> 32 == 0) return BitWidth.width32;
return BitWidth.width64;
}
return value == _toF32(value) ? BitWidth.width32 : BitWidth.width64;
}
static BitWidth fromByteWidth(int value) { static BitWidth fromByteWidth(int value) {
if (value == 1) { if (value == 1) {
return BitWidth.width8; return BitWidth.width8;

20
dart/test/flex_reader_test.dart
View File

@@ -1,6 +1,6 @@
import 'dart:typed_data'; import 'dart:typed_data';
import 'package:flat_buffers/flex_buffers.dart' show Reference; import 'package:flat_buffers/flex_buffers.dart' show Reference, Builder;
import 'package:test/test.dart'; import 'package:test/test.dart';
void main() { void main() {
@@ -261,6 +261,24 @@ void main() {
expect(flx.mapValueIterable.map((e) => e.json).toList(), [flx['address'].json, flx['age'].json, flx['flags'].json, flx['name'].json, flx['weight'].json]); expect(flx.mapValueIterable.map((e) => e.json).toList(), [flx['address'].json, flx['age'].json, flx['flags'].json, flx['name'].json, flx['weight'].json]);
expect(flx['flags'].vectorIterable.map((e) => e.boolValue).toList(), [true, false, true, true]); expect(flx['flags'].vectorIterable.map((e) => e.boolValue).toList(), [true, false, true, true]);
}); });
test('bug where offest were stored as int instead of uint', (){
const data = [99, 104, 97, 110, 110, 101, 108, 115, 95, 105, 110, 0,
100, 105, 108, 97, 116, 105, 111, 110, 95, 104, 101, 105, 103, 104, 116, 95, 102, 97, 99, 116, 111, 114, 0,
100, 105, 108, 97, 116, 105, 111, 110, 95, 119, 105, 100, 116, 104, 95, 102, 97, 99, 116, 111, 114, 0,
102, 117, 115, 101, 100, 95, 97, 99, 116, 105, 118, 97, 116, 105, 111, 110, 95, 102, 117, 110, 99, 116, 105, 111, 110, 0,
112, 97, 100, 95, 118, 97, 108, 117, 101, 115, 0, 112, 97, 100, 100, 105, 110, 103, 0,
115, 116, 114, 105, 100, 101, 95, 104, 101, 105, 103, 104, 116, 0,
115, 116, 114, 105, 100, 101, 95, 119, 105, 100, 116, 104, 0,
8, 130, 119, 97, 76, 51, 41, 34, 21, 8, 1, 8, 64, 1, 1, 1, 1, 0, 1, 1, 4, 4, 4, 4, 4, 4, 4, 4, 16, 36, 1];
var flx = Reference.fromBuffer(b(data));
expect(flx.json, '{"channels_in":64,"dilation_height_factor":1,"dilation_width_factor":1,"fused_activation_function":1,"pad_values":1,"padding":0,"stride_height":1,"stride_width":1}');
const object = {"channels_in":64,"dilation_height_factor":1,"dilation_width_factor":1,"fused_activation_function":1,"pad_values":1,"padding":0,"stride_height":1,"stride_width":1};
var data1 = Builder.buildFromObject(object).asUint8List();
expect(data1.length, data.length);
var flx1 = Reference.fromBuffer(b(data1));
expect(flx1.json, '{"channels_in":64,"dilation_height_factor":1,"dilation_width_factor":1,"fused_activation_function":1,"pad_values":1,"padding":0,"stride_height":1,"stride_width":1}');
});
} }
ByteBuffer b(List<int> values) { ByteBuffer b(List<int> values) {

59
js/flexbuffers.js
View File

@@ -202,7 +202,7 @@ flexbuffers.toReference = (buffer) => {
} }
function indirect(dataView, offset, width) { function indirect(dataView, offset, width) {
const step = readInt(dataView, offset, width); const step = readUInt(dataView, offset, width);
return offset - step; return offset - step;
} }
@@ -210,8 +210,8 @@ flexbuffers.toReference = (buffer) => {
const input = toUTF8Array(key); const input = toUTF8Array(key);
const keysVectorOffset = indirect(dataView, offset, parentWidth) - byteWidth * 3; const keysVectorOffset = indirect(dataView, offset, parentWidth) - byteWidth * 3;
const bitWidth = flexbuffers.BitWidthUtil.fromByteWidth(byteWidth); const bitWidth = flexbuffers.BitWidthUtil.fromByteWidth(byteWidth);
const indirectOffset = keysVectorOffset - readInt(dataView, keysVectorOffset, bitWidth); const indirectOffset = keysVectorOffset - readUInt(dataView, keysVectorOffset, bitWidth);
const _byteWidth = readInt(dataView, keysVectorOffset + byteWidth, bitWidth); const _byteWidth = readUInt(dataView, keysVectorOffset + byteWidth, bitWidth);
let low = 0; let low = 0;
let high = length - 1; let high = length - 1;
while (low <= high) { while (low <= high) {
@@ -229,7 +229,7 @@ flexbuffers.toReference = (buffer) => {
function diffKeys(input, index, dataView, offset, width) { function diffKeys(input, index, dataView, offset, width) {
const keyOffset = offset + index * width; const keyOffset = offset + index * width;
const keyIndirectOffset = keyOffset - readInt(dataView, keyOffset, flexbuffers.BitWidthUtil.fromByteWidth(width)); const keyIndirectOffset = keyOffset - readUInt(dataView, keyOffset, flexbuffers.BitWidthUtil.fromByteWidth(width));
for (let i = 0; i < input.length; i++) { for (let i = 0; i < input.length; i++) {
const dif = input[i] - dataView.getUint8(keyIndirectOffset + i); const dif = input[i] - dataView.getUint8(keyIndirectOffset + i);
if (dif !== 0) { if (dif !== 0) {
@@ -248,12 +248,12 @@ flexbuffers.toReference = (buffer) => {
function keyForIndex(index, dataView, offset, parentWidth, byteWidth) { function keyForIndex(index, dataView, offset, parentWidth, byteWidth) {
const keysVectorOffset = indirect(dataView, offset, parentWidth) - byteWidth * 3; const keysVectorOffset = indirect(dataView, offset, parentWidth) - byteWidth * 3;
const bitWidth = flexbuffers.BitWidthUtil.fromByteWidth(byteWidth) const bitWidth = flexbuffers.BitWidthUtil.fromByteWidth(byteWidth);
const indirectOffset = keysVectorOffset - readInt(dataView, keysVectorOffset, bitWidth); const indirectOffset = keysVectorOffset - readUInt(dataView, keysVectorOffset, bitWidth);
const _byteWidth = readInt(dataView, keysVectorOffset + byteWidth, bitWidth); const _byteWidth = readUInt(dataView, keysVectorOffset + byteWidth, bitWidth);
const keyOffset = indirectOffset + index * _byteWidth; const keyOffset = indirectOffset + index * _byteWidth;
const keyIndirectOffset = keyOffset - readInt(dataView, keyOffset, flexbuffers.BitWidthUtil.fromByteWidth(_byteWidth)); const keyIndirectOffset = keyOffset - readUInt(dataView, keyOffset, flexbuffers.BitWidthUtil.fromByteWidth(_byteWidth));
var length = 0; let length = 0;
while (dataView.getUint8(keyIndirectOffset + length) !== 0) { while (dataView.getUint8(keyIndirectOffset + length) !== 0) {
length++; length++;
} }
@@ -363,16 +363,16 @@ flexbuffers.toReference = (buffer) => {
} else if (valueType === flexbuffers.ValueType.BLOB } else if (valueType === flexbuffers.ValueType.BLOB
|| valueType === flexbuffers.ValueType.MAP || valueType === flexbuffers.ValueType.MAP
|| flexbuffers.ValueTypeUtil.isAVector(valueType)) { || flexbuffers.ValueTypeUtil.isAVector(valueType)) {
length = readInt(dataView, indirect(dataView, offset, parentWidth) - byteWidth, flexbuffers.BitWidthUtil.fromByteWidth(byteWidth)) length = readUInt(dataView, indirect(dataView, offset, parentWidth) - byteWidth, flexbuffers.BitWidthUtil.fromByteWidth(byteWidth))
} else if (valueType === flexbuffers.ValueType.NULL) { } else if (valueType === flexbuffers.ValueType.NULL) {
length = 0; length = 0;
} else if (valueType === flexbuffers.ValueType.STRING) { } else if (valueType === flexbuffers.ValueType.STRING) {
const _indirect = indirect(dataView, offset, parentWidth); const _indirect = indirect(dataView, offset, parentWidth);
let sizeByteWidth = byteWidth; let sizeByteWidth = byteWidth;
size = readInt(dataView, _indirect - sizeByteWidth, flexbuffers.BitWidthUtil.fromByteWidth(byteWidth)); size = readUInt(dataView, _indirect - sizeByteWidth, flexbuffers.BitWidthUtil.fromByteWidth(byteWidth));
while (dataView.getInt8(_indirect + size) !== 0) { while (dataView.getInt8(_indirect + size) !== 0) {
sizeByteWidth <<= 1; sizeByteWidth <<= 1;
size = readInt(dataView, _indirect - sizeByteWidth, flexbuffers.BitWidthUtil.fromByteWidth(byteWidth)); size = readUInt(dataView, _indirect - sizeByteWidth, flexbuffers.BitWidthUtil.fromByteWidth(byteWidth));
} }
length = size; length = size;
} else if (valueType === flexbuffers.ValueType.KEY) { } else if (valueType === flexbuffers.ValueType.KEY) {
@@ -411,7 +411,10 @@ flexbuffers.toReference = (buffer) => {
if (this.isBool()) { if (this.isBool()) {
return this.boolValue(); return this.boolValue();
} }
return this.numericValue() || this.blobValue() || this.stringValue(); if (this.isNumber()) {
return this.numericValue();
}
return this.blobValue() || this.stringValue();
} }
} }
} }
@@ -509,11 +512,17 @@ flexbuffers.builder = (size = 2048, deduplicateString = true, deduplicateKeys =
offset = newOffset; offset = newOffset;
} }
function writeUInt(value, byteWidth) {
const newOffset = computeOffset(byteWidth);
pushUInt(value, flexbuffers.BitWidthUtil.fromByteWidth(byteWidth));
offset = newOffset;
}
function writeBlob(arrayBuffer) { function writeBlob(arrayBuffer) {
const length = arrayBuffer.byteLength; const length = arrayBuffer.byteLength;
const bitWidth = flexbuffers.BitWidthUtil.iwidth(length); const bitWidth = flexbuffers.BitWidthUtil.uwidth(length);
const byteWidth = align(bitWidth); const byteWidth = align(bitWidth);
writeInt(length, byteWidth); writeUInt(length, byteWidth);
const blobOffset = offset; const blobOffset = offset;
const newOffset = computeOffset(length); const newOffset = computeOffset(length);
new Uint8Array(buffer).set(new Uint8Array(arrayBuffer), blobOffset); new Uint8Array(buffer).set(new Uint8Array(arrayBuffer), blobOffset);
@@ -528,9 +537,9 @@ flexbuffers.builder = (size = 2048, deduplicateString = true, deduplicateKeys =
} }
const utf8 = toUTF8Array(str); const utf8 = toUTF8Array(str);
const length = utf8.length; const length = utf8.length;
const bitWidth = flexbuffers.BitWidthUtil.iwidth(length); const bitWidth = flexbuffers.BitWidthUtil.uwidth(length);
const byteWidth = align(bitWidth); const byteWidth = align(bitWidth);
writeInt(length, byteWidth); writeUInt(length, byteWidth);
const stringOffset = offset; const stringOffset = offset;
const newOffset = computeOffset(length + 1); const newOffset = computeOffset(length + 1);
new Uint8Array(buffer).set(utf8, stringOffset); new Uint8Array(buffer).set(utf8, stringOffset);
@@ -564,7 +573,7 @@ flexbuffers.builder = (size = 2048, deduplicateString = true, deduplicateKeys =
if (value.isOffset) { if (value.isOffset) {
const relativeOffset = offset - value.offset; const relativeOffset = offset - value.offset;
if (byteWidth === 8 || BigInt(relativeOffset) < (1n << BigInt(byteWidth * 8))) { if (byteWidth === 8 || BigInt(relativeOffset) < (1n << BigInt(byteWidth * 8))) {
writeInt(relativeOffset, byteWidth); writeUInt(relativeOffset, byteWidth);
} else { } else {
throw `Unexpected size ${byteWidth}. This might be a bug. Please create an issue https://github.com/google/flatbuffers/issues/new` throw `Unexpected size ${byteWidth}. This might be a bug. Please create an issue https://github.com/google/flatbuffers/issues/new`
} }
@@ -744,7 +753,7 @@ flexbuffers.builder = (size = 2048, deduplicateString = true, deduplicateKeys =
} }
function createVector(start, vecLength, step, keys = null) { function createVector(start, vecLength, step, keys = null) {
let bitWidth = flexbuffers.BitWidthUtil.iwidth(vecLength); let bitWidth = flexbuffers.BitWidthUtil.uwidth(vecLength);
let prefixElements = 1; let prefixElements = 1;
if (keys !== null) { if (keys !== null) {
const elementWidth = keys.elementWidth(offset, 0); const elementWidth = keys.elementWidth(offset, 0);
@@ -773,10 +782,10 @@ flexbuffers.builder = (size = 2048, deduplicateString = true, deduplicateKeys =
const fix = typed && flexbuffers.ValueTypeUtil.isNumber(vectorType) && vecLength >= 2 && vecLength <= 4; const fix = typed && flexbuffers.ValueTypeUtil.isNumber(vectorType) && vecLength >= 2 && vecLength <= 4;
if (keys !== null) { if (keys !== null) {
writeStackValue(keys, byteWidth); writeStackValue(keys, byteWidth);
writeInt(1 << keys.width, byteWidth); writeUInt(1 << keys.width, byteWidth);
} }
if (!fix) { if (!fix) {
writeInt(vecLength, byteWidth); writeUInt(vecLength, byteWidth);
} }
const vecOffset = offset; const vecOffset = offset;
for (let i = start; i < stack.length; i += step) { for (let i = start; i < stack.length; i += step) {
@@ -784,7 +793,7 @@ flexbuffers.builder = (size = 2048, deduplicateString = true, deduplicateKeys =
} }
if (!typed) { if (!typed) {
for (let i = start; i < stack.length; i += step) { for (let i = start; i < stack.length; i += step) {
writeInt(stack[i].storedPackedType(), 1); writeUInt(stack[i].storedPackedType(), 1);
} }
} }
if (keys !== null) { if (keys !== null) {
@@ -809,7 +818,7 @@ flexbuffers.builder = (size = 2048, deduplicateString = true, deduplicateKeys =
const width = 1 << i; const width = 1 << i;
const offsetLoc = size + flexbuffers.BitWidthUtil.paddingSize(size, width) + index * width; const offsetLoc = size + flexbuffers.BitWidthUtil.paddingSize(size, width) + index * width;
const offset = offsetLoc - this.offset; const offset = offsetLoc - this.offset;
const bitWidth = flexbuffers.BitWidthUtil.iwidth(offset); const bitWidth = flexbuffers.BitWidthUtil.uwidth(offset);
if (1 << bitWidth === width) { if (1 << bitWidth === width) {
return bitWidth; return bitWidth;
} }
@@ -880,8 +889,8 @@ flexbuffers.builder = (size = 2048, deduplicateString = true, deduplicateKeys =
const value = stack[0]; const value = stack[0];
const byteWidth = align(value.elementWidth(offset, 0)); const byteWidth = align(value.elementWidth(offset, 0));
writeStackValue(value, byteWidth); writeStackValue(value, byteWidth);
writeInt(value.storedPackedType(), 1); writeUInt(value.storedPackedType(), 1);
writeInt(byteWidth, 1); writeUInt(byteWidth, 1);
finished = true; finished = true;
} }

25
tests/JavaScriptFlexBuffersTest.js
View File

@@ -15,6 +15,7 @@ function main() {
testRoundTrip(); testRoundTrip();
testRoundTripWithBuilder(); testRoundTripWithBuilder();
testDeduplicationOff(); testDeduplicationOff();
testBugWhereOffestWereStoredAsIntInsteadOfUInt();
} }
function testSingleValueBuffers() { function testSingleValueBuffers() {
@@ -360,5 +361,29 @@ function testGoldBuffer() {
}); });
} }
function testBugWhereOffestWereStoredAsIntInsteadOfUInt() {
// Reported in https://github.com/google/flatbuffers/issues/5949#issuecomment-688421193
const object = {'channels_in': 64, 'dilation_height_factor': 1, 'dilation_width_factor': 1, 'fused_activation_function': 1, 'pad_values': 1, 'padding': 0, 'stride_height': 1, 'stride_width': 1};
let data1 = flexbuffers.encode(object);
const data = [99, 104, 97, 110, 110, 101, 108, 115, 95, 105, 110, 0,
100, 105, 108, 97, 116, 105, 111, 110, 95, 104, 101, 105, 103, 104, 116, 95, 102, 97, 99, 116, 111, 114, 0,
100, 105, 108, 97, 116, 105, 111, 110, 95, 119, 105, 100, 116, 104, 95, 102, 97, 99, 116, 111, 114, 0,
102, 117, 115, 101, 100, 95, 97, 99, 116, 105, 118, 97, 116, 105, 111, 110, 95, 102, 117, 110, 99, 116, 105, 111, 110, 0,
112, 97, 100, 95, 118, 97, 108, 117, 101, 115, 0, 112, 97, 100, 100, 105, 110, 103, 0,
115, 116, 114, 105, 100, 101, 95, 104, 101, 105, 103, 104, 116, 0,
115, 116, 114, 105, 100, 101, 95, 119, 105, 100, 116, 104, 0,
8, 130, 119, 97, 76, 51, 41, 34, 21, 8, 1, 8, 64, 1, 1, 1, 1, 0, 1, 1, 4, 4, 4, 4, 4, 4, 4, 4, 16, 36, 1];
let object2 = flexbuffers.toObject(new Uint8Array(data).buffer);
let object1 = flexbuffers.toObject(new Uint8Array(data1).buffer);
assert.deepStrictEqual(object, object2);
assert.deepStrictEqual(object, object1);
assert.strictEqual(data.length, data1.length);
let ref = flexbuffers.toReference(new Uint8Array(data).buffer);
assert.strictEqual(ref.isMap(), true);
assert.strictEqual(ref.length(), 8);
assert.strictEqual(ref.get("channels_in").numericValue(), 64);
assert.strictEqual(ref.get("padding").isNumber(), true);
}
main(); main();