diff --git a/java/com/google/flatbuffers/FlexBuffers.java b/java/com/google/flatbuffers/FlexBuffers.java new file mode 100644 index 000000000..07e162d9c --- /dev/null +++ b/java/com/google/flatbuffers/FlexBuffers.java @@ -0,0 +1,869 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.flatbuffers; + + +import static com.google.flatbuffers.FlexBuffers.Unsigned.byteToUnsignedInt; +import static com.google.flatbuffers.FlexBuffers.Unsigned.intToUnsignedLong; +import static com.google.flatbuffers.FlexBuffers.Unsigned.shortToUnsignedInt; + +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +/** + * This class can be used to parse FlexBuffer messages. + *

+ * For generating FlexBuffer messages, use {@link FlexBuffersBuilder}. + *

+ * Example of usage: + * ByteBuffer bb = ... // load message from file or network + * FlexBuffers.Reference r = FlexBuffers.getRoot(bb); // Reads the root element + * FlexBuffers.Map map = r.asMap(); // We assumed root object is a map + * System.out.println(map.get("name").asString()); // prints element with key "name" + */ +public class FlexBuffers { + + // These are used as the upper 6 bits of a type field to indicate the actual + // type. + public static final int FBT_NULL = 0; + public static final int FBT_INT = 1; + public static final int FBT_UINT = 2; + public static final int FBT_FLOAT = 3; // Types above stored inline, types below store an offset. + public static final int FBT_KEY = 4; + public static final int FBT_STRING = 5; + public static final int FBT_INDIRECT_INT = 6; + public static final int FBT_INDIRECT_UINT = 7; + public static final int FBT_INDIRECT_FLOAT = 8; + public static final int FBT_MAP = 9; + public static final int FBT_VECTOR = 10; // Untyped. + public static final int FBT_VECTOR_INT = 11; // Typed any size = stores no type table). + public static final int FBT_VECTOR_UINT = 12; + public static final int FBT_VECTOR_FLOAT = 13; + public static final int FBT_VECTOR_KEY = 14; + public static final int FBT_VECTOR_STRING = 15; + public static final int FBT_VECTOR_INT2 = 16; // Typed tuple = no type table; no size field). + public static final int FBT_VECTOR_UINT2 = 17; + public static final int FBT_VECTOR_FLOAT2 = 18; + public static final int FBT_VECTOR_INT3 = 19; // Typed triple = no type table; no size field). + public static final int FBT_VECTOR_UINT3 = 20; + public static final int FBT_VECTOR_FLOAT3 = 21; + public static final int FBT_VECTOR_INT4 = 22; // Typed quad = no type table; no size field). + public static final int FBT_VECTOR_UINT4 = 23; + public static final int FBT_VECTOR_FLOAT4 = 24; + public static final int FBT_BLOB = 25; + public static final int FBT_BOOL = 26; + public static final int FBT_VECTOR_BOOL = 36; // To Allow the same type of conversion of type to vector type + private static final ByteBuffer EMPTY_BB = ByteBuffer.allocate(0).asReadOnlyBuffer(); + + /** + * Checks where a type is a typed vector + * + * @param type type to be checked + * @return true if typed vector + */ + static boolean isTypedVector(int type) { + return (type >= FBT_VECTOR_INT && type <= FBT_VECTOR_STRING) || type == FBT_VECTOR_BOOL; + } + + /** + * Check whether you can access type directly (no indirection) or not. + * + * @param type type to be checked + * @return true if inline type + */ + static boolean isTypeInline(int type) { + return type <= FBT_FLOAT || type == FBT_BOOL; + } + + static int toTypedVectorElementType(int original_type) { + return original_type - FBT_VECTOR_INT + FBT_INT; + } + + /** + * Return a vector type our of a original element type + * + * @param type element type + * @param fixedLength size of elment + * @return typed vector type + */ + static int toTypedVector(int type, int fixedLength) { + assert (isTypedVectorElementType(type)); + switch (fixedLength) { + case 0: return type - FBT_INT + FBT_VECTOR_INT; + case 2: return type - FBT_INT + FBT_VECTOR_INT2; + case 3: return type - FBT_INT + FBT_VECTOR_INT3; + case 4: return type - FBT_INT + FBT_VECTOR_INT4; + default: + assert (false); + return FBT_NULL; + } + } + + static boolean isTypedVectorElementType(int type) { + return (type >= FBT_INT && type <= FBT_STRING) || type == FBT_BOOL; + } + + // return position of the element that the offset is pointing to + private static int indirect(ByteBuffer bb, int offset, int byteWidth) { + //TODO: we assume all offset fits on a int, since ByteBuffer operates with that assumption + return (int) (offset - readUInt(bb, offset, byteWidth)); + } + + // read unsigned int with size byteWidth and return as a 64-bit integer + private static long readUInt(ByteBuffer buff, int end, int byteWidth) { + switch (byteWidth) { + case 1: return byteToUnsignedInt(buff.get(end)); + case 2: return shortToUnsignedInt(buff.getShort(end)); + case 4: return intToUnsignedLong(buff.getInt(end)); + case 8: return buff.getLong(end); // We are passing signed long here. Losing information (user should know) + default: return -1; // we should never reach here + } + } + + // read signed int of size byteWidth and return as 32-bit int + private static int readInt(ByteBuffer buff, int end, int byteWidth) { + return (int) readLong(buff, end, byteWidth); + } + + // read signed int of size byteWidth and return as 64-bit int + private static long readLong(ByteBuffer buff, int end, int byteWidth) { + switch (byteWidth) { + case 1: return buff.get(end); + case 2: return buff.getShort(end); + case 4: return buff.getInt(end); + case 8: return buff.getLong(end); + default: return -1; // we should never reach here + } + } + + private static double readDouble(ByteBuffer buff, int end, int byteWidth) { + switch (byteWidth) { + case 4: return buff.getFloat(end); + case 8: return buff.getDouble(end); + default: return -1; // we should never reach here + } + } + + /** + * Reads a FlexBuffer message in ByteBuffer and returns {@link Reference} to + * the root element. + * @param buffer ByteBuffer containing FlexBuffer message + * @return {@link Reference} to the root object + */ + public static Reference getRoot(ByteBuffer buffer) { + // See Finish() below for the serialization counterpart of this. + // The root ends at the end of the buffer, so we parse backwards from there. + int end = buffer.limit(); + int byteWidth = buffer.get(--end); + int packetType = byteToUnsignedInt(buffer.get(--end)); + end -= byteWidth; // The root data item. + return new Reference(buffer, end, byteWidth, packetType); + } + + public static class Reference { + + private static final Reference NULL_REFERENCE = new Reference(EMPTY_BB, 0, 1, 0); + private ByteBuffer bb; + private int end; + private int parentWidth; + private int byteWidth; + private int type; + + Reference(ByteBuffer bb, int end, int parentWidth, int packedType) { + this(bb, end, parentWidth, (1 << (packedType & 3)), packedType >> 2); + } + + Reference(ByteBuffer bb, int end, int parentWidth, int byteWidth, int type) { + this.bb = bb; + this.end = end; + this.parentWidth = parentWidth; + this.byteWidth = byteWidth; + this.type = type; + } + + public int getType() { + return type; + } + + public boolean isNull() { + return type == FBT_NULL; + } + + public boolean isBoolean() { + return type == FBT_BOOL; + } + + public boolean isNumeric() { + return isIntOrUInt() || isFloat(); + } + + public boolean isIntOrUInt() { + return isInt() || isUInt(); + } + + public boolean isFloat() { + return type == FBT_FLOAT || type == FBT_INDIRECT_FLOAT; + } + + public boolean isInt() { + return type == FBT_INT || type == FBT_INDIRECT_INT; + } + + public boolean isUInt() { + return type == FBT_UINT || type == FBT_INDIRECT_UINT; + } + + public boolean isString() { + return type == FBT_STRING; + } + + public boolean isKey() { + return type == FBT_KEY; + } + + public boolean isVector() { + return type == FBT_VECTOR || type == FBT_MAP; + } + + public boolean isTypedVector() { + return (type >= FBT_VECTOR_INT && type <= FBT_VECTOR_STRING) || + type == FBT_VECTOR_BOOL; + } + + public boolean isMap() { + return type == FBT_MAP; + } + + public boolean isBlob() { + return type == FBT_BLOB; + } + + public int asInt() { + if (type == FBT_INT) { + // A fast path for the common case. + return readInt(bb, end, parentWidth); + } else + switch (type) { + case FBT_INDIRECT_INT: return readInt(bb, indirect(bb, end, parentWidth), byteWidth); + case FBT_UINT: return (int) readUInt(bb, end, parentWidth); + case FBT_INDIRECT_UINT: return (int) readUInt(bb, indirect(bb, end, parentWidth), parentWidth); + case FBT_FLOAT: return (int) readDouble(bb, end, parentWidth); + case FBT_INDIRECT_FLOAT: return (int) readDouble(bb, indirect(bb, end, parentWidth), byteWidth); + case FBT_NULL: return 0; + case FBT_STRING: return Integer.parseInt(asString()); + case FBT_VECTOR: return asVector().size(); + case FBT_BOOL: return readInt(bb, end, parentWidth); + default: + // Convert other things to int. + return 0; + } + } + + public long asUInt() { + if (type == FBT_UINT) { + // A fast path for the common case. + return readUInt(bb, end, parentWidth); + } else + switch (type) { + case FBT_INDIRECT_UINT: return readUInt(bb, indirect(bb, end, parentWidth), byteWidth); + case FBT_INT: return readLong(bb, end, parentWidth); + case FBT_INDIRECT_INT: return readLong(bb, indirect(bb, end, parentWidth), byteWidth); + case FBT_FLOAT: return (long) readDouble(bb, end, parentWidth); + case FBT_INDIRECT_FLOAT: return (long) readDouble(bb, indirect(bb, end, parentWidth), parentWidth); + case FBT_NULL: return 0; + case FBT_STRING: return Long.parseLong(asString()); + case FBT_VECTOR: return asVector().size(); + case FBT_BOOL: readInt(bb, end, parentWidth); + default: + // Convert other things to uint. + return 0; + } + } + + public long asLong() { + if (type == FBT_INT) { + // A fast path for the common case. + return readLong(bb, end, parentWidth); + } else + switch (type) { + case FBT_INDIRECT_INT: return readLong(bb, indirect(bb, end, parentWidth), byteWidth); + case FBT_UINT: return readUInt(bb, end, parentWidth); + case FBT_INDIRECT_UINT: return readUInt(bb, indirect(bb, end, parentWidth), parentWidth); + case FBT_FLOAT: return (long) readDouble(bb, end, parentWidth); + case FBT_INDIRECT_FLOAT: return (long) readDouble(bb, indirect(bb, end, parentWidth), byteWidth); + case FBT_NULL: return 0; + case FBT_STRING: { + try { + return Long.parseLong(asString()); + } catch (NumberFormatException nfe) { + return 0; //same as C++ implementation + } + } + case FBT_VECTOR: return asVector().size(); + case FBT_BOOL: return readInt(bb, end, parentWidth); + default: + // Convert other things to int. + return 0; + } + } + + public double asFloat() { + if (type == FBT_FLOAT) { + // A fast path for the common case. + return readDouble(bb, end, parentWidth); + } else + switch (type) { + case FBT_INDIRECT_FLOAT: return readDouble(bb, indirect(bb, end, parentWidth), byteWidth); + case FBT_INT: return readInt(bb, end, parentWidth); + case FBT_UINT: + case FBT_BOOL: + return readUInt(bb, end, parentWidth); + case FBT_INDIRECT_INT: return readInt(bb, indirect(bb, end, parentWidth), byteWidth); + case FBT_INDIRECT_UINT: return readUInt(bb, indirect(bb, end, parentWidth), byteWidth); + case FBT_NULL: return 0.0; + case FBT_STRING: return Double.parseDouble(asString()); + case FBT_VECTOR: return asVector().size(); + default: + // Convert strings and other things to float. + return 0; + } + } + + public Key asKey() { + if (isKey()) { + return new Key(bb, indirect(bb, end, parentWidth), byteWidth); + } else { + return Key.empty(); + } + } + + public String asString() { + if (isString()) { + int start = indirect(bb, end, byteWidth); + int size = readInt(bb, start - byteWidth, byteWidth); + return Utf8.getDefault().decodeUtf8(bb, start, size); + } + else if (isKey()){ + int start = indirect(bb, end, byteWidth); + for (int i = start; ; i++) { + if (bb.get(i) == 0) { + return Utf8.getDefault().decodeUtf8(bb, start, i - start); + } + } + } else { + return ""; + } + } + + public Map asMap() { + if (isMap()) { + return new Map(bb, indirect(bb, end, parentWidth), byteWidth); + } else { + return Map.empty(); + } + } + + public Vector asVector() { + if (isVector()) { + return new Vector(bb, indirect(bb, end, parentWidth), byteWidth); + } else if (FlexBuffers.isTypedVector(type)) { + return new TypedVector(bb, indirect(bb, end, parentWidth), byteWidth, FlexBuffers.toTypedVectorElementType(type)); + } else { + return Vector.empty(); + } + } + + public Blob asBlob() { + if (isBlob() || isString()) { + return new Blob(bb, indirect(bb, end, parentWidth), byteWidth); + } else { + return Blob.empty(); + } + } + + public boolean asBoolean() { + if (isBoolean()) { + return bb.get(end) != 0; + } + return asUInt() != 0; + } + + @Override + public String toString() { + return toString(new StringBuilder(128)).toString(); + } + + StringBuilder toString(StringBuilder sb) { + //TODO: Original C++ implementation escape strings. + // probably we should do it as well. + switch (type) { + case FBT_NULL: + return sb.append("null"); + case FBT_INT: + case FBT_INDIRECT_INT: + return sb.append(asLong()); + case FBT_UINT: + case FBT_INDIRECT_UINT: + return sb.append(asUInt()); + case FBT_INDIRECT_FLOAT: + case FBT_FLOAT: + return sb.append(asFloat()); + case FBT_KEY: + return asKey().toString(sb.append('"')).append('"'); + case FBT_STRING: + return sb.append('"').append(asString()).append('"'); + case FBT_MAP: + return asMap().toString(sb); + case FBT_VECTOR: + return asVector().toString(sb); + case FBT_BLOB: + return asBlob().toString(sb); + case FBT_BOOL: + return sb.append(asBoolean()); + case FBT_VECTOR_INT: + case FBT_VECTOR_UINT: + case FBT_VECTOR_FLOAT: + case FBT_VECTOR_KEY: + case FBT_VECTOR_STRING: + case FBT_VECTOR_BOOL: + return sb.append(asVector()); + case FBT_VECTOR_INT2: + case FBT_VECTOR_UINT2: + case FBT_VECTOR_FLOAT2: + case FBT_VECTOR_INT3: + case FBT_VECTOR_UINT3: + case FBT_VECTOR_FLOAT3: + case FBT_VECTOR_INT4: + case FBT_VECTOR_UINT4: + case FBT_VECTOR_FLOAT4: + + throw new FlexBufferException("not_implemented:" + type); + default: + return sb; + } + } + } + + /** + * Base class of all types below. + * Points into the data buffer and allows access to one type. + */ + private static abstract class Object { + ByteBuffer bb; + int end; + int byteWidth; + + Object(ByteBuffer buff, int end, int byteWidth) { + this.bb = buff; + this.end = end; + this.byteWidth = byteWidth; + } + + @Override + public String toString() { + return toString(new StringBuilder(128)).toString(); + } + + public abstract StringBuilder toString(StringBuilder sb); + } + + // Stores size in `byte_width_` bytes before end position. + private static abstract class Sized extends Object { + Sized(ByteBuffer buff, int end, int byteWidth) { + super(buff, end, byteWidth); + } + + public int size() { + return readInt(bb, end - byteWidth, byteWidth); + } + } + + public static class Blob extends Sized { + static final Blob EMPTY = new Blob(EMPTY_BB, 0, 1); + + Blob(ByteBuffer buff, int end, int byteWidth) { + super(buff, end, byteWidth); + } + + public static Blob empty() { + return EMPTY; + } + + /** + * @return blob as a {@link ByteBuffer} + */ + public ByteBuffer data() { + ByteBuffer dup = bb.duplicate(); + dup.position(end); + dup.limit(end + size()); + return dup.asReadOnlyBuffer().slice(); + } + + /** + * @return blob as a byte array + */ + public byte[] getBytes() { + int size = size(); + byte[] result = new byte[size]; + for (int i = 0; i < size; i++) { + result[i] = bb.get(end + i); + } + return result; + } + + public byte get(int pos) { + assert pos >=0 && pos <= size(); + return bb.get(end + pos); + } + + @Override + public String toString() { + return Utf8.getDefault().decodeUtf8(bb, end, size()); + } + + @Override + public StringBuilder toString(StringBuilder sb) { + sb.append('"'); + sb.append(Utf8.getDefault().decodeUtf8(bb, end, size())); + return sb.append('"'); + } + } + + public static class Key extends Object { + + private static final Key EMPTY = new Key(EMPTY_BB, 0, 0); + + Key(ByteBuffer buff, int end, int byteWidth) { + super(buff, end, byteWidth); + } + + public static Key empty() { + return Key.EMPTY; + } + + @Override + public StringBuilder toString(StringBuilder sb) { + int size; + for (int i = end; ; i++) { + if (bb.get(i) == 0) { + size = i - end; + break; + } + } + sb.append(Utf8.getDefault().decodeUtf8(bb, end, size)); + return sb; + } + + int compareTo(byte[] other) { + int ia = end; + int io = 0; + byte c1, c2; + do { + c1 = bb.get(ia); + c2 = other[io]; + if (c1 == '\0') + return c1 - c2; + ia++; + io++; + if (io == other.length) { + // in our buffer we have an additional \0 byte + // but this does not exist in regular Java strings, so we return now + return c1 - c2; + } + } + while (c1 == c2); + return c1 - c2; + } + + @Override + public boolean equals(java.lang.Object obj) { + if (!(obj instanceof Key)) + return false; + + return ((Key) obj).end == end && ((Key) obj).byteWidth == byteWidth; + } + } + + /** + * Map object representing a set of key-value pairs. + */ + public static class Map extends Vector { + private static final Map EMPTY_MAP = new Map(EMPTY_BB, 0, 0); + + Map(ByteBuffer bb, int end, int byteWidth) { + super(bb, end, byteWidth); + } + + public static Map empty() { + return EMPTY_MAP; + } + + /** + * @param key access key to element on map + * @return reference to value in map + */ + public Reference get(String key) { + return get(key.getBytes(StandardCharsets.UTF_8)); + } + + /** + * @param key access key to element on map. Keys are assumed to be encoded in UTF-8 + * @return reference to value in map + */ + public Reference get(byte[] key) { + KeyVector keys = keys(); + int size = keys.size(); + int index = binarySearch(keys, key); + if (index >= 0 && index < size) { + return get(index); + } + return Reference.NULL_REFERENCE; + } + + /** + * Get a vector or keys in the map + * + * @return vector of keys + */ + public KeyVector keys() { + final int num_prefixed_fields = 3; + int keysOffset = end - (byteWidth * num_prefixed_fields); + return new KeyVector(new TypedVector(bb, + indirect(bb, keysOffset, byteWidth), + readInt(bb, keysOffset + byteWidth, byteWidth), + FBT_KEY)); + } + + /** + * @return {@code Vector} of values from map + */ + public Vector values() { + return new Vector(bb, end, byteWidth); + } + + /** + * Writes text (json) representation of map in a {@code StringBuilder}. + * + * @param builder {@code StringBuilder} to be appended to + * @return Same {@code StringBuilder} with appended text + */ + public StringBuilder toString(StringBuilder builder) { + builder.append("{ "); + KeyVector keys = keys(); + int size = size(); + Vector vals = values(); + for (int i = 0; i < size; i++) { + builder.append('"') + .append(keys.get(i).toString()) + .append("\" : "); + builder.append(vals.get(i).toString()); + if (i != size - 1) + builder.append(", "); + } + builder.append(" }"); + return builder; + } + + // Performs a binary search on a key vector and return index of the key in key vector + private int binarySearch(KeyVector keys, byte[] searchedKey) { + int low = 0; + int high = keys.size() - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; + Key k = keys.get(mid); + int cmp = k.compareTo(searchedKey); + if (cmp < 0) + low = mid + 1; + else if (cmp > 0) + high = mid - 1; + else + return mid; // key found + } + return -(low + 1); // key not found + } + } + + /** + * Object that represents a set of elements in the buffer + */ + public static class Vector extends Sized { + + private static final Vector EMPTY_VECTOR = new Vector(ByteBuffer.allocate(0), 1, 1); + + Vector(ByteBuffer bb, int end, int byteWidth) { + super(bb, end, byteWidth); + } + + public static Vector empty() { + return EMPTY_VECTOR; + } + + public boolean isEmpty() { + return this == EMPTY_VECTOR; + } + + @Override + public StringBuilder toString(StringBuilder sb) { + sb.append("[ "); + int size = size(); + for (int i = 0; i < size; i++) { + get(i).toString(sb); + if (i != size - 1) { + sb.append(", "); + } + } + sb.append(" ]"); + return sb; + } + + /** + * Get a element in a vector by index + * + * @param index position of the element + * @return {@code Reference} to the element + */ + public Reference get(int index) { + long len = size(); + if (index >= len) { + return Reference.NULL_REFERENCE; + } + int packedType = byteToUnsignedInt(bb.get((int) (end + (len * byteWidth) + index))); + int obj_end = end + index * byteWidth; + return new Reference(bb, obj_end, byteWidth, packedType); + } + } + + /** + * Object that represents a set of elements with the same type + */ + public static class TypedVector extends Vector { + + private static final TypedVector EMPTY_VECTOR = new TypedVector(EMPTY_BB, 0, 1, FBT_INT); + + private final int elemType; + + TypedVector(ByteBuffer bb, int end, int byteWidth, int elemType) { + super(bb, end, byteWidth); + this.elemType = elemType; + } + + public static TypedVector empty() { + return EMPTY_VECTOR; + } + + /** + * Returns whether the vector is empty + * + * @return true if empty + */ + public boolean isEmptyVector() { + return this == EMPTY_VECTOR; + } + + /** + * Return element type for all elements in the vector + * + * @return element type + */ + public int getElemType() { + return elemType; + } + + /** + * Get reference to an object in the {@code Vector} + * + * @param pos position of the object in {@code Vector} + * @return reference to element + */ + @Override + public Reference get(int pos) { + int len = size(); + if (pos >= len) return Reference.NULL_REFERENCE; + int childPos = end + pos * byteWidth; + return new Reference(bb, childPos, byteWidth, 1, elemType); + } + } + + /** + * Represent a vector of keys in a map + */ + public static class KeyVector { + + private final TypedVector vec; + + KeyVector(TypedVector vec) { + this.vec = vec; + } + + /** + * Return key + * + * @param pos position of the key in key vector + * @return key + */ + public Key get(int pos) { + int len = size(); + if (pos >= len) return Key.EMPTY; + int childPos = vec.end + pos * vec.byteWidth; + return new Key(vec.bb, indirect(vec.bb, childPos, vec.byteWidth), 1); + } + + /** + * Returns size of key vector + * + * @return size + */ + public int size() { + return vec.size(); + } + + public String toString() { + StringBuilder b = new StringBuilder(); + b.append('['); + for (int i = 0; i < vec.size(); i++) { + vec.get(i).toString(b); + if (i != vec.size() - 1) { + b.append(", "); + } + } + return b.append("]").toString(); + } + } + + public static class FlexBufferException extends RuntimeException { + FlexBufferException(String msg) { + super(msg); + } + } + + static class Unsigned { + + static int byteToUnsignedInt(byte x) { + return ((int) x) & 0xff; + } + + static int shortToUnsignedInt(short x) { + return ((int) x) & 0xffff; + } + + static long intToUnsignedLong(int x) { + return ((long) x) & 0xffffffffL; + } + } +} diff --git a/java/com/google/flatbuffers/FlexBuffersBuilder.java b/java/com/google/flatbuffers/FlexBuffersBuilder.java new file mode 100644 index 000000000..36e8544ab --- /dev/null +++ b/java/com/google/flatbuffers/FlexBuffersBuilder.java @@ -0,0 +1,692 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.flatbuffers; + +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; + +import static com.google.flatbuffers.FlexBuffers.*; +import static com.google.flatbuffers.FlexBuffers.Unsigned.byteToUnsignedInt; +import static com.google.flatbuffers.FlexBuffers.Unsigned.intToUnsignedLong; +import static com.google.flatbuffers.FlexBuffers.Unsigned.shortToUnsignedInt; + +/** + * A class that generates FlexBuffers + *

+ * This class presents all necessary APIs to create FlexBuffers. The {@link ByteBuffer } buffer used to store the + * data can be created internally, or passed down in the constructor. + *

+ * Because it uses {@link ByteBuffer} internally, this impose some limitations in generating FlexBuffers. Mostly noted, + * the maximum size limitation on FlexBuffer message, which is {@link Integer#MAX_VALUE}. + * + *

There is also some differences from the original implementation in C++. It can changed in future updates. + *

+ *

+ */ +public class FlexBuffersBuilder { + + private static final int WIDTH_8 = 0; + private static final int WIDTH_16 = 1; + private static final int WIDTH_32 = 2; + private static final int WIDTH_64 = 3; + + /** + * No keys or strings will be shared + */ + public static final int BUILDER_FLAG_NONE = 0; + /** + * Keys will be shared between elements. Identical keys will only be serialized once, thus possibly saving space. + * But serialization performance might be slower and consumes more memory. + */ + public static final int BUILDER_FLAG_SHARE_KEYS = 1; + /** + * Strings will be shared between elements. Identical strings will only be serialized once, thus possibly saving space. + * But serialization performance might be slower and consumes more memory. This is ideal if you expect many repeated + * strings on the message. + */ + public static final int BUILDER_FLAG_SHARE_STRINGS = 1; + /** + * Strings and keys will be shared between elements. + */ + public static final int BUILDER_FLAG_SHARE_KEYS_AND_STRINGS = 3; + /** + * Reserved for the future. + */ + public static final int BUILDER_FLAG_SHARE_KEY_VECTORS = 4; + /** + * Reserved for the future. + */ + public static final int BUILDER_FLAG_SHARE_ALL = 7; + + private final ByteBuffer bb; + private final ArrayList stack = new ArrayList<>(); + private final HashMap keyPool = new HashMap<>(); + private final HashMap stringPool = new HashMap<>(); + private final int flags; + private boolean finished = false; + + private Comparator valueComparator = new Comparator() { + @Override + public int compare(Value o1, Value o2) { + int ia = o1.key; + int io = o2.key; + byte c1, c2; + do { + c1 = bb.get(ia); + c2 = bb.get(io); + if (c1 == 0) + return c1 - c2; + ia++; + io++; + } + while (c1 == c2); + return c1 - c2; + } + }; + + /** + * Constructs a newly allocated {@code FlexBuffersBuilder} with {@link #BUILDER_FLAG_SHARE_KEYS} set. + */ + public FlexBuffersBuilder() { + this(ByteBuffer.allocate(256), BUILDER_FLAG_SHARE_KEYS); + } + + /** + * Constructs a newly allocated {@code FlexBuffersBuilder}. + * + * @param bb ByteBuffer that will hold the message + * @param flags Share flags + */ + public FlexBuffersBuilder(ByteBuffer bb, int flags) { + this.bb = bb; + this.flags = flags; + bb.order(ByteOrder.LITTLE_ENDIAN); + bb.position(0); + } + + /** + * Constructs a newly allocated {@code FlexBuffersBuilder}. + * + * @param bb ByteBuffer that will hold the message + */ + public FlexBuffersBuilder(ByteBuffer bb) { + this(bb, BUILDER_FLAG_SHARE_KEYS); + } + + /** + * Return {@code ByteBuffer} containing FlexBuffer message. {@code #finish()} must be called before calling this + * function otherwise an assert will trigger. + * + * @return {@code ByteBuffer} with finished message + */ + public ByteBuffer getBuffer() { + assert (finished); + return bb; + } + + /** + * Insert a single boolean into the buffer + * @param val true or false + */ + public void putBoolean(boolean val) { + putBoolean(null, val); + } + + public void putBoolean(String key, boolean val) { + stack.add(Value.bool(putKey(key), val)); + } + + private int putKey(String key) { + if (key == null) { + return -1; + } + int pos = bb.position(); + if ((flags & BUILDER_FLAG_SHARE_KEYS) != 0) { + if (keyPool.get(key) == null) { + bb.put(key.getBytes(StandardCharsets.UTF_8)); + bb.put((byte) 0); + keyPool.put(key, pos); + } else { + pos = keyPool.get(key); + } + } else { + bb.put(key.getBytes(StandardCharsets.UTF_8)); + bb.put((byte) 0); + keyPool.put(key, pos); + } + return pos; + } + + /** + * Adds a integer into the buff + * @param val integer + */ + public void putInt(int val) { + putInt(null, val); + } + + public void putInt(String key, int val) { + putInt(key, (long) val); + } + + public void putInt(String key, long val) { + int iKey = putKey(key); + if (Byte.MIN_VALUE <= val && val <= Byte.MAX_VALUE) { + stack.add(Value.int8(iKey, (int) val)); + } else if (Short.MIN_VALUE <= val && val <= Short.MAX_VALUE) { + stack.add(Value.int16(iKey, (int) val)); + } else if (Integer.MIN_VALUE <= val && val <= Integer.MAX_VALUE) { + stack.add(Value.int32(iKey, (int) val)); + } else { + stack.add(Value.int64(iKey, val)); + } + } + + /** + * Adds a 64-bit integer into the buff + * @param value integer + */ + public void putInt(long value) { + putInt(null, value); + } + + /** + * Adds a unsigned integer into the buff. + * @param value integer representing unsigned value + */ + public void putUInt(int value) { + putUInt(null, (long) value); + } + + /** + * Adds a unsigned integer (stored in a signed 64-bit integer) into the buff. + * @param value integer representing unsigned value + */ + public void putUInt(long value) { + putUInt(null, value); + } + + /** + * Adds a 64-bit unsigned integer (stored as {@link BigInteger}) into the buff. + * Warning: This operation might be very slow. + * @param value integer representing unsigned value + */ + public void putUInt64(BigInteger value) { + putUInt64(null, value.longValue()); + } + + private void putUInt64(String key, long value) { + stack.add(Value.uInt64(putKey(key), value)); + } + + private void putUInt(String key, long value) { + int iKey = putKey(key); + Value vVal; + + int width = widthUInBits(value); + + if (width == WIDTH_8) { + vVal = Value.uInt8(iKey, (int)value); + } else if (width == WIDTH_16) { + vVal = Value.uInt16(iKey, (int)value); + } else if (width == WIDTH_32) { + vVal = Value.uInt32(iKey, (int)value); + } else { + vVal = Value.uInt64(iKey, value); + } + stack.add(vVal); + } + + /** + * Adds a 32-bit float into the buff. + * @param value float representing value + */ + public void putFloat(float value) { + putFloat(null, value); + } + + public void putFloat(String key, float val) { + stack.add(Value.float32(putKey(key), val)); + } + + /** + * Adds a 64-bit float into the buff. + * @param value float representing value + */ + public void putFloat(double value) { + putFloat(null, value); + } + + public void putFloat(String key, double val) { + stack.add(Value.float64(putKey(key), val)); + } + + /** + * Adds a String into the buffer + * @param value string + * @return start position of string in the buffer + */ + public int putString(String value) { + return putString(null, value); + } + + public int putString(String key, String val) { + int iKey = putKey(key); + if ((flags & FlexBuffersBuilder.BUILDER_FLAG_SHARE_STRINGS) != 0) { + Integer i = stringPool.get(val); + if (i == null) { + Value value = writeString(iKey, val); + stringPool.put(val, (int) value.iValue); + stack.add(value); + return (int) value.iValue; + } else { + int bitWidth = widthUInBits(val.length()); + stack.add(Value.blob(iKey, i, FBT_STRING, bitWidth)); + return i; + } + } else { + Value value = writeString(iKey, val); + stack.add(value); + return (int) value.iValue; + } + } + + private Value writeString(int key, String s) { + return writeBlob(key, s.getBytes(StandardCharsets.UTF_8), FBT_STRING); + } + + // in bits to fit a unsigned int + private static int widthUInBits(long len) { + if (len <= byteToUnsignedInt((byte)0xff)) return WIDTH_8; + if (len <= shortToUnsignedInt((short)0xffff)) return WIDTH_16; + if (len <= intToUnsignedLong(0xffff_ffff)) return WIDTH_32; + return WIDTH_64; + } + + private Value writeBlob(int key, byte[] blob, int type) { + int bitWidth = widthUInBits(blob.length); + int byteWidth = align(bitWidth); + writeInt(blob.length, byteWidth); + int sloc = bb.position(); + bb.put(blob); + if (type == FBT_STRING) { + bb.put((byte) 0); + } + return Value.blob(key, sloc, type, bitWidth); + } + + // Align to prepare for writing a scalar with a certain size. + private int align(int alignment) { + int byteWidth = 1 << alignment; + int padBytes = Value.paddingBytes(bb.capacity(), byteWidth); + while (padBytes-- != 0) { + bb.put((byte) 0); + } + return byteWidth; + } + + private void writeInt(long value, int byteWidth) { + switch (byteWidth) { + case 1: bb.put((byte) value); break; + case 2: bb.putShort((short) value); break; + case 4: bb.putInt((int) value); break; + case 8: bb.putLong(value); break; + } + } + + /** + * Adds a byte array into the message + * @param value byte array + * @return position in buffer as the start of byte array + */ + public int putBlob(byte[] value) { + return putBlob(null, value); + } + + public int putBlob(String key, byte[] val) { + int iKey = putKey(key); + Value value = writeBlob(iKey, val, FBT_BLOB); + stack.add(value); + return (int) value.iValue; + } + + public int startVector() { + return stack.size(); + } + + public int endVector(String key, int start, boolean typed, boolean fixed) { + int iKey = putKey(key); + Value vec = createVector(iKey, start, stack.size() - start, typed, fixed, null); + // Remove temp elements and return vector. + while (stack.size() > start) { + stack.remove(stack.size() - 1); + } + stack.add(vec); + return (int) vec.iValue; + } + + /** + * Finish writing the message into the buffer. After that no other element must + * be inserted into the buffer. Also, you must call this function before start using the + * FlexBuffer message + * @return ByteBuffer containing the FlexBuffer message + */ + public ByteBuffer finish() { + // If you hit this assert, you likely have objects that were never included + // in a parent. You need to have exactly one root to finish a buffer. + // Check your Start/End calls are matched, and all objects are inside + // some other object. + assert (stack.size() == 1); + // Write root value. + int byteWidth = align(stack.get(0).elemWidth(bb.position(), 0)); + writeAny(stack.get(0), byteWidth); + // Write root type. + bb.put(stack.get(0).storedPackedType()); + // Write root size. Normally determined by parent, but root has no parent :) + bb.put((byte) byteWidth); + bb.limit(bb.position()); + this.finished = true; + return bb; + } + + /* + * Create a vector based on the elements stored in the stack + * + * @param key reference to its key + * @param start element in the stack + * @param length size of the vector + * @param typed whether is TypedVector or not + * @param fixed whether is Fixed vector or not + * @param keys Value representing key vector + * @return Value representing the created vector + */ + private Value createVector(int key, int start, int length, boolean typed, boolean fixed, Value keys) { + assert (!fixed || typed); // typed=false, fixed=true combination is not supported. + // Figure out smallest bit width we can store this vector with. + int bitWidth = Math.max(WIDTH_8, widthUInBits(length)); + int prefixElems = 1; + if (keys != null) { + // If this vector is part of a map, we will pre-fix an offset to the keys + // to this vector. + bitWidth = Math.max(bitWidth, keys.elemWidth(bb.position(), 0)); + prefixElems += 2; + } + int vectorType = FBT_KEY; + // Check bit widths and types for all elements. + for (int i = start; i < stack.size(); i++) { + int elemWidth = stack.get(i).elemWidth(bb.position(), i + prefixElems); + bitWidth = Math.max(bitWidth, elemWidth); + if (typed) { + if (i == start) { + vectorType = stack.get(i).type; + } else { + // If you get this assert, you are writing a typed vector with + // elements that are not all the same type. + assert (vectorType == stack.get(i).type); + } + } + } + // If you get this assert, your fixed types are not one of: + // Int / UInt / Float / Key. + assert (!fixed || FlexBuffers.isTypedVectorElementType(vectorType)); + + int byteWidth = align(bitWidth); + // Write vector. First the keys width/offset if available, and size. + if (keys != null) { + writeOffset(keys.iValue, byteWidth); + writeInt(1L << keys.minBitWidth, byteWidth); + } + if (!fixed) { + writeInt(length, byteWidth); + } + // Then the actual data. + int vloc = bb.position(); + for (int i = start; i < stack.size(); i++) { + writeAny(stack.get(i), byteWidth); + } + // Then the types. + if (!typed) { + for (int i = start; i < stack.size(); i++) { + bb.put(stack.get(i).storedPackedType(bitWidth)); + } + } + return new Value(key, keys != null ? FBT_MAP + : (typed ? FlexBuffers.toTypedVector(vectorType, fixed ? length : 0) + : FBT_VECTOR), bitWidth, vloc); + } + + private void writeOffset(long val, int byteWidth) { + int reloff = (int) (bb.position() - val); + assert (byteWidth == 8 || reloff < 1L << (byteWidth * 8)); + writeInt(reloff, byteWidth); + } + + private void writeAny(final Value val, int byteWidth) { + switch (val.type) { + case FBT_NULL: + case FBT_BOOL: + case FBT_INT: + case FBT_UINT: + writeInt(val.iValue, byteWidth); + break; + case FBT_FLOAT: + writeDouble(val.dValue, byteWidth); + break; + default: + writeOffset(val.iValue, byteWidth); + break; + } + } + + private void writeDouble(double val, int byteWidth) { + if (byteWidth == 4) { + bb.putFloat((float) val); + } else if (byteWidth == 8) { + bb.putDouble(val); + } + } + + public int startMap() { + return stack.size(); + } + + public int endMap(String key, int start) { + int iKey = putKey(key); + + Collections.sort(stack.subList(start, stack.size()), valueComparator); + + Value keys = createKeyVector(start, stack.size() - start); + Value vec = createVector(iKey, start, stack.size() - start, false, false, keys); + // Remove temp elements and return map. + while (stack.size() > start) { + stack.remove(stack.size() - 1); + } + stack.add(vec); + return (int) vec.iValue; + } + + private Value createKeyVector(int start, int length) { + // Figure out smallest bit width we can store this vector with. + int bitWidth = Math.max(WIDTH_8, widthUInBits(length)); + int prefixElems = 1; + // Check bit widths and types for all elements. + for (int i = start; i < stack.size(); i++) { + int elemWidth = Value.elemWidth(FBT_KEY, WIDTH_8, stack.get(i).key, bb.position(), i + prefixElems); + bitWidth = Math.max(bitWidth, elemWidth); + } + + int byteWidth = align(bitWidth); + // Write vector. First the keys width/offset if available, and size. + writeInt(length, byteWidth); + // Then the actual data. + int vloc = bb.position(); + for (int i = start; i < stack.size(); i++) { + int pos = stack.get(i).key; + assert(pos != -1); + writeOffset(stack.get(i).key, byteWidth); + } + // Then the types. + return new Value(-1, FlexBuffers.toTypedVector(FBT_KEY,0), bitWidth, vloc); + } + + public static class Value { + final int type; + // for scalars, represents scalar size in bytes + // for vectors, represents the size + // for string, length + final int minBitWidth; + // float value + final double dValue; + // integer value + long iValue; + // position of the key associated with this value in buffer + int key; + + Value(int key, int type, int bitWidth, long iValue) { + this.key = key; + this.type = type; + this.minBitWidth = bitWidth; + this.iValue = iValue; + this.dValue = Double.MIN_VALUE; + } + + Value(int key, int type, int bitWidth, double dValue) { + this.key = key; + this.type = type; + this.minBitWidth = bitWidth; + this.dValue = dValue; + this.iValue = Long.MIN_VALUE; + } + + static Value bool(int key, boolean b) { + return new Value(key, FBT_BOOL, WIDTH_8, b ? 1 : 0); + } + + static Value blob(int key, int position, int type, int bitWidth) { + return new Value(key, type, WIDTH_8, position); + } + + static Value int8(int key, int value) { + return new Value(key, FBT_INT, WIDTH_8, value); + } + + static Value int16(int key, int value) { + return new Value(key, FBT_INT, WIDTH_16, value); + } + + static Value int32(int key, int value) { + return new Value(key, FBT_INT, WIDTH_32, value); + } + + static Value int64(int key, long value) { + return new Value(key, FBT_INT, WIDTH_64, value); + } + + static Value uInt8(int key, int value) { + return new Value(key, FBT_UINT, WIDTH_8, value); + } + + static Value uInt16(int key, int value) { + return new Value(key, FBT_UINT, WIDTH_16, value); + } + + static Value uInt32(int key, int value) { + return new Value(key, FBT_UINT, WIDTH_32, value); + } + + static Value uInt64(int key, long value) { + return new Value(key, FBT_UINT, WIDTH_64, value); + } + + static Value float32(int key, float value) { + return new Value(key, FBT_FLOAT, WIDTH_32, value); + } + + static Value float64(int key, double value) { + return new Value(key, FBT_FLOAT, WIDTH_64, value); + } + + private byte storedPackedType() { + return storedPackedType(WIDTH_8); + } + + private byte storedPackedType(int parentBitWidth) { + return packedType(storedWidth(parentBitWidth), type); + } + + private static byte packedType(int bitWidth, int type) { + return (byte) (bitWidth | (type << 2)); + } + + private int storedWidth(int parentBitWidth) { + if (FlexBuffers.isTypeInline(type)) { + return Math.max(minBitWidth, parentBitWidth); + } else { + return minBitWidth; + } + } + + private int elemWidth(int bufSize, int elemIndex) { + return elemWidth(type, minBitWidth, iValue, bufSize, elemIndex); + } + + private static int elemWidth(int type, int minBitWidth, long iValue, int bufSize, int elemIndex) { + if (FlexBuffers.isTypeInline(type)) { + return minBitWidth; + } else { + // We have an absolute offset, but want to store a relative offset + // elem_index elements beyond the current buffer end. Since whether + // the relative offset fits in a certain byte_width depends on + // the size of the elements before it (and their alignment), we have + // to test for each size in turn. + + // Original implementation checks for largest scalar + // which is long unsigned int + for (int byteWidth = 1; byteWidth <= 32; byteWidth *= 2) { + // Where are we going to write this offset? + int offsetLoc = bufSize + paddingBytes(bufSize, byteWidth) + (elemIndex * byteWidth); + // Compute relative offset. + long offset = offsetLoc - iValue; + // Does it fit? + int bitWidth = widthUInBits((int) offset); + if (((1L) << bitWidth) == byteWidth) + return bitWidth; + } + assert (false); // Must match one of the sizes above. + return WIDTH_64; + } + } + + private static int paddingBytes(int bufSize, int scalarSize) { + return ((~bufSize) + 1) & (scalarSize - 1); + } + } +} diff --git a/tests/JavaTest.java b/tests/JavaTest.java index f31e65420..fe308c6fd 100644 --- a/tests/JavaTest.java +++ b/tests/JavaTest.java @@ -14,6 +14,8 @@ * limitations under the License. */ +import java.util.Arrays; +import java.math.BigInteger; import java.io.*; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -24,6 +26,8 @@ import NamespaceA.NamespaceB.*; import com.google.flatbuffers.ByteBufferUtil; import static com.google.flatbuffers.Constants.*; import com.google.flatbuffers.FlatBufferBuilder; +import com.google.flatbuffers.FlexBuffersBuilder; +import com.google.flatbuffers.FlexBuffers; import MyGame.MonsterExtra; class JavaTest { @@ -77,6 +81,8 @@ class JavaTest { TestFixedLengthArrays(); + TestFlexBuffers(); + System.out.println("FlatBuffers test: completed successfully"); } @@ -242,7 +248,9 @@ class JavaTest { public ByteBuffer newByteBuffer(int capacity) { ByteBuffer bb; try { - bb = new RandomAccessFile("javatest.bin", "rw").getChannel().map(FileChannel.MapMode.READ_WRITE, 0, capacity).order(ByteOrder.LITTLE_ENDIAN); + RandomAccessFile f = new RandomAccessFile("javatest.bin", "rw"); + bb = f.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, capacity).order(ByteOrder.LITTLE_ENDIAN); + f.close(); } catch(Throwable e) { System.out.println("FlatBuffers test: couldn't map ByteBuffer to a file"); bb = null; @@ -523,10 +531,336 @@ class JavaTest { TestEq(table.a().f(1), (long)1); } + public static void testFlexBuffersTest() { + FlexBuffersBuilder builder = new FlexBuffersBuilder(ByteBuffer.allocate(512), + FlexBuffersBuilder.BUILDER_FLAG_SHARE_KEYS_AND_STRINGS); + + // Write the equivalent of: + // { vec: [ -100, "Fred", 4.0, false ], bar: [ 1, 2, 3 ], bar3: [ 1, 2, 3 ], + // foo: 100, bool: true, mymap: { foo: "Fred" } } + // It's possible to do this without std::function support as well. + int map1 = builder.startMap(); + + int vec1 = builder.startVector(); + builder.putInt(-100); + builder.putString("Fred"); + builder.putBlob(new byte[]{(byte) 77}); + builder.putBoolean(false); + builder.putInt(Long.MAX_VALUE); + + int map2 = builder.startMap(); + builder.putInt("test", 200); + builder.endMap(null, map2); + + builder.putFloat(150.9); + builder.putFloat(150.9999998); + builder.endVector("vec", vec1, false, false); + + vec1 = builder.startVector(); + builder.putInt(1); + builder.putInt(2); + builder.putInt(3); + builder.endVector("bar", vec1, true, false); + + vec1 = builder.startVector(); + builder.putBoolean(true); + builder.putBoolean(false); + builder.putBoolean(true); + builder.putBoolean(false); + builder.endVector("bools", vec1, true, false); + + builder.putBoolean("bool", true); + builder.putFloat("foo", 100); + + map2 = builder.startMap(); + builder.putString("bar", "Fred"); // Testing key and string reuse. + builder.putInt("int", -120); + builder.putFloat("float", -123.0f); + builder.putBlob("blob", new byte[]{ 65, 67 }); + builder.endMap("mymap", map2); + + builder.endMap(null, map1); + builder.finish(); + + FlexBuffers.Map m = FlexBuffers.getRoot(builder.getBuffer()).asMap(); + + TestEq(m.size(), 6); + + // test empty (an null) + TestEq(m.get("no_key").asString(), ""); // empty if fail + TestEq(m.get("no_key").asMap(), FlexBuffers.Map.empty()); // empty if fail + TestEq(m.get("no_key").asKey(), FlexBuffers.Key.empty()); // empty if fail + TestEq(m.get("no_key").asVector(), FlexBuffers.Vector.empty()); // empty if fail + TestEq(m.get("no_key").asBlob(), FlexBuffers.Blob.empty()); // empty if fail + assert(m.get("no_key").asVector().isEmpty()); // empty if fail + + // testing "vec" field + FlexBuffers.Vector vec = m.get("vec").asVector(); + TestEq(vec.size(), 8); + TestEq(vec.get(0).asLong(), (long) -100); + TestEq(vec.get(1).asString(), "Fred"); + TestEq(vec.get(2).isBlob(), true); + TestEq(vec.get(2).asBlob().size(), 1); + TestEq(vec.get(2).asBlob().data().get(0), (byte) 77); + TestEq(vec.get(3).isBoolean(), true); // Check if type is a bool + TestEq(vec.get(3).asBoolean(), false); // Check if value is false + TestEq(vec.get(4).asLong(), Long.MAX_VALUE); + TestEq(vec.get(5).isMap(), true); + TestEq(vec.get(5).asMap().get("test").asInt(), 200); + TestEq(Float.compare((float)vec.get(6).asFloat(), 150.9f), 0); + TestEq(Double.compare(vec.get(7).asFloat(), 150.9999998), 0); + TestEq((long)0, (long)vec.get(1).asLong()); //conversion fail returns 0 as C++ + + // bar vector + FlexBuffers.Vector tvec = m.get("bar").asVector(); + TestEq(tvec.size(), 3); + TestEq(tvec.get(0).asInt(), 1); + TestEq(tvec.get(1).asInt(), 2); + TestEq(tvec.get(2).asInt(), 3); + TestEq(((FlexBuffers.TypedVector) tvec).getElemType(), FlexBuffers.FBT_INT); + + // bools vector + FlexBuffers.Vector bvec = m.get("bools").asVector(); + TestEq(bvec.size(), 4); + TestEq(bvec.get(0).asBoolean(), true); + TestEq(bvec.get(1).asBoolean(), false); + TestEq(bvec.get(2).asBoolean(), true); + TestEq(bvec.get(3).asBoolean(), false); + TestEq(((FlexBuffers.TypedVector) bvec).getElemType(), FlexBuffers.FBT_BOOL); + + + TestEq((float)m.get("foo").asFloat(), (float) 100); + TestEq(m.get("unknown").isNull(), true); + + // mymap vector + FlexBuffers.Map mymap = m.get("mymap").asMap(); + TestEq(mymap.keys().get(0), m.keys().get(0)); // These should be equal by pointer equality, since key and value are shared. + TestEq(mymap.values().get(0).asString(), vec.get(1).asString()); + TestEq(mymap.get("int").asInt(), -120); + TestEq((float)mymap.get("float").asFloat(), -123.0f); + TestEq(Arrays.equals(mymap.get("blob").asBlob().getBytes(), new byte[]{ 65, 67 }), true); + TestEq(mymap.get("blob").asBlob().toString(), "AC"); + TestEq(mymap.get("blob").toString(), "\"AC\""); + } + + public static void testSingleElementBoolean() { + FlexBuffersBuilder builder = new FlexBuffersBuilder(ByteBuffer.allocate(100)); + builder.putBoolean(true); + ByteBuffer b = builder.finish(); + assert(FlexBuffers.getRoot(b).asBoolean()); + } + + public static void testSingleElementByte() { + FlexBuffersBuilder builder = new FlexBuffersBuilder(); + builder.putInt(10); + ByteBuffer b = builder.finish(); + TestEq(10, FlexBuffers.getRoot(b).asInt()); + } + + public static void testSingleElementShort() { + FlexBuffersBuilder builder = new FlexBuffersBuilder(); + builder.putInt(Short.MAX_VALUE); + ByteBuffer b = builder.finish(); + TestEq(Short.MAX_VALUE, (short)FlexBuffers.getRoot(b).asInt()); + } + + public static void testSingleElementInt() { + FlexBuffersBuilder builder = new FlexBuffersBuilder(); + builder.putInt(Integer.MIN_VALUE); + ByteBuffer b = builder.finish(); + TestEq(Integer.MIN_VALUE, FlexBuffers.getRoot(b).asInt()); + } + + public static void testSingleElementLong() { + FlexBuffersBuilder builder = new FlexBuffersBuilder(); + builder.putInt(Long.MAX_VALUE); + ByteBuffer b = builder.finish(); + TestEq(Long.MAX_VALUE, FlexBuffers.getRoot(b).asLong()); + } + + public static void testSingleElementFloat() { + FlexBuffersBuilder builder = new FlexBuffersBuilder(); + builder.putFloat(Float.MAX_VALUE); + ByteBuffer b = builder.finish(); + TestEq(Float.compare(Float.MAX_VALUE, (float) FlexBuffers.getRoot(b).asFloat()), 0); + } + + public static void testSingleElementDouble() { + FlexBuffersBuilder builder = new FlexBuffersBuilder(); + builder.putFloat(Double.MAX_VALUE); + ByteBuffer b = builder.finish(); + TestEq(Double.compare(Double.MAX_VALUE, FlexBuffers.getRoot(b).asFloat()), 0); + } + + public static void testSingleElementString() { + FlexBuffersBuilder builder = new FlexBuffersBuilder(); + builder.putString("wow"); + ByteBuffer b = builder.finish(); + FlexBuffers.Reference r = FlexBuffers.getRoot(b); + TestEq(FlexBuffers.FBT_STRING, r.getType()); + TestEq("wow", r.asString()); + } + + public static void testSingleElementBlob() { + FlexBuffersBuilder builder = new FlexBuffersBuilder(); + builder.putBlob(new byte[]{5, 124, 118, -1}); + ByteBuffer b = builder.finish(); + FlexBuffers.Reference r = FlexBuffers.getRoot(b); + byte[] result = r.asBlob().getBytes(); + TestEq((byte)5, result[0]); + TestEq((byte)124, result[1]); + TestEq((byte)118, result[2]); + TestEq((byte)-1, result[3]); + } + + public static void testSingleElementUByte() { + FlexBuffersBuilder builder = new FlexBuffersBuilder(); + builder.putUInt(0xFF); + ByteBuffer b = builder.finish(); + FlexBuffers.Reference r = FlexBuffers.getRoot(b); + TestEq(255, (int)r.asUInt()); + } + + public static void testSingleElementUShort() { + FlexBuffersBuilder builder = new FlexBuffersBuilder(); + builder.putUInt(0xFFFF); + ByteBuffer b = builder.finish(); + FlexBuffers.Reference r = FlexBuffers.getRoot(b); + TestEq(65535, (int)r.asUInt()); + } + + public static void testSingleElementUInt() { + FlexBuffersBuilder builder = new FlexBuffersBuilder(); + builder.putUInt(0xFFFF_FFFFL); + ByteBuffer b = builder.finish(); + FlexBuffers.Reference r = FlexBuffers.getRoot(b); + TestEq(4294967295L, r.asUInt()); + } + + public static void testSingleFixedTypeVector() { + + int[] ints = new int[]{5, 124, 118, -1}; + float[] floats = new float[]{5.5f, 124.124f, 118.118f, -1.1f}; + String[] strings = new String[]{"This", "is", "a", "typed", "array"}; + boolean[] booleans = new boolean[]{false, true, true, false}; + + + FlexBuffersBuilder builder = new FlexBuffersBuilder(ByteBuffer.allocate(512), + FlexBuffersBuilder.BUILDER_FLAG_NONE); + + int mapPos = builder.startMap(); + + int vecPos = builder.startVector(); + for (final int i : ints) { + builder.putInt(i); + } + builder.endVector("ints", vecPos, true, false); + + vecPos = builder.startVector(); + for (final float i : floats) { + builder.putFloat(i); + } + builder.endVector("floats", vecPos, true, false); + + vecPos = builder.startVector(); + for (final String i : strings) { + builder.putString(i); + } + builder.endVector("strings", vecPos, true, false); + + vecPos = builder.startVector(); + for (final boolean i : booleans) { + builder.putBoolean(i); + } + builder.endVector("booleans", vecPos, true, false); + + builder.endMap(null, mapPos); + + + ByteBuffer b = builder.finish(); + FlexBuffers.Reference r = FlexBuffers.getRoot(b); + assert(r.asMap().get("ints").isTypedVector()); + assert(r.asMap().get("floats").isTypedVector()); + assert(r.asMap().get("strings").isTypedVector()); + assert(r.asMap().get("booleans").isTypedVector()); + } + + public static void testSingleElementVector() { + FlexBuffersBuilder b = new FlexBuffersBuilder(); + + int vecPos = b.startVector(); + b.putInt(99); + b.putString("wow"); + int vecpos2 = b.startVector(); + b.putInt(99); + b.putString("wow"); + b.endVector(null, vecpos2, false, false); + b.endVector(null, vecPos, false, false); + b.finish(); + + FlexBuffers.Reference r = FlexBuffers.getRoot(b.getBuffer()); + TestEq(FlexBuffers.FBT_VECTOR, r.getType()); + FlexBuffers.Vector vec = FlexBuffers.getRoot(b.getBuffer()).asVector(); + TestEq(3, vec.size()); + TestEq(99, vec.get(0).asInt()); + TestEq("wow", vec.get(1).asString()); + TestEq("[ 99, \"wow\" ]", vec.get(2).toString()); + TestEq("[ 99, \"wow\", [ 99, \"wow\" ] ]", FlexBuffers.getRoot(b.getBuffer()).toString()); + } + + public static void testSingleElementMap() { + FlexBuffersBuilder b = new FlexBuffersBuilder(); + + int mapPost = b.startMap(); + b.putInt("myInt", 0x7fffffbbbfffffffL); + b.putString("myString", "wow"); + b.putString("myString2", "incredible"); + int start = b.startVector(); + b.putInt(99); + b.putString("wow"); + b.endVector("myVec", start, false, false); + + b.putFloat("double", 0x1.ffffbbbffffffP+1023); + b.endMap(null, mapPost); + b.finish(); + + FlexBuffers.Reference r = FlexBuffers.getRoot(b.getBuffer()); + TestEq(FlexBuffers.FBT_MAP, r.getType()); + FlexBuffers.Map map = FlexBuffers.getRoot(b.getBuffer()).asMap(); + TestEq(5, map.size()); + TestEq(0x7fffffbbbfffffffL, map.get("myInt").asLong()); + TestEq("wow", map.get("myString").asString()); + TestEq("incredible", map.get("myString2").asString()); + TestEq(99, map.get("myVec").asVector().get(0).asInt()); + TestEq("wow", map.get("myVec").asVector().get(1).asString()); + TestEq(Double.compare(0x1.ffffbbbffffffP+1023, map.get("double").asFloat()), 0); + TestEq("{ \"double\" : 1.7976894783391937E308, \"myInt\" : 9223371743723257855, \"myString\" : \"wow\", \"myString2\" : \"incredible\", \"myVec\" : [ 99, \"wow\" ] }", + FlexBuffers.getRoot(b.getBuffer()).toString()); + } + + public static void TestFlexBuffers() { + testSingleElementByte(); + testSingleElementShort(); + testSingleElementInt(); + testSingleElementLong(); + testSingleElementFloat(); + testSingleElementDouble(); + testSingleElementString(); + testSingleElementBlob(); + testSingleElementVector(); + testSingleFixedTypeVector(); + testSingleElementUShort(); + testSingleElementUInt(); + testSingleElementUByte(); + testSingleElementMap(); + testFlexBuffersTest(); + } + static void TestEq(T a, T b) { if (!a.equals(b)) { System.out.println("" + a.getClass().getName() + " " + b.getClass().getName()); System.out.println("FlatBuffers test FAILED: \'" + a + "\' != \'" + b + "\'"); + new Throwable().printStackTrace(); assert false; System.exit(1); } diff --git a/tests/JavaTest.sh b/tests/JavaTest.sh index 58d84429e..9ec6933c3 100755 --- a/tests/JavaTest.sh +++ b/tests/JavaTest.sh @@ -20,7 +20,7 @@ echo Compile then run the Java test. java -version -testdir="$(readlink -fn "$(dirname "$0")")" +testdir=$(dirname $0) targetdir="${testdir}/target"