BigfootDev/flatbuffers
1
0
Fork 1
mirror of https://github.com/google/flatbuffers.git synced 2026-06-08 14:15:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve LookupByKey , update docs

Browse Source
This commit is contained in:
TGIshib
2016-08-22 18:10:52 +03:00
parent fa74ce6d16
commit 9f16090f90
9 changed files with 173 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
docs/source/JavaCsharpUsage.md
View File

@@ -131,6 +131,36 @@ object are prefixed with `Get`, e.g.:
monster.GetPos(preconstructedPos);
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
## Storing dictionaries in a FlatBuffer
FlatBuffers doesn't support dictionaries natively, but there is support to
emulate their behavior with vectors and binary search, which means you
can have fast lookups directly from a FlatBuffer without having to unpack
your data into a `Dictionary` or similar.
To use it:
- Designate one of the fields in a table as they "key" field. You do this
by setting the `key` attribute on this field, e.g.
`name:string (key)`.
You may only have one key field, and it must be of string or scalar type.
- Write out tables of this type as usual, collect their offsets in an
array.
- Instead of calling standard generated method,
e.g.: `Monster.createTestarrayoftablesVector`,
call `CreateMySortedVectorOfTables` in C# or
`createSortedVectorOfTables` (from the `FlatBufferBuilder` object) in Java,
which will first sort all offsets such that the tables they refer to
are sorted by the key field, then serialize it.
- Now when you're accessing the FlatBuffer, you can use `LookupByKey`
to access elements of the vector, e.g.:
`Monster.lookupByKey(tablesVectorOffset, "Frodo", dataBuffer)`,
which returns an object of the corresponding table type,
or `null` if not found.
`LookupByKey` performs a binary search, so should have a similar speed to
`Dictionary`, though may be faster because of better caching. `LookupByKey`
only works if the vector has been sorted, it will likely not find elements
if it hasn't been sorted.
## Text parsing
There currently is no support for parsing text (Schema's and JSON) directly

4
java/com/google/flatbuffers/FlatBufferBuilder.java
View File

@@ -383,11 +383,11 @@ public class FlatBufferBuilder {
/**
* Create a vector of sorted by the key tables.
*
* @param obj Instance of the table class.
* @param obj Instance of the table subclass.
* @param offsets Offsets of the tables.
* @return Returns offset of the sorted vector.
*/
public <T extends Table> int createSortedTableVector(T obj, int[] offsets) {
public <T extends Table> int createSortedVectorOfTables(T obj, int[] offsets) {
obj.sortTables(offsets, bb);
return createVectorOfTables(offsets);
}

32
java/com/google/flatbuffers/Table.java
View File

@@ -75,6 +75,10 @@ public class Table {
protected int __indirect(int offset) {
return offset + bb.getInt(offset);
}
protected static int __indirect(int offset, ByteBuffer bb) {
return offset + bb.getInt(offset);
}
/**
* Create a Java `String` from UTF-8 data stored inside the FlatBuffer.
@@ -197,7 +201,7 @@ public class Table {
/**
* Sort tables by the key.
*
* @param offsets An 'int' indexes of the tables into the _bb.
* @param offsets An 'int' indexes of the tables into the bb.
* @param bb A {@code ByteBuffer} to get the tables.
*/
protected void sortTables(int[] offsets, ByteBuffer bb) {
@@ -210,8 +214,8 @@ public class Table {
/**
* Compare two tables by the key.
*
* @param o1 An 'Integer' index of the first key into the _bb.
* @param o2 An 'Integer' index of the second key into the _bb.
* @param o1 An 'Integer' index of the first key into the bb.
* @param o2 An 'Integer' index of the second key into the bb.
* @param bb A {@code ByteBuffer} to get the keys.
*/
protected int keysCompare(Integer o1, Integer o2, ByteBuffer bb) { return 0; }
@@ -239,6 +243,28 @@ public class Table {
if (len_1 > len_2) return 1;
return 0;
}
/**
* Compare string from the buffer with the 'String' object.
*
* @param offset_1 An 'int' index of the first string into the bb.
* @param key Second string.
* @param bb A {@code ByteBuffer} to get the first string.
*/
protected static int compareStrings(int offset_1, String key, ByteBuffer bb) {
offset_1 += bb.getInt(offset_1);
int len_1 = bb.getInt(offset_1);
int len_2 = key.length();
int startPos_1 = offset_1 + Constants.SIZEOF_INT;
int len = Math.min(len_1, len_2);
for (int i = 0; i < len; i++) {
if (bb.array()[i + startPos_1] != key.charAt(i))
return bb.array()[i + startPos_1] - key.charAt(i);
}
if (len_1 < len_2) return -1;
if (len_1 > len_2) return 1;
return 0;
}
}
/// @endcond

22
net/FlatBuffers/Table.cs
View File

@@ -48,6 +48,11 @@ namespace FlatBuffers
{
return offset + bb.GetInt(offset);
}
protected static int __indirect(int offset, ByteBuffer bb)
{
return offset + bb.GetInt(offset);
}
// Create a .NET String from UTF-8 data stored inside the flatbuffer.
protected string __string(int offset)
@@ -128,6 +133,23 @@ namespace FlatBuffers
if (len_1 > len_2) return 1;
return 0;
}
// Compare string from the ByteBuffer with the string object
protected static int CompareStrings(int offset_1, string key, ByteBuffer bb)
{
offset_1 += bb.GetInt(offset_1);
var len_1 = bb.GetInt(offset_1);
var len_2 = key.Length;
var startPos_1 = offset_1 + sizeof(int);
var len = Math.Min(len_1, len_2);
for (int i = 0; i < len; i++) {
if (bb.Data[i + startPos_1] != key[i])
return bb.Data[i + startPos_1] - key[i];
}
if (len_1 < len_2) return -1;
if (len_1 > len_2) return 1;
return 0;
}
}
}

105
src/idl_gen_general.cpp
View File

@@ -669,43 +669,84 @@ void GenStructBody(const StructDef &struct_def, std::string *code_ptr, const cha
}
}
std::string GenOffsetGetter(flatbuffers::FieldDef *key_field, const char &num) {
return "__offset(" +
NumToString(key_field->value.offset) + ", o" + num +
(lang_.language == IDLOptions::kCSharp ?
".Value, builder.DataBuffer)" : ", _bb)");
std::string GenByteBufferLength(const char *bb_name) {
std::string bb_len = bb_name;
if (lang_.language == IDLOptions::kCSharp) bb_len += ".Length";
else bb_len += ".array().length";
return bb_len;
}
std::string GenOffsetGetter(flatbuffers::FieldDef *key_field, const char *num = nullptr) {
std::string key_offset = "";
key_offset += "__offset(" +
NumToString(key_field->value.offset) + ", ";
if (num) {
key_offset += num;
key_offset += (lang_.language == IDLOptions::kCSharp ?
".Value, builder.DataBuffer)" : ", _bb)");
}
else {
key_offset += GenByteBufferLength("bb");
key_offset += " - tableOffset, bb)";
}
return key_offset;
}
std::string GenLookupKeyGetter(flatbuffers::FieldDef *key_field) {
std::string key_getter = " ";
key_getter += "tableOffset = __indirect(vectorLocation + 4 * (start + middle)";
key_getter += ", bb);\n ";
if (key_field->value.type.base_type == BASE_TYPE_STRING) {
key_getter += "comp = " + FunctionStart('C') + "ompareStrings(";
key_getter += GenOffsetGetter(key_field);
key_getter += ", key, bb);\n";
}
else {
auto get_val = GenGetter(key_field->value.type) +
"(" + GenOffsetGetter(key_field) + ")";
if (lang_.language == IDLOptions::kCSharp) {
key_getter += "comp = " + get_val + ".CompateTo(key);\n";
}
else {
key_getter += GenTypeGet(key_field->value.type) + " val = ";
key_getter += get_val + ";\n";
key_getter += " comp = val > key ? 1 : val < key ? -1 : 0;\n";
}
}
return key_getter;
}
std::string GenKeyGetter(flatbuffers::FieldDef *key_field) {
std::string key_getter = "";
auto data_buffer = (lang_.language == IDLOptions::kCSharp) ?
"builder.DataBuffer" : "_bb";
std::string key_getter = "";
if (key_field->value.type.base_type == BASE_TYPE_STRING) {
if (lang_.language == IDLOptions::kJava)
key_getter += " return ";
key_getter += FunctionStart('C') + "ompareStrings(";
key_getter += GenOffsetGetter(key_field, '1') + ", ";
key_getter += GenOffsetGetter(key_field, '2') + ", " + data_buffer + ")";
key_getter += GenOffsetGetter(key_field, "o1") + ", ";
key_getter += GenOffsetGetter(key_field, "o2") + ", " + data_buffer + ")";
if (lang_.language == IDLOptions::kJava)
key_getter += ";";
}
else {
auto field_getter = data_buffer + GenGetter(key_field->value.type).substr(2) +
"(" + GenOffsetGetter(key_field, '1') + ")";
"(" + GenOffsetGetter(key_field, "o1") + ")";
if (lang_.language == IDLOptions::kCSharp) {
key_getter += field_getter;
field_getter = data_buffer + GenGetter(key_field->value.type).substr(2) +
"(" + GenOffsetGetter(key_field, '2') + ")";
"(" + GenOffsetGetter(key_field, "o2") + ")";
key_getter += ".CompareTo(" + field_getter + ")";
}
else {
key_getter += "\n " + GenTypeGet(key_field->value.type) + " off1 = ";
key_getter += "\n " + GenTypeGet(key_field->value.type) + " val_1 = ";
key_getter += field_getter + ";\n " + GenTypeGet(key_field->value.type);
key_getter += " off2 = ";
key_getter += " val_2 = ";
field_getter = data_buffer + GenGetter(key_field->value.type).substr(2) +
"(" + GenOffsetGetter(key_field, '2') + ")";
"(" + GenOffsetGetter(key_field, "o2") + ")";
key_getter += field_getter + ";\n";
key_getter += " return off1 > off2 ? 1 : off1 < off2 ? -1 : 0;\n";
key_getter += " return val_1 > val_2 ? 1 : val_1 < val_2 ? -1 : 0;\n ";
}
}
return key_getter;
@@ -1175,25 +1216,15 @@ void GenStruct(StructDef &struct_def, std::string *code_ptr) {
}
}
if (struct_def.has_key) {
auto is_string = key_field->value.type.base_type == BASE_TYPE_STRING;
auto key_name = "table." +
(lang_.language == IDLOptions::kCSharp ?
MakeCamel(key_field->name) : key_field->name + "()");
if (lang_.language == IDLOptions::kJava) {
code += "\n @Override\n protected int keysCompare(";
code += "Integer o1, Integer o2, ByteBuffer _bb) {";
code += GenKeyGetter(key_field);
if (!is_string) {
code += " if (off1 < off2) return -1;\n";
code += " else if (off1 == off2) return 0;\n";
code += " else return 1;\n ";
}
code += " }\n";
}
else {
code += "\n public static VectorOffset ";
code += "CreateMySortedTableVector(FlatBufferBuilder builder, ";
code += "CreateMySortedVectorOfTables(FlatBufferBuilder builder, ";
code += "Offset<" + struct_def.name + ">";
code += "[] offsets) {\n";
code += " Array.Sort(offsets, (Offset<" + struct_def.name +
@@ -1203,27 +1234,25 @@ void GenStruct(StructDef &struct_def, std::string *code_ptr) {
}
code += "\n public static " + struct_def.name + " " + FunctionStart('L');
code += "ookupByKey(" + struct_def.name;
code += "[] tables, " + GenTypeGet(key_field->value.type) + " key) {\n";
code += " int span = tables." + FunctionStart('L') + "ength, start = 0;\n";
code += "ookupByKey(" + GenVectorOffsetType();
code += " vectorOffset, " + GenTypeGet(key_field->value.type);
code += " key, ByteBuffer bb) {\n";
code += " int vectorLocation = " + GenByteBufferLength("bb");
code += " - vectorOffset.Value;\n int span = ";
code += "bb." + FunctionStart('G') + "etInt(vectorLocation), ";
code += "middle, start = 0, comp, tableOffset; \n";
code += " vectorLocation += 4;\n";
code += " while (span != 0) {\n";
code += " int middle = span / 2;\n";
code += " " + struct_def.name + " table = tables[start + middle];\n";
if (lang_.language == IDLOptions::kCSharp || is_string) {
code += " int comp = " + key_name + "." + FunctionStart('C');
code += "ompareTo(key);\n";
}
else {
code += " int comp = " + key_name + " > key ? 1 : " + key_name;
code += " < key ? -1 : 0;\n";
}
code += GenLookupKeyGetter(key_field);
code += " if (comp > 0) span = middle;\n";
code += " else if (comp < 0) {\n";
code += " middle++;\n";
code += " start += middle;\n";
code += " span -= middle;\n";
code += " }\n";
code += " else return table;\n";
code += " else return new " + struct_def.name;
code += "().__init(tableOffset, bb);\n";
code += " }\n";
code += " return null;\n";
code += " }\n";

9
tests/FlatBuffers.Test/FlatBuffersExampleTests.cs
View File

@@ -50,7 +50,7 @@ namespace FlatBuffers.Test
Monster.StartMonster(fbb);
Monster.AddName(fbb, names[2]);
off[2] = Monster.EndMonster(fbb);
var sortMons = Monster.CreateMySortedTableVector(fbb, off);
var sortMons = Monster.CreateMySortedVectorOfTables(fbb, off);
// We set up the same values as monsterdata.json:
@@ -118,15 +118,14 @@ namespace FlatBuffers.Test
Assert.AreEqual(monster.Mana, (short)150);
// Accessing a vector of sorted by the key tables
Monster[] monsters = { monster.GetTestarrayoftables(0), monster.GetTestarrayoftables(1), monster.GetTestarrayoftables(2) };
Assert.AreEqual(monster.GetTestarrayoftables(0).Name, "Barney");
Assert.AreEqual(monster.GetTestarrayoftables(1).Name, "Frodo");
Assert.AreEqual(monster.GetTestarrayoftables(2).Name, "Wilma");
// Example of searching for a table by the key
Assert.IsTrue(Monster.LookupByKey(monsters, "Frodo") != null);
Assert.IsTrue(Monster.LookupByKey(monsters, "Barney") != null);
Assert.IsTrue(Monster.LookupByKey(monsters, "Wilma") != null);
Assert.IsTrue(Monster.LookupByKey(sortMons, "Frodo", fbb.DataBuffer) != null);
Assert.IsTrue(Monster.LookupByKey(sortMons, "Barney", fbb.DataBuffer) != null);
Assert.IsTrue(Monster.LookupByKey(sortMons, "Wilma", fbb.DataBuffer)!= null);
// testType is an existing field and mutating it should succeed
Assert.AreEqual(monster.TestType, Any.Monster);

9
tests/JavaTest.java
View File

@@ -62,7 +62,7 @@ class JavaTest {
Monster.startMonster(fbb);
Monster.addName(fbb, names[2]);
off[2] = Monster.endMonster(fbb);
int sortMons = fbb.createSortedTableVector(new Monster(), off);
int sortMons = fbb.createSortedVectorOfTables(new Monster(), off);
// We set up the same values as monsterdata.json:
@@ -137,15 +137,14 @@ class JavaTest {
TestEq(monster.mana(), (short)150);
// Accessing a vector of sorted by the key tables
Monster[] monsters = { monster.testarrayoftables(0), monster.testarrayoftables(1), monster.testarrayoftables(2) };
TestEq(monster.testarrayoftables(0).name(), "Barney");
TestEq(monster.testarrayoftables(1).name(), "Frodo");
TestEq(monster.testarrayoftables(2).name(), "Wilma");
// Example of searching for a table by the key
TestEq(Monster.lookupByKey(monsters, "Frodo").name(), "Frodo");
TestEq(Monster.lookupByKey(monsters, "Barney").name(), "Barney");
TestEq(Monster.lookupByKey(monsters, "Wilma").name(), "Wilma");
TestEq(Monster.lookupByKey(sortMons, "Frodo", fbb.dataBuffer()).name(), "Frodo");
TestEq(Monster.lookupByKey(sortMons, "Barney", fbb.dataBuffer()).name(), "Barney");
TestEq(Monster.lookupByKey(sortMons, "Wilma", fbb.dataBuffer()).name(), "Wilma");
// testType is an existing field and mutating it should succeed
TestEq(monster.testType(), (byte)Any.Monster);

14
tests/MyGame/Example/Monster.cs
View File

@@ -130,24 +130,26 @@ public sealed class Monster : Table {
}
public static void FinishMonsterBuffer(FlatBufferBuilder builder, Offset<Monster> offset) { builder.Finish(offset.Value, "MONS"); }
public static VectorOffset CreateMySortedTableVector(FlatBufferBuilder builder, Offset<Monster>[] offsets) {
public static VectorOffset CreateMySortedVectorOfTables(FlatBufferBuilder builder, Offset<Monster>[] offsets) {
Array.Sort(offsets, (Offset<Monster> o1, Offset<Monster> o2) => CompareStrings(__offset(10, o1.Value, builder.DataBuffer), __offset(10, o2.Value, builder.DataBuffer), builder.DataBuffer));
return builder.CreateVectorOfTables(offsets);
}
public static Monster LookupByKey(Monster[] tables, string key) {
int span = tables.Length, start = 0;
public static Monster LookupByKey(VectorOffset vectorOffset, string key, ByteBuffer bb) {
int vectorLocation = bb.Length - vectorOffset.Value;
int span = bb.GetInt(vectorLocation), middle, start = 0, comp, tableOffset;
vectorLocation += 4;
while (span != 0) {
int middle = span / 2;
Monster table = tables[start + middle];
int comp = table.Name.CompareTo(key);
tableOffset = __indirect(vectorLocation + 4 * (start + middle), bb);
comp = CompareStrings(__offset(10, bb.Length - tableOffset, bb), key, bb);
if (comp > 0) span = middle;
else if (comp < 0) {
middle++;
start += middle;
span -= middle;
}
else return table;
else return new Monster().__init(tableOffset, bb);
}
return null;
}

12
tests/MyGame/Example/Monster.java
View File

@@ -139,19 +139,21 @@ public final class Monster extends Table {
@Override
protected int keysCompare(Integer o1, Integer o2, ByteBuffer _bb) { return compareStrings(__offset(10, o1, _bb), __offset(10, o2, _bb), _bb); }
public static Monster lookupByKey(Monster[] tables, String key) {
int span = tables.length, start = 0;
public static Monster lookupByKey(int vectorOffset, String key, ByteBuffer bb) {
int vectorLocation = bb.array().length - vectorOffset.Value;
int span = bb.getInt(vectorLocation), middle, start = 0, comp, tableOffset;
vectorLocation += 4;
while (span != 0) {
int middle = span / 2;
Monster table = tables[start + middle];
int comp = table.name().compareTo(key);
tableOffset = __indirect(vectorLocation + 4 * (start + middle), bb);
comp = compareStrings(__offset(10, bb.array().length - tableOffset, bb), key, bb);
if (comp > 0) span = middle;
else if (comp < 0) {
middle++;
start += middle;
span -= middle;
}
else return table;
else return new Monster().__init(tableOffset, bb);
}
return null;
}