Improve LookupByKey , update docs

docs/source/JavaCsharpUsage.md

@@ -131,6 +131,36 @@ object are prefixed with `Get`, e.g.:
     monster.GetPos(preconstructedPos);
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+## Storing dictionaries in a FlatBuffer
+
+FlatBuffers doesn't support dictionaries natively, but there is support to
+emulate their behavior with vectors and binary search, which means you
+can have fast lookups directly from a FlatBuffer without having to unpack
+your data into a `Dictionary` or similar.
+
+To use it:
+-   Designate one of the fields in a table as they "key" field. You do this
+    by setting the `key` attribute on this field, e.g.
+    `name:string (key)`.
+    You may only have one key field, and it must be of string or scalar type.
+-   Write out tables of this type as usual, collect their offsets in an
+    array.
+-   Instead of calling standard generated method,
+    e.g.: `Monster.createTestarrayoftablesVector`,
+    call `CreateMySortedVectorOfTables` in C# or
+    `createSortedVectorOfTables` (from the `FlatBufferBuilder` object) in Java,
+    which will first sort all offsets such that the tables they refer to
+    are sorted by the key field, then serialize it.
+-   Now when you're accessing the FlatBuffer, you can use `LookupByKey`
+    to access elements of the vector, e.g.:
+    `Monster.lookupByKey(tablesVectorOffset, "Frodo", dataBuffer)`,
+    which returns an object of the corresponding table type,
+    or `null` if not found.
+    `LookupByKey` performs a binary search, so should have a similar speed to
+    `Dictionary`, though may be faster because of better caching. `LookupByKey`
+    only works if the vector has been sorted, it will likely not find elements
+    if it hasn't been sorted.
+
 ## Text parsing
 
 There currently is no support for parsing text (Schema's and JSON) directly

java/com/google/flatbuffers/FlatBufferBuilder.java

@@ -383,11 +383,11 @@ public class FlatBufferBuilder {
     /**
      * Create a vector of sorted by the key tables.
      *
-     * @param obj Instance of the table class.
+     * @param obj Instance of the table subclass.
      * @param offsets Offsets of the tables.
      * @return Returns offset of the sorted vector.
      */
-    public <T extends Table> int createSortedTableVector(T obj, int[] offsets) {
+    public <T extends Table> int createSortedVectorOfTables(T obj, int[] offsets) {
         obj.sortTables(offsets, bb);
         return createVectorOfTables(offsets);
     }

java/com/google/flatbuffers/Table.java

@@ -76,6 +76,10 @@ public class Table {
     return offset + bb.getInt(offset);
   }
   
+  protected static int __indirect(int offset, ByteBuffer bb) {
+    return offset + bb.getInt(offset);
+  }
+
   /**
    * Create a Java `String` from UTF-8 data stored inside the FlatBuffer.
    *
@@ -197,7 +201,7 @@ public class Table {
   /**
    * Sort tables by the key.
    *
-   * @param offsets An 'int' indexes of the tables into the _bb.
+   * @param offsets An 'int' indexes of the tables into the bb.
    * @param bb A {@code ByteBuffer} to get the tables.
    */
   protected void sortTables(int[] offsets, ByteBuffer bb) {
@@ -210,8 +214,8 @@ public class Table {
   /**
    * Compare two tables by the key.
    *
-   * @param o1 An 'Integer' index of the first key into the _bb.
+   * @param o1 An 'Integer' index of the first key into the bb.
-   * @param o2 An 'Integer' index of the second key into the _bb.
+   * @param o2 An 'Integer' index of the second key into the bb.
    * @param bb A {@code ByteBuffer} to get the keys.
    */
   protected int keysCompare(Integer o1, Integer o2, ByteBuffer bb) { return 0; }
@@ -239,6 +243,28 @@ public class Table {
     if (len_1 > len_2) return 1;
     return 0;
   }
+  
+  /**
+   * Compare string from the buffer with the 'String' object.
+   *
+   * @param offset_1 An 'int' index of the first string into the bb.
+   * @param key Second string.
+   * @param bb A {@code ByteBuffer} to get the first string.
+   */
+  protected static int compareStrings(int offset_1, String key, ByteBuffer bb) {
+    offset_1 += bb.getInt(offset_1);
+    int len_1 = bb.getInt(offset_1);
+    int len_2 = key.length();
+    int startPos_1 = offset_1 + Constants.SIZEOF_INT;
+    int len = Math.min(len_1, len_2);
+    for (int i = 0; i < len; i++) {
+      if (bb.array()[i + startPos_1] != key.charAt(i))
+        return bb.array()[i + startPos_1] - key.charAt(i);
+    }
+    if (len_1 < len_2) return -1;
+    if (len_1 > len_2) return 1;
+    return 0;
+  }
 }
 
 /// @endcond

net/FlatBuffers/Table.cs

@@ -49,6 +49,11 @@ namespace FlatBuffers
             return offset + bb.GetInt(offset);
         }
 		
+        protected static int __indirect(int offset, ByteBuffer bb)
+        {
+            return offset + bb.GetInt(offset);
+        }
+
         // Create a .NET String from UTF-8 data stored inside the flatbuffer.
         protected string __string(int offset)
         {
@@ -129,5 +134,22 @@ namespace FlatBuffers
             return 0;
         }
 		
+        // Compare string from the ByteBuffer with the string object
+        protected static int CompareStrings(int offset_1, string key, ByteBuffer bb)
+        {
+            offset_1 += bb.GetInt(offset_1);
+            var len_1 = bb.GetInt(offset_1);
+            var len_2 = key.Length;
+            var startPos_1 = offset_1 + sizeof(int);
+            var len = Math.Min(len_1, len_2);
+            for (int i = 0; i < len; i++) {
+                if (bb.Data[i + startPos_1] != key[i])
+                    return bb.Data[i + startPos_1] - key[i];
+            }
+            if (len_1 < len_2) return -1;
+            if (len_1 > len_2) return 1;
+            return 0;
+        }
+
     }
 }

src/idl_gen_general.cpp

@@ -669,43 +669,84 @@ void GenStructBody(const StructDef &struct_def, std::string *code_ptr, const cha
   }
 }
 
-std::string GenOffsetGetter(flatbuffers::FieldDef *key_field, const char &num) {
+std::string GenByteBufferLength(const char *bb_name) {
-  return "__offset(" +
+  std::string bb_len = bb_name;
-    NumToString(key_field->value.offset) + ", o" + num +
+  if (lang_.language == IDLOptions::kCSharp) bb_len += ".Length";
-    (lang_.language == IDLOptions::kCSharp ?
+  else bb_len += ".array().length";
-      ".Value, builder.DataBuffer)" : ", _bb)");
+  return bb_len;
 }
 
+std::string GenOffsetGetter(flatbuffers::FieldDef *key_field, const char *num = nullptr) {
+  std::string key_offset = "";
+  key_offset += "__offset(" +
+    NumToString(key_field->value.offset) + ", ";
+  if (num) {
+    key_offset += num;
+    key_offset += (lang_.language == IDLOptions::kCSharp ?
+      ".Value, builder.DataBuffer)" : ", _bb)");
+  }
+  else {
+    key_offset += GenByteBufferLength("bb");
+    key_offset += " - tableOffset, bb)";
+  }
+  return key_offset;
+}
+
+std::string GenLookupKeyGetter(flatbuffers::FieldDef *key_field) {
+  std::string key_getter = "      ";
+  key_getter += "tableOffset = __indirect(vectorLocation + 4 * (start + middle)";
+  key_getter += ", bb);\n      ";
+  if (key_field->value.type.base_type == BASE_TYPE_STRING) {
+    key_getter += "comp = " + FunctionStart('C') + "ompareStrings(";
+    key_getter += GenOffsetGetter(key_field);
+    key_getter += ", key, bb);\n";
+  }
+  else {
+    auto get_val = GenGetter(key_field->value.type) +
+      "(" + GenOffsetGetter(key_field) + ")";
+    if (lang_.language == IDLOptions::kCSharp) {
+      key_getter += "comp = " + get_val + ".CompateTo(key);\n";
+    }
+    else {
+      key_getter += GenTypeGet(key_field->value.type) + " val = ";
+      key_getter += get_val + ";\n";
+      key_getter += "      comp = val > key ? 1 : val < key ? -1 : 0;\n";
+    }
+  }
+  return key_getter;
+}
+
+
 std::string GenKeyGetter(flatbuffers::FieldDef *key_field) {
+  std::string key_getter = "";
   auto data_buffer = (lang_.language == IDLOptions::kCSharp) ?
     "builder.DataBuffer" : "_bb";
-  std::string key_getter = "";
   if (key_field->value.type.base_type == BASE_TYPE_STRING) {
     if (lang_.language == IDLOptions::kJava)
       key_getter += " return ";
     key_getter += FunctionStart('C') + "ompareStrings(";
-    key_getter += GenOffsetGetter(key_field, '1') + ", ";
+    key_getter += GenOffsetGetter(key_field, "o1") + ", ";
-    key_getter += GenOffsetGetter(key_field, '2') + ", " + data_buffer + ")";
+    key_getter += GenOffsetGetter(key_field, "o2") + ", " + data_buffer + ")";
     if (lang_.language == IDLOptions::kJava)
       key_getter += ";";
   }
   else {
     auto field_getter = data_buffer + GenGetter(key_field->value.type).substr(2) +
-      "(" + GenOffsetGetter(key_field, '1') + ")";
+      "(" + GenOffsetGetter(key_field, "o1") + ")";
     if (lang_.language == IDLOptions::kCSharp) {
       key_getter += field_getter;
       field_getter = data_buffer + GenGetter(key_field->value.type).substr(2) +
-        "(" + GenOffsetGetter(key_field, '2') + ")";
+        "(" + GenOffsetGetter(key_field, "o2") + ")";
       key_getter += ".CompareTo(" + field_getter + ")";
     }
     else {
-      key_getter += "\n    " + GenTypeGet(key_field->value.type) + " off1 = ";
+      key_getter += "\n    " + GenTypeGet(key_field->value.type) + " val_1 = ";
       key_getter += field_getter + ";\n    " + GenTypeGet(key_field->value.type);
-      key_getter += " off2 = ";
+      key_getter += " val_2 = ";
       field_getter = data_buffer + GenGetter(key_field->value.type).substr(2) +
-        "(" + GenOffsetGetter(key_field, '2') + ")";
+        "(" + GenOffsetGetter(key_field, "o2") + ")";
       key_getter += field_getter + ";\n";
-      key_getter += "    return off1 > off2 ? 1 : off1 < off2 ? -1 : 0;\n";
+      key_getter += "    return val_1 > val_2 ? 1 : val_1 < val_2 ? -1 : 0;\n ";
     }
   }
   return key_getter;
@@ -1175,25 +1216,15 @@ void GenStruct(StructDef &struct_def, std::string *code_ptr) {
     }
   }
   if (struct_def.has_key) {
-    auto is_string = key_field->value.type.base_type == BASE_TYPE_STRING;
-    auto key_name = "table." +
-      (lang_.language == IDLOptions::kCSharp ?
-        MakeCamel(key_field->name) : key_field->name + "()");
-
     if (lang_.language == IDLOptions::kJava) {
       code += "\n  @Override\n  protected int keysCompare(";
       code += "Integer o1, Integer o2, ByteBuffer _bb) {";
       code += GenKeyGetter(key_field);
-      if (!is_string) {
-        code += "    if (off1 < off2) return -1;\n";
-        code += "    else if (off1 == off2) return 0;\n";
-        code += "    else return 1;\n ";
-      }
       code += " }\n";
     }
     else {
       code += "\n  public static VectorOffset ";
-      code += "CreateMySortedTableVector(FlatBufferBuilder builder, ";
+      code += "CreateMySortedVectorOfTables(FlatBufferBuilder builder, ";
       code += "Offset<" + struct_def.name + ">";
       code += "[] offsets) {\n";
       code += "    Array.Sort(offsets, (Offset<" + struct_def.name +
@@ -1203,27 +1234,25 @@ void GenStruct(StructDef &struct_def, std::string *code_ptr) {
     }
 
     code += "\n  public static " + struct_def.name + " " + FunctionStart('L');
-    code += "ookupByKey(" + struct_def.name;
+    code += "ookupByKey(" + GenVectorOffsetType();
-    code += "[] tables, " + GenTypeGet(key_field->value.type) + " key) {\n";
+    code += " vectorOffset, " + GenTypeGet(key_field->value.type);
-    code += "    int span = tables." + FunctionStart('L') + "ength, start = 0;\n";
+    code += " key, ByteBuffer bb) {\n";
+    code += "    int vectorLocation = " + GenByteBufferLength("bb");
+    code += " - vectorOffset.Value;\n    int span = ";
+    code += "bb." + FunctionStart('G') + "etInt(vectorLocation), ";
+    code += "middle, start = 0, comp, tableOffset; \n";
+    code += "    vectorLocation += 4;\n";
     code += "    while (span != 0) {\n";
     code += "      int middle = span / 2;\n";
-    code += "      " + struct_def.name + " table = tables[start + middle];\n";
+    code += GenLookupKeyGetter(key_field);
-    if (lang_.language == IDLOptions::kCSharp || is_string) {
-      code += "      int comp = " + key_name + "." + FunctionStart('C');
-      code += "ompareTo(key);\n";
-    }
-    else {
-      code += "      int comp = " + key_name + " > key ? 1 : " + key_name;
-      code += " < key ? -1 : 0;\n";
-    }
     code += "      if (comp > 0) span = middle;\n";
     code += "      else if (comp < 0) {\n";
     code += "        middle++;\n";
     code += "        start += middle;\n";
     code += "        span -= middle;\n";
     code += "      }\n";
-    code += "      else return table;\n";
+    code += "      else return new " + struct_def.name;
+    code += "().__init(tableOffset, bb);\n";
     code += "    }\n";
     code += "    return null;\n";
     code += "  }\n";

tests/FlatBuffers.Test/FlatBuffersExampleTests.cs

@@ -50,7 +50,7 @@ namespace FlatBuffers.Test
             Monster.StartMonster(fbb);
             Monster.AddName(fbb, names[2]);
             off[2] = Monster.EndMonster(fbb);
-            var sortMons = Monster.CreateMySortedTableVector(fbb, off);
+            var sortMons = Monster.CreateMySortedVectorOfTables(fbb, off);
 			
             // We set up the same values as monsterdata.json:
 
@@ -118,15 +118,14 @@ namespace FlatBuffers.Test
             Assert.AreEqual(monster.Mana, (short)150);
 			
             // Accessing a vector of sorted by the key tables
-            Monster[] monsters = { monster.GetTestarrayoftables(0), monster.GetTestarrayoftables(1), monster.GetTestarrayoftables(2) };
             Assert.AreEqual(monster.GetTestarrayoftables(0).Name, "Barney");
             Assert.AreEqual(monster.GetTestarrayoftables(1).Name, "Frodo");
             Assert.AreEqual(monster.GetTestarrayoftables(2).Name, "Wilma");
 			
             // Example of searching for a table by the key
-            Assert.IsTrue(Monster.LookupByKey(monsters, "Frodo") != null);
+            Assert.IsTrue(Monster.LookupByKey(sortMons, "Frodo", fbb.DataBuffer) != null);
-            Assert.IsTrue(Monster.LookupByKey(monsters, "Barney") != null);
+            Assert.IsTrue(Monster.LookupByKey(sortMons, "Barney", fbb.DataBuffer) != null);
-            Assert.IsTrue(Monster.LookupByKey(monsters, "Wilma") != null);
+            Assert.IsTrue(Monster.LookupByKey(sortMons, "Wilma", fbb.DataBuffer)!= null);
 
             // testType is an existing field and mutating it should succeed
             Assert.AreEqual(monster.TestType, Any.Monster);

tests/JavaTest.java

@@ -62,7 +62,7 @@ class JavaTest {
         Monster.startMonster(fbb);
         Monster.addName(fbb, names[2]);
         off[2] = Monster.endMonster(fbb);
-        int sortMons = fbb.createSortedTableVector(new Monster(), off);
+        int sortMons = fbb.createSortedVectorOfTables(new Monster(), off);
 		
         // We set up the same values as monsterdata.json:
 
@@ -137,15 +137,14 @@ class JavaTest {
         TestEq(monster.mana(), (short)150);
 		
 		// Accessing a vector of sorted by the key tables
-        Monster[] monsters = { monster.testarrayoftables(0), monster.testarrayoftables(1), monster.testarrayoftables(2) };
         TestEq(monster.testarrayoftables(0).name(), "Barney");
         TestEq(monster.testarrayoftables(1).name(), "Frodo");
         TestEq(monster.testarrayoftables(2).name(), "Wilma");
 		
 		// Example of searching for a table by the key
-        TestEq(Monster.lookupByKey(monsters, "Frodo").name(), "Frodo");
+        TestEq(Monster.lookupByKey(sortMons, "Frodo", fbb.dataBuffer()).name(), "Frodo");
-        TestEq(Monster.lookupByKey(monsters, "Barney").name(), "Barney");
+        TestEq(Monster.lookupByKey(sortMons, "Barney", fbb.dataBuffer()).name(), "Barney");
-        TestEq(Monster.lookupByKey(monsters, "Wilma").name(), "Wilma");
+        TestEq(Monster.lookupByKey(sortMons, "Wilma", fbb.dataBuffer()).name(), "Wilma");
 
         // testType is an existing field and mutating it should succeed
         TestEq(monster.testType(), (byte)Any.Monster);

tests/MyGame/Example/Monster.cs

@@ -130,24 +130,26 @@ public sealed class Monster : Table {
   }
   public static void FinishMonsterBuffer(FlatBufferBuilder builder, Offset<Monster> offset) { builder.Finish(offset.Value, "MONS"); }
   
-  public static VectorOffset CreateMySortedTableVector(FlatBufferBuilder builder, Offset<Monster>[] offsets) {
+  public static VectorOffset CreateMySortedVectorOfTables(FlatBufferBuilder builder, Offset<Monster>[] offsets) {
     Array.Sort(offsets, (Offset<Monster> o1, Offset<Monster> o2) => CompareStrings(__offset(10, o1.Value, builder.DataBuffer), __offset(10, o2.Value, builder.DataBuffer), builder.DataBuffer));
     return builder.CreateVectorOfTables(offsets);
   }
 
-  public static Monster LookupByKey(Monster[] tables, string key) {
+  public static Monster LookupByKey(VectorOffset vectorOffset, string key, ByteBuffer bb) {
-    int span = tables.Length, start = 0;
+    int vectorLocation = bb.Length - vectorOffset.Value;
+    int span = bb.GetInt(vectorLocation), middle, start = 0, comp, tableOffset; 
+    vectorLocation += 4;
     while (span != 0) {
       int middle = span / 2;
-      Monster table = tables[start + middle];
+      tableOffset = __indirect(vectorLocation + 4 * (start + middle), bb);
-      int comp = table.Name.CompareTo(key);
+      comp = CompareStrings(__offset(10, bb.Length - tableOffset, bb), key, bb);
       if (comp > 0) span = middle;
       else if (comp < 0) {
         middle++;
         start += middle;
         span -= middle;
       }
-      else return table;
+      else return new Monster().__init(tableOffset, bb);
     }
     return null;
   }

tests/MyGame/Example/Monster.java

@@ -139,19 +139,21 @@ public final class Monster extends Table {
   @Override
   protected int keysCompare(Integer o1, Integer o2, ByteBuffer _bb) { return compareStrings(__offset(10, o1, _bb), __offset(10, o2, _bb), _bb); }
 
-  public static Monster lookupByKey(Monster[] tables, String key) {
+  public static Monster lookupByKey(int vectorOffset, String key, ByteBuffer bb) {
-    int span = tables.length, start = 0;
+    int vectorLocation = bb.array().length - vectorOffset.Value;
+    int span = bb.getInt(vectorLocation), middle, start = 0, comp, tableOffset; 
+    vectorLocation += 4;
     while (span != 0) {
       int middle = span / 2;
-      Monster table = tables[start + middle];
+      tableOffset = __indirect(vectorLocation + 4 * (start + middle), bb);
-      int comp = table.name().compareTo(key);
+      comp = compareStrings(__offset(10, bb.array().length - tableOffset, bb), key, bb);
       if (comp > 0) span = middle;
       else if (comp < 0) {
         middle++;
         start += middle;
         span -= middle;
       }
-      else return table;
+      else return new Monster().__init(tableOffset, bb);
     }
     return null;
   }