diff --git a/.gitignore b/.gitignore index eab9d79f8..ff7fcbf18 100755 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ proguard-project.txt linklint_results Makefile flatc +flathash flattests flatsamplebinary flatsampletext diff --git a/CMakeLists.txt b/CMakeLists.txt index 823de9c10..a746a646a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,7 @@ option(FLATBUFFERS_CODE_COVERAGE "Enable the code coverage build option." OFF) option(FLATBUFFERS_BUILD_TESTS "Enable the build of tests and samples." ON) option(FLATBUFFERS_INSTALL "Enable the installation of targets." ON) option(FLATBUFFERS_BUILD_FLATC "Enable the build of the flatbuffers compiler" ON) +option(FLATBUFFERS_BUILD_FLATHASH "Enable the build of flathash" ON) if(NOT FLATBUFFERS_BUILD_FLATC AND FLATBUFFERS_BUILD_TESTS) message(WARNING @@ -16,6 +17,7 @@ endif() set(FlatBuffers_Compiler_SRCS include/flatbuffers/flatbuffers.h + include/flatbuffers/hash.h include/flatbuffers/idl.h include/flatbuffers/util.h src/idl_parser.cpp @@ -27,8 +29,14 @@ set(FlatBuffers_Compiler_SRCS src/flatc.cpp ) +set(FlatHash_SRCS + include/flatbuffers/hash.h + src/flathash.cpp +) + set(FlatBuffers_Tests_SRCS include/flatbuffers/flatbuffers.h + include/flatbuffers/hash.h include/flatbuffers/idl.h include/flatbuffers/util.h src/idl_parser.cpp @@ -48,6 +56,7 @@ set(FlatBuffers_Sample_Binary_SRCS set(FlatBuffers_Sample_Text_SRCS include/flatbuffers/flatbuffers.h + include/flatbuffers/hash.h include/flatbuffers/idl.h include/flatbuffers/util.h src/idl_parser.cpp @@ -83,7 +92,11 @@ endif(BIICODE) include_directories(include) if(FLATBUFFERS_BUILD_FLATC) -add_executable(flatc ${FlatBuffers_Compiler_SRCS}) + add_executable(flatc ${FlatBuffers_Compiler_SRCS}) +endif() + +if(FLATBUFFERS_BUILD_FLATC) + add_executable(flathash ${FlatHash_SRCS}) endif() function(compile_flatbuffers_schema_to_cpp SRC_FBS) diff --git a/include/flatbuffers/hash.h b/include/flatbuffers/hash.h new file mode 100644 index 000000000..134d17517 --- /dev/null +++ b/include/flatbuffers/hash.h @@ -0,0 +1,105 @@ +/* + * Copyright 2015 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_HASH_H_ +#define FLATBUFFERS_HASH_H_ + +#include +#include + +namespace flatbuffers { + +template +struct FnvTraits { + static const T kFnvPrime; + static const T kOffsetBasis; +}; + +template <> +struct FnvTraits { + static const uint32_t kFnvPrime = 0x01000193; + static const uint32_t kOffsetBasis = 0x811C9DC5; +}; + +template <> +struct FnvTraits { + static const uint64_t kFnvPrime = 0x00000100000001b3; + static const uint64_t kOffsetBasis = 0xcbf29ce484222645; +}; + +template +T HashFnv1(const char *input) { + T hash = FnvTraits::kOffsetBasis; + for (const char *c = input; *c; ++c) { + hash *= FnvTraits::kFnvPrime; + hash ^= static_cast(*c); + } + return hash; +} + +template +T HashFnv1a(const char *input) { + T hash = FnvTraits::kOffsetBasis; + for (const char *c = input; *c; ++c) { + hash ^= static_cast(*c); + hash *= FnvTraits::kFnvPrime; + } + return hash; +} + +template +struct NamedHashFunction { + const char *name; + + typedef T (*HashFunction)(const char*); + HashFunction function; +}; + +const NamedHashFunction kHashFunctions32[] = { + { "fnv1_32", HashFnv1 }, + { "fnv1a_32", HashFnv1a }, +}; + +const NamedHashFunction kHashFunctions64[] = { + { "fnv1_64", HashFnv1 }, + { "fnv1a_64", HashFnv1a }, +}; + +inline NamedHashFunction::HashFunction FindHashFunction32( + const char *name) { + std::size_t size = sizeof(kHashFunctions32) / sizeof(kHashFunctions32[0]); + for (std::size_t i = 0; i < size; ++i) { + if (std::strcmp(name, kHashFunctions32[i].name) == 0) { + return kHashFunctions32[i].function; + } + } + return nullptr; +} + +inline NamedHashFunction::HashFunction FindHashFunction64( + const char *name) { + std::size_t size = sizeof(kHashFunctions64) / sizeof(kHashFunctions64[0]); + for (std::size_t i = 0; i < size; ++i) { + if (std::strcmp(name, kHashFunctions64[i].name) == 0) { + return kHashFunctions64[i].function; + } + } + return nullptr; +} + +} // namespace flatbuffers + +#endif // FLATBUFFERS_HASH_H_ diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 4ed789e37..6437362e6 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -24,6 +24,7 @@ #include #include "flatbuffers/flatbuffers.h" +#include "flatbuffers/hash.h" // This file defines the data types representing a parsed IDL (Interface // Definition Language) / schema file. @@ -280,6 +281,7 @@ class Parser { known_attributes_.insert("deprecated"); known_attributes_.insert("required"); known_attributes_.insert("key"); + known_attributes_.insert("hash"); known_attributes_.insert("id"); known_attributes_.insert("force_align"); known_attributes_.insert("bit_flags"); @@ -334,6 +336,7 @@ class Parser { uoffset_t ParseVector(const Type &type); void ParseMetaData(Definition &def); bool TryTypedValue(int dtoken, bool check, Value &e, BaseType req); + void ParseHash(Value &e, FieldDef* field); void ParseSingleValue(Value &e); int64_t ParseIntegerFromString(Type &type); StructDef *LookupCreateStruct(const std::string &name); diff --git a/src/flathash.cpp b/src/flathash.cpp new file mode 100644 index 000000000..bbd865e2a --- /dev/null +++ b/src/flathash.cpp @@ -0,0 +1,102 @@ +/* + * Copyright 2015 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "flatbuffers/hash.h" + +enum OutputFormat { + kDecimal, + kHexadecimal, + kHexadecimal0x, +}; + +int main(int argc, char* argv[]) { + const char* name = argv[0]; + if (argc <= 1) { + printf("%s HASH [OPTION]... STRING... [-- STRING...]\n", name); + printf("Available hashing algorithms:\n 32 bit:\n"); + size_t size = sizeof(flatbuffers::kHashFunctions32) / + sizeof(flatbuffers::kHashFunctions32[0]); + for (size_t i = 0; i < size; ++i) { + printf(" * %s\n", flatbuffers::kHashFunctions32[i].name); + } + printf(" 64 bit:\n"); + size = sizeof(flatbuffers::kHashFunctions64) / + sizeof(flatbuffers::kHashFunctions64[0]); + for (size_t i = 0; i < size; ++i) { + printf(" * %s\n", flatbuffers::kHashFunctions64[i].name); + } + printf( + " -d Output hash in decimal.\n" + " -x Output hash in hexadecimal.\n" + " -0x Output hash in hexadecimal and prefix with 0x.\n" + " -c Append the string to the output in a c-style comment.\n"); + return 0; + } + + const char* hash_algorithm = argv[1]; + + flatbuffers::NamedHashFunction::HashFunction hash_function32 = + flatbuffers::FindHashFunction32(hash_algorithm); + flatbuffers::NamedHashFunction::HashFunction hash_function64 = + flatbuffers::FindHashFunction64(hash_algorithm); + + if (!hash_function32 && !hash_function64) { + printf("\"%s\" is not a known hash algorithm.\n", hash_algorithm); + return 0; + } + + OutputFormat output_format = kHexadecimal; + bool annotate = false; + bool escape_dash = false; + for (int i = 2; i < argc; i++) { + const char* arg = argv[i]; + if (!escape_dash && arg[0] == '-') { + std::string opt = arg; + if (opt == "-d") output_format = kDecimal; + else if (opt == "-x") output_format = kHexadecimal; + else if (opt == "-0x") output_format = kHexadecimal0x; + else if (opt == "-c") annotate = true; + else if (opt == "--") escape_dash = true; + else printf("Unrecognized argument: \"%s\"\n", arg); + } else { + std::stringstream ss; + if (output_format == kDecimal) { + ss << std::dec; + } else if (output_format == kHexadecimal) { + ss << std::hex; + } else if (output_format == kHexadecimal0x) { + ss << std::hex; + ss << "0x"; + } + if (hash_function32) + ss << hash_function32(arg); + else if (hash_function64) + ss << hash_function64(arg); + + if (annotate) + ss << " /* \"" << arg << "\" */"; + + ss << "\n"; + + std::cout << ss.str(); + } + } + return 0; +} + diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index d3deef7f9..968ad017f 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -18,6 +18,7 @@ #include #include "flatbuffers/flatbuffers.h" +#include "flatbuffers/hash.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" @@ -392,6 +393,27 @@ void Parser::ParseField(StructDef &struct_def) { field.doc_comment = dc; ParseMetaData(field); field.deprecated = field.attributes.Lookup("deprecated") != nullptr; + auto hash_name = field.attributes.Lookup("hash"); + if (hash_name) { + switch (type.base_type) { + case BASE_TYPE_INT: + case BASE_TYPE_UINT: { + if (FindHashFunction32(hash_name->constant.c_str()) == nullptr) + Error("Unknown hashing algorithm for 32 bit types: " + + hash_name->constant); + break; + } + case BASE_TYPE_LONG: + case BASE_TYPE_ULONG: { + if (FindHashFunction64(hash_name->constant.c_str()) == nullptr) + Error("Unknown hashing algorithm for 64 bit types: " + + hash_name->constant); + break; + } + default: + Error("only int, uint, long and ulong data types support hashing."); + } + } if (field.deprecated && struct_def.fixed) Error("can't deprecate fields in a struct"); field.required = field.attributes.Lookup("required") != nullptr; @@ -465,6 +487,18 @@ void Parser::ParseAnyValue(Value &val, FieldDef *field) { val.constant = NumToString(ParseVector(val.type.VectorType())); break; } + case BASE_TYPE_INT: + case BASE_TYPE_UINT: + case BASE_TYPE_LONG: + case BASE_TYPE_ULONG: { + if (field && field->attributes.Lookup("hash") && + (token_ == kTokenIdentifier || token_ == kTokenStringConstant)) { + ParseHash(val, field); + } else { + ParseSingleValue(val); + } + break; + } default: ParseSingleValue(val); break; @@ -583,7 +617,7 @@ uoffset_t Parser::ParseVector(const Type &type) { if ((!strict_json_ || !count) && IsNext(']')) break; Value val; val.type = type; - ParseAnyValue(val, NULL); + ParseAnyValue(val, nullptr); field_stack_.push_back(std::make_pair(val, nullptr)); count++; if (IsNext(']')) break; @@ -689,6 +723,31 @@ int64_t Parser::ParseIntegerFromString(Type &type) { return result; } + +void Parser::ParseHash(Value &e, FieldDef* field) { + assert(field); + Value *hash_name = field->attributes.Lookup("hash"); + switch (e.type.base_type) { + case BASE_TYPE_INT: + case BASE_TYPE_UINT: { + auto hash = FindHashFunction32(hash_name->constant.c_str()); + uint32_t hashed_value = hash(attribute_.c_str()); + e.constant = NumToString(hashed_value); + break; + } + case BASE_TYPE_LONG: + case BASE_TYPE_ULONG: { + auto hash = FindHashFunction64(hash_name->constant.c_str()); + uint64_t hashed_value = hash(attribute_.c_str()); + e.constant = NumToString(hashed_value); + break; + } + default: + assert(0); + } + Next(); +} + void Parser::ParseSingleValue(Value &e) { // First check if this could be a string/identifier enum value: if (e.type.base_type != BASE_TYPE_STRING && diff --git a/tests/monster_test.fbs b/tests/monster_test.fbs index 01b9e5965..26106574d 100755 --- a/tests/monster_test.fbs +++ b/tests/monster_test.fbs @@ -44,6 +44,14 @@ table Monster { testnestedflatbuffer:[ubyte] (id:13, nested_flatbuffer: "Monster"); testempty:Stat (id:14); testbool:bool (id:15); + testhashs32_fnv1:int (id:16, hash:"fnv1_32"); + testhashu32_fnv1:uint (id:17, hash:"fnv1_32"); + testhashs64_fnv1:long (id:18, hash:"fnv1_64"); + testhashu64_fnv1:ulong (id:19, hash:"fnv1_64"); + testhashs32_fnv1a:int (id:20, hash:"fnv1a_32"); + testhashu32_fnv1a:uint (id:21, hash:"fnv1a_32"); + testhashs64_fnv1a:long (id:22, hash:"fnv1a_64"); + testhashu64_fnv1a:ulong (id:23, hash:"fnv1a_64"); } root_type Monster; diff --git a/tests/monster_test_generated.h b/tests/monster_test_generated.h index 74bd0d884..2364c01d7 100755 --- a/tests/monster_test_generated.h +++ b/tests/monster_test_generated.h @@ -141,6 +141,10 @@ struct Monster FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { const Monster *testnestedflatbuffer_nested_root() const { return flatbuffers::GetRoot(testnestedflatbuffer()->Data()); } const Stat *testempty() const { return GetPointer(32); } uint8_t testbool() const { return GetField(34, 0); } + int32_t testhashs32_fnv1() const { return GetField(36, 0); } + uint32_t testhashu32_fnv1() const { return GetField(38, 0); } + int64_t testhashs64_fnv1() const { return GetField(40, 0); } + uint64_t testhashu64_fnv1() const { return GetField(42, 0); } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && VerifyField(verifier, 4 /* pos */) && @@ -169,6 +173,10 @@ struct Monster FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { VerifyField(verifier, 32 /* testempty */) && verifier.VerifyTable(testempty()) && VerifyField(verifier, 34 /* testbool */) && + VerifyField(verifier, 36 /* testhashs32_fnv1 */) && + VerifyField(verifier, 38 /* testhashu32_fnv1 */) && + VerifyField(verifier, 40 /* testhashs64_fnv1 */) && + VerifyField(verifier, 42 /* testhashu64_fnv1 */) && verifier.EndTable(); } }; @@ -191,10 +199,14 @@ struct MonsterBuilder { void add_testnestedflatbuffer(flatbuffers::Offset> testnestedflatbuffer) { fbb_.AddOffset(30, testnestedflatbuffer); } void add_testempty(flatbuffers::Offset testempty) { fbb_.AddOffset(32, testempty); } void add_testbool(uint8_t testbool) { fbb_.AddElement(34, testbool, 0); } + void add_testhashs32_fnv1(int32_t testhashs32_fnv1) { fbb_.AddElement(36, testhashs32_fnv1, 0); } + void add_testhashu32_fnv1(uint32_t testhashu32_fnv1) { fbb_.AddElement(38, testhashu32_fnv1, 0); } + void add_testhashs64_fnv1(int64_t testhashs64_fnv1) { fbb_.AddElement(40, testhashs64_fnv1, 0); } + void add_testhashu64_fnv1(uint64_t testhashu64_fnv1) { fbb_.AddElement(42, testhashu64_fnv1, 0); } MonsterBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); } MonsterBuilder &operator=(const MonsterBuilder &); flatbuffers::Offset Finish() { - auto o = flatbuffers::Offset(fbb_.EndTable(start_, 16)); + auto o = flatbuffers::Offset(fbb_.EndTable(start_, 20)); fbb_.Required(o, 10); // name return o; } @@ -215,8 +227,16 @@ inline flatbuffers::Offset CreateMonster(flatbuffers::FlatBufferBuilder flatbuffers::Offset enemy = 0, flatbuffers::Offset> testnestedflatbuffer = 0, flatbuffers::Offset testempty = 0, - uint8_t testbool = 0) { + uint8_t testbool = 0, + int32_t testhashs32_fnv1 = 0, + uint32_t testhashu32_fnv1 = 0, + int64_t testhashs64_fnv1 = 0, + uint64_t testhashu64_fnv1 = 0) { MonsterBuilder builder_(_fbb); + builder_.add_testhashu64_fnv1(testhashu64_fnv1); + builder_.add_testhashs64_fnv1(testhashs64_fnv1); + builder_.add_testhashu32_fnv1(testhashu32_fnv1); + builder_.add_testhashs32_fnv1(testhashs32_fnv1); builder_.add_testempty(testempty); builder_.add_testnestedflatbuffer(testnestedflatbuffer); builder_.add_enemy(enemy); diff --git a/tests/monsterdata_test.golden b/tests/monsterdata_test.golden index a79095a13..73afc4241 100644 --- a/tests/monsterdata_test.golden +++ b/tests/monsterdata_test.golden @@ -36,5 +36,13 @@ testarrayofstring: [ "test1", "test2" - ] + ], + testhashs32_fnv1: -579221183, + testhashu32_fnv1: 3715746113, + testhashs64_fnv1: 7930699090847568257, + testhashu64_fnv1: 7930699090847568257, + testhashs32_fnv1a: -1904106383, + testhashu32_fnv1a: 2390860913, + testhashs64_fnv1a: 4898026182817603057, + testhashu64_fnv1a: 4898026182817603057 } diff --git a/tests/monsterdata_test.json b/tests/monsterdata_test.json index a79095a13..6525d5458 100755 --- a/tests/monsterdata_test.json +++ b/tests/monsterdata_test.json @@ -36,5 +36,13 @@ testarrayofstring: [ "test1", "test2" - ] + ], + testhashs32_fnv1: "This string is being hashed!", + testhashu32_fnv1: "This string is being hashed!", + testhashs64_fnv1: "This string is being hashed!", + testhashu64_fnv1: "This string is being hashed!", + testhashs32_fnv1a: "This string is being hashed!", + testhashu32_fnv1a: "This string is being hashed!", + testhashs64_fnv1a: "This string is being hashed!", + testhashu64_fnv1a: "This string is being hashed!", } diff --git a/tests/monsterdata_test.mon b/tests/monsterdata_test.mon index 1b79b7eb4..feb29e83d 100644 Binary files a/tests/monsterdata_test.mon and b/tests/monsterdata_test.mon differ