Added Google benchmarks (and gtests) (#6920)

* Added Google benchmarks (and gtests)

* Default building benchmarks to OFF as it requires c++11

* Separate benchmark CMakeLists.txt to its own file

* Move output directory to target just flatbenchmark

benchmarks/cpp/bench.h

@@ -0,0 +1,19 @@
+#ifndef BENCHMARKS_CPP_BENCH_H_
+#define BENCHMARKS_CPP_BENCH_H_
+
+#include <cstdint>
+
+struct Bench {
+  virtual ~Bench() {}
+
+  inline void Add(int64_t value) { sum += value; }
+
+  virtual uint8_t *Encode(void *buf, int64_t &len) = 0;
+  virtual void *Decode(void *buf, int64_t len) = 0;
+  virtual int64_t Use(void *decoded) = 0;
+  virtual void Dealloc(void *decoded) = 0;
+
+  int64_t sum = 0;
+};
+
+#endif // BENCHMARKS_CPP_BENCH_H_

benchmarks/cpp/benchmark_main.cpp

@@ -0,0 +1,61 @@
+#include <benchmark/benchmark.h>
+#include <gtest/gtest.h>
+
+#include "benchmarks/cpp/bench.h"
+#include "benchmarks/cpp/flatbuffers/fb_bench.h"
+
+static void BM_Flatbuffers_Encode(benchmark::State &state) {
+  const int64_t kBufferLength = 1024;
+  uint8_t buffer[kBufferLength];
+
+  int64_t length;
+
+  StaticAllocator allocator(&buffer[0]);
+  std::unique_ptr<Bench> bench = NewFlatBuffersBench(kBufferLength, &allocator);
+
+  for (auto _ : state) {
+    bench->Encode(buffer, length);
+    benchmark::DoNotOptimize(length);
+  }
+}
+BENCHMARK(BM_Flatbuffers_Encode);
+
+static void BM_Flatbuffers_Decode(benchmark::State &state) {
+  const int64_t kBufferLength = 1024;
+  uint8_t buffer[kBufferLength];
+
+  int64_t length;
+
+  StaticAllocator allocator(&buffer[0]);
+  std::unique_ptr<Bench> bench = NewFlatBuffersBench(kBufferLength, &allocator);
+
+  uint8_t* encoded = bench->Encode(buffer, length);
+  
+  for (auto _ : state) {
+    void* decoded = bench->Decode(encoded, length);
+    benchmark::DoNotOptimize(decoded);
+  }
+}
+BENCHMARK(BM_Flatbuffers_Decode);
+
+static void BM_Flatbuffers_Use(benchmark::State &state) {
+  const int64_t kBufferLength = 1024;
+  uint8_t buffer[kBufferLength];
+
+  int64_t length;
+
+  StaticAllocator allocator(&buffer[0]);
+  std::unique_ptr<Bench> bench = NewFlatBuffersBench(kBufferLength, &allocator);
+
+  uint8_t* encoded = bench->Encode(buffer, length);
+  void* decoded = bench->Decode(encoded, length);
+
+  int64_t sum = 0;
+
+  for (auto _ : state) {
+    sum = bench->Use(decoded);
+  }
+
+  EXPECT_EQ(sum , 218812692406581874);
+}
+BENCHMARK(BM_Flatbuffers_Use);

benchmarks/cpp/flatbuffers/bench.fbs

@@ -0,0 +1,52 @@
+// Copyright 2021 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+// trying to represent a typical mix of datatypes:
+// 1 array of 3 elements, each element: 1 string, 3 nested objects, 9 scalars
+// root element has the array, additional string and an enum
+
+namespace benchmarks_flatbuffers;
+
+enum Enum : short { Apples, Pears, Bananas}
+
+struct Foo {
+  id:ulong;
+  count:short;
+  prefix:byte;
+  length:uint;
+}
+
+struct Bar {
+  parent:Foo;
+  time:int;
+  ratio:float;
+  size:ushort;
+}
+
+table FooBar {
+  sibling:Bar;
+  name:string;
+  rating:double;
+  postfix:ubyte;
+}
+
+table FooBarContainer {
+  list:[FooBar];  // 3 copies of the above
+  initialized:bool;
+  fruit:Enum;
+  location:string;
+}
+
+root_type FooBarContainer;

benchmarks/cpp/flatbuffers/bench_generated.h

@@ -0,0 +1,347 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+
+#ifndef FLATBUFFERS_GENERATED_BENCH_BENCHMARKS_FLATBUFFERS_H_
+#define FLATBUFFERS_GENERATED_BENCH_BENCHMARKS_FLATBUFFERS_H_
+
+#include "flatbuffers/flatbuffers.h"
+
+namespace benchmarks_flatbuffers {
+
+struct Foo;
+
+struct Bar;
+
+struct FooBar;
+struct FooBarBuilder;
+
+struct FooBarContainer;
+struct FooBarContainerBuilder;
+
+enum Enum : int16_t {
+  Enum_Apples = 0,
+  Enum_Pears = 1,
+  Enum_Bananas = 2,
+  Enum_MIN = Enum_Apples,
+  Enum_MAX = Enum_Bananas
+};
+
+inline const Enum (&EnumValuesEnum())[3] {
+  static const Enum values[] = {
+    Enum_Apples,
+    Enum_Pears,
+    Enum_Bananas
+  };
+  return values;
+}
+
+inline const char * const *EnumNamesEnum() {
+  static const char * const names[4] = {
+    "Apples",
+    "Pears",
+    "Bananas",
+    nullptr
+  };
+  return names;
+}
+
+inline const char *EnumNameEnum(Enum e) {
+  if (flatbuffers::IsOutRange(e, Enum_Apples, Enum_Bananas)) return "";
+  const size_t index = static_cast<size_t>(e);
+  return EnumNamesEnum()[index];
+}
+
+FLATBUFFERS_MANUALLY_ALIGNED_STRUCT(8) Foo FLATBUFFERS_FINAL_CLASS {
+ private:
+  uint64_t id_;
+  int16_t count_;
+  int8_t prefix_;
+  int8_t padding0__;
+  uint32_t length_;
+
+ public:
+  Foo()
+      : id_(0),
+        count_(0),
+        prefix_(0),
+        padding0__(0),
+        length_(0) {
+    (void)padding0__;
+  }
+  Foo(uint64_t _id, int16_t _count, int8_t _prefix, uint32_t _length)
+      : id_(flatbuffers::EndianScalar(_id)),
+        count_(flatbuffers::EndianScalar(_count)),
+        prefix_(flatbuffers::EndianScalar(_prefix)),
+        padding0__(0),
+        length_(flatbuffers::EndianScalar(_length)) {
+    (void)padding0__;
+  }
+  uint64_t id() const {
+    return flatbuffers::EndianScalar(id_);
+  }
+  int16_t count() const {
+    return flatbuffers::EndianScalar(count_);
+  }
+  int8_t prefix() const {
+    return flatbuffers::EndianScalar(prefix_);
+  }
+  uint32_t length() const {
+    return flatbuffers::EndianScalar(length_);
+  }
+};
+FLATBUFFERS_STRUCT_END(Foo, 16);
+
+FLATBUFFERS_MANUALLY_ALIGNED_STRUCT(8) Bar FLATBUFFERS_FINAL_CLASS {
+ private:
+  benchmarks_flatbuffers::Foo parent_;
+  int32_t time_;
+  float ratio_;
+  uint16_t size_;
+  int16_t padding0__;  int32_t padding1__;
+
+ public:
+  Bar()
+      : parent_(),
+        time_(0),
+        ratio_(0),
+        size_(0),
+        padding0__(0),
+        padding1__(0) {
+    (void)padding0__;
+    (void)padding1__;
+  }
+  Bar(const benchmarks_flatbuffers::Foo &_parent, int32_t _time, float _ratio, uint16_t _size)
+      : parent_(_parent),
+        time_(flatbuffers::EndianScalar(_time)),
+        ratio_(flatbuffers::EndianScalar(_ratio)),
+        size_(flatbuffers::EndianScalar(_size)),
+        padding0__(0),
+        padding1__(0) {
+    (void)padding0__;
+    (void)padding1__;
+  }
+  const benchmarks_flatbuffers::Foo &parent() const {
+    return parent_;
+  }
+  int32_t time() const {
+    return flatbuffers::EndianScalar(time_);
+  }
+  float ratio() const {
+    return flatbuffers::EndianScalar(ratio_);
+  }
+  uint16_t size() const {
+    return flatbuffers::EndianScalar(size_);
+  }
+};
+FLATBUFFERS_STRUCT_END(Bar, 32);
+
+struct FooBar FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef FooBarBuilder Builder;
+  enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
+    VT_SIBLING = 4,
+    VT_NAME = 6,
+    VT_RATING = 8,
+    VT_POSTFIX = 10
+  };
+  const benchmarks_flatbuffers::Bar *sibling() const {
+    return GetStruct<const benchmarks_flatbuffers::Bar *>(VT_SIBLING);
+  }
+  const flatbuffers::String *name() const {
+    return GetPointer<const flatbuffers::String *>(VT_NAME);
+  }
+  double rating() const {
+    return GetField<double>(VT_RATING, 0.0);
+  }
+  uint8_t postfix() const {
+    return GetField<uint8_t>(VT_POSTFIX, 0);
+  }
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           VerifyField<benchmarks_flatbuffers::Bar>(verifier, VT_SIBLING) &&
+           VerifyOffset(verifier, VT_NAME) &&
+           verifier.VerifyString(name()) &&
+           VerifyField<double>(verifier, VT_RATING) &&
+           VerifyField<uint8_t>(verifier, VT_POSTFIX) &&
+           verifier.EndTable();
+  }
+};
+
+struct FooBarBuilder {
+  typedef FooBar Table;
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  void add_sibling(const benchmarks_flatbuffers::Bar *sibling) {
+    fbb_.AddStruct(FooBar::VT_SIBLING, sibling);
+  }
+  void add_name(flatbuffers::Offset<flatbuffers::String> name) {
+    fbb_.AddOffset(FooBar::VT_NAME, name);
+  }
+  void add_rating(double rating) {
+    fbb_.AddElement<double>(FooBar::VT_RATING, rating, 0.0);
+  }
+  void add_postfix(uint8_t postfix) {
+    fbb_.AddElement<uint8_t>(FooBar::VT_POSTFIX, postfix, 0);
+  }
+  explicit FooBarBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  flatbuffers::Offset<FooBar> Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset<FooBar>(end);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset<FooBar> CreateFooBar(
+    flatbuffers::FlatBufferBuilder &_fbb,
+    const benchmarks_flatbuffers::Bar *sibling = nullptr,
+    flatbuffers::Offset<flatbuffers::String> name = 0,
+    double rating = 0.0,
+    uint8_t postfix = 0) {
+  FooBarBuilder builder_(_fbb);
+  builder_.add_rating(rating);
+  builder_.add_name(name);
+  builder_.add_sibling(sibling);
+  builder_.add_postfix(postfix);
+  return builder_.Finish();
+}
+
+inline flatbuffers::Offset<FooBar> CreateFooBarDirect(
+    flatbuffers::FlatBufferBuilder &_fbb,
+    const benchmarks_flatbuffers::Bar *sibling = nullptr,
+    const char *name = nullptr,
+    double rating = 0.0,
+    uint8_t postfix = 0) {
+  auto name__ = name ? _fbb.CreateString(name) : 0;
+  return benchmarks_flatbuffers::CreateFooBar(
+      _fbb,
+      sibling,
+      name__,
+      rating,
+      postfix);
+}
+
+struct FooBarContainer FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef FooBarContainerBuilder Builder;
+  enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
+    VT_LIST = 4,
+    VT_INITIALIZED = 6,
+    VT_FRUIT = 8,
+    VT_LOCATION = 10
+  };
+  const flatbuffers::Vector<flatbuffers::Offset<benchmarks_flatbuffers::FooBar>> *list() const {
+    return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<benchmarks_flatbuffers::FooBar>> *>(VT_LIST);
+  }
+  bool initialized() const {
+    return GetField<uint8_t>(VT_INITIALIZED, 0) != 0;
+  }
+  benchmarks_flatbuffers::Enum fruit() const {
+    return static_cast<benchmarks_flatbuffers::Enum>(GetField<int16_t>(VT_FRUIT, 0));
+  }
+  const flatbuffers::String *location() const {
+    return GetPointer<const flatbuffers::String *>(VT_LOCATION);
+  }
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           VerifyOffset(verifier, VT_LIST) &&
+           verifier.VerifyVector(list()) &&
+           verifier.VerifyVectorOfTables(list()) &&
+           VerifyField<uint8_t>(verifier, VT_INITIALIZED) &&
+           VerifyField<int16_t>(verifier, VT_FRUIT) &&
+           VerifyOffset(verifier, VT_LOCATION) &&
+           verifier.VerifyString(location()) &&
+           verifier.EndTable();
+  }
+};
+
+struct FooBarContainerBuilder {
+  typedef FooBarContainer Table;
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  void add_list(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<benchmarks_flatbuffers::FooBar>>> list) {
+    fbb_.AddOffset(FooBarContainer::VT_LIST, list);
+  }
+  void add_initialized(bool initialized) {
+    fbb_.AddElement<uint8_t>(FooBarContainer::VT_INITIALIZED, static_cast<uint8_t>(initialized), 0);
+  }
+  void add_fruit(benchmarks_flatbuffers::Enum fruit) {
+    fbb_.AddElement<int16_t>(FooBarContainer::VT_FRUIT, static_cast<int16_t>(fruit), 0);
+  }
+  void add_location(flatbuffers::Offset<flatbuffers::String> location) {
+    fbb_.AddOffset(FooBarContainer::VT_LOCATION, location);
+  }
+  explicit FooBarContainerBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  flatbuffers::Offset<FooBarContainer> Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset<FooBarContainer>(end);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset<FooBarContainer> CreateFooBarContainer(
+    flatbuffers::FlatBufferBuilder &_fbb,
+    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<benchmarks_flatbuffers::FooBar>>> list = 0,
+    bool initialized = false,
+    benchmarks_flatbuffers::Enum fruit = benchmarks_flatbuffers::Enum_Apples,
+    flatbuffers::Offset<flatbuffers::String> location = 0) {
+  FooBarContainerBuilder builder_(_fbb);
+  builder_.add_location(location);
+  builder_.add_list(list);
+  builder_.add_fruit(fruit);
+  builder_.add_initialized(initialized);
+  return builder_.Finish();
+}
+
+inline flatbuffers::Offset<FooBarContainer> CreateFooBarContainerDirect(
+    flatbuffers::FlatBufferBuilder &_fbb,
+    const std::vector<flatbuffers::Offset<benchmarks_flatbuffers::FooBar>> *list = nullptr,
+    bool initialized = false,
+    benchmarks_flatbuffers::Enum fruit = benchmarks_flatbuffers::Enum_Apples,
+    const char *location = nullptr) {
+  auto list__ = list ? _fbb.CreateVector<flatbuffers::Offset<benchmarks_flatbuffers::FooBar>>(*list) : 0;
+  auto location__ = location ? _fbb.CreateString(location) : 0;
+  return benchmarks_flatbuffers::CreateFooBarContainer(
+      _fbb,
+      list__,
+      initialized,
+      fruit,
+      location__);
+}
+
+inline const benchmarks_flatbuffers::FooBarContainer *GetFooBarContainer(const void *buf) {
+  return flatbuffers::GetRoot<benchmarks_flatbuffers::FooBarContainer>(buf);
+}
+
+inline const benchmarks_flatbuffers::FooBarContainer *GetSizePrefixedFooBarContainer(const void *buf) {
+  return flatbuffers::GetSizePrefixedRoot<benchmarks_flatbuffers::FooBarContainer>(buf);
+}
+
+inline bool VerifyFooBarContainerBuffer(
+    flatbuffers::Verifier &verifier) {
+  return verifier.VerifyBuffer<benchmarks_flatbuffers::FooBarContainer>(nullptr);
+}
+
+inline bool VerifySizePrefixedFooBarContainerBuffer(
+    flatbuffers::Verifier &verifier) {
+  return verifier.VerifySizePrefixedBuffer<benchmarks_flatbuffers::FooBarContainer>(nullptr);
+}
+
+inline void FinishFooBarContainerBuffer(
+    flatbuffers::FlatBufferBuilder &fbb,
+    flatbuffers::Offset<benchmarks_flatbuffers::FooBarContainer> root) {
+  fbb.Finish(root);
+}
+
+inline void FinishSizePrefixedFooBarContainerBuffer(
+    flatbuffers::FlatBufferBuilder &fbb,
+    flatbuffers::Offset<benchmarks_flatbuffers::FooBarContainer> root) {
+  fbb.FinishSizePrefixed(root);
+}
+
+}  // namespace benchmarks_flatbuffers
+
+#endif  // FLATBUFFERS_GENERATED_BENCH_BENCHMARKS_FLATBUFFERS_H_

benchmarks/cpp/flatbuffers/fb_bench.cpp

@@ -0,0 +1,81 @@
+#include "benchmarks/cpp/flatbuffers/fb_bench.h"
+
+#include <cstdint>
+#include <memory>
+
+#include "benchmarks/cpp/bench.h"
+#include "benchmarks/cpp/flatbuffers/bench_generated.h"
+#include "flatbuffers/flatbuffers.h"
+
+using namespace flatbuffers;
+using namespace benchmarks_flatbuffers;
+
+namespace {
+
+struct FlatBufferBench : Bench {
+  explicit FlatBufferBench(int64_t initial_size,
+                           Allocator *allocator)
+      : fbb(initial_size, allocator, false) {}
+
+  uint8_t *Encode(void *, int64_t &len) override {
+    fbb.Clear();
+
+    const int kVectorLength = 3;
+    Offset<FooBar> vec[kVectorLength];
+
+    for(int i = 0; i < kVectorLength; ++i) {
+      Foo foo(0xABADCAFEABADCAFE + i, 10000 + i, '@' + i, 1000000 + i);
+      Bar bar(foo, 123456 + i, 3.14159f + i, 10000 + i);
+      auto name = fbb.CreateString("Hello, World!");
+      auto foobar = CreateFooBar(fbb, &bar, name, 3.1415432432445543543 + i,
+                                 '!' + i);
+      vec[i] = foobar;
+    }
+    auto location = fbb.CreateString("http://google.com/flatbuffers/");
+    auto foobarvec = fbb.CreateVector(vec, kVectorLength);
+    auto foobarcontainer = CreateFooBarContainer(fbb, foobarvec, true,
+                                                 Enum_Bananas, location);
+    fbb.Finish(foobarcontainer);
+
+    len = fbb.GetSize();
+    return fbb.GetBufferPointer();
+  }
+
+  int64_t Use(void *decoded) override {
+    sum = 0;
+    auto foobarcontainer = GetFooBarContainer(decoded);
+    sum = 0;
+    Add(foobarcontainer->initialized());
+    Add(foobarcontainer->location()->Length());
+    Add(foobarcontainer->fruit());
+    for (unsigned int i = 0; i < foobarcontainer->list()->Length(); i++) {
+      auto foobar = foobarcontainer->list()->Get(i);
+      Add(foobar->name()->Length());
+      Add(foobar->postfix());
+      Add(static_cast<int64_t>(foobar->rating()));
+      auto bar = foobar->sibling();
+      Add(static_cast<int64_t>(bar->ratio()));
+      Add(bar->size());
+      Add(bar->time());
+      auto &foo = bar->parent();
+      Add(foo.count());
+      Add(foo.id());
+      Add(foo.length());
+      Add(foo.prefix());
+    }
+    return sum;
+  }
+
+  void *Decode(void *buffer, int64_t) override { return buffer; }
+  void Dealloc(void *) override{};
+
+  FlatBufferBuilder fbb;
+};
+
+}  // namespace
+
+std::unique_ptr<Bench> NewFlatBuffersBench(int64_t initial_size,
+                                           Allocator *allocator) {
+  return std::unique_ptr<FlatBufferBench>(
+      new FlatBufferBench(initial_size, allocator));
+}

benchmarks/cpp/flatbuffers/fb_bench.h

@@ -0,0 +1,23 @@
+#ifndef BENCHMARKS_CPP_FLATBUFFERS_FB_BENCH_H_
+#define BENCHMARKS_CPP_FLATBUFFERS_FB_BENCH_H_
+
+#include <cstdint>
+#include <memory>
+
+#include "benchmarks/cpp/bench.h"
+#include "include/flatbuffers/flatbuffers.h"
+
+struct StaticAllocator : public flatbuffers::Allocator {
+  explicit StaticAllocator(uint8_t *buffer) : buffer_(buffer) {}
+
+  uint8_t *allocate(size_t) override { return buffer_; }
+
+  void deallocate(uint8_t *, size_t) override {}
+
+  uint8_t *buffer_;
+};
+
+std::unique_ptr<Bench> NewFlatBuffersBench(
+    int64_t initial_size = 1024, flatbuffers::Allocator *allocator = nullptr);
+
+#endif  // BENCHMARKS_CPP_FLATBUFFERS_FB_BENCH_H_