Cleanup Benchmarks

Bigfoot/Benchmarks/Engine/BigFile/BigFile.cpp

@@ -15,6 +15,7 @@
 #include <AssetsForBenchmarking/Asset/BenchmarkAssetReferred.hpp>
 
 #include <EASTL/optional.h>
+#include <EASTL/span.h>
 #include <EASTL/vector.h>
 #include <benchmark/benchmark.h>
 #include <effolkronium/random.hpp>
@@ -34,18 +35,17 @@ class BigFileFixture: public benchmark::Fixture
     static constexpr std::uint32_t REFS_PER_ASSET = 4;
     static constexpr std::size_t ASSET_PAYLOAD_SIZE = 256;
 
-    void SetUp(benchmark::State&) override
+    void SetUp([[maybe_unused]] benchmark::State& p_state) override
     {
         // The DB is populated exactly once for the entire benchmark process.
         // SetUp is called before every calibration round (Google Benchmark doubles
         // iteration counts until it reaches a stable measurement), so rebuilding
         // the DB each time would dominate the total benchmark time.
-        static const eastl::vector<UUID> s_uuids = BuildData();
-        m_uuids = &s_uuids;
+        m_uuids = BuildData();
         m_bigFile.emplace(File {BIGFILE_ENGINEBENCHMARKS_LOCATION});
     }
 
-    void TearDown(benchmark::State&) override
+    void TearDown([[maybe_unused]] benchmark::State& p_state) override
     {
         m_bigFile.reset();
     }
@@ -57,7 +57,7 @@ class BigFileFixture: public benchmark::Fixture
         batch.reserve(p_batchSize);
         for (std::size_t i = 0; i < p_batchSize; ++i)
         {
-            batch.push_back((*m_uuids)[Random::get<std::size_t>(0, m_uuids->size() - 1)]);
+            batch.push_back(m_uuids[Random::get<std::size_t>(0, m_uuids.size() - 1)]);
         }
 
         return batch;
@@ -78,42 +78,47 @@ class BigFileFixture: public benchmark::Fixture
     }
 
     eastl::optional<BigFile> m_bigFile;
-    const eastl::vector<UUID>* m_uuids = nullptr;
+    eastl::span<const UUID> m_uuids;
 
   private:
-    static eastl::vector<UUID> BuildData()
+    static const eastl::vector<UUID>& BuildData()
     {
-        eastl::vector<UUID> uuids;
-        eastl::vector<eastl::vector<std::byte>> flatbuffers;
-        eastl::vector<eastl::vector<UUID>> hardReferences;
-        uuids.reserve(ASSET_COUNT);
-        flatbuffers.reserve(ASSET_COUNT);
-        hardReferences.reserve(ASSET_COUNT);
-
-        for (std::uint32_t i = 0; i < ASSET_COUNT; ++i)
+        static const eastl::vector<UUID> s_uuids = []()
         {
-            BenchmarkAsset asset;
-            asset.SetName(asset.GetHeader().uuid.ToString().c_str());
-            asset.GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB});
-            for (std::uint32_t j = 0; j < REFS_PER_ASSET; ++j)
-            {
-                BenchmarkAssetReferred assetReferred;
-                assetReferred.SetName(assetReferred.GetHeader().uuid.ToString().c_str());
+            eastl::vector<UUID> uuids;
+            eastl::vector<eastl::vector<std::byte>> flatbuffers;
+            eastl::vector<eastl::vector<UUID>> hardReferences;
+            uuids.reserve(ASSET_COUNT);
+            flatbuffers.reserve(ASSET_COUNT);
+            hardReferences.reserve(ASSET_COUNT);
 
-                asset.GetAsset().refs.emplace_back(assetReferred.GetHeader().uuid);
+            for (std::uint32_t i = 0; i < ASSET_COUNT; ++i)
+            {
+                BenchmarkAsset asset;
+                asset.SetName(asset.GetHeader().uuid.ToString().c_str());
+                asset.GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB});
+                for (std::uint32_t j = 0; j < REFS_PER_ASSET; ++j)
+                {
+                    BenchmarkAssetReferred assetReferred;
+                    assetReferred.SetName(assetReferred.GetHeader().uuid.ToString().c_str());
+
+                    asset.GetAsset().refs.emplace_back(assetReferred.GetHeader().uuid);
+                }
+
+                eastl::vector<std::byte> flatbuffer = asset.Pack();
+                hardReferences.push_back(asset.GetHardReferences(flatbuffer));
+                flatbuffers.push_back(std::move(flatbuffer));
+                uuids.push_back(asset.GetHeader().uuid);
             }
 
-            eastl::vector<std::byte> flatbuffer = asset.Pack();
-            hardReferences.push_back(asset.GetHardReferences(flatbuffer));
-            flatbuffers.push_back(std::move(flatbuffer));
-            uuids.push_back(asset.GetHeader().uuid);
-        }
+            BigFile bigFile {File {BIGFILE_ENGINEBENCHMARKS_LOCATION}};
+            bigFile.Clear();
+            bigFile.SaveAssetFlatbuffers(uuids, flatbuffers, hardReferences);
 
-        BigFile bigFile {File {BIGFILE_ENGINEBENCHMARKS_LOCATION}};
-        bigFile.Clear();
-        bigFile.SaveAssetFlatbuffers(uuids, flatbuffers, hardReferences);
+            return uuids;
+        }();
 
-        return uuids;
+        return s_uuids;
     }
 };
 
@@ -126,7 +131,7 @@ BENCHMARK_DEFINE_F(BigFileFixture, GetAssetFlatbuffer)(benchmark::State& state)
     for (auto _: state)
     {
         eastl::vector<std::byte> flatbuffer = m_bigFile->GetAssetFlatbuffer(
-            (*m_uuids)[Random::get<std::size_t>(0, m_uuids->size() - 1)]);
+            m_uuids[Random::get<std::size_t>(0, m_uuids.size() - 1)]);
         benchmark::DoNotOptimize(flatbuffer);
 
         bytesProcessed += static_cast<std::int64_t>(flatbuffer.size());
@@ -183,10 +188,11 @@ BENCHMARK_DEFINE_F(BigFileFixture, GetAssetHardReferences)(benchmark::State& sta
     for (auto _: state)
     {
         benchmark::DoNotOptimize(
-            m_bigFile->GetAssetHardReferences((*m_uuids)[Random::get<std::size_t>(0, m_uuids->size() - 1)]));
+            m_bigFile->GetAssetHardReferences(m_uuids[Random::get<std::size_t>(0, m_uuids.size() - 1)]));
     }
 
     state.SetItemsProcessed(state.iterations());
+    state.SetBytesProcessed(static_cast<std::int64_t>(state.iterations()) * REFS_PER_ASSET * UUID::UUID_BYTE_SIZE);
 }
 
 BENCHMARK_REGISTER_F(BigFileFixture, GetAssetHardReferences)->Name("BigFile/GetAssetHardReferences")->UseRealTime();
@@ -208,6 +214,8 @@ BENCHMARK_DEFINE_F(BigFileFixture, GetAssetsHardReferences)(benchmark::State& st
     }
 
     state.SetItemsProcessed(state.iterations() * batchSize);
+    state.SetBytesProcessed(static_cast<std::int64_t>(state.iterations()) * batchSize * REFS_PER_ASSET *
+                            UUID::UUID_BYTE_SIZE);
 }
 
 BENCHMARK_REGISTER_F(BigFileFixture, GetAssetsHardReferences)
@@ -231,6 +239,10 @@ BENCHMARK_DEFINE_F(BigFileFixture, SaveAssetFlatbuffer)(benchmark::State& state)
         BenchmarkAsset asset;
         asset.SetName(asset.GetHeader().uuid.ToString().c_str());
         asset.GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB});
+        for (std::uint32_t j = 0; j < REFS_PER_ASSET; ++j)
+        {
+            asset.GetAsset().refs.emplace_back(m_uuids[Random::get<std::size_t>(0, m_uuids.size() - 1)]);
+        }
 
         const eastl::vector<std::byte> flatbuffer = asset.Pack();
         const eastl::vector<UUID> hardReferences = asset.GetHardReferences(flatbuffer);
@@ -268,6 +280,10 @@ BENCHMARK_DEFINE_F(BigFileFixture, SaveAssetFlatbuffers)(benchmark::State& state
             BenchmarkAsset asset;
             asset.SetName(asset.GetHeader().uuid.ToString().c_str());
             asset.GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB});
+            for (std::uint32_t j = 0; j < REFS_PER_ASSET; ++j)
+            {
+                asset.GetAsset().refs.emplace_back(m_uuids[Random::get<std::size_t>(0, m_uuids.size() - 1)]);
+            }
 
             eastl::vector<std::byte> flatbuffer = asset.Pack();
             uuids.push_back(asset.GetHeader().uuid);

Bigfoot/Benchmarks/Engine/Sasquatch/Sasquatch.cpp

@@ -15,6 +15,7 @@
 #include <AssetsForBenchmarking/Asset/BenchmarkAssetReferred.hpp>
 
 #include <EASTL/optional.h>
+#include <EASTL/span.h>
 #include <EASTL/vector.h>
 #include <benchmark/benchmark.h>
 #include <effolkronium/random.hpp>
@@ -40,12 +41,11 @@ class SasquatchFixture: public benchmark::Fixture
         // SetUp is called before every calibration round (Google Benchmark doubles
         // iteration counts until it reaches a stable measurement), so rebuilding
         // the file each time would dominate the total benchmark time.
-        static const eastl::vector<UUID> s_uuids = BuildData();
-        m_uuids = &s_uuids;
+        m_uuids = BuildData();
         m_sasquatch.emplace(File {SASQUATCH_ENGINEBENCHMARKS_LOCATION});
     }
 
-    void TearDown(benchmark::State&) override
+    void TearDown([[maybe_unused]] benchmark::State& p_state) override
     {
         m_sasquatch.reset();
     }
@@ -57,7 +57,7 @@ class SasquatchFixture: public benchmark::Fixture
         batch.reserve(p_batchSize);
         for (std::size_t i = 0; i < p_batchSize; ++i)
         {
-            batch.push_back((*m_uuids)[Random::get<std::size_t>(0, m_uuids->size() - 1)]);
+            batch.push_back(m_uuids[Random::get<std::size_t>(0, m_uuids.size() - 1)]);
         }
 
         return batch;
@@ -78,57 +78,62 @@ class SasquatchFixture: public benchmark::Fixture
     }
 
     eastl::optional<Sasquatch> m_sasquatch;
-    const eastl::vector<UUID>* m_uuids = nullptr;
+    eastl::span<const UUID> m_uuids;
 
   private:
-    static eastl::vector<UUID> BuildData()
+    static const eastl::vector<UUID>& BuildData()
     {
-        File {SASQUATCH_ENGINEBENCHMARKS_LOCATION}.Delete();
-
-        eastl::vector<UUID> uuids;
-        eastl::vector<eastl::vector<std::byte>> flatbuffers;
-        eastl::vector<eastl::vector<UUID>> hardReferences;
-        uuids.reserve(ASSET_COUNT);
-        flatbuffers.reserve(ASSET_COUNT);
-        hardReferences.reserve(ASSET_COUNT);
-
-        for (std::uint32_t i = 0; i < ASSET_COUNT; ++i)
+        static const eastl::vector<UUID> s_uuids = []()
         {
-            BenchmarkAsset asset;
-            asset.SetName(asset.GetHeader().uuid.ToString().c_str());
-            asset.GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB});
-            for (std::uint32_t j = 0; j < REFS_PER_ASSET; ++j)
-            {
-                BenchmarkAssetReferred assetReferred;
-                assetReferred.SetName(assetReferred.GetHeader().uuid.ToString().c_str());
+            File {SASQUATCH_ENGINEBENCHMARKS_LOCATION}.Delete();
 
-                asset.GetAsset().refs.emplace_back(assetReferred.GetHeader().uuid);
+            eastl::vector<UUID> uuids;
+            eastl::vector<eastl::vector<std::byte>> flatbuffers;
+            eastl::vector<eastl::vector<UUID>> hardReferences;
+            uuids.reserve(ASSET_COUNT);
+            flatbuffers.reserve(ASSET_COUNT);
+            hardReferences.reserve(ASSET_COUNT);
+
+            for (std::uint32_t i = 0; i < ASSET_COUNT; ++i)
+            {
+                BenchmarkAsset asset;
+                asset.SetName(asset.GetHeader().uuid.ToString().c_str());
+                asset.GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB});
+                for (std::uint32_t j = 0; j < REFS_PER_ASSET; ++j)
+                {
+                    BenchmarkAssetReferred assetReferred;
+                    assetReferred.SetName(assetReferred.GetHeader().uuid.ToString().c_str());
+
+                    asset.GetAsset().refs.emplace_back(assetReferred.GetHeader().uuid);
+                }
+
+                eastl::vector<std::byte> flatbuffer = asset.Pack();
+                hardReferences.push_back(asset.GetHardReferences(flatbuffer));
+                flatbuffers.push_back(std::move(flatbuffer));
+                uuids.push_back(asset.GetHeader().uuid);
             }
 
-            eastl::vector<std::byte> flatbuffer = asset.Pack();
-            hardReferences.push_back(asset.GetHardReferences(flatbuffer));
-            flatbuffers.push_back(std::move(flatbuffer));
-            uuids.push_back(asset.GetHeader().uuid);
-        }
+            std::size_t totalFlatbufferBytes = 0;
+            std::size_t totalHardReferencesBytes = 0;
+            for (std::size_t i = 0; i < uuids.size(); ++i)
+            {
+                totalFlatbufferBytes += flatbuffers[i].size();
+                totalHardReferencesBytes += hardReferences[i].size() * UUID::UUID_BYTE_SIZE;
+            }
 
-        std::size_t totalFlatbufferBytes = 0;
-        std::size_t totalHardReferencesBytes = 0;
-        for (std::size_t i = 0; i < uuids.size(); ++i)
-        {
-            totalFlatbufferBytes += flatbuffers[i].size();
-            totalHardReferencesBytes += hardReferences[i].size() * UUID::UUID_BYTE_SIZE;
-        }
+            SasquatchWriter writer {File {SASQUATCH_ENGINEBENCHMARKS_LOCATION},
+                                    uuids.size(),
+                                    totalFlatbufferBytes,
+                                    totalHardReferencesBytes};
+            for (std::size_t i = 0; i < uuids.size(); ++i)
+            {
+                writer.WriteAsset(uuids[i], flatbuffers[i], hardReferences[i]);
+            }
 
-        SasquatchWriter writer {File {SASQUATCH_ENGINEBENCHMARKS_LOCATION},
-                                uuids.size(),
-                                totalFlatbufferBytes,
-                                totalHardReferencesBytes};
-        for (std::size_t i = 0; i < uuids.size(); ++i)
-        {
-            writer.WriteAsset(uuids[i], flatbuffers[i], hardReferences[i]);
-        }
+            return uuids;
+        }();
 
-        return uuids;
+        return s_uuids;
     }
 };
 
@@ -139,10 +144,11 @@ BENCHMARK_DEFINE_F(SasquatchFixture, GetAssetHardReferences)(benchmark::State& s
     for (auto _: state)
     {
         benchmark::DoNotOptimize(
-            m_sasquatch->GetAssetHardReferences((*m_uuids)[Random::get<std::size_t>(0, m_uuids->size() - 1)]));
+            m_sasquatch->GetAssetHardReferences(m_uuids[Random::get<std::size_t>(0, m_uuids.size() - 1)]));
     }
 
     state.SetItemsProcessed(state.iterations());
+    state.SetBytesProcessed(static_cast<std::int64_t>(state.iterations()) * REFS_PER_ASSET * UUID::UUID_BYTE_SIZE);
 }
 
 BENCHMARK_REGISTER_F(SasquatchFixture, GetAssetHardReferences)->Name("Sasquatch/GetAssetHardReferences")->UseRealTime();
@@ -164,6 +170,8 @@ BENCHMARK_DEFINE_F(SasquatchFixture, GetAssetsHardReferences)(benchmark::State&
     }
 
     state.SetItemsProcessed(state.iterations() * batchSize);
+    state.SetBytesProcessed(static_cast<std::int64_t>(state.iterations()) * batchSize * REFS_PER_ASSET *
+                            UUID::UUID_BYTE_SIZE);
 }
 
 BENCHMARK_REGISTER_F(SasquatchFixture, GetAssetsHardReferences)
@@ -184,7 +192,7 @@ BENCHMARK_DEFINE_F(SasquatchFixture, GetAssetFlatbuffer)(benchmark::State& state
     for (auto _: state)
     {
         eastl::vector<std::byte> flatbuffer = m_sasquatch->GetAssetFlatbuffer(
-            (*m_uuids)[Random::get<std::size_t>(0, m_uuids->size() - 1)]);
+            m_uuids[Random::get<std::size_t>(0, m_uuids.size() - 1)]);
         benchmark::DoNotOptimize(flatbuffer);
 
         bytesProcessed += static_cast<std::int64_t>(flatbuffer.size());

Bigfoot/Benchmarks/Engine/SerializationManager/SerializationManager.cpp

@@ -26,6 +26,7 @@
 #include <AssetsForBenchmarking/Asset/BenchmarkAssetReferred.hpp>
 
 #include <EASTL/optional.h>
+#include <EASTL/span.h>
 #include <EASTL/vector.h>
 #include <ankerl/unordered_dense.h>
 #include <benchmark/benchmark.h>
@@ -46,7 +47,7 @@ class SerializationManagerFixture: public benchmark::Fixture
     static constexpr std::uint32_t REFS_PER_ROOT = 4;
     static constexpr std::size_t ASSET_PAYLOAD_SIZE = 256;
 
-    void SetUp(benchmark::State&) override
+    void SetUp([[maybe_unused]] benchmark::State& p_state) override
     {
         m_previousDataAccessor = g_dataAccessor;
         m_previousAssetLibrary = g_activeAssetLibrary;
@@ -56,8 +57,8 @@ class SerializationManagerFixture: public benchmark::Fixture
         // iteration counts until it reaches a stable measurement), so rebuilding
         // the data store each time would dominate the total benchmark time.
         static const SharedData& s_data = BuildData();
-        m_rootUuids = &s_data.rootUuids;
-        m_leafUuids = &s_data.leafUuids;
+        m_rootUuids = s_data.rootUuids;
+        m_leafUuids = s_data.leafUuids;
         m_rootTotalBytes = &s_data.rootTotalBytes;
 
 #if defined BIGFOOT_TOOLMODE
@@ -69,7 +70,7 @@ class SerializationManagerFixture: public benchmark::Fixture
 #endif
     }
 
-    void TearDown(benchmark::State&) override
+    void TearDown([[maybe_unused]] benchmark::State& p_state) override
     {
 #if defined BIGFOOT_TOOLMODE
         m_bigFile.reset();
@@ -103,7 +104,7 @@ class SerializationManagerFixture: public benchmark::Fixture
         batch.reserve(p_batchSize);
         for (std::size_t i = 0; i < p_batchSize; ++i)
         {
-            batch.push_back((*m_rootUuids)[Random::get<std::size_t>(0, m_rootUuids->size() - 1)]);
+            batch.push_back(m_rootUuids[Random::get<std::size_t>(0, m_rootUuids.size() - 1)]);
         }
 
         return batch;
@@ -138,8 +139,8 @@ class SerializationManagerFixture: public benchmark::Fixture
     DataAccessor* m_previousDataAccessor = nullptr;
     AssetLibrary* m_previousAssetLibrary = nullptr;
 
-    const eastl::vector<UUID>* m_rootUuids = nullptr;
-    const eastl::vector<UUID>* m_leafUuids = nullptr;
+    eastl::span<const UUID> m_rootUuids;
+    eastl::span<const UUID> m_leafUuids;
     const ankerl::unordered_dense::map<UUID, std::size_t>* m_rootTotalBytes = nullptr;
 
 #if defined BIGFOOT_TOOLMODE
@@ -204,7 +205,7 @@ class SerializationManagerFixture: public benchmark::Fixture
             }
 
 #if defined BIGFOOT_TOOLMODE
-            BigFile bigFile {File {BIGFILE_ENGINEBENCHMARKS_LOCATION}};
+            BigFile bigFile {File { BIGFILE_ENGINEBENCHMARKS_LOCATION }};
             bigFile.Clear();
             bigFile.SaveAssetFlatbuffers(uuids, flatbuffers, hardReferences);
 #elif defined BIGFOOT_GAME
@@ -250,7 +251,7 @@ BENCHMARK_DEFINE_F(SerializationManagerFixture, LoadAsset)(benchmark::State& sta
         AssetContainer<BenchmarkAsset> rootContainer;
         ResetActiveLibrary(library, leafContainer, rootContainer);
 
-        const UUID uuid = (*m_rootUuids)[Random::get<std::size_t>(0, m_rootUuids->size() - 1)];
+        const UUID uuid = m_rootUuids[Random::get<std::size_t>(0, m_rootUuids.size() - 1)];
         const std::size_t uuidBytes = m_rootTotalBytes->at(uuid);
         state.ResumeTiming();
 
@@ -321,14 +322,14 @@ BENCHMARK_DEFINE_F(SerializationManagerFixture, SaveAsset)(benchmark::State& sta
         asset->GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB});
         for (std::uint32_t j = 0; j < REFS_PER_ROOT; ++j)
         {
-            asset->GetAsset().refs.emplace_back((*m_leafUuids)[Random::get<std::size_t>(0, m_leafUuids->size() - 1)]);
+            asset->GetAsset().refs.emplace_back(m_leafUuids[Random::get<std::size_t>(0, m_leafUuids.size() - 1)]);
         }
         const UUID uuid = asset->GetHeader().uuid;
         const std::size_t assetBytes = asset->Pack().size();
+        bytesProcessed += static_cast<std::int64_t>(assetBytes);
         state.ResumeTiming();
 
         benchmark::DoNotOptimize(SerializationManager::SaveAsset(uuid));
-        bytesProcessed += static_cast<std::int64_t>(assetBytes);
     }
 
     state.SetItemsProcessed(state.iterations());
@@ -363,8 +364,7 @@ BENCHMARK_DEFINE_F(SerializationManagerFixture, SaveAssets)(benchmark::State& st
             asset->GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB});
             for (std::uint32_t j = 0; j < REFS_PER_ROOT; ++j)
             {
-                asset->GetAsset().refs.emplace_back(
-                    (*m_leafUuids)[Random::get<std::size_t>(0, m_leafUuids->size() - 1)]);
+                asset->GetAsset().refs.emplace_back(m_leafUuids[Random::get<std::size_t>(0, m_leafUuids.size() - 1)]);
             }
 
             batchBytes += asset->Pack().size();