BigfootDev/Bigfoot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

BigFile/Sasquatch #1

Merged
rboullard merged 100 commits from BigFile into Development 2026-09-01 16:32:02 +00:00
Conversation 5 Commits 100 Files Changed 207
+118 -94
3 changed files with 118 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 92f4d184d5 - Show all commits
Bigfoot/Benchmarks/Engine/BigFile/BigFile.cpp
+50 -34
View File
@@ -15,6 +15,7 @@
#include <AssetsForBenchmarking/Asset/BenchmarkAssetReferred.hpp>
#include <EASTL/optional.h>
#include <EASTL/span.h>
#include <EASTL/vector.h>
#include <benchmark/benchmark.h>
#include <effolkronium/random.hpp>
@@ -34,18 +35,17 @@ class BigFileFixture: public benchmark::Fixture
static constexpr std::uint32_t REFS_PER_ASSET = 4;
static constexpr std::size_t ASSET_PAYLOAD_SIZE = 256;
void SetUp(benchmark::State&) override
void SetUp([[maybe_unused]] benchmark::State& p_state) override
{
// The DB is populated exactly once for the entire benchmark process.
// SetUp is called before every calibration round (Google Benchmark doubles
// iteration counts until it reaches a stable measurement), so rebuilding
// the DB each time would dominate the total benchmark time.
static const eastl::vector<UUID> s_uuids = BuildData();
m_uuids = &s_uuids;
m_uuids = BuildData();
m_bigFile.emplace(File {BIGFILE_ENGINEBENCHMARKS_LOCATION});
}
void TearDown(benchmark::State&) override
void TearDown([[maybe_unused]] benchmark::State& p_state) override
{
m_bigFile.reset();
}
@@ -57,7 +57,7 @@ class BigFileFixture: public benchmark::Fixture
batch.reserve(p_batchSize);
for (std::size_t i = 0; i < p_batchSize; ++i)
{
batch.push_back((*m_uuids)[Random::get<std::size_t>(0, m_uuids->size() - 1)]);
batch.push_back(m_uuids[Random::get<std::size_t>(0, m_uuids.size() - 1)]);
}
return batch;
@@ -78,42 +78,47 @@ class BigFileFixture: public benchmark::Fixture
}
eastl::optional<BigFile> m_bigFile;
const eastl::vector<UUID>* m_uuids = nullptr;
eastl::span<const UUID> m_uuids;
private:
static eastl::vector<UUID> BuildData()
static const eastl::vector<UUID>& BuildData()
{
eastl::vector<UUID> uuids;
eastl::vector<eastl::vector<std::byte>> flatbuffers;
eastl::vector<eastl::vector<UUID>> hardReferences;
uuids.reserve(ASSET_COUNT);
flatbuffers.reserve(ASSET_COUNT);
hardReferences.reserve(ASSET_COUNT);
for (std::uint32_t i = 0; i < ASSET_COUNT; ++i)
static const eastl::vector<UUID> s_uuids = []()
{
BenchmarkAsset asset;
asset.SetName(asset.GetHeader().uuid.ToString().c_str());
asset.GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB});
for (std::uint32_t j = 0; j < REFS_PER_ASSET; ++j)
{
BenchmarkAssetReferred assetReferred;
assetReferred.SetName(assetReferred.GetHeader().uuid.ToString().c_str());
eastl::vector<UUID> uuids;
eastl::vector<eastl::vector<std::byte>> flatbuffers;
eastl::vector<eastl::vector<UUID>> hardReferences;
uuids.reserve(ASSET_COUNT);
flatbuffers.reserve(ASSET_COUNT);
hardReferences.reserve(ASSET_COUNT);
asset.GetAsset().refs.emplace_back(assetReferred.GetHeader().uuid);
for (std::uint32_t i = 0; i < ASSET_COUNT; ++i)
{
BenchmarkAsset asset;
asset.SetName(asset.GetHeader().uuid.ToString().c_str());
asset.GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB});
for (std::uint32_t j = 0; j < REFS_PER_ASSET; ++j)
{
BenchmarkAssetReferred assetReferred;
assetReferred.SetName(assetReferred.GetHeader().uuid.ToString().c_str());
asset.GetAsset().refs.emplace_back(assetReferred.GetHeader().uuid);
}
eastl::vector<std::byte> flatbuffer = asset.Pack();
hardReferences.push_back(asset.GetHardReferences(flatbuffer));
flatbuffers.push_back(std::move(flatbuffer));
uuids.push_back(asset.GetHeader().uuid);
}
eastl::vector<std::byte> flatbuffer = asset.Pack();
hardReferences.push_back(asset.GetHardReferences(flatbuffer));
flatbuffers.push_back(std::move(flatbuffer));
uuids.push_back(asset.GetHeader().uuid);
}
BigFile bigFile {File {BIGFILE_ENGINEBENCHMARKS_LOCATION}};
bigFile.Clear();
bigFile.SaveAssetFlatbuffers(uuids, flatbuffers, hardReferences);
BigFile bigFile {File {BIGFILE_ENGINEBENCHMARKS_LOCATION}};
bigFile.Clear();
bigFile.SaveAssetFlatbuffers(uuids, flatbuffers, hardReferences);
return uuids;
}();
return uuids;
return s_uuids;
}
};
@@ -126,7 +131,7 @@ BENCHMARK_DEFINE_F(BigFileFixture, GetAssetFlatbuffer)(benchmark::State& state)
for (auto _: state)
{
eastl::vector<std::byte> flatbuffer = m_bigFile->GetAssetFlatbuffer(
(*m_uuids)[Random::get<std::size_t>(0, m_uuids->size() - 1)]);
m_uuids[Random::get<std::size_t>(0, m_uuids.size() - 1)]);
benchmark::DoNotOptimize(flatbuffer);
bytesProcessed += static_cast<std::int64_t>(flatbuffer.size());
@@ -183,10 +188,11 @@ BENCHMARK_DEFINE_F(BigFileFixture, GetAssetHardReferences)(benchmark::State& sta
for (auto _: state)
{
benchmark::DoNotOptimize(
m_bigFile->GetAssetHardReferences((*m_uuids)[Random::get<std::size_t>(0, m_uuids->size() - 1)]));
m_bigFile->GetAssetHardReferences(m_uuids[Random::get<std::size_t>(0, m_uuids.size() - 1)]));
}
state.SetItemsProcessed(state.iterations());
state.SetBytesProcessed(static_cast<std::int64_t>(state.iterations()) * REFS_PER_ASSET * UUID::UUID_BYTE_SIZE);
}
BENCHMARK_REGISTER_F(BigFileFixture, GetAssetHardReferences)->Name("BigFile/GetAssetHardReferences")->UseRealTime();
@@ -208,6 +214,8 @@ BENCHMARK_DEFINE_F(BigFileFixture, GetAssetsHardReferences)(benchmark::State& st
}
state.SetItemsProcessed(state.iterations() * batchSize);
state.SetBytesProcessed(static_cast<std::int64_t>(state.iterations()) * batchSize * REFS_PER_ASSET *
UUID::UUID_BYTE_SIZE);
}
BENCHMARK_REGISTER_F(BigFileFixture, GetAssetsHardReferences)
@@ -231,6 +239,10 @@ BENCHMARK_DEFINE_F(BigFileFixture, SaveAssetFlatbuffer)(benchmark::State& state)
BenchmarkAsset asset;
asset.SetName(asset.GetHeader().uuid.ToString().c_str());
asset.GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB});
for (std::uint32_t j = 0; j < REFS_PER_ASSET; ++j)
{
asset.GetAsset().refs.emplace_back(m_uuids[Random::get<std::size_t>(0, m_uuids.size() - 1)]);
}
const eastl::vector<std::byte> flatbuffer = asset.Pack();
const eastl::vector<UUID> hardReferences = asset.GetHardReferences(flatbuffer);
@@ -268,6 +280,10 @@ BENCHMARK_DEFINE_F(BigFileFixture, SaveAssetFlatbuffers)(benchmark::State& state
BenchmarkAsset asset;
asset.SetName(asset.GetHeader().uuid.ToString().c_str());
asset.GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB});
for (std::uint32_t j = 0; j < REFS_PER_ASSET; ++j)
{
asset.GetAsset().refs.emplace_back(m_uuids[Random::get<std::size_t>(0, m_uuids.size() - 1)]);
}
eastl::vector<std::byte> flatbuffer = asset.Pack();
uuids.push_back(asset.GetHeader().uuid);
Bigfoot/Benchmarks/Engine/Sasquatch/Sasquatch.cpp
+55 -47
View File
@@ -15,6 +15,7 @@
#include <AssetsForBenchmarking/Asset/BenchmarkAssetReferred.hpp>
#include <EASTL/optional.h>
#include <EASTL/span.h>
#include <EASTL/vector.h>
#include <benchmark/benchmark.h>
#include <effolkronium/random.hpp>
@@ -40,12 +41,11 @@ class SasquatchFixture: public benchmark::Fixture
// SetUp is called before every calibration round (Google Benchmark doubles
// iteration counts until it reaches a stable measurement), so rebuilding
// the file each time would dominate the total benchmark time.
static const eastl::vector<UUID> s_uuids = BuildData();
m_uuids = &s_uuids;
m_uuids = BuildData();
m_sasquatch.emplace(File {SASQUATCH_ENGINEBENCHMARKS_LOCATION});
}
void TearDown(benchmark::State&) override
void TearDown([[maybe_unused]] benchmark::State& p_state) override
{
m_sasquatch.reset();
}
@@ -57,7 +57,7 @@ class SasquatchFixture: public benchmark::Fixture
batch.reserve(p_batchSize);
for (std::size_t i = 0; i < p_batchSize; ++i)
{
batch.push_back((*m_uuids)[Random::get<std::size_t>(0, m_uuids->size() - 1)]);
batch.push_back(m_uuids[Random::get<std::size_t>(0, m_uuids.size() - 1)]);
}
return batch;
@@ -78,57 +78,62 @@ class SasquatchFixture: public benchmark::Fixture
}
eastl::optional<Sasquatch> m_sasquatch;
const eastl::vector<UUID>* m_uuids = nullptr;
eastl::span<const UUID> m_uuids;
private:
static eastl::vector<UUID> BuildData()
static const eastl::vector<UUID>& BuildData()
{
File {SASQUATCH_ENGINEBENCHMARKS_LOCATION}.Delete();
eastl::vector<UUID> uuids;
eastl::vector<eastl::vector<std::byte>> flatbuffers;
eastl::vector<eastl::vector<UUID>> hardReferences;
uuids.reserve(ASSET_COUNT);
flatbuffers.reserve(ASSET_COUNT);
hardReferences.reserve(ASSET_COUNT);
for (std::uint32_t i = 0; i < ASSET_COUNT; ++i)
static const eastl::vector<UUID> s_uuids = []()
{
BenchmarkAsset asset;
asset.SetName(asset.GetHeader().uuid.ToString().c_str());
asset.GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB});
for (std::uint32_t j = 0; j < REFS_PER_ASSET; ++j)
{
BenchmarkAssetReferred assetReferred;
assetReferred.SetName(assetReferred.GetHeader().uuid.ToString().c_str());
File {SASQUATCH_ENGINEBENCHMARKS_LOCATION}.Delete();
asset.GetAsset().refs.emplace_back(assetReferred.GetHeader().uuid);
eastl::vector<UUID> uuids;
eastl::vector<eastl::vector<std::byte>> flatbuffers;
eastl::vector<eastl::vector<UUID>> hardReferences;
uuids.reserve(ASSET_COUNT);
flatbuffers.reserve(ASSET_COUNT);
hardReferences.reserve(ASSET_COUNT);
for (std::uint32_t i = 0; i < ASSET_COUNT; ++i)
{
BenchmarkAsset asset;
asset.SetName(asset.GetHeader().uuid.ToString().c_str());
asset.GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB});
for (std::uint32_t j = 0; j < REFS_PER_ASSET; ++j)
{
BenchmarkAssetReferred assetReferred;
assetReferred.SetName(assetReferred.GetHeader().uuid.ToString().c_str());
asset.GetAsset().refs.emplace_back(assetReferred.GetHeader().uuid);
}
eastl::vector<std::byte> flatbuffer = asset.Pack();
hardReferences.push_back(asset.GetHardReferences(flatbuffer));
flatbuffers.push_back(std::move(flatbuffer));
uuids.push_back(asset.GetHeader().uuid);
}
eastl::vector<std::byte> flatbuffer = asset.Pack();
hardReferences.push_back(asset.GetHardReferences(flatbuffer));
flatbuffers.push_back(std::move(flatbuffer));
uuids.push_back(asset.GetHeader().uuid);
}
std::size_t totalFlatbufferBytes = 0;
std::size_t totalHardReferencesBytes = 0;
for (std::size_t i = 0; i < uuids.size(); ++i)
{
totalFlatbufferBytes += flatbuffers[i].size();
totalHardReferencesBytes += hardReferences[i].size() * UUID::UUID_BYTE_SIZE;
}
std::size_t totalFlatbufferBytes = 0;
std::size_t totalHardReferencesBytes = 0;
for (std::size_t i = 0; i < uuids.size(); ++i)
{
totalFlatbufferBytes += flatbuffers[i].size();
totalHardReferencesBytes += hardReferences[i].size() * UUID::UUID_BYTE_SIZE;
}
SasquatchWriter writer {File {SASQUATCH_ENGINEBENCHMARKS_LOCATION},
uuids.size(),
totalFlatbufferBytes,
totalHardReferencesBytes};
for (std::size_t i = 0; i < uuids.size(); ++i)
{
writer.WriteAsset(uuids[i], flatbuffers[i], hardReferences[i]);
}
SasquatchWriter writer {File {SASQUATCH_ENGINEBENCHMARKS_LOCATION},
uuids.size(),
totalFlatbufferBytes,
totalHardReferencesBytes};
for (std::size_t i = 0; i < uuids.size(); ++i)
{
writer.WriteAsset(uuids[i], flatbuffers[i], hardReferences[i]);
}
return uuids;
}();
return uuids;
return s_uuids;
}
};
@@ -139,10 +144,11 @@ BENCHMARK_DEFINE_F(SasquatchFixture, GetAssetHardReferences)(benchmark::State& s
for (auto _: state)
{
benchmark::DoNotOptimize(
m_sasquatch->GetAssetHardReferences((*m_uuids)[Random::get<std::size_t>(0, m_uuids->size() - 1)]));
m_sasquatch->GetAssetHardReferences(m_uuids[Random::get<std::size_t>(0, m_uuids.size() - 1)]));
}
state.SetItemsProcessed(state.iterations());
state.SetBytesProcessed(static_cast<std::int64_t>(state.iterations()) * REFS_PER_ASSET * UUID::UUID_BYTE_SIZE);
}
BENCHMARK_REGISTER_F(SasquatchFixture, GetAssetHardReferences)->Name("Sasquatch/GetAssetHardReferences")->UseRealTime();
@@ -164,6 +170,8 @@ BENCHMARK_DEFINE_F(SasquatchFixture, GetAssetsHardReferences)(benchmark::State&
}
state.SetItemsProcessed(state.iterations() * batchSize);
state.SetBytesProcessed(static_cast<std::int64_t>(state.iterations()) * batchSize * REFS_PER_ASSET *
UUID::UUID_BYTE_SIZE);
}
BENCHMARK_REGISTER_F(SasquatchFixture, GetAssetsHardReferences)
@@ -184,7 +192,7 @@ BENCHMARK_DEFINE_F(SasquatchFixture, GetAssetFlatbuffer)(benchmark::State& state
for (auto _: state)
{
eastl::vector<std::byte> flatbuffer = m_sasquatch->GetAssetFlatbuffer(
(*m_uuids)[Random::get<std::size_t>(0, m_uuids->size() - 1)]);
m_uuids[Random::get<std::size_t>(0, m_uuids.size() - 1)]);
benchmark::DoNotOptimize(flatbuffer);
bytesProcessed += static_cast<std::int64_t>(flatbuffer.size());
Bigfoot/Benchmarks/Engine/SerializationManager/SerializationManager.cpp
+13 -13
View File
@@ -26,6 +26,7 @@
#include <AssetsForBenchmarking/Asset/BenchmarkAssetReferred.hpp>
#include <EASTL/optional.h>
#include <EASTL/span.h>
#include <EASTL/vector.h>
#include <ankerl/unordered_dense.h>
#include <benchmark/benchmark.h>
@@ -46,7 +47,7 @@ class SerializationManagerFixture: public benchmark::Fixture
static constexpr std::uint32_t REFS_PER_ROOT = 4;
static constexpr std::size_t ASSET_PAYLOAD_SIZE = 256;
void SetUp(benchmark::State&) override
void SetUp([[maybe_unused]] benchmark::State& p_state) override
{
m_previousDataAccessor = g_dataAccessor;
m_previousAssetLibrary = g_activeAssetLibrary;
@@ -56,8 +57,8 @@ class SerializationManagerFixture: public benchmark::Fixture
// iteration counts until it reaches a stable measurement), so rebuilding
// the data store each time would dominate the total benchmark time.
static const SharedData& s_data = BuildData();
m_rootUuids = &s_data.rootUuids;
m_leafUuids = &s_data.leafUuids;
m_rootUuids = s_data.rootUuids;
m_leafUuids = s_data.leafUuids;
m_rootTotalBytes = &s_data.rootTotalBytes;
#if defined BIGFOOT_TOOLMODE
@@ -69,7 +70,7 @@ class SerializationManagerFixture: public benchmark::Fixture
#endif
}
void TearDown(benchmark::State&) override
void TearDown([[maybe_unused]] benchmark::State& p_state) override
{
#if defined BIGFOOT_TOOLMODE
m_bigFile.reset();
@@ -103,7 +104,7 @@ class SerializationManagerFixture: public benchmark::Fixture
batch.reserve(p_batchSize);
for (std::size_t i = 0; i < p_batchSize; ++i)
{
batch.push_back((*m_rootUuids)[Random::get<std::size_t>(0, m_rootUuids->size() - 1)]);
batch.push_back(m_rootUuids[Random::get<std::size_t>(0, m_rootUuids.size() - 1)]);
}
return batch;
@@ -138,8 +139,8 @@ class SerializationManagerFixture: public benchmark::Fixture
DataAccessor* m_previousDataAccessor = nullptr;
AssetLibrary* m_previousAssetLibrary = nullptr;
const eastl::vector<UUID>* m_rootUuids = nullptr;
const eastl::vector<UUID>* m_leafUuids = nullptr;
eastl::span<const UUID> m_rootUuids;
eastl::span<const UUID> m_leafUuids;
const ankerl::unordered_dense::map<UUID, std::size_t>* m_rootTotalBytes = nullptr;
#if defined BIGFOOT_TOOLMODE
@@ -204,7 +205,7 @@ class SerializationManagerFixture: public benchmark::Fixture
}
#if defined BIGFOOT_TOOLMODE
BigFile bigFile {File {BIGFILE_ENGINEBENCHMARKS_LOCATION}};
BigFile bigFile {File { BIGFILE_ENGINEBENCHMARKS_LOCATION }};
bigFile.Clear();
bigFile.SaveAssetFlatbuffers(uuids, flatbuffers, hardReferences);
#elif defined BIGFOOT_GAME
@@ -250,7 +251,7 @@ BENCHMARK_DEFINE_F(SerializationManagerFixture, LoadAsset)(benchmark::State& sta
AssetContainer<BenchmarkAsset> rootContainer;
ResetActiveLibrary(library, leafContainer, rootContainer);
const UUID uuid = (*m_rootUuids)[Random::get<std::size_t>(0, m_rootUuids->size() - 1)];
const UUID uuid = m_rootUuids[Random::get<std::size_t>(0, m_rootUuids.size() - 1)];
const std::size_t uuidBytes = m_rootTotalBytes->at(uuid);
state.ResumeTiming();
@@ -321,14 +322,14 @@ BENCHMARK_DEFINE_F(SerializationManagerFixture, SaveAsset)(benchmark::State& sta
asset->GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB});
for (std::uint32_t j = 0; j < REFS_PER_ROOT; ++j)
{
asset->GetAsset().refs.emplace_back((*m_leafUuids)[Random::get<std::size_t>(0, m_leafUuids->size() - 1)]);
asset->GetAsset().refs.emplace_back(m_leafUuids[Random::get<std::size_t>(0, m_leafUuids.size() - 1)]);
}
const UUID uuid = asset->GetHeader().uuid;
const std::size_t assetBytes = asset->Pack().size();
bytesProcessed += static_cast<std::int64_t>(assetBytes);
state.ResumeTiming();
benchmark::DoNotOptimize(SerializationManager::SaveAsset(uuid));
bytesProcessed += static_cast<std::int64_t>(assetBytes);
}
state.SetItemsProcessed(state.iterations());
@@ -363,8 +364,7 @@ BENCHMARK_DEFINE_F(SerializationManagerFixture, SaveAssets)(benchmark::State& st
asset->GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB});
for (std::uint32_t j = 0; j < REFS_PER_ROOT; ++j)
{
asset->GetAsset().refs.emplace_back(
(*m_leafUuids)[Random::get<std::size_t>(0, m_leafUuids->size() - 1)]);
asset->GetAsset().refs.emplace_back(m_leafUuids[Random::get<std::size_t>(0, m_leafUuids.size() - 1)]);
}
batchBytes += asset->Pack().size();