From ae18a062b7931c9acf68250e37d1ce9c12c9b7e4 Mon Sep 17 00:00:00 2001 From: Romain BOULLARD Date: Sat, 22 Aug 2026 17:43:21 +0200 Subject: [PATCH] Add Sasquatch: mmap'd, load-only Game-mode asset backend, plus parallel batch loading Introduces a dual-backend SerializationManager: Tool mode keeps reading/writing BigFile over SQL as before, Game mode now reads a Sasquatch file instead - a flattened, mmap'd counterpart to a BigFile produced ahead of time by Sasquatch::WriteFrom or the new SasquatchPacker CLI tool. The backend is picked at compile time via a new BIGFOOT_TOOL_MODE option (mirrors the existing BIGFOOT_OPTIMIZED/BIGFOOT_NOT_OPTIMIZED pattern); SaveAsset only exists in Tool mode, since a running game never writes back to its own asset pack. Also adds SerializationManager::LoadAssets (Game-mode only): loads a batch of UUIDs by decoding them across multiple threads instead of one at a time, layered by HardReference depth so a reference target is always fully registered before anything decodes a reference to it. This required making HardReference/SoftReference resolution genuinely thread-safe (it mutates ContainerHandle ref counts and AssetLibrary's UUID index/unreferenced-set from inside Unpack()) via atomic ref counts and targeted mutexes, validated by dedicated concurrency tests exercising real thread contention. BigFile itself is faster too: prepared statements are now pooled per query text instead of recompiled on every Request (pooled rather than cached-as-one because LoadAsset recurses and a naive cache would corrupt an in-flight statement), and PRAGMA mmap_size is enabled. Benchmarked against real Asset objects (AssetsForBenchmarking) through the full Pack/Unpack/HardReference pipeline: Sasquatch's raw storage layer hits ~31 GiB/s (mmap ceiling), and the new parallel LoadAssets reaches ~18 GiB/s end-to-end versus ~1.3 GiB/s for the existing sequential LoadAsset loop. Co-Authored-By: Claude Sonnet 5 --- Bigfoot/Benchmarks/CMakeLists.txt | 4 +- .../Benchmarks/Engine/BigFile/Sasquatch.cpp | 465 +++++++++++++++++ .../Engine/BigFile/SasquatchLoadAssets.cpp | 182 +++++++ Bigfoot/Benchmarks/Engine/CMakeLists.txt | 11 + .../BigFileInfo_generated.hpp | 19 + .../Asset/BenchAssetLeaf.cpp | 11 + .../Asset/BenchAssetRoot.cpp | 11 + .../Misc/AssetsForBenchmarking/CMakeLists.txt | 15 + .../Asset/BenchAssetLeaf.fbs | 8 + .../Asset/BenchAssetLeaf.hpp | 40 ++ .../Asset/BenchAssetLeaf_generated.hpp | 192 +++++++ .../Asset/BenchAssetRoot.fbs | 10 + .../Asset/BenchAssetRoot.hpp | 40 ++ .../Asset/BenchAssetRoot_generated.hpp | 221 ++++++++ Bigfoot/Benchmarks/Misc/CMakeLists.txt | 1 + Bigfoot/Sources/CMakeLists.txt | 5 +- .../Sources/Engine/Asset/AssetContainer.cpp | 45 ++ .../Engine/Asset/SerializationManager.cpp | 479 ++++++++++++++++++ Bigfoot/Sources/Engine/BigFile/BigFile.cpp | 89 +++- .../Sources/Engine/BigFile/BigFileSchema.sql | 39 +- .../Engine/Include/Engine/Asset/Asset.hpp | 95 +++- .../Include/Engine/Asset/AssetContainer.hpp | 237 +++++++-- .../Include/Engine/Asset/AssetLibrary.hpp | 142 +++++- .../Engine/Include/Engine/Asset/Reference.hpp | 48 +- .../Engine/Asset/SerializationManager.hpp | 226 +++++++++ .../Engine/Include/Engine/BigFile/BigFile.hpp | 47 ++ .../Include/Engine/Sasquatch/Sasquatch.hpp | 106 ++++ .../Sources/Engine/Sasquatch/Sasquatch.cpp | 298 +++++++++++ Bigfoot/Sources/Tools/CMakeLists.txt | 1 + .../Tools/SasquatchPacker/CMakeLists.txt | 25 + .../Sources/Tools/SasquatchPacker/main.cpp | 30 ++ Bigfoot/Tests/Engine/Asset/AssetContainer.cpp | 196 +++++++ Bigfoot/Tests/Engine/Asset/Reference.cpp | 128 +++++ .../Engine/Asset/SerializationManager.cpp | 357 +++++++++++++ .../Include/AssetsForTesting/Asset/AssetA.hpp | 5 - .../Include/AssetsForTesting/Asset/AssetB.hpp | 5 - .../Include/AssetsForTesting/Asset/AssetC.hpp | 5 - CMake/CustomTargets.cmake | 4 +- CMakeLists.txt | 1 + 39 files changed, 3728 insertions(+), 115 deletions(-) create mode 100644 Bigfoot/Benchmarks/Engine/BigFile/Sasquatch.cpp create mode 100644 Bigfoot/Benchmarks/Engine/BigFile/SasquatchLoadAssets.cpp create mode 100644 Bigfoot/Benchmarks/Engine/CMakeLists.txt create mode 100644 Bigfoot/Benchmarks/Engine/Include/EngineBenchmarks/BigFileInfo_generated.hpp create mode 100644 Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Asset/BenchAssetLeaf.cpp create mode 100644 Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Asset/BenchAssetRoot.cpp create mode 100644 Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/CMakeLists.txt create mode 100644 Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetLeaf.fbs create mode 100644 Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetLeaf.hpp create mode 100644 Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetLeaf_generated.hpp create mode 100644 Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetRoot.fbs create mode 100644 Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetRoot.hpp create mode 100644 Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetRoot_generated.hpp create mode 100644 Bigfoot/Benchmarks/Misc/CMakeLists.txt create mode 100644 Bigfoot/Sources/Engine/Asset/AssetContainer.cpp create mode 100644 Bigfoot/Sources/Engine/Asset/SerializationManager.cpp create mode 100644 Bigfoot/Sources/Engine/Include/Engine/Asset/SerializationManager.hpp create mode 100644 Bigfoot/Sources/Engine/Include/Engine/Sasquatch/Sasquatch.hpp create mode 100644 Bigfoot/Sources/Engine/Sasquatch/Sasquatch.cpp create mode 100644 Bigfoot/Sources/Tools/CMakeLists.txt create mode 100644 Bigfoot/Sources/Tools/SasquatchPacker/CMakeLists.txt create mode 100644 Bigfoot/Sources/Tools/SasquatchPacker/main.cpp create mode 100644 Bigfoot/Tests/Engine/Asset/SerializationManager.cpp diff --git a/Bigfoot/Benchmarks/CMakeLists.txt b/Bigfoot/Benchmarks/CMakeLists.txt index 992910a..cdcdf4e 100644 --- a/Bigfoot/Benchmarks/CMakeLists.txt +++ b/Bigfoot/Benchmarks/CMakeLists.txt @@ -1 +1,3 @@ -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/Utils) \ No newline at end of file +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/Utils) +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/Misc) +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/Engine) \ No newline at end of file diff --git a/Bigfoot/Benchmarks/Engine/BigFile/Sasquatch.cpp b/Bigfoot/Benchmarks/Engine/BigFile/Sasquatch.cpp new file mode 100644 index 0000000..ceab208 --- /dev/null +++ b/Bigfoot/Benchmarks/Engine/BigFile/Sasquatch.cpp @@ -0,0 +1,465 @@ +/********************************************************************* + * \file Sasquatch.cpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Tool-mode only: populates via SerializationManager::SaveAsset, which only exists in Tool mode (see +// SerializationManager.hpp). Building EngineBenchmarks against BIGFOOT_GAME (see +// SasquatchLoadAssets.cpp, in this same directory) compiles this file down to nothing. +#if defined BIGFOOT_TOOL + +// Compares the production BigFile/SQLite path against the production Sasquatch path, loading real +// Asset objects (see AssetsForBenchmarking) through the real AssetContainer/SerializationManager +// pipeline - Pack()/Unpack(), HardReference resolution and all - at asset sizes big enough that per-asset +// fixed overhead (hashing, SQL parsing, small allocations) stops dominating and storage throughput does. +// Sasquatch_RawRead skips the AssetContainer/flatbuffers layer entirely to show the ceiling that layer +// costs against - see the class comment on SerializationManager for why Sasquatch has no BigFile-style +// prepared-statement-reuse concern (it's a single mmap, not a series of SQL round-trips). +namespace Bigfoot +{ +namespace +{ +[[nodiscard]] +eastl::string UniqueName(const UUID& p_uuid) +{ + const std::string uuidString = p_uuid.ToString(); + return eastl::string {uuidString.data(), uuidString.size()}; +} + +void ClearBigFile(BigFile& p_bigFile) +{ + // Cascades to Asset/AssetHardReference/AssetSoftReference (see BigFileSchema.sql's ON DELETE CASCADE): + // the fixture BigFile is shared and persistent across every Arg() this benchmark runs with, so each + // run clears out whatever the previous one left behind. VACUUM reclaims the freed pages: without it, + // repeated DELETE+repopulate cycles across every Arg()/repetition only ever grow the file, and query + // planning cost on a several-hundred-MB file swamps everything else being measured. + BigFile::Request deleteRequest {p_bigFile, "DELETE FROM AssetHeader;"}; + std::ignore = deleteRequest.Execute(); + + BigFile::Request vacuumRequest {p_bigFile, "VACUUM;"}; + std::ignore = vacuumRequest.Execute(); +} + +[[nodiscard]] +std::filesystem::path TempPath(const std::string_view p_name) +{ + return std::filesystem::temp_directory_path() / p_name; +} + +[[nodiscard]] +File ToFile(const std::filesystem::path& p_path) +{ + const std::string pathString = p_path.string(); + return File {eastl::string_view {pathString.c_str(), pathString.size()}}; +} + +// Sums every 8 bytes of the payload, so the loop can't be optimized away and every byte actually gets +// faulted in/read - a stride of 8 rather than 1 so the reduction itself stays cheap enough not to become +// the bottleneck we're trying to measure around it. +[[nodiscard]] +std::uint64_t Touch(const eastl::span p_bytes) +{ + std::uint64_t sum = 0; + + std::size_t i = 0; + for (; i + sizeof(std::uint64_t) <= p_bytes.size(); i += sizeof(std::uint64_t)) + { + std::uint64_t chunk; + std::memcpy(&chunk, p_bytes.data() + i, sizeof(chunk)); + sum += chunk; + } + for (; i < p_bytes.size(); ++i) + { + sum += static_cast(p_bytes[i]); + } + + return sum; +} + +[[nodiscard]] +std::uint64_t Touch(const eastl::vector& p_payload) +{ + return Touch(eastl::span {reinterpret_cast(p_payload.data()), + p_payload.size()}); +} + +// Builds p_count BenchAssetRoot/BenchAssetLeaf pairs, each leaf carrying a p_payloadSize-byte payload, and +// saves each one via SerializationManager::SaveAsset - the real Pack()/BigFile write path - then unloads +// them (SaveAndUnload, same as Tests/Engine/Asset/SerializationManager.cpp's fixture) so the benchmarked +// LoadAsset calls have real work to do instead of trivially finding everything already Has()-true. +void PopulateAssets(SerializationManager& p_manager, + AssetContainer& p_leafContainer, + AssetContainer& p_rootContainer, + const std::uint32_t p_count, + const std::size_t p_payloadSize, + std::mt19937_64& p_rng, + std::vector& p_outRootUUIDs, + std::vector& p_outLeafUUIDs) +{ + std::uniform_int_distribution byteDist {0, 255}; + + p_outRootUUIDs.reserve(p_count); + p_outLeafUUIDs.reserve(p_count); + + for (std::uint32_t i = 0; i < p_count; ++i) + { + BenchAssetLeaf leaf; + leaf.SetName(UniqueName(leaf.GetHeader().uuid)); + leaf.GetAsset().payload.resize(p_payloadSize); + for (std::uint8_t& byte: leaf.GetAsset().payload) + { + byte = static_cast(byteDist(p_rng)); + } + const UUID leafUUID = leaf.GetHeader().uuid; + std::ignore = p_leafContainer.Insert(std::move(leaf)); + p_manager.SaveAsset(leafUUID); + p_leafContainer.Remove(leafUUID); + + BenchAssetRoot root; + root.SetName(UniqueName(root.GetHeader().uuid)); + root.GetAsset().leaf = HardReference {leafUUID}; + const UUID rootUUID = root.GetHeader().uuid; + std::ignore = p_rootContainer.Insert(std::move(root)); + p_manager.SaveAsset(rootUUID); + p_rootContainer.Remove(rootUUID); + + p_outRootUUIDs.push_back(rootUUID); + p_outLeafUUIDs.push_back(leafUUID); + } +} + +// SerializationManager's LoadAsset body is picked at compile time by BIGFOOT_TOOL/BIGFOOT_GAME (see +// SerializationManager.hpp): this benchmark binary is built Tool-mode, so only the BigFile-backed body +// exists in it - there is no compiled Game-mode/Sasquatch LoadAsset to call here. This mirrors just enough +// of it (no cycle guard: this fixture's data is a strict two-level root->leaf tree, never cyclic) to +// exercise Sasquatch::Find() and AssetContainerBase::Insert() the same way a Game build would. +void LoadFromSasquatch(Sasquatch& p_sasquatch, AssetLibrary& p_library, const UUID& p_uuid) +{ + if (!p_uuid) + { + return; + } + + const std::optional entry = p_sasquatch.Find(p_uuid); + if (!entry.has_value()) + { + return; + } + + AssetContainerBase* container = p_library.GetAssetContainer(entry->m_typeID); + if (container == nullptr || container->Has(p_uuid)) + { + return; + } + + for (std::size_t offset = 0; offset < entry->m_hardReferenceUUIDs.size(); offset += UUID::UUID_BYTE_SIZE) + { + const eastl::span uuidBytes = entry->m_hardReferenceUUIDs.subspan(offset, UUID::UUID_BYTE_SIZE); + LoadFromSasquatch( + p_sasquatch, p_library, + UUID {std::span {uuidBytes.data(), UUID::UUID_BYTE_SIZE}}); + } + + container->Insert(entry->m_bytes); +} + +[[nodiscard]] +std::uint64_t TouchLoadedRoot(AssetContainer& p_rootContainer, const UUID& p_uuid) +{ + const auto* root = static_cast(p_rootContainer.AcquireContainerHandle(p_uuid)->GetAsset()); + if (root == nullptr) + { + return 0; + } + + const BenchAssetLeaf* leaf = root->GetAsset().leaf.Get(); + return leaf == nullptr ? 0 : Touch(leaf->GetAsset().payload); +} + +void RunBigFileBenchmark(benchmark::State& state, const bool p_shuffleLookupOrder) +{ + const auto count = static_cast(state.range(0)); + const auto payloadSize = static_cast(state.range(1)); + + BigFile bigFile {File {BIGFILE_ENGINEBENCHMARKS_LOCATION}}; + SerializationManager manager {bigFile}; + + AssetLibrary library; + AssetContainer leafContainer; + AssetContainer rootContainer; + SetActiveAssetLibrary(&library); + std::ignore = library.Insert(&leafContainer); + std::ignore = library.Insert(&rootContainer); + + ClearBigFile(bigFile); + + std::mt19937_64 rng {0x5EEDULL}; + std::vector rootUUIDs; + std::vector leafUUIDs; + PopulateAssets(manager, leafContainer, rootContainer, count, payloadSize, rng, rootUUIDs, leafUUIDs); + + std::vector lookupOrder = rootUUIDs; + if (p_shuffleLookupOrder) + { + std::shuffle(lookupOrder.begin(), lookupOrder.end(), rng); + } + + for (auto _: state) + { + std::uint64_t sum = 0; + for (const UUID& uuid: lookupOrder) + { + manager.LoadAsset(uuid); + sum += TouchLoadedRoot(rootContainer, uuid); + } + benchmark::DoNotOptimize(sum); + + state.PauseTiming(); + for (const UUID& uuid: rootUUIDs) + { + rootContainer.Remove(uuid); + } + for (const UUID& uuid: leafUUIDs) + { + leafContainer.Remove(uuid); + } + state.ResumeTiming(); + } + + state.SetItemsProcessed(state.iterations() * lookupOrder.size()); + state.SetBytesProcessed(static_cast(state.iterations()) * count * static_cast(payloadSize)); + + SetActiveAssetLibrary(&g_assetLibrary); +} + +void RunSasquatchBenchmark(benchmark::State& state, const bool p_shuffleLookupOrder) +{ + const auto count = static_cast(state.range(0)); + const auto payloadSize = static_cast(state.range(1)); + const std::filesystem::path sasquatchPath = TempPath("BigfootAssetBench.sasquatch"); + + BigFile bigFile {File {BIGFILE_ENGINEBENCHMARKS_LOCATION}}; + SerializationManager manager {bigFile}; + + AssetLibrary library; + AssetContainer leafContainer; + AssetContainer rootContainer; + SetActiveAssetLibrary(&library); + std::ignore = library.Insert(&leafContainer); + std::ignore = library.Insert(&rootContainer); + + ClearBigFile(bigFile); + + std::mt19937_64 rng {0x5EEDULL}; + std::vector rootUUIDs; + std::vector leafUUIDs; + PopulateAssets(manager, leafContainer, rootContainer, count, payloadSize, rng, rootUUIDs, leafUUIDs); + Sasquatch::WriteFrom(bigFile, ToFile(sasquatchPath)); + + std::vector lookupOrder = rootUUIDs; + if (p_shuffleLookupOrder) + { + std::shuffle(lookupOrder.begin(), lookupOrder.end(), rng); + } + + { + Sasquatch sasquatch {ToFile(sasquatchPath)}; + + for (auto _: state) + { + std::uint64_t sum = 0; + for (const UUID& uuid: lookupOrder) + { + LoadFromSasquatch(sasquatch, library, uuid); + sum += TouchLoadedRoot(rootContainer, uuid); + } + benchmark::DoNotOptimize(sum); + + state.PauseTiming(); + for (const UUID& uuid: rootUUIDs) + { + rootContainer.Remove(uuid); + } + for (const UUID& uuid: leafUUIDs) + { + leafContainer.Remove(uuid); + } + state.ResumeTiming(); + } + + state.SetItemsProcessed(state.iterations() * lookupOrder.size()); + state.SetBytesProcessed(static_cast(state.iterations()) * count * + static_cast(payloadSize)); + } + + SetActiveAssetLibrary(&g_assetLibrary); + std::filesystem::remove(sasquatchPath); +} + +// Bypasses AssetContainer/flatbuffers entirely: just Sasquatch::Find() (hash lookup into the index built +// at construction) plus a direct read of the returned span. This is the ceiling the full pipeline above +// (Unpack + HardReference resolution + AssetContainer::Insert) is measured against - the gap between the +// two is exactly what that layer costs. +void Sasquatch_RawRead(benchmark::State& state) +{ + const auto count = static_cast(state.range(0)); + const auto payloadSize = static_cast(state.range(1)); + const std::filesystem::path sasquatchPath = TempPath("BigfootAssetBench.sasquatch"); + + BigFile bigFile {File {BIGFILE_ENGINEBENCHMARKS_LOCATION}}; + SerializationManager manager {bigFile}; + + AssetLibrary library; + AssetContainer leafContainer; + AssetContainer rootContainer; + SetActiveAssetLibrary(&library); + std::ignore = library.Insert(&leafContainer); + std::ignore = library.Insert(&rootContainer); + + ClearBigFile(bigFile); + + std::mt19937_64 rng {0x5EEDULL}; + std::vector rootUUIDs; + std::vector leafUUIDs; + PopulateAssets(manager, leafContainer, rootContainer, count, payloadSize, rng, rootUUIDs, leafUUIDs); + Sasquatch::WriteFrom(bigFile, ToFile(sasquatchPath)); + + SetActiveAssetLibrary(&g_assetLibrary); + + std::vector lookupOrder = leafUUIDs; + std::shuffle(lookupOrder.begin(), lookupOrder.end(), rng); + + { + Sasquatch sasquatch {ToFile(sasquatchPath)}; + + for (auto _: state) + { + std::uint64_t sum = 0; + for (const UUID& uuid: lookupOrder) + { + if (const std::optional entry = sasquatch.Find(uuid)) + { + sum += Touch(entry->m_bytes); + } + } + benchmark::DoNotOptimize(sum); + } + + state.SetItemsProcessed(state.iterations() * lookupOrder.size()); + state.SetBytesProcessed(static_cast(state.iterations()) * count * + static_cast(payloadSize)); + } + + std::filesystem::remove(sasquatchPath); +} + +void BigFile_RandomLookup(benchmark::State& state) +{ + RunBigFileBenchmark(state, true); +} + +void BigFile_FullScan(benchmark::State& state) +{ + RunBigFileBenchmark(state, false); +} + +void Sasquatch_RandomLookup(benchmark::State& state) +{ + RunSasquatchBenchmark(state, true); +} + +void Sasquatch_FullScan(benchmark::State& state) +{ + RunSasquatchBenchmark(state, false); +} + +// Not a real benchmark: writes a persistent .sasquatch file plus a manifest of its root/leaf UUID pairs +// (one "root leaf" line per pair) to well-known temp paths, for +// Benchmarks/Engine/BigFile/SasquatchLoadAssets.cpp's Game-mode-only LoadAsset/LoadAssets comparison to +// read - Game mode never writes, so it can't produce its own fixture data or know the UUIDs without being +// told. Run once via --benchmark_filter=GenerateLoadAssetsFixture before running that Game-mode build. +void Sasquatch_GenerateLoadAssetsFixture(benchmark::State& state) +{ + static constexpr std::uint32_t COUNT = 5'000; + static constexpr std::size_t PAYLOAD_SIZE = 262'144; + + for (auto _: state) + { + BigFile bigFile {File {BIGFILE_ENGINEBENCHMARKS_LOCATION}}; + SerializationManager manager {bigFile}; + + AssetLibrary library; + AssetContainer leafContainer; + AssetContainer rootContainer; + SetActiveAssetLibrary(&library); + std::ignore = library.Insert(&leafContainer); + std::ignore = library.Insert(&rootContainer); + + ClearBigFile(bigFile); + + std::mt19937_64 rng {0x5EEDULL}; + std::vector rootUUIDs; + std::vector leafUUIDs; + PopulateAssets(manager, leafContainer, rootContainer, COUNT, PAYLOAD_SIZE, rng, rootUUIDs, leafUUIDs); + + Sasquatch::WriteFrom(bigFile, ToFile(TempPath("BigfootLoadAssetsFixture.sasquatch"))); + + std::ofstream manifest {TempPath("BigfootLoadAssetsFixture.manifest")}; + for (std::size_t i = 0; i < rootUUIDs.size(); ++i) + { + manifest << rootUUIDs[i].ToString() << ' ' << leafUUIDs[i].ToString() << '\n'; + } + + SetActiveAssetLibrary(&g_assetLibrary); + } +} +} // namespace + +// {count, payloadSize} - total data volume kept in the low hundreds of MB so population (real, +// transactional SaveAsset calls on the BigFile side) stays fast: 2000x64KiB=128MiB, 200x1MiB=200MiB, +// 20x16MiB=320MiB. Small-to-large so per-asset fixed overhead's share of the total is visible shrinking. +#define BIGFOOT_ASSET_SIZE_ARGS \ + Args({2'000, 65'536})->Args({200, 1'048'576})->Args({20, 16'777'216}) + +BENCHMARK(BigFile_RandomLookup)->Name("BigFile/RandomLookup")->BIGFOOT_ASSET_SIZE_ARGS; +BENCHMARK(Sasquatch_RandomLookup)->Name("Sasquatch/RandomLookup")->BIGFOOT_ASSET_SIZE_ARGS; +BENCHMARK(BigFile_FullScan)->Name("BigFile/FullScan")->BIGFOOT_ASSET_SIZE_ARGS; +BENCHMARK(Sasquatch_FullScan)->Name("Sasquatch/FullScan")->BIGFOOT_ASSET_SIZE_ARGS; +BENCHMARK(Sasquatch_RawRead)->Name("Sasquatch/RawRead")->BIGFOOT_ASSET_SIZE_ARGS; +BENCHMARK(Sasquatch_GenerateLoadAssetsFixture)->Name("GenerateLoadAssetsFixture")->Iterations(1); + +#undef BIGFOOT_ASSET_SIZE_ARGS +} // namespace Bigfoot +#endif diff --git a/Bigfoot/Benchmarks/Engine/BigFile/SasquatchLoadAssets.cpp b/Bigfoot/Benchmarks/Engine/BigFile/SasquatchLoadAssets.cpp new file mode 100644 index 0000000..8475167 --- /dev/null +++ b/Bigfoot/Benchmarks/Engine/BigFile/SasquatchLoadAssets.cpp @@ -0,0 +1,182 @@ +/********************************************************************* + * \file SasquatchLoadAssets.cpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#if !defined BIGFOOT_TOOL +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +// Game-mode-only: compares the existing, single-threaded LoadAsset (called once per root, in a loop) +// against the new, parallel-decode LoadAssets (one call, the whole batch) - see SerializationManager - +// against real Asset objects. Reads the fixture Sasquatch_GenerateLoadAssetsFixture (Tool mode, see +// Sasquatch.cpp in this directory) writes out, since a Game build never saves and so can't produce its +// own: run that once first via a Tool-mode EngineBenchmarks build. +namespace Bigfoot +{ +namespace +{ +[[nodiscard]] +File ToFile(const std::filesystem::path& p_path) +{ + const std::string pathString = p_path.string(); + return File {eastl::string_view {pathString.c_str(), pathString.size()}}; +} + +struct ManifestEntry +{ + UUID m_root; + UUID m_leaf; +}; + +[[nodiscard]] +std::vector ReadManifest() +{ + const std::filesystem::path path = std::filesystem::temp_directory_path() / "BigfootLoadAssetsFixture.manifest"; + std::ifstream manifest {path}; + CRITICAL_ASSERT(EngineAssertHandler, manifest.is_open(), + "Missing LoadAssets fixture manifest at {} - run " + "EngineBenchmarks --benchmark_filter=GenerateLoadAssetsFixture from a Tool-mode build first", + path.string()); + + std::vector entries; + std::string rootString; + std::string leafString; + while (manifest >> rootString >> leafString) + { + entries.push_back({UUID {rootString}, UUID {leafString}}); + } + return entries; +} + +[[nodiscard]] +std::uint64_t Touch(const eastl::vector& p_payload) +{ + return p_payload.empty() ? 0 : p_payload.back(); +} + +[[nodiscard]] +std::uint64_t TouchLoadedRoot(AssetContainer& p_rootContainer, const UUID& p_uuid) +{ + const auto* root = static_cast(p_rootContainer.AcquireContainerHandle(p_uuid)->GetAsset()); + if (root == nullptr) + { + return 0; + } + + const BenchAssetLeaf* leaf = root->GetAsset().leaf.Get(); + return leaf == nullptr ? 0 : Touch(leaf->GetAsset().payload); +} + +void RunLoadAssetsBenchmark(benchmark::State& state, const bool p_parallel) +{ + Sasquatch sasquatch {ToFile(std::filesystem::temp_directory_path() / "BigfootLoadAssetsFixture.sasquatch")}; + const std::vector manifest = ReadManifest(); + + SerializationManager manager {sasquatch}; + + AssetLibrary library; + AssetContainer leafContainer; + AssetContainer rootContainer; + SetActiveAssetLibrary(&library); + std::ignore = library.Insert(&leafContainer); + std::ignore = library.Insert(&rootContainer); + + eastl::vector rootUUIDs; + rootUUIDs.reserve(manifest.size()); + for (const ManifestEntry& entry: manifest) + { + rootUUIDs.push_back(entry.m_root); + } + + std::uintmax_t totalBytes = 0; + for (const ManifestEntry& entry: manifest) + { + if (const std::optional leaf = sasquatch.Find(entry.m_leaf)) + { + totalBytes += leaf->m_bytes.size(); + } + } + + for (auto _: state) + { + std::uint64_t sum = 0; + + if (p_parallel) + { + manager.LoadAssets(eastl::span {rootUUIDs.data(), rootUUIDs.size()}); + for (const UUID& uuid: rootUUIDs) + { + sum += TouchLoadedRoot(rootContainer, uuid); + } + } + else + { + for (const UUID& uuid: rootUUIDs) + { + manager.LoadAsset(uuid); + sum += TouchLoadedRoot(rootContainer, uuid); + } + } + benchmark::DoNotOptimize(sum); + + state.PauseTiming(); + for (const ManifestEntry& entry: manifest) + { + rootContainer.Remove(entry.m_root); + leafContainer.Remove(entry.m_leaf); + } + state.ResumeTiming(); + } + + state.SetItemsProcessed(state.iterations() * manifest.size()); + state.SetBytesProcessed(static_cast(state.iterations()) * static_cast(totalBytes)); + + SetActiveAssetLibrary(&g_assetLibrary); +} + +void Sasquatch_LoadAsset_Sequential(benchmark::State& state) +{ + RunLoadAssetsBenchmark(state, false); +} + +void Sasquatch_LoadAssets_Parallel(benchmark::State& state) +{ + RunLoadAssetsBenchmark(state, true); +} +} // namespace + +// UseRealTime(): LoadAssets does its decoding on worker threads, not the calling one, so the default CPU- +// time measurement (which only counts the calling thread) would wildly understate the real cost - wall +// time is what actually matters for "how long until these assets are usable." +BENCHMARK(Sasquatch_LoadAsset_Sequential) + ->Name("Sasquatch/LoadAsset_Sequential") + ->Unit(benchmark::kMillisecond) + ->UseRealTime(); +BENCHMARK(Sasquatch_LoadAssets_Parallel) + ->Name("Sasquatch/LoadAssets_Parallel") + ->Unit(benchmark::kMillisecond) + ->UseRealTime(); +} // namespace Bigfoot +#endif diff --git a/Bigfoot/Benchmarks/Engine/CMakeLists.txt b/Bigfoot/Benchmarks/Engine/CMakeLists.txt new file mode 100644 index 0000000..8b793f3 --- /dev/null +++ b/Bigfoot/Benchmarks/Engine/CMakeLists.txt @@ -0,0 +1,11 @@ +get_filename_component(PackageName ${CMAKE_CURRENT_SOURCE_DIR} NAME) +project(${PackageName}Benchmarks) + +bigfoot_create_package_benchmarks( + "") + +target_link_libraries(${PROJECT_NAME} + PRIVATE + $) + +bigfoot_create_bigfile("Benchmarks/Bigfoot") diff --git a/Bigfoot/Benchmarks/Engine/Include/EngineBenchmarks/BigFileInfo_generated.hpp b/Bigfoot/Benchmarks/Engine/Include/EngineBenchmarks/BigFileInfo_generated.hpp new file mode 100644 index 0000000..4d2f05f --- /dev/null +++ b/Bigfoot/Benchmarks/Engine/Include/EngineBenchmarks/BigFileInfo_generated.hpp @@ -0,0 +1,19 @@ +// AUTO-GENERATED DO NOT TOUCH + +/********************************************************************* + * \file BigFileInfo.generated.hpp + * + *********************************************************************/ +#ifndef BIGFOOT_ENGINEBENCHMARKS_BIGFILEINFO_GENERATED_HPP +#define BIGFOOT_ENGINEBENCHMARKS_BIGFILEINFO_GENERATED_HPP + +#include + +namespace Bigfoot +{ +/* + * BigFile location + */ +constexpr eastl::string_view BIGFILE_ENGINEBENCHMARKS_LOCATION {"./EngineBenchmarks-bigfile.db"}; +} // namespace Bigfoot +#endif diff --git a/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Asset/BenchAssetLeaf.cpp b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Asset/BenchAssetLeaf.cpp new file mode 100644 index 0000000..6146551 --- /dev/null +++ b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Asset/BenchAssetLeaf.cpp @@ -0,0 +1,11 @@ +/********************************************************************* + * \file BenchAssetLeaf.cpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#include + +#include + +BIGFOOT_ASSET(BenchAssetLeaf, ::Bigfoot::BenchAssetLeaf) diff --git a/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Asset/BenchAssetRoot.cpp b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Asset/BenchAssetRoot.cpp new file mode 100644 index 0000000..8187745 --- /dev/null +++ b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Asset/BenchAssetRoot.cpp @@ -0,0 +1,11 @@ +/********************************************************************* + * \file BenchAssetRoot.cpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#include + +#include + +BIGFOOT_ASSET(BenchAssetRoot, ::Bigfoot::BenchAssetRoot) diff --git a/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/CMakeLists.txt b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/CMakeLists.txt new file mode 100644 index 0000000..a5f8610 --- /dev/null +++ b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/CMakeLists.txt @@ -0,0 +1,15 @@ +get_filename_component(PackageName ${CMAKE_CURRENT_SOURCE_DIR} NAME) +project(${PackageName}) + +set(PublicDependencies) +set(PrivateDependencies) +set(BigfootPublicDependencies) +set(BigfootPrivateDependencies + Engine) + +bigfoot_create_package_tests_helper_lib( + "${PublicDependencies}" + "${PrivateDependencies}" + "${BigfootPublicDependencies}" + "${BigfootPrivateDependencies}" + "") diff --git a/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetLeaf.fbs b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetLeaf.fbs new file mode 100644 index 0000000..fa83e2e --- /dev/null +++ b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetLeaf.fbs @@ -0,0 +1,8 @@ +namespace Flat.Bigfoot; + +table BenchAssetLeaf +{ + payload: [ubyte]; +} + +root_type BenchAssetLeaf; diff --git a/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetLeaf.hpp b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetLeaf.hpp new file mode 100644 index 0000000..66f1de2 --- /dev/null +++ b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetLeaf.hpp @@ -0,0 +1,40 @@ +/********************************************************************* + * \file BenchAssetLeaf.hpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#ifndef BIGFOOT_ASSETSFORBENCHMARKING_ASSET_BENCHASSETLEAF_HPP +#define BIGFOOT_ASSETSFORBENCHMARKING_ASSET_BENCHASSETLEAF_HPP +#include + +#include + +namespace Bigfoot +{ +struct BenchAssetLeafTraits +{ + using FLAT = ::Flat::Bigfoot::BenchAssetLeaf; +}; + +/* + * A leaf asset with a variable-size byte payload - stands in for a self-contained game asset (a texture, + * a mesh, ...) so BigFile/Sasquatch benchmarks measure real Asset Pack()/Unpack() round-trips instead + * of hand-rolled blobs. + */ +class BenchAssetLeaf: public Asset +{ + public: + BenchAssetLeaf() = default; + + BenchAssetLeaf(const BenchAssetLeaf& p_asset) = delete; + BenchAssetLeaf(BenchAssetLeaf&& p_asset) = default; + + ~BenchAssetLeaf() = default; + + BenchAssetLeaf& operator=(const BenchAssetLeaf& p_asset) = delete; + BenchAssetLeaf& operator=(BenchAssetLeaf&& p_asset) = default; +}; +} // namespace Bigfoot + +#endif diff --git a/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetLeaf_generated.hpp b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetLeaf_generated.hpp new file mode 100644 index 0000000..7e3b2db --- /dev/null +++ b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetLeaf_generated.hpp @@ -0,0 +1,192 @@ +// automatically generated by the FlatBuffers compiler, do not modify + + +#ifndef FLATBUFFERS_GENERATED_BENCHASSETLEAF_FLAT_BIGFOOT_H_ +#define FLATBUFFERS_GENERATED_BENCHASSETLEAF_FLAT_BIGFOOT_H_ + +#include "flatbuffers/flatbuffers.h" + +// Ensure the included flatbuffers.h is the same version as when this file was +// generated, otherwise it may not be compatible. +static_assert(FLATBUFFERS_VERSION_MAJOR == 25 && + FLATBUFFERS_VERSION_MINOR == 12 && + FLATBUFFERS_VERSION_REVISION == 19, + "Non-compatible flatbuffers version included"); +// Ensure the included flatbuffers.h is Bigfoot's fork - stock +// flatbuffers (or a differently versioned Bigfoot fork) is not compatible. +#ifndef FLATBUFFERS_BIGFOOT_VERSION +#error "This file requires Bigfoot's flatbuffers fork - stock flatbuffers is not compatible" +#endif +static_assert(FLATBUFFERS_BIGFOOT_VERSION == 1, + "Non-compatible Bigfoot flatbuffers fork version included"); + +#include "EASTL/unique_ptr.h" +#include "EASTL/string.h" +#include "EASTL/vector.h" + +namespace Flat { +namespace Bigfoot { + +struct BenchAssetLeaf; +struct BenchAssetLeafBuilder; +struct BenchAssetLeafT; + +inline const ::flatbuffers::TypeTable *BenchAssetLeafTypeTable(); + +struct BenchAssetLeafT : public ::flatbuffers::NativeTable { + typedef BenchAssetLeaf TableType; + static FLATBUFFERS_CONSTEXPR_CPP11 const char *GetFullyQualifiedName() { + return "Flat.Bigfoot.BenchAssetLeafT"; + } + eastl::vector payload{}; +}; + +struct BenchAssetLeaf FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { + typedef BenchAssetLeafT NativeTableType; + typedef BenchAssetLeafBuilder Builder; + struct Traits; + static const ::flatbuffers::TypeTable *MiniReflectTypeTable() { + return BenchAssetLeafTypeTable(); + } + static FLATBUFFERS_CONSTEXPR_CPP11 const char *GetFullyQualifiedName() { + return "Flat.Bigfoot.BenchAssetLeaf"; + } + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_PAYLOAD = 4 + }; + const ::flatbuffers::Vector *payload() const { + return GetPointer *>(VT_PAYLOAD); + } + template + bool Verify(::flatbuffers::VerifierTemplate &verifier) const { + return VerifyTableStart(verifier) && + VerifyOffset(verifier, VT_PAYLOAD) && + verifier.VerifyVector(payload()) && + verifier.EndTable(); + } + BenchAssetLeafT *UnPack(const ::flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(BenchAssetLeafT *_o, const ::flatbuffers::resolver_function_t *_resolver = nullptr) const; + static ::flatbuffers::Offset Pack(::flatbuffers::FlatBufferBuilder &_fbb, const BenchAssetLeafT* _o, const ::flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct BenchAssetLeafBuilder { + typedef BenchAssetLeaf Table; + ::flatbuffers::FlatBufferBuilder &fbb_; + ::flatbuffers::uoffset_t start_; + void add_payload(::flatbuffers::Offset<::flatbuffers::Vector> payload) { + fbb_.AddOffset(BenchAssetLeaf::VT_PAYLOAD, payload); + } + explicit BenchAssetLeafBuilder(::flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + ::flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = ::flatbuffers::Offset(end); + return o; + } +}; + +inline ::flatbuffers::Offset CreateBenchAssetLeaf( + ::flatbuffers::FlatBufferBuilder &_fbb, + ::flatbuffers::Offset<::flatbuffers::Vector> payload = 0) { + BenchAssetLeafBuilder builder_(_fbb); + builder_.add_payload(payload); + return builder_.Finish(); +} + +struct BenchAssetLeaf::Traits { + using type = BenchAssetLeaf; + static auto constexpr Create = CreateBenchAssetLeaf; +}; + +::flatbuffers::Offset CreateBenchAssetLeaf(::flatbuffers::FlatBufferBuilder &_fbb, const BenchAssetLeafT *_o, const ::flatbuffers::rehasher_function_t *_rehasher = nullptr); + +inline BenchAssetLeafT *BenchAssetLeaf::UnPack(const ::flatbuffers::resolver_function_t *_resolver) const { + auto _o = std::make_unique(); + UnPackTo(_o.get(), _resolver); + return _o.release(); +} + +inline void BenchAssetLeaf::UnPackTo(BenchAssetLeafT *_o, const ::flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; + { auto _e = payload(); if (_e) { _o->payload.resize(_e->size()); std::copy(_e->begin(), _e->end(), _o->payload.begin()); } } +} + +inline ::flatbuffers::Offset CreateBenchAssetLeaf(::flatbuffers::FlatBufferBuilder &_fbb, const BenchAssetLeafT *_o, const ::flatbuffers::rehasher_function_t *_rehasher) { + return BenchAssetLeaf::Pack(_fbb, _o, _rehasher); +} + +inline ::flatbuffers::Offset BenchAssetLeaf::Pack(::flatbuffers::FlatBufferBuilder &_fbb, const BenchAssetLeafT* _o, const ::flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { ::flatbuffers::FlatBufferBuilder *__fbb; const BenchAssetLeafT* __o; const ::flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + auto _payload = _fbb.CreateVector(_o->payload.data(), _o->payload.size()); + return Flat::Bigfoot::CreateBenchAssetLeaf( + _fbb, + _payload); +} + +inline const ::flatbuffers::TypeTable *BenchAssetLeafTypeTable() { + static const ::flatbuffers::TypeCode type_codes[] = { + { ::flatbuffers::ET_UCHAR, 1, -1 } + }; + static const ::flatbuffers::TypeTable tt = { + ::flatbuffers::ST_TABLE, 1, type_codes, nullptr, nullptr, nullptr, nullptr + }; + return &tt; +} + +inline const Flat::Bigfoot::BenchAssetLeaf *GetBenchAssetLeaf(const void *buf) { + return ::flatbuffers::GetRoot(buf); +} + +inline const Flat::Bigfoot::BenchAssetLeaf *GetSizePrefixedBenchAssetLeaf(const void *buf) { + return ::flatbuffers::GetSizePrefixedRoot(buf); +} + +template +inline bool VerifyBenchAssetLeafBuffer( + ::flatbuffers::VerifierTemplate &verifier) { + return verifier.template VerifyBuffer(nullptr); +} + +template +inline bool VerifySizePrefixedBenchAssetLeafBuffer( + ::flatbuffers::VerifierTemplate &verifier) { + return verifier.template VerifySizePrefixedBuffer(nullptr); +} + +inline const char *BenchAssetLeafExtension() { + return "bfbs"; +} + +inline void FinishBenchAssetLeafBuffer( + ::flatbuffers::FlatBufferBuilder &fbb, + ::flatbuffers::Offset root) { + fbb.Finish(root); +} + +inline void FinishSizePrefixedBenchAssetLeafBuffer( + ::flatbuffers::FlatBufferBuilder &fbb, + ::flatbuffers::Offset root) { + fbb.FinishSizePrefixed(root); +} + +inline eastl::unique_ptr UnPackBenchAssetLeaf( + const void *buf, + const ::flatbuffers::resolver_function_t *res = nullptr) { + return eastl::unique_ptr(GetBenchAssetLeaf(buf)->UnPack(res)); +} + +inline eastl::unique_ptr UnPackSizePrefixedBenchAssetLeaf( + const void *buf, + const ::flatbuffers::resolver_function_t *res = nullptr) { + return eastl::unique_ptr(GetSizePrefixedBenchAssetLeaf(buf)->UnPack(res)); +} + +} // namespace Bigfoot +} // namespace Flat + +#endif // FLATBUFFERS_GENERATED_BENCHASSETLEAF_FLAT_BIGFOOT_H_ diff --git a/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetRoot.fbs b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetRoot.fbs new file mode 100644 index 0000000..d7dd29f --- /dev/null +++ b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetRoot.fbs @@ -0,0 +1,10 @@ +include "Engine/Asset/Reference.fbs"; + +namespace Flat.Bigfoot; + +table BenchAssetRoot +{ + leaf: HardReference (native_inline, bigfoot_ref: "::Bigfoot::BenchAssetLeaf"); +} + +root_type BenchAssetRoot; diff --git a/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetRoot.hpp b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetRoot.hpp new file mode 100644 index 0000000..93d4957 --- /dev/null +++ b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetRoot.hpp @@ -0,0 +1,40 @@ +/********************************************************************* + * \file BenchAssetRoot.hpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#ifndef BIGFOOT_ASSETSFORBENCHMARKING_ASSET_BENCHASSETROOT_HPP +#define BIGFOOT_ASSETSFORBENCHMARKING_ASSET_BENCHASSETROOT_HPP +#include + +#include + +namespace Bigfoot +{ +struct BenchAssetRootTraits +{ + using FLAT = ::Flat::Bigfoot::BenchAssetRoot; +}; + +/* + * A small "pointer" asset holding a HardReference to a BenchAssetLeaf - stands in for e.g. a Material + * referencing a Texture, so LoadAsset's recursive HardReference walk is actually exercised (a + * BenchAssetLeaf alone never triggers it). + */ +class BenchAssetRoot: public Asset +{ + public: + BenchAssetRoot() = default; + + BenchAssetRoot(const BenchAssetRoot& p_asset) = delete; + BenchAssetRoot(BenchAssetRoot&& p_asset) = default; + + ~BenchAssetRoot() = default; + + BenchAssetRoot& operator=(const BenchAssetRoot& p_asset) = delete; + BenchAssetRoot& operator=(BenchAssetRoot&& p_asset) = default; +}; +} // namespace Bigfoot + +#endif diff --git a/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetRoot_generated.hpp b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetRoot_generated.hpp new file mode 100644 index 0000000..c38ca46 --- /dev/null +++ b/Bigfoot/Benchmarks/Misc/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchAssetRoot_generated.hpp @@ -0,0 +1,221 @@ +// automatically generated by the FlatBuffers compiler, do not modify + + +#ifndef FLATBUFFERS_GENERATED_BENCHASSETROOT_FLAT_BIGFOOT_H_ +#define FLATBUFFERS_GENERATED_BENCHASSETROOT_FLAT_BIGFOOT_H_ + +#include "flatbuffers/flatbuffers.h" + +// Ensure the included flatbuffers.h is the same version as when this file was +// generated, otherwise it may not be compatible. +static_assert(FLATBUFFERS_VERSION_MAJOR == 25 && + FLATBUFFERS_VERSION_MINOR == 12 && + FLATBUFFERS_VERSION_REVISION == 19, + "Non-compatible flatbuffers version included"); +// Ensure the included flatbuffers.h is Bigfoot's fork - stock +// flatbuffers (or a differently versioned Bigfoot fork) is not compatible. +#ifndef FLATBUFFERS_BIGFOOT_VERSION +#error "This file requires Bigfoot's flatbuffers fork - stock flatbuffers is not compatible" +#endif +static_assert(FLATBUFFERS_BIGFOOT_VERSION == 1, + "Non-compatible Bigfoot flatbuffers fork version included"); + +#include "System/UUID/UUID.hpp" +#include "Engine/Asset/Reference.hpp" +#include "Engine/Asset/Reference_generated.hpp" + +#include "EASTL/unique_ptr.h" +#include "EASTL/string.h" +#include "EASTL/vector.h" + +namespace Flat { +namespace Bigfoot { + +struct BenchAssetRoot; +struct BenchAssetRootBuilder; +struct BenchAssetRootT; + +} // namespace Bigfoot +} // namespace Flat + +namespace Bigfoot { +class BenchAssetLeaf; +} // namespace Bigfoot + +namespace flatbuffers { +Flat::Bigfoot::HardReference PackHardReferenceBenchAssetLeaf(const ::Bigfoot::HardReference<::Bigfoot::BenchAssetLeaf>& p_asset); +::Bigfoot::HardReference<::Bigfoot::BenchAssetLeaf> UnPackHardReferenceBenchAssetLeaf(const Flat::Bigfoot::HardReference& p_asset); +} // namespace flatbuffers + +namespace Flat { +namespace Bigfoot { + +inline const ::flatbuffers::TypeTable *BenchAssetRootTypeTable(); + +struct BenchAssetRootT : public ::flatbuffers::NativeTable { + typedef BenchAssetRoot TableType; + static FLATBUFFERS_CONSTEXPR_CPP11 const char *GetFullyQualifiedName() { + return "Flat.Bigfoot.BenchAssetRootT"; + } + ::Bigfoot::HardReference<::Bigfoot::BenchAssetLeaf> leaf{}; +}; + +struct BenchAssetRoot FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { + typedef BenchAssetRootT NativeTableType; + typedef BenchAssetRootBuilder Builder; + struct Traits; + static const ::flatbuffers::TypeTable *MiniReflectTypeTable() { + return BenchAssetRootTypeTable(); + } + static FLATBUFFERS_CONSTEXPR_CPP11 const char *GetFullyQualifiedName() { + return "Flat.Bigfoot.BenchAssetRoot"; + } + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_LEAF = 4 + }; + const Flat::Bigfoot::HardReference *leaf() const { + return GetStruct(VT_LEAF); + } + template + bool Verify(::flatbuffers::VerifierTemplate &verifier) const { + return VerifyTableStart(verifier) && + VerifyField(verifier, VT_LEAF, 1) && + verifier.EndTable(); + } + BenchAssetRootT *UnPack(const ::flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(BenchAssetRootT *_o, const ::flatbuffers::resolver_function_t *_resolver = nullptr) const; + static ::flatbuffers::Offset Pack(::flatbuffers::FlatBufferBuilder &_fbb, const BenchAssetRootT* _o, const ::flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct BenchAssetRootBuilder { + typedef BenchAssetRoot Table; + ::flatbuffers::FlatBufferBuilder &fbb_; + ::flatbuffers::uoffset_t start_; + void add_leaf(const Flat::Bigfoot::HardReference *leaf) { + fbb_.AddStruct(BenchAssetRoot::VT_LEAF, leaf); + } + explicit BenchAssetRootBuilder(::flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + ::flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = ::flatbuffers::Offset(end); + return o; + } +}; + +inline ::flatbuffers::Offset CreateBenchAssetRoot( + ::flatbuffers::FlatBufferBuilder &_fbb, + const Flat::Bigfoot::HardReference *leaf = nullptr) { + BenchAssetRootBuilder builder_(_fbb); + builder_.add_leaf(leaf); + return builder_.Finish(); +} + +struct BenchAssetRoot::Traits { + using type = BenchAssetRoot; + static auto constexpr Create = CreateBenchAssetRoot; +}; + +::flatbuffers::Offset CreateBenchAssetRoot(::flatbuffers::FlatBufferBuilder &_fbb, const BenchAssetRootT *_o, const ::flatbuffers::rehasher_function_t *_rehasher = nullptr); + +inline BenchAssetRootT *BenchAssetRoot::UnPack(const ::flatbuffers::resolver_function_t *_resolver) const { + auto _o = std::make_unique(); + UnPackTo(_o.get(), _resolver); + return _o.release(); +} + +inline void BenchAssetRoot::UnPackTo(BenchAssetRootT *_o, const ::flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; + { auto _e = leaf(); if (_e) _o->leaf = ::flatbuffers::UnPackHardReferenceBenchAssetLeaf(*_e); } +} + +inline ::flatbuffers::Offset CreateBenchAssetRoot(::flatbuffers::FlatBufferBuilder &_fbb, const BenchAssetRootT *_o, const ::flatbuffers::rehasher_function_t *_rehasher) { + return BenchAssetRoot::Pack(_fbb, _o, _rehasher); +} + +inline ::flatbuffers::Offset BenchAssetRoot::Pack(::flatbuffers::FlatBufferBuilder &_fbb, const BenchAssetRootT* _o, const ::flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { ::flatbuffers::FlatBufferBuilder *__fbb; const BenchAssetRootT* __o; const ::flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + auto _leaf = ::flatbuffers::PackHardReferenceBenchAssetLeaf(_o->leaf); + return Flat::Bigfoot::CreateBenchAssetRoot( + _fbb, + &_leaf); +} + +inline const ::flatbuffers::TypeTable *BenchAssetRootTypeTable() { + static const ::flatbuffers::TypeCode type_codes[] = { + { ::flatbuffers::ET_SEQUENCE, 0, 0 } + }; + static const ::flatbuffers::TypeFunction type_refs[] = { + Flat::Bigfoot::HardReferenceTypeTable + }; + static const ::flatbuffers::TypeTable tt = { + ::flatbuffers::ST_TABLE, 1, type_codes, type_refs, nullptr, nullptr, nullptr + }; + return &tt; +} + +inline const Flat::Bigfoot::BenchAssetRoot *GetBenchAssetRoot(const void *buf) { + return ::flatbuffers::GetRoot(buf); +} + +inline const Flat::Bigfoot::BenchAssetRoot *GetSizePrefixedBenchAssetRoot(const void *buf) { + return ::flatbuffers::GetSizePrefixedRoot(buf); +} + +template +inline bool VerifyBenchAssetRootBuffer( + ::flatbuffers::VerifierTemplate &verifier) { + return verifier.template VerifyBuffer(nullptr); +} + +template +inline bool VerifySizePrefixedBenchAssetRootBuffer( + ::flatbuffers::VerifierTemplate &verifier) { + return verifier.template VerifySizePrefixedBuffer(nullptr); +} + +inline const char *BenchAssetRootExtension() { + return "bfbs"; +} + +inline void FinishBenchAssetRootBuffer( + ::flatbuffers::FlatBufferBuilder &fbb, + ::flatbuffers::Offset root) { + fbb.Finish(root); +} + +inline void FinishSizePrefixedBenchAssetRootBuffer( + ::flatbuffers::FlatBufferBuilder &fbb, + ::flatbuffers::Offset root) { + fbb.FinishSizePrefixed(root); +} + +inline eastl::unique_ptr UnPackBenchAssetRoot( + const void *buf, + const ::flatbuffers::resolver_function_t *res = nullptr) { + return eastl::unique_ptr(GetBenchAssetRoot(buf)->UnPack(res)); +} + +inline eastl::unique_ptr UnPackSizePrefixedBenchAssetRoot( + const void *buf, + const ::flatbuffers::resolver_function_t *res = nullptr) { + return eastl::unique_ptr(GetSizePrefixedBenchAssetRoot(buf)->UnPack(res)); +} + +} // namespace Bigfoot +} // namespace Flat + +namespace flatbuffers { +#ifndef FLATBUFFERS_BIGFOOT_REF_HARDREFERENCEBENCHASSETLEAF +#define FLATBUFFERS_BIGFOOT_REF_HARDREFERENCEBENCHASSETLEAF +inline Flat::Bigfoot::HardReference PackHardReferenceBenchAssetLeaf(const ::Bigfoot::HardReference<::Bigfoot::BenchAssetLeaf>& p_asset) { return {Pack(p_asset.GetUUID())}; } +inline ::Bigfoot::HardReference<::Bigfoot::BenchAssetLeaf> UnPackHardReferenceBenchAssetLeaf(const Flat::Bigfoot::HardReference& p_asset) { return {UnPack(p_asset.uuid())}; } +#endif // FLATBUFFERS_BIGFOOT_REF_HARDREFERENCEBENCHASSETLEAF +} // namespace flatbuffers + +#endif // FLATBUFFERS_GENERATED_BENCHASSETROOT_FLAT_BIGFOOT_H_ diff --git a/Bigfoot/Benchmarks/Misc/CMakeLists.txt b/Bigfoot/Benchmarks/Misc/CMakeLists.txt new file mode 100644 index 0000000..5d47403 --- /dev/null +++ b/Bigfoot/Benchmarks/Misc/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/AssetsForBenchmarking) diff --git a/Bigfoot/Sources/CMakeLists.txt b/Bigfoot/Sources/CMakeLists.txt index 8eca193..c489b24 100644 --- a/Bigfoot/Sources/CMakeLists.txt +++ b/Bigfoot/Sources/CMakeLists.txt @@ -1,3 +1,6 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/Utils) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/System) -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/Engine) \ No newline at end of file +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/Engine) +if(BIGFOOT_TOOL_MODE) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/Tools) +endif() \ No newline at end of file diff --git a/Bigfoot/Sources/Engine/Asset/AssetContainer.cpp b/Bigfoot/Sources/Engine/Asset/AssetContainer.cpp new file mode 100644 index 0000000..4535010 --- /dev/null +++ b/Bigfoot/Sources/Engine/Asset/AssetContainer.cpp @@ -0,0 +1,45 @@ +/********************************************************************* + * \file AssetContainer.cpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#include +#include + +namespace Bigfoot +{ +AssetContainerBase::Entry& AssetContainerBase::GetOrCreateEntry(const UUID& p_uuid) +{ + const std::lock_guard lock {m_entryMutex}; + + auto [it, inserted] = m_uuidToEntry.try_emplace(p_uuid); + if (inserted) + { + it->second.m_handleSlotKey = m_handles.Insert(); + + if (m_owningLibrary != nullptr) + { + m_owningLibrary->RegisterUUID(p_uuid, this); + } + } + + return it->second; +} + +/****************************************************************************************/ + +void AssetContainerBase::Remove(const UUID& p_uuid) +{ + if (Entry* entry = FindEntry(p_uuid); entry != nullptr && entry->m_handleSlotKey.Valid()) + { + m_handles.Remove(entry->m_handleSlotKey); + m_uuidToEntry.erase(p_uuid); + + if (m_owningLibrary != nullptr) + { + m_owningLibrary->UnregisterUUID(p_uuid); + } + } +} +} // namespace Bigfoot diff --git a/Bigfoot/Sources/Engine/Asset/SerializationManager.cpp b/Bigfoot/Sources/Engine/Asset/SerializationManager.cpp new file mode 100644 index 0000000..5595499 --- /dev/null +++ b/Bigfoot/Sources/Engine/Asset/SerializationManager.cpp @@ -0,0 +1,479 @@ +/********************************************************************* + * \file SerializationManager.cpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#include + +#include + +#include + +#if defined BIGFOOT_TOOL +#include + +#include +#else +#include +#include +#include +#endif + +namespace Bigfoot +{ +#if defined BIGFOOT_TOOL +namespace +{ +// Walks a flatbuffer generically (via its TypeTable) and collects the UUID of every HardReference and +// SoftReference struct field it finds, at any depth, sorted into the two lists by comparing each field's +// TypeTable against Flat::Bigfoot::HardReferenceTypeTable()/SoftReferenceTypeTable(). +// +// This works uniformly for scalar reference fields, vectors of references, and polymorphic +// HardReference/SoftReference fields alike: on the wire, a HardReference (or +// SoftReference) is always the same { uuid: UUID } struct regardless of the C++ template argument used on +// the native side. +class ReferenceCollector: public flatbuffers::IterationVisitor +{ + public: + void Field(const std::size_t /*p_fieldIndex*/, + const std::size_t /*p_setIndex*/, + const flatbuffers::ElementaryType p_type, + const bool p_isVector, + const flatbuffers::TypeTable* p_typeTable, + const char* /*p_name*/, + const std::uint8_t* p_value) override + { + // Vector fields are handled per-element via Element(): p_value here would only be the address + // of the field's offset slot, not usable as reference data. + if (!p_isVector) + { + TryCollect(p_type, p_typeTable, p_value); + } + } + + void Element(const std::size_t /*p_index*/, + const flatbuffers::ElementaryType p_type, + const flatbuffers::TypeTable* p_typeTable, + const std::uint8_t* p_value) override + { + TryCollect(p_type, p_typeTable, p_value); + } + + SerializationManager::ReferenceUUIDs m_references; + + private: + void TryCollect(const flatbuffers::ElementaryType p_type, + const flatbuffers::TypeTable* p_typeTable, + const std::uint8_t* p_value) + { + if (p_value == nullptr || p_type != flatbuffers::ET_SEQUENCE) + { + return; + } + + // A HardReference/SoftReference struct is exactly a UUID (uuid: UUID, at struct offset 0), stored + // inline: its address directly holds the 16 raw UUID bytes. + const std::span uuidBytes { + reinterpret_cast(p_value), UUID::UUID_BYTE_SIZE}; + + if (p_typeTable == ::Flat::Bigfoot::HardReferenceTypeTable()) + { + m_references.m_hard.push_back(UUID {uuidBytes}); + } + else if (p_typeTable == ::Flat::Bigfoot::SoftReferenceTypeTable()) + { + m_references.m_soft.push_back(UUID {uuidBytes}); + } + } +}; +} // namespace + +SerializationManager::SerializationManager(BigFile& p_bigFile): + m_bigFile(&p_bigFile) +{ +} +#else +SerializationManager::SerializationManager(Sasquatch& p_sasquatch): + m_sasquatch(&p_sasquatch) +{ +} +#endif + +/****************************************************************************************/ + +void SerializationManager::LoadAsset(const UUID& p_uuid) +{ + if (!p_uuid) + { + return; + } + +#if defined BIGFOOT_TOOL + BigFile::Request assetRequest {*m_bigFile, + "SELECT AssetHeader.TypeID, Asset.Asset FROM Asset " + "JOIN AssetHeader ON AssetHeader.UUID = Asset.UUID " + "WHERE Asset.UUID = ?1;"}; + assetRequest.Bind(1, UUIDToSpan(p_uuid)); + + if (!assetRequest.Step()) + { + return; + } + + const AssetTypeID typeID = assetRequest.Get(0); + const eastl::span bytes = assetRequest.Get(1); +#else + const std::optional entry = m_sasquatch->Find(p_uuid); + if (!entry.has_value()) + { + return; + } + + const AssetTypeID typeID = entry->m_typeID; + const eastl::span bytes = entry->m_bytes; +#endif + + AssetContainerBase* container = g_activeAssetLibrary->GetAssetContainer(typeID); + CRITICAL_ASSERT(EngineAssertHandler, container != nullptr, "No AssetContainer registered for AssetTypeID {}", typeID); + + if (container->Has(p_uuid) || !m_loadingInProgress.insert(p_uuid).second) + { + // Already loaded, or a HardReference cycle led back to a UUID an ancestor call is still loading: + // skip either way, the ancestor call already owns finishing that load. + return; + } + + // Load every hard-referenced asset before constructing this one, so that this asset's own reference + // fields - notably HardReference/SoftReference, which can only resolve to an + // already-loaded asset - find their target immediately when they're constructed below. +#if defined BIGFOOT_TOOL + BigFile::Request hardReferenceRequest {*m_bigFile, "SELECT ReferencedUUID FROM AssetHardReference WHERE UUID = ?1;"}; + hardReferenceRequest.Bind(1, UUIDToSpan(p_uuid)); + while (hardReferenceRequest.Step()) + { + LoadAsset(SpanToUUID(hardReferenceRequest.Get(0))); + } +#else + for (std::size_t offset = 0; offset < entry->m_hardReferenceUUIDs.size(); offset += UUID::UUID_BYTE_SIZE) + { + LoadAsset(SpanToUUID(entry->m_hardReferenceUUIDs.subspan(offset, UUID::UUID_BYTE_SIZE))); + } +#endif + + container->Insert(bytes); + + m_loadingInProgress.erase(p_uuid); +} + +/****************************************************************************************/ + +#if !defined BIGFOOT_TOOL +namespace +{ +struct PendingLoad +{ + AssetContainerBase* m_container; + eastl::span m_bytes; + eastl::vector m_hardReferences; +}; + +// Decodes every UUID in p_layer in parallel - striped across workers so per-item cost variance (a big +// texture next to a tiny material) balances out rather than one worker being left with all the heavy +// items - then registers them one at a time. See LoadAssets for why decode and register are split like +// this at all. +void DecodeAndInsertLayer(const ankerl::unordered_dense::segmented_map& p_pending, + const eastl::vector& p_layer) +{ + eastl::vector> decoded(p_layer.size()); + + const auto decodeOne = [&](const std::size_t p_index) + { + const PendingLoad& load = p_pending.find(p_layer[p_index])->second; + decoded[p_index] = load.m_container->DecodeOffContainer(load.m_bytes); + }; + + // Not worth spinning up threads to decode a handful of assets - the spawn/join overhead would dwarf + // the work itself. + static constexpr std::size_t PARALLEL_THRESHOLD = 32; + const std::size_t threadCount = + std::min(std::max(1, std::thread::hardware_concurrency()), p_layer.size()); + + if (threadCount <= 1 || p_layer.size() < PARALLEL_THRESHOLD) + { + for (std::size_t i = 0; i < p_layer.size(); ++i) + { + decodeOne(i); + } + } + else + { + std::vector workers; + workers.reserve(threadCount); + for (std::size_t worker = 0; worker < threadCount; ++worker) + { + workers.emplace_back( + [&, worker]() + { + for (std::size_t i = worker; i < p_layer.size(); i += threadCount) + { + decodeOne(i); + } + }); + } + for (std::thread& worker: workers) + { + worker.join(); + } + } + + for (std::size_t i = 0; i < p_layer.size(); ++i) + { + p_pending.find(p_layer[i])->second.m_container->InsertDecoded(std::move(decoded[i])); + } +} +} // namespace + +void SerializationManager::LoadAssets(const eastl::span p_uuids) +{ + // Phase 1: discover every UUID that needs loading - p_uuids plus everything they transitively hold a + // HardReference to - skipping anything already loaded, exactly like LoadAsset's own recursion (just + // breadth-first with an explicit stack instead of the call stack, since layering needs the whole set + // up front). + ankerl::unordered_dense::segmented_map pending; + eastl::vector toVisit {p_uuids.begin(), p_uuids.end()}; + + while (!toVisit.empty()) + { + const UUID uuid = toVisit.back(); + toVisit.pop_back(); + + if (!uuid || pending.contains(uuid)) + { + continue; + } + + const std::optional entry = m_sasquatch->Find(uuid); + if (!entry.has_value()) + { + continue; + } + + AssetContainerBase* container = g_activeAssetLibrary->GetAssetContainer(entry->m_typeID); + CRITICAL_ASSERT(EngineAssertHandler, container != nullptr, "No AssetContainer registered for AssetTypeID {}", + entry->m_typeID); + + if (container->Has(uuid)) + { + continue; + } + + PendingLoad load; + load.m_container = container; + load.m_bytes = entry->m_bytes; + for (std::size_t offset = 0; offset < entry->m_hardReferenceUUIDs.size(); offset += UUID::UUID_BYTE_SIZE) + { + const UUID referenced = SpanToUUID(entry->m_hardReferenceUUIDs.subspan(offset, UUID::UUID_BYTE_SIZE)); + load.m_hardReferences.push_back(referenced); + toVisit.push_back(referenced); + } + + pending.emplace(uuid, std::move(load)); + } + + // Phase 2: repeatedly peel off a "ready" layer - every still-pending UUID whose HardReferences are all + // no longer pending (either outside this batch entirely and already loaded, or registered in an + // earlier layer this call) - decode that layer in parallel, then register it serially before moving to + // the next layer. A cycle within the batch would mean no UUID is ever ready; that should never happen + // (see LoadAssets' doc comment) so it asserts rather than looping forever. + ankerl::unordered_dense::segmented_set remaining; + for (const auto& [uuid, load]: pending) + { + remaining.insert(uuid); + } + + eastl::vector layer; + while (!remaining.empty()) + { + layer.clear(); + for (const UUID& uuid: remaining) + { + bool ready = true; + for (const UUID& referenced: pending.find(uuid)->second.m_hardReferences) + { + if (remaining.contains(referenced)) + { + ready = false; + break; + } + } + + if (ready) + { + layer.push_back(uuid); + } + } + + CRITICAL_ASSERT(EngineAssertHandler, !layer.empty(), "Cycle detected among HardReferences during LoadAssets"); + + DecodeAndInsertLayer(pending, layer); + + for (const UUID& uuid: layer) + { + remaining.erase(uuid); + } + } +} +#endif + +/****************************************************************************************/ + +#if defined BIGFOOT_TOOL +void SerializationManager::SaveAsset(const UUID& p_uuid) +{ + if (!p_uuid) + { + return; + } + + AssetContainerBase* container = g_activeAssetLibrary->FindContainerForUUID(p_uuid); + AssetBase* asset = container != nullptr ? container->AcquireContainerHandle(p_uuid)->GetAsset() : nullptr; + CRITICAL_ASSERT(EngineAssertHandler, asset != nullptr, "SaveAsset: no asset currently loaded for UUID {}", p_uuid); + + const ::Flat::Bigfoot::AssetHeaderT& header = asset->GetHeader(); + const eastl::vector bytes = asset->Pack(); + + const ::Flat::Bigfoot::Asset* outerAsset = flatbuffers::GetRoot<::Flat::Bigfoot::Asset>(bytes.data()); + const ReferenceUUIDs references = + CollectReferenceUUIDs(outerAsset->inner_asset()->data(), asset->GetInnerTypeTable()); + + const eastl::span uuidBytes = UUIDToSpan(header.uuid); + + m_bigFile->BeginTransaction(); + + BigFile::Request headerRequest {*m_bigFile, + "INSERT INTO AssetHeader (UUID, Name, TypeID, TypeName, Version) " + "VALUES (?1, ?2, ?3, ?4, ?5) " + "ON CONFLICT(UUID) DO UPDATE SET " + "Name = excluded.Name, TypeID = excluded.TypeID, TypeName = excluded.TypeName, " + "Version = excluded.Version;"}; + headerRequest.Bind(1, uuidBytes); + headerRequest.Bind(2, eastl::string_view {header.name.data(), header.name.size()}); + headerRequest.Bind(3, header.type_id); + headerRequest.Bind(4, eastl::string_view {header.type_name.data(), header.type_name.size()}); + headerRequest.Bind(5, header.version); + std::ignore = headerRequest.Execute(); + + BigFile::Request assetRequest {*m_bigFile, + "INSERT INTO Asset (UUID, Asset) VALUES (?1, ?2) " + "ON CONFLICT(UUID) DO UPDATE SET Asset = excluded.Asset;"}; + assetRequest.Bind(1, uuidBytes); + assetRequest.Bind(2, eastl::span {bytes.data(), bytes.size()}); + std::ignore = assetRequest.Execute(); + + BigFile::Request clearHardReferencesRequest {*m_bigFile, "DELETE FROM AssetHardReference WHERE UUID = ?1;"}; + clearHardReferencesRequest.Bind(1, uuidBytes); + std::ignore = clearHardReferencesRequest.Execute(); + + BigFile::Request clearSoftReferencesRequest {*m_bigFile, "DELETE FROM AssetSoftReference WHERE UUID = ?1;"}; + clearSoftReferencesRequest.Bind(1, uuidBytes); + std::ignore = clearSoftReferencesRequest.Execute(); + + InsertReferences("INSERT INTO AssetHardReference (UUID, ReferencedUUID) VALUES (?1, ?2) ON CONFLICT DO NOTHING;", + uuidBytes, + references.m_hard); + InsertReferences("INSERT INTO AssetSoftReference (UUID, ReferencedUUID) VALUES (?1, ?2) ON CONFLICT DO NOTHING;", + uuidBytes, + references.m_soft); + + m_bigFile->CommitTransaction(); +} + +/****************************************************************************************/ + +eastl::unique_ptr<::Flat::Bigfoot::AssetHeaderT> SerializationManager::GetAssetHeader(const UUID& p_uuid) const +{ + if (!p_uuid) + { + return nullptr; + } + + BigFile::Request request {*m_bigFile, "SELECT Name, TypeID, TypeName, Version FROM AssetHeader WHERE UUID = ?1;"}; + request.Bind(1, UUIDToSpan(p_uuid)); + + if (!request.Step()) + { + return nullptr; + } + + auto header = eastl::make_unique<::Flat::Bigfoot::AssetHeaderT>(); + header->uuid = p_uuid; + + const eastl::string_view name = request.Get(0); + header->name = eastl::string {name.data(), name.size()}; + header->type_id = request.Get(1); + const eastl::string_view typeName = request.Get(2); + header->type_name = eastl::string {typeName.data(), typeName.size()}; + header->version = request.Get(3); + + return header; +} + +/****************************************************************************************/ +#endif + +void SerializationManager::RemoveUnreferencedAssets() +{ + for (const UUID& uuid: g_activeAssetLibrary->TakeUnreferencedUUIDs()) + { + AssetContainerBase* container = g_activeAssetLibrary->FindContainerForUUID(uuid); + if (container != nullptr && container->AcquireContainerHandle(uuid)->GetHardRefCount() == 0) + { + container->Remove(uuid); + } + } +} + +/****************************************************************************************/ + +eastl::span SerializationManager::UUIDToSpan(const UUID& p_uuid) +{ + const std::span bytes = p_uuid.ToBytes(); + return eastl::span {bytes.data(), bytes.size()}; +} + +/****************************************************************************************/ + +UUID SerializationManager::SpanToUUID(const eastl::span p_bytes) +{ + ASSERT(EngineAssertHandler, p_bytes.size() == UUID::UUID_BYTE_SIZE, "Invalid UUID blob size"); + return UUID {std::span {p_bytes.data(), UUID::UUID_BYTE_SIZE}}; +} + +#if defined BIGFOOT_TOOL +/****************************************************************************************/ + +SerializationManager::ReferenceUUIDs SerializationManager::CollectReferenceUUIDs( + const std::uint8_t* p_buffer, const flatbuffers::TypeTable* p_typeTable) +{ + ReferenceCollector collector; + flatbuffers::IterateFlatBuffer(p_buffer, p_typeTable, &collector); + return std::move(collector.m_references); +} + +/****************************************************************************************/ + +void SerializationManager::InsertReferences(const eastl::string_view p_insertQuery, + const eastl::span p_uuidBytes, + const eastl::vector& p_referencedUUIDs) +{ + for (const UUID& referencedUUID: p_referencedUUIDs) + { + BigFile::Request request {*m_bigFile, p_insertQuery}; + request.Bind(1, p_uuidBytes); + request.Bind(2, UUIDToSpan(referencedUUID)); + std::ignore = request.Execute(); + } +} +#endif +} // namespace Bigfoot diff --git a/Bigfoot/Sources/Engine/BigFile/BigFile.cpp b/Bigfoot/Sources/Engine/BigFile/BigFile.cpp index e2f8bc4..f5239eb 100644 --- a/Bigfoot/Sources/Engine/BigFile/BigFile.cpp +++ b/Bigfoot/Sources/Engine/BigFile/BigFile.cpp @@ -19,6 +19,13 @@ BigFile::BigFile(const File& p_file) "Failed to open BigFile {} DB: {}", p_file.ToAbsolute().GetPath(), sqlite3_errmsg(m_db)); + + // Lets SQLite read pages straight out of a memory mapping instead of read()-ing them into its own page + // cache - a plain read-perf win for the Asset blob column in particular, which dwarfs every other + // column. Not persisted (PRAGMA, not a schema change): every connection has to set this itself. + [[maybe_unused]] + const int mmapResult = sqlite3_exec(m_db, "PRAGMA mmap_size=1073741824;", nullptr, nullptr, nullptr); + ASSERT(EngineAssertHandler, mmapResult == SQLITE_OK, "Failed to set BigFile mmap_size: {}", sqlite3_errmsg(m_db)); } /****************************************************************************************/ @@ -52,6 +59,17 @@ void BigFile::RollbackTransaction() BigFile::~BigFile() { + for (auto& [query, statements]: m_statementPool) + { + for (sqlite3_stmt* const statement: statements) + { + [[maybe_unused]] + const int result = sqlite3_finalize(statement); + CRITICAL_ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to finalize pooled statement: {}", + sqlite3_errmsg(m_db)); + } + } + [[maybe_unused]] const int result = sqlite3_close_v2(m_db); CRITICAL_ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to close BigFile DB: {}", sqlite3_errmsg(m_db)); @@ -59,19 +77,52 @@ BigFile::~BigFile() /****************************************************************************************/ -BigFile::Request::Request(const BigFile& p_bigFile, const eastl::string_view p_request): - m_db(p_bigFile.m_db) +sqlite3_stmt* BigFile::AcquireStatement(const eastl::string_view p_query) const { + eastl::vector& pool = m_statementPool[std::string {p_query.data(), p_query.size()}]; + if (!pool.empty()) + { + sqlite3_stmt* const statement = pool.back(); + pool.pop_back(); + + [[maybe_unused]] + const int resetResult = sqlite3_reset(statement); + ASSERT(EngineAssertHandler, resetResult == SQLITE_OK, "Failed to reset pooled statement: {}", + sqlite3_errmsg(m_db)); + + [[maybe_unused]] + const int clearResult = sqlite3_clear_bindings(statement); + ASSERT(EngineAssertHandler, clearResult == SQLITE_OK, "Failed to clear pooled statement bindings: {}", + sqlite3_errmsg(m_db)); + + return statement; + } + + sqlite3_stmt* statement = nullptr; [[maybe_unused]] - const int result = sqlite3_prepare_v2(m_db, - p_request.data(), - static_cast(p_request.size()), - &m_statement, - nullptr); - CRITICAL_ASSERT(EngineAssertHandler, - result == SQLITE_OK, - "Failed to create statement from BigFile DB: {}", + const int result = + sqlite3_prepare_v2(m_db, p_query.data(), static_cast(p_query.size()), &statement, nullptr); + CRITICAL_ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to create statement from BigFile DB: {}", sqlite3_errmsg(m_db)); + + return statement; +} + +/****************************************************************************************/ + +void BigFile::ReleaseStatement(const eastl::string_view p_query, sqlite3_stmt* const p_statement) const +{ + m_statementPool[std::string {p_query.data(), p_query.size()}].push_back(p_statement); +} + +/****************************************************************************************/ + +BigFile::Request::Request(const BigFile& p_bigFile, const eastl::string_view p_request): + m_db(p_bigFile.m_db), + m_owner(&p_bigFile), + m_queryText(p_request), + m_statement(p_bigFile.AcquireStatement(p_request)) +{ } /****************************************************************************************/ @@ -179,12 +230,16 @@ void BigFile::Request::Bind(const std::uint32_t p_index, (p_index >= 1) && (p_index <= static_cast(sqlite3_bind_parameter_count(m_statement))), "Invalid index for statement"); + // sqlite3_bind_blob treats a NULL data pointer as binding SQL NULL rather than an empty blob, even + // when the size is 0 - which eastl::span::data() commonly is for an empty span. Bind through a + // dummy non-null address in that case so an empty span always binds an empty (non-NULL) blob. + static constexpr std::byte EMPTY_BLOB_SENTINEL {}; + const std::byte* data = p_value.empty() ? &EMPTY_BLOB_SENTINEL : p_value.data(); + [[maybe_unused]] - const int result = sqlite3_bind_blob(m_statement, - p_index, - p_value.data(), - static_cast(p_value.size()), - p_copy ? SQLITE_TRANSIENT : SQLITE_STATIC); + const int result = + sqlite3_bind_blob(m_statement, p_index, data, static_cast(p_value.size()), + p_copy ? SQLITE_TRANSIENT : SQLITE_STATIC); ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db)); } @@ -292,8 +347,6 @@ BigFile::Request::Column BigFile::Request::Get(const std::uint32_t p_index) cons BigFile::Request::~Request() { - [[maybe_unused]] - const int result = sqlite3_finalize(m_statement); - CRITICAL_ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to finalize statement: {}", sqlite3_errmsg(m_db)); + m_owner->ReleaseStatement(m_queryText, m_statement); } } // namespace Bigfoot diff --git a/Bigfoot/Sources/Engine/BigFile/BigFileSchema.sql b/Bigfoot/Sources/Engine/BigFile/BigFileSchema.sql index e329324..1f219ce 100644 --- a/Bigfoot/Sources/Engine/BigFile/BigFileSchema.sql +++ b/Bigfoot/Sources/Engine/BigFile/BigFileSchema.sql @@ -6,6 +6,7 @@ CREATE TABLE IF NOT EXISTS AssetHeader ( Name TEXT NOT NULL UNIQUE, TypeID INTEGER NOT NULL, TypeName TEXT NOT NULL, + Version INTEGER NOT NULL DEFAULT 0, CreateTime INTEGER NOT NULL DEFAULT(CAST(unixepoch('subsec') AS INTEGER) * 1000000), ModificationTime INTEGER NOT NULL DEFAULT(CAST(unixepoch('subsec') AS INTEGER) * 1000000), @@ -14,7 +15,7 @@ CREATE TABLE IF NOT EXISTS AssetHeader ( ); CREATE TRIGGER IF NOT EXISTS AssetHeader_UpdateTime -AFTER UPDATE OF Name, TypeID, TypeName ON AssetHeader FOR EACH ROW +AFTER UPDATE OF Name, TypeID, TypeName, Version ON AssetHeader FOR EACH ROW BEGIN UPDATE AssetHeader SET ModificationTime = CAST(unixepoch('subsec') AS INTEGER) * 1000000 @@ -36,4 +37,38 @@ BEGIN UPDATE AssetHeader SET ModificationTime = CAST(unixepoch('subsec') AS INTEGER) * 1000000 WHERE UUID = NEW.UUID; -END; \ No newline at end of file +END; + +-- Every asset another asset points at via a HardReference field, one row per (referrer, referenced) pair. +-- Populated by SerializationManager on save (discovered generically via flatbuffers mini-reflection, see +-- SerializationManager.cpp), so that SerializationManager::LoadAsset can recursively load an asset's +-- HardReferences without having to deserialize the asset's own bytes first. Also directly queryable/ +-- debuggable in SQL, e.g.: +-- SELECT ReferencedUUID FROM AssetHardReference WHERE UUID = ? -- what does this asset reference? +-- SELECT UUID FROM AssetHardReference WHERE ReferencedUUID = ? -- what references this asset? +-- SELECT AssetHeader.Name FROM AssetHardReference JOIN AssetHeader +-- ON AssetHeader.UUID = AssetHardReference.ReferencedUUID WHERE ... -- with human-readable names +CREATE TABLE IF NOT EXISTS AssetHardReference ( + UUID BLOB NOT NULL, + ReferencedUUID BLOB NOT NULL, + + PRIMARY KEY(UUID, ReferencedUUID), + FOREIGN KEY(UUID) REFERENCES AssetHeader(UUID) ON DELETE CASCADE +); + +-- For "what references this asset" lookups: the primary key above only indexes UUID -> ReferencedUUID +-- efficiently, not the reverse. +CREATE INDEX IF NOT EXISTS AssetHardReference_ReferencedUUID ON AssetHardReference(ReferencedUUID); + +-- Same as AssetHardReference, but for SoftReference fields. Purely informational: SoftReferences are +-- never eagerly loaded, so LoadAsset never reads this table - it exists so the full reference graph, +-- including optional/lazy links, is queryable and debuggable directly in SQL too. +CREATE TABLE IF NOT EXISTS AssetSoftReference ( + UUID BLOB NOT NULL, + ReferencedUUID BLOB NOT NULL, + + PRIMARY KEY(UUID, ReferencedUUID), + FOREIGN KEY(UUID) REFERENCES AssetHeader(UUID) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS AssetSoftReference_ReferencedUUID ON AssetSoftReference(ReferencedUUID); diff --git a/Bigfoot/Sources/Engine/Include/Engine/Asset/Asset.hpp b/Bigfoot/Sources/Engine/Include/Engine/Asset/Asset.hpp index c4b945d..2b0f219 100644 --- a/Bigfoot/Sources/Engine/Include/Engine/Asset/Asset.hpp +++ b/Bigfoot/Sources/Engine/Include/Engine/Asset/Asset.hpp @@ -22,6 +22,34 @@ class AssetBase public: virtual ~AssetBase() = default; + /* + * Packs this asset's header and data into the flatbuffer byte representation used throughout the + * BigFile. + * + * \return The packed bytes + */ + [[nodiscard]] + virtual eastl::vector Pack() const = 0; + + /* + * Get the asset's header + * + * \return The header + */ + [[nodiscard]] + virtual const ::Flat::Bigfoot::AssetHeaderT& GetHeader() const = 0; + + /* + * Get the TypeTable describing this asset's own (inner) flatbuffer type, as generated by flatc's + * --reflect-types (see FLAT_ASSET::MiniReflectTypeTable()). Lets code that only has an AssetBase& + * (e.g. SerializationManager::SaveAsset) walk an asset's packed bytes generically - to find its + * HardReferences, for instance - without needing to know its concrete type at compile time. + * + * \return The TypeTable for this asset's own flatbuffer type + */ + [[nodiscard]] + virtual const flatbuffers::TypeTable* GetInnerTypeTable() const = 0; + protected: AssetBase() = default; @@ -44,8 +72,20 @@ class Asset: public AssetBase m_header.type_name = eastl::string {GetTypeName()}; } - explicit Asset(const eastl::span p_flatbuffer): - Asset() + Asset(const Asset& p_asset) = default; + Asset(Asset&& p_asset) = default; + + ~Asset() = default; + + /* + * Overwrites this asset's header and data by unpacking p_flatbuffer, as produced by Pack(). Meant to + * be called right after default-construction (see AssetContainer::Insert(span)): unlike a + * "from bytes" constructor, this lets the caller construct the asset in place first and only register + * it (by its real UUID, read from p_flatbuffer) once its data is actually in place. + * + * \param p_flatbuffer The flatbuffer to unpack, as produced by Pack() + */ + void Unpack(const eastl::span p_flatbuffer) { #ifdef BIGFOOT_NOT_OPTIMIZED flatbuffers::Verifier verifier {std::bit_cast(p_flatbuffer.data()), p_flatbuffer.size()}; @@ -65,17 +105,42 @@ class Asset: public AssetBase flatbuffers::GetRoot(asset->inner_asset()->data())->UnPackTo(&m_asset, nullptr); } - Asset(const Asset& p_asset) = default; - Asset(Asset&& p_asset) = default; + /* + * Packs this asset's header and data into the flatbuffer byte representation used throughout the + * BigFile - the format Unpack() (and AssetContainer::Insert(span)) expects. + * + * \return The packed bytes + */ + [[nodiscard]] + eastl::vector Pack() const override + { + flatbuffers::FlatBufferBuilder innerAssetFbb; + innerAssetFbb.Finish(FLAT_ASSET::Pack(innerAssetFbb, &m_asset)); - ~Asset() = default; + ::Flat::Bigfoot::AssetT asset; + asset.header = eastl::make_unique<::Flat::Bigfoot::AssetHeaderT>(m_header); + asset.inner_asset = eastl::vector {innerAssetFbb.GetBufferPointer(), + innerAssetFbb.GetBufferPointer() + innerAssetFbb.GetSize()}; + + flatbuffers::FlatBufferBuilder assetFbb; + assetFbb.Finish(::Flat::Bigfoot::Asset::Pack(assetFbb, &asset)); + + return eastl::vector {std::bit_cast(assetFbb.GetBufferPointer()), + std::bit_cast(assetFbb.GetBufferPointer() + assetFbb.GetSize())}; + } [[nodiscard]] - const ::Flat::Bigfoot::AssetHeaderT& GetHeader() const + const ::Flat::Bigfoot::AssetHeaderT& GetHeader() const override { return m_header; } + [[nodiscard]] + const flatbuffers::TypeTable* GetInnerTypeTable() const override + { + return FLAT_ASSET::MiniReflectTypeTable(); + } + void SetName(const eastl::string_view p_name) { m_header.name = p_name; @@ -98,24 +163,6 @@ class Asset: public AssetBase return m_asset; } - [[nodiscard]] - eastl::vector Save() const - { - flatbuffers::FlatBufferBuilder innerAssetFbb; - innerAssetFbb.Finish(FLAT_ASSET::Pack(innerAssetFbb, &m_asset)); - - ::Flat::Bigfoot::AssetT asset; - asset.header = eastl::make_unique<::Flat::Bigfoot::AssetHeaderT>(m_header); - asset.inner_asset = eastl::vector {innerAssetFbb.GetBufferPointer(), - innerAssetFbb.GetBufferPointer() + innerAssetFbb.GetSize()}; - - flatbuffers::FlatBufferBuilder assetFbb; - assetFbb.Finish(::Flat::Bigfoot::Asset::Pack(assetFbb, &asset)); - - return eastl::vector {std::bit_cast(assetFbb.GetBufferPointer()), - std::bit_cast(assetFbb.GetBufferPointer() + assetFbb.GetSize())}; - } - [[nodiscard]] static std::uint64_t GetTypeID() { diff --git a/Bigfoot/Sources/Engine/Include/Engine/Asset/AssetContainer.hpp b/Bigfoot/Sources/Engine/Include/Engine/Asset/AssetContainer.hpp index f516c8f..893a3d7 100644 --- a/Bigfoot/Sources/Engine/Include/Engine/Asset/AssetContainer.hpp +++ b/Bigfoot/Sources/Engine/Include/Engine/Asset/AssetContainer.hpp @@ -11,19 +11,40 @@ #include +#include +#include #include +#include #include +#include +#include namespace Bigfoot { +class AssetLibrary; + class ContainerHandle { public: ContainerHandle() = default; - ContainerHandle(const ContainerHandle& p_handle) = default; - ContainerHandle(ContainerHandle&& p_handle) = default; + // Copying/moving an atomic isn't allowed by default (and never actually happens: every ContainerHandle + // is default-constructed once in place by StableSlotMap::Insert() and never relocated afterwards - + // StableSlotMap's pages are exactly what makes that a stable guarantee) - these snapshot the counts + // rather than moving the atomics themselves, which would only be wrong if a copy/move genuinely raced + // with a concurrent Increment/Decrement, which nothing in this codebase does. + ContainerHandle(const ContainerHandle& p_handle): + m_hardRefCount(p_handle.m_hardRefCount.load(std::memory_order_relaxed)), + m_softRefCount(p_handle.m_softRefCount.load(std::memory_order_relaxed)), + m_asset(p_handle.m_asset) + { + } + + ContainerHandle(ContainerHandle&& p_handle) noexcept: + ContainerHandle(p_handle) + { + } ~ContainerHandle() = default; @@ -35,25 +56,27 @@ class ContainerHandle [[nodiscard]] std::uint32_t GetHardRefCount() const { - return m_hardRefCount; + return m_hardRefCount.load(std::memory_order_relaxed); } /* - * Increment the HardRef count + * Increment the HardRef count. Atomic: LoadAssets decodes a batch's HardReference fields (which + * increment the referenced asset's count) from multiple threads at once, possibly the same asset from + * more than one of them (e.g. several assets sharing one referenced texture). * */ void IncrementHardRefCount() { - ++m_hardRefCount; + m_hardRefCount.fetch_add(1, std::memory_order_relaxed); } /* - * Decrement the HardRef count + * Decrement the HardRef count. See IncrementHardRefCount for why this is atomic. * */ void DecrementHardRefCount() { - --m_hardRefCount; + m_hardRefCount.fetch_sub(1, std::memory_order_relaxed); } /* @@ -64,25 +87,25 @@ class ContainerHandle [[nodiscard]] std::uint32_t GetSoftRefCount() const { - return m_softRefCount; + return m_softRefCount.load(std::memory_order_relaxed); } /* - * Increment the SoftRef count + * Increment the SoftRef count. See IncrementHardRefCount for why this is atomic. * */ void IncrementSoftRefCount() { - ++m_softRefCount; + m_softRefCount.fetch_add(1, std::memory_order_relaxed); } /* - * Decrement the SoftRef count + * Decrement the SoftRef count. See IncrementHardRefCount for why this is atomic. * */ void DecrementSoftRefCount() { - --m_softRefCount; + m_softRefCount.fetch_sub(1, std::memory_order_relaxed); } /* @@ -117,19 +140,32 @@ class ContainerHandle m_asset = p_asset; } - ContainerHandle& operator=(const ContainerHandle& p_handle) = default; - ContainerHandle& operator=(ContainerHandle&& p_handle) = default; + ContainerHandle& operator=(const ContainerHandle& p_handle) + { + if (this != &p_handle) + { + m_hardRefCount.store(p_handle.m_hardRefCount.load(std::memory_order_relaxed), std::memory_order_relaxed); + m_softRefCount.store(p_handle.m_softRefCount.load(std::memory_order_relaxed), std::memory_order_relaxed); + m_asset = p_handle.m_asset; + } + return *this; + } + + ContainerHandle& operator=(ContainerHandle&& p_handle) noexcept + { + return *this = p_handle; + } private: /* - * The HardRef count + * The HardRef count. Atomic - see IncrementHardRefCount. */ - std::uint32_t m_hardRefCount = 0; + std::atomic m_hardRefCount = 0; /* - * The SoftRef count + * The SoftRef count. Atomic - see IncrementHardRefCount. */ - std::uint32_t m_softRefCount = 0; + std::atomic m_softRefCount = 0; /* * The asset @@ -178,13 +214,54 @@ class AssetContainerBase * * \param p_uuid The UUID to remove */ - virtual void Remove(const UUID& p_uuid) + virtual void Remove(const UUID& p_uuid); + + /* + * Type-erased overload of AssetContainer::Insert, used by SerializationManager (which only has an + * AssetContainerBase*, the concrete ASSET being resolved at runtime via AssetTypeID) to construct and + * insert an asset of this container's concrete type from its serialized bytes. The asset's UUID is + * read from the bytes themselves (its header), same as AssetContainer::Insert. + * + * \param p_bytes The serialized bytes of the asset, as produced by Asset::Save() + */ + virtual void Insert(const eastl::span p_bytes) = 0; + + /* + * Type-erased, container-state-free half of Insert: constructs an asset of this container's concrete + * type and unpacks p_bytes into it, but never touches this container (no SlotMap/map mutation) - safe + * to call concurrently, from any number of threads, on the same AssetContainerBase, including + * concurrently with other DecodeOffContainer calls on completely different containers. Pairs with + * InsertDecoded, which does the (cheap, must-stay-single-threaded) other half. See + * SerializationManager::LoadAssets, which uses this pair to decode a batch of assets in parallel while + * only ever mutating a container from one thread at a time. + * + * \param p_bytes The serialized bytes of the asset, as produced by Asset::Save() + * \return The decoded asset, not yet part of this container + */ + [[nodiscard]] + virtual eastl::unique_ptr DecodeOffContainer(const eastl::span p_bytes) const = 0; + + /* + * Moves an asset decoded by DecodeOffContainer into this container, exactly as Insert(bytes) would have + * - not safe to call concurrently with anything else touching this container (Insert, Remove, + * DecodeOffContainer is fine, InsertDecoded is not). + * + * \param p_decoded An asset produced by this same container's DecodeOffContainer + */ + virtual void InsertDecoded(eastl::unique_ptr p_decoded) = 0; + + /* + * Set the AssetLibrary this container is registered into. Called once by AssetLibrary::Insert. + * + * Needed so that newly created UUID entries (see GetOrCreateEntry) can be published into the + * library's UUID -> AssetContainerBase index, which HardReference/SoftReference + * use to resolve a bare UUID to its container without knowing its concrete type ahead of time. + * + * \param p_library The owning AssetLibrary + */ + void SetOwningLibrary(AssetLibrary* p_library) { - if (Entry* entry = FindEntry(p_uuid); entry != nullptr && entry->m_handleSlotKey.Valid()) - { - m_handles.Remove(entry->m_handleSlotKey); - m_uuidToEntry.erase(p_uuid); - } + m_owningLibrary = p_library; } AssetContainerBase& operator=(const AssetContainerBase& p_container) = delete; @@ -239,20 +316,17 @@ class AssetContainerBase * Finds an entry matching the Asset * Creates it if it does not exist * + * Thread-safe against concurrent calls to itself (guarded by m_entryMutex): SerializationManager:: + * LoadAssets resolves Hard/SoftReference fields - which call this indirectly via + * AcquireContainerHandle - from multiple threads at once while decoding a batch, and a reference can + * turn out to be the first thing that ever touches its target's entry (e.g. an unloaded + * SoftReference), so two threads can race to create the same entry. + * * \param p_uuid The UUID to find * \return The entry matching the Asset */ [[nodiscard]] - Entry& GetOrCreateEntry(const UUID& p_uuid) - { - auto [it, inserted] = m_uuidToEntry.try_emplace(p_uuid); - if (inserted) - { - it->second.m_handleSlotKey = m_handles.Insert(); - } - - return it->second; - } + Entry& GetOrCreateEntry(const UUID& p_uuid); /* * Get a ContainerHandle @@ -276,6 +350,16 @@ class AssetContainerBase * Every Asset added to the map */ ankerl::unordered_dense::segmented_map m_uuidToEntry; + + /* + * Guards m_uuidToEntry/m_handles against concurrent GetOrCreateEntry calls - see its comment. + */ + std::mutex m_entryMutex; + + /* + * The AssetLibrary this container is registered into, if any. Set via SetOwningLibrary. + */ + AssetLibrary* m_owningLibrary = nullptr; }; /****************************************************************************************/ @@ -305,22 +389,7 @@ class AssetContainer: public AssetContainerBase [[nodiscard]] SlotMapKey Insert(ARGS&&... p_args) { - SlotMapKey slotKey = m_assets.Insert(std::forward(p_args)...); - ASSET* asset = m_assets.Get(slotKey); - - Entry& entry = GetOrCreateEntry(asset->GetHeader().uuid); - if (entry.m_assetSlotKey.Valid()) - { - m_assets.Remove(slotKey); - slotKey = entry.m_assetSlotKey; - } - else - { - entry.m_assetSlotKey = slotKey; - FindContainerHandle(entry.m_handleSlotKey).SetAsset(asset); - } - - return slotKey; + return Register(EmplaceUnregistered(std::forward(p_args)...)); } /* @@ -364,10 +433,78 @@ class AssetContainer: public AssetContainerBase AssetContainerBase::Remove(p_uuid); } + void Insert(const eastl::span p_bytes) override + { + // Construct a default asset in place first (a placeholder, random UUID), fill it in from p_bytes + // (which sets its real UUID), and only then register it - registering before Unpack would index + // the entry under the placeholder UUID instead of the asset's real one. + const SlotMapKey slotKey = EmplaceUnregistered(); + m_assets.Get(slotKey)->Unpack(p_bytes); + std::ignore = Register(slotKey); + } + + [[nodiscard]] + eastl::unique_ptr DecodeOffContainer(const eastl::span p_bytes) const override + { + auto asset = eastl::make_unique(); + asset->Unpack(p_bytes); + return asset; + } + + void InsertDecoded(eastl::unique_ptr p_decoded) override + { + // Move rather than re-construct: the caller already paid for Unpack() via DecodeOffContainer, this + // should be nothing more than placing an already-built object into a slot. + eastl::unique_ptr decoded {static_cast(p_decoded.release())}; + std::ignore = Register(EmplaceUnregistered(std::move(*decoded))); + } + AssetContainer& operator=(const AssetContainer& p_container) = delete; AssetContainer& operator=(AssetContainer&& p_container) = delete; private: + /* + * Constructs an Asset in place from p_args, without registering it under any UUID yet. Split out from + * Insert so that Insert(span) can fill in a default-constructed asset (via Asset::Unpack, + * which may change its UUID) before registration happens - see Register. + * + * \tparam ARGS Arguments to construct the asset + * \param p_args The arguments + * \return The key of the constructed, not-yet-registered asset + */ + template + [[nodiscard]] + SlotMapKey EmplaceUnregistered(ARGS&&... p_args) + { + return m_assets.Insert(std::forward(p_args)...); + } + + /* + * Registers an asset previously constructed via EmplaceUnregistered under its current UUID: creates + * or reuses the entry for that UUID and wires up its ContainerHandle. If another asset is already + * registered under the same UUID, p_slotKey's asset is discarded and the existing slot's key is + * returned instead. + * + * \param p_slotKey The key of a not-yet-registered asset, from EmplaceUnregistered + * \return The key of the now-registered asset (p_slotKey, or an existing slot on UUID collision) + */ + [[nodiscard]] + SlotMapKey Register(const SlotMapKey p_slotKey) + { + ASSET* asset = m_assets.Get(p_slotKey); + + Entry& entry = GetOrCreateEntry(asset->GetHeader().uuid); + if (entry.m_assetSlotKey.Valid()) + { + m_assets.Remove(p_slotKey); + return entry.m_assetSlotKey; + } + + entry.m_assetSlotKey = p_slotKey; + FindContainerHandle(entry.m_handleSlotKey).SetAsset(asset); + return p_slotKey; + } + /* * The assets */ diff --git a/Bigfoot/Sources/Engine/Include/Engine/Asset/AssetLibrary.hpp b/Bigfoot/Sources/Engine/Include/Engine/Asset/AssetLibrary.hpp index fa34daf..b03c140 100644 --- a/Bigfoot/Sources/Engine/Include/Engine/Asset/AssetLibrary.hpp +++ b/Bigfoot/Sources/Engine/Include/Engine/Asset/AssetLibrary.hpp @@ -11,6 +11,9 @@ #include +#include +#include + namespace Bigfoot { class AssetLibrary @@ -18,7 +21,16 @@ class AssetLibrary public: AssetLibrary() = default; - AssetLibrary(const AssetLibrary& p_library) = default; + // std::mutex isn't copyable, so this can't be = default: copies every member except the mutexes, which + // a copy just default-constructs its own instances of (a mutex protects one specific object's + // internals, copying a lock along with the data it guards would be meaningless). + AssetLibrary(const AssetLibrary& p_library): + m_assetContainers(p_library.m_assetContainers), + m_containerByUUID(p_library.m_containerByUUID), + m_unreferencedUUIDs(p_library.m_unreferencedUUIDs) + { + } + AssetLibrary(AssetLibrary&& p_library) = delete; ~AssetLibrary() = default; @@ -41,6 +53,7 @@ class AssetLibrary } const auto inserted = m_assetContainers.emplace(typeID, p_assetContainer); + p_assetContainer->SetOwningLibrary(this); return inserted.second; } @@ -79,12 +92,137 @@ class AssetLibrary return nullptr; } - AssetLibrary& operator=(const AssetLibrary& p_library) = default; + /** + * Find whichever registered AssetContainer currently owns a given UUID, regardless of its concrete + * type. Used to resolve HardReference/SoftReference (polymorphic references, + * which only carry a UUID - the concrete type isn't known statically) generically. + * + * Only finds UUIDs that already have an entry in some container (see AssetContainerBase:: + * GetOrCreateEntry / RegisterUUID) - unlike GetAssetContainer(AssetTypeID), there is no "reserve a + * slot for a UUID we haven't seen yet" here, since which container to reserve it in isn't knowable + * without the concrete type. + * + * \param p_uuid The UUID to find + * \return The AssetContainer currently holding p_uuid. Nullptr if none does + */ + [[nodiscard]] + AssetContainerBase* FindContainerForUUID(const UUID& p_uuid) + { + const std::lock_guard lock {m_containerByUUIDMutex}; + if (const auto container = m_containerByUUID.find(p_uuid); container != m_containerByUUID.end()) + { + return container->second; + } + + return nullptr; + } + + /** + * Publish that p_uuid now lives in p_container, so that FindContainerForUUID can find it. Called by + * AssetContainerBase::GetOrCreateEntry whenever it creates a new entry for a UUID - not meant to be + * called directly. + * + * \param p_uuid The UUID that now has an entry in p_container + * \param p_container The container the UUID's entry lives in + */ + void RegisterUUID(const UUID& p_uuid, AssetContainerBase* p_container) + { + const std::lock_guard lock {m_containerByUUIDMutex}; + m_containerByUUID[p_uuid] = p_container; + } + + /** + * Undo RegisterUUID. Called by AssetContainerBase::Remove - not meant to be called directly. + * + * \param p_uuid The UUID to forget + */ + void UnregisterUUID(const UUID& p_uuid) + { + const std::lock_guard lock {m_containerByUUIDMutex}; + m_containerByUUID.erase(p_uuid); + } + + /** + * Mark p_uuid as a candidate for removal: nothing holds a HardReference to it anymore (its + * ContainerHandle's hard ref count just reached zero). Called by HardReference when the last + * HardReference pointing at an asset is released - not meant to be called directly. See + * SerializationManager::RemoveUnreferencedAssets, which consumes these via TakeUnreferencedUUIDs. + * + * \param p_uuid The UUID whose HardReference count just reached zero + */ + void MarkUnreferenced(const UUID& p_uuid) + { + const std::lock_guard lock {m_unreferencedMutex}; + m_unreferencedUUIDs.insert(p_uuid); + } + + /** + * Undo MarkUnreferenced: p_uuid is no longer a removal candidate. Called by HardReference whenever a + * new HardReference to p_uuid is acquired (even if it was never marked - erasing an absent key is a + * harmless no-op) - not meant to be called directly. + * + * \param p_uuid The UUID that just gained a HardReference + */ + void UnmarkUnreferenced(const UUID& p_uuid) + { + const std::lock_guard lock {m_unreferencedMutex}; + m_unreferencedUUIDs.erase(p_uuid); + } + + /** + * Take (and clear) every UUID marked via MarkUnreferenced since the last call. A UUID coming out of + * here is only a candidate: it may have gained a new HardReference since being marked, so callers + * should re-check before actually removing anything (see SerializationManager::RemoveUnreferencedAssets). + * + * \return The UUIDs marked as unreferenced since the last call + */ + [[nodiscard]] + ankerl::unordered_dense::segmented_set TakeUnreferencedUUIDs() + { + const std::lock_guard lock {m_unreferencedMutex}; + ankerl::unordered_dense::segmented_set uuids = std::move(m_unreferencedUUIDs); + m_unreferencedUUIDs.clear(); + return uuids; + } + + AssetLibrary& operator=(const AssetLibrary& p_library) + { + if (this != &p_library) + { + m_assetContainers = p_library.m_assetContainers; + { + const std::lock_guard lock {m_containerByUUIDMutex}; + m_containerByUUID = p_library.m_containerByUUID; + } + const std::lock_guard lock {m_unreferencedMutex}; + m_unreferencedUUIDs = p_library.m_unreferencedUUIDs; + } + return *this; + } AssetLibrary& operator=(AssetLibrary&& p_library) = delete; private: // All the asset containers ankerl::unordered_dense::segmented_map m_assetContainers; + + // Guards m_containerByUUID: SerializationManager::LoadAssets resolves Hard/SoftReference + // fields (which call RegisterUUID/FindContainerForUUID) from multiple threads at once while decoding a + // batch, and different AssetContainers - each with their own AssetContainerBase::m_entryMutex - can + // both be publishing a newly-seen UUID into this single shared map at the same time. + std::mutex m_containerByUUIDMutex; + + // Every UUID that has an entry in some container, and which container that is. See FindContainerForUUID. + ankerl::unordered_dense::segmented_map m_containerByUUID; + + // Guards m_unreferencedUUIDs only: SerializationManager::LoadAssets resolves HardReference fields + // (which call MarkUnreferenced/UnmarkUnreferenced) from multiple threads at once while decoding a + // batch. m_assetContainers/m_containerByUUID need no such guard - they only ever mutate during + // AssetLibrary::Insert/RegisterUUID/UnregisterUUID, which LoadAssets always does single-threaded (see + // AssetContainerBase::InsertDecoded). + std::mutex m_unreferencedMutex; + + // UUIDs whose HardReference count reached zero since the last TakeUnreferencedUUIDs. See MarkUnreferenced. + ankerl::unordered_dense::segmented_set m_unreferencedUUIDs; }; /****************************************************************************************/ diff --git a/Bigfoot/Sources/Engine/Include/Engine/Asset/Reference.hpp b/Bigfoot/Sources/Engine/Include/Engine/Asset/Reference.hpp index dcf0281..7aa3876 100644 --- a/Bigfoot/Sources/Engine/Include/Engine/Asset/Reference.hpp +++ b/Bigfoot/Sources/Engine/Include/Engine/Asset/Reference.hpp @@ -10,6 +10,7 @@ #include +#include #include namespace Bigfoot @@ -103,13 +104,28 @@ class ReferenceBase return; } - if (AssetContainerBase* assetContainer = g_activeAssetLibrary->GetAssetContainer(GetAssetTypeID())) + // AssetBase has no AssetContainer of its own (it's not a concrete, loadable type): a + // Hard/SoftReference can point at an asset of any type, so its container has to be + // found from the UUID alone rather than via the usual static GetAssetTypeID() lookup. This + // only finds assets that are already loaded somewhere - see AssetLibrary::FindContainerForUUID. + AssetContainerBase* assetContainer; + if constexpr (std::is_same_v) + { + assetContainer = g_activeAssetLibrary->FindContainerForUUID(m_uuid); + } + else + { + assetContainer = g_activeAssetLibrary->GetAssetContainer(GetAssetTypeID()); + } + + if (assetContainer) { if (!m_handle) { m_handle = assetContainer->AcquireContainerHandle(m_uuid); } DERIVED::IncrementCount(*m_handle); + DERIVED::OnCountIncremented(m_uuid, *m_handle); } } @@ -118,6 +134,7 @@ class ReferenceBase if (m_handle) { DERIVED::DecrementCount(*m_handle); + DERIVED::OnCountDecremented(m_uuid, *m_handle); m_handle = nullptr; } } @@ -150,6 +167,24 @@ class HardReference: public ReferenceBase, ASSET> p_handle.DecrementHardRefCount(); } + // Registers p_uuid as a candidate for SerializationManager::RemoveUnreferencedAssets once the last + // HardReference pointing at it is released - instead of that having to scan every loaded asset to + // find the ones nothing holds a HardReference to anymore. + static void OnCountDecremented(const UUID& p_uuid, const ContainerHandle& p_handle) + { + if (p_handle.GetHardRefCount() == 0) + { + g_activeAssetLibrary->MarkUnreferenced(p_uuid); + } + } + + // A new HardReference to p_uuid means it's no longer a candidate for removal, even if it was + // previously marked (its count had reached zero before, but has since been acquired again). + static void OnCountIncremented(const UUID& p_uuid, const ContainerHandle&) + { + g_activeAssetLibrary->UnmarkUnreferenced(p_uuid); + } + friend Base; }; @@ -177,6 +212,17 @@ class SoftReference: public ReferenceBase, ASSET> p_handle.DecrementSoftRefCount(); } + // SoftReferences don't keep an asset alive, so releasing one is never a reason to consider the asset + // for removal. + static void OnCountDecremented(const UUID&, const ContainerHandle&) + { + } + + // SoftReferences don't affect removal eligibility - only HardReference does. + static void OnCountIncremented(const UUID&, const ContainerHandle&) + { + } + friend Base; }; } // namespace Bigfoot diff --git a/Bigfoot/Sources/Engine/Include/Engine/Asset/SerializationManager.hpp b/Bigfoot/Sources/Engine/Include/Engine/Asset/SerializationManager.hpp new file mode 100644 index 0000000..976babc --- /dev/null +++ b/Bigfoot/Sources/Engine/Include/Engine/Asset/SerializationManager.hpp @@ -0,0 +1,226 @@ +/********************************************************************* + * \file SerializationManager.hpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#ifndef BIGFOOT_ENGINE_ASSET_SERIALIZATIONMANAGER_HPP +#define BIGFOOT_ENGINE_ASSET_SERIALIZATIONMANAGER_HPP +#include +#include +#if defined BIGFOOT_TOOL +#include +#else +#include +#endif + +#include + +#include +#include +#include +#include +#include + +#include + +namespace Bigfoot +{ +/* + * Bridges the in-memory AssetLibrary/AssetContainer world with an on-disk asset store: loads assets (and + * their hard-referenced assets, recursively) into the active AssetLibrary, and - Tool mode only - saves + * assets (along with the set of assets they hold a HardReference to) back out. + * + * The backend is picked at compile time: Tool mode (BIGFOOT_TOOL) reads/writes a BigFile directly over + * SQL; Game mode (BIGFOOT_GAME) only ever reads a Sasquatch, the mmap'd, load-only flattening of a BigFile + * produced ahead of time by Sasquatch::WriteFrom or the SasquatchPacker tool. SaveAsset does not exist in + * Game mode - a running game never writes back to its own asset pack. + */ +class SerializationManager +{ + public: +#if defined BIGFOOT_TOOL + /* + * Constructor + * + * \param p_bigFile The BigFile to load assets from / save assets to + */ + explicit SerializationManager(BigFile& p_bigFile); +#else + /* + * Constructor + * + * \param p_sasquatch The Sasquatch to load assets from + */ + explicit SerializationManager(Sasquatch& p_sasquatch); +#endif + + SerializationManager(const SerializationManager& p_manager) = default; + SerializationManager(SerializationManager&& p_manager) = default; + + ~SerializationManager() = default; + + /* + * Loads an asset from the BigFile into the active AssetLibrary. Every asset it (transitively) holds a + * HardReference to is loaded first, recursively, so that by the time this asset itself is constructed + * its own reference fields - including polymorphic HardReference/SoftReference, + * which can only resolve to an already-loaded asset - immediately find their target. SoftReferences + * are never eagerly loaded. + * + * Does nothing if the UUID is null, is not present in the BigFile, or has already been loaded. Also + * safe against HardReference cycles: a UUID already in the middle of being loaded (an ancestor in the + * current recursion) is skipped rather than re-entered. + * + * \param p_uuid The UUID of the asset to load + */ + void LoadAsset(const UUID& p_uuid); + +#if !defined BIGFOOT_TOOL + /* + * Loads every asset in p_uuids into the active AssetLibrary, and every asset they (transitively) hold + * a HardReference to - same end result as calling LoadAsset(uuid) once per uuid, except the CPU-heavy + * part (Asset::Unpack, which deserializes the flatbuffer and resolves the asset's own reference + * fields) runs across multiple threads instead of one at a time. + * + * Assets are grouped into dependency layers - an asset's layer is one past the deepest layer among + * what it HardReferences, so every HardReference target is fully registered before its referrer is + * even decoded - each layer is decoded in parallel (AssetContainerBase::DecodeOffContainer, which + * touches nothing but the asset being decoded) and then that layer is registered serially + * (AssetContainerBase::InsertDecoded) before moving on to the next. HardReference/SoftReference + * resolution during Unpack is safe to run concurrently across a layer (see AssetContainer.hpp's + * m_entryMutex and AssetLibrary.hpp's mutexes) but container registration itself is not, hence the + * layering. + * + * Same skip rules as LoadAsset: a null, not-present, or already-loaded UUID is silently skipped, and a + * UUID reachable through more than one path (shared by several requested assets, or requested directly + * as well as referenced) is only ever decoded once. Unlike LoadAsset, a HardReference cycle across the + * batch asserts rather than being silently skipped - Sasquatch-packed data should never contain one + * (WriteFrom always runs against a BigFile, and nothing writes a cyclic HardReference graph into a + * BigFile in the first place). + * + * \param p_uuids The UUIDs to load + */ + void LoadAssets(const eastl::span p_uuids); +#endif + +#if defined BIGFOOT_TOOL + /* + * Saves the asset currently loaded under p_uuid (in the active AssetLibrary, found the same way + * HardReference/SoftReference resolve a bare UUID - see + * AssetLibrary::FindContainerForUUID) to the BigFile: inserts it if its UUID is not present in the + * BigFile yet, overwrites it otherwise. + * + * Every asset it holds a HardReference or SoftReference to is discovered generically via flatbuffers' + * mini-reflection (walking the packed bytes using AssetBase::GetInnerTypeTable()) and (re)written to + * the AssetHardReference/AssetSoftReference tables, so that LoadAsset can later recursively load its + * HardReferences without having to deserialize the asset itself again, and so the full reference graph + * (Hard and Soft alike) is queryable directly in SQL for debugging. + * + * Does nothing if the UUID is null. Asserts if no asset is currently loaded under p_uuid: unlike + * LoadAsset (which treats "nothing to load" as a legitimate, silent case), asking to save a UUID + * nothing is loaded for is a programmer error. + * + * Tool-only: a running game never writes back to its own asset pack, see the class comment. + * + * \param p_uuid The UUID of the (currently loaded) asset to save + */ + void SaveAsset(const UUID& p_uuid); + + /* + * Reads just an asset's header/metadata (name, type, version) straight from the BigFile's AssetHeader + * table, without touching the (often much bigger) Asset blob column and without unpacking anything - + * cheap enough to call for e.g. an asset browser or a dependency listing, unlike LoadAsset. + * + * Tool-only: Sasquatch doesn't carry this metadata (Game mode never needs an asset browser), so this + * has no Game-mode equivalent. + * + * \param p_uuid The UUID of the asset whose header to read + * \return The header, or nullptr if p_uuid is null or not present in the BigFile + */ + [[nodiscard]] + eastl::unique_ptr<::Flat::Bigfoot::AssetHeaderT> GetAssetHeader(const UUID& p_uuid) const; +#endif + + /* + * Removes every asset currently marked as unreferenced - nothing holds a HardReference to it anymore, + * see HardReference/AssetLibrary::MarkUnreferenced - from the active AssetLibrary. Each candidate's + * hard ref count is re-checked before actually removing it, since a new HardReference could have been + * acquired after it was marked, so this is safe to call at any time. + * + * This only unloads assets from memory: it does not touch the BigFile, an unreferenced asset can + * still be found again later via LoadAsset. + */ + void RemoveUnreferencedAssets(); + + SerializationManager& operator=(const SerializationManager& p_manager) = default; + SerializationManager& operator=(SerializationManager&& p_manager) = default; + +#if defined BIGFOOT_TOOL + /* + * The UUIDs a single asset's HardReference and SoftReference fields point at, as discovered by + * CollectReferenceUUIDs. Public only so the reflection visitor implementing CollectReferenceUUIDs + * (an unrelated class, in SerializationManager.cpp's anonymous namespace) can fill one in directly. + */ + struct ReferenceUUIDs + { + eastl::vector m_hard; + eastl::vector m_soft; + }; +#endif + + private: + [[nodiscard]] + static eastl::span UUIDToSpan(const UUID& p_uuid); + + [[nodiscard]] + static UUID SpanToUUID(const eastl::span p_bytes); + +#if defined BIGFOOT_TOOL + /* + * Generically collects the UUIDs of every HardReference and SoftReference struct found while walking + * p_buffer (a standalone flatbuffer, e.g. the inner bytes of an Asset::Pack() buffer) using + * p_typeTable (obtained via AssetBase::GetInnerTypeTable()). + * + * \param p_buffer The flatbuffer to walk + * \param p_typeTable The TypeTable describing p_buffer's root type + * \return The UUIDs of every HardReference/SoftReference found + */ + [[nodiscard]] + static ReferenceUUIDs CollectReferenceUUIDs(const std::uint8_t* p_buffer, + const flatbuffers::TypeTable* p_typeTable); + + /* + * Writes one AssetHardReference/AssetSoftReference row per UUID in p_referencedUUIDs, all under + * p_uuidBytes. p_insertQuery picks which of the two tables - it must be an INSERT INTO + * AssetHardReference/AssetSoftReference (UUID, ReferencedUUID) VALUES (?1, ?2) ... statement, ?1 bound + * to the referrer's UUID and ?2 to each referenced UUID in turn. + * + * \param p_insertQuery The INSERT statement to run once per referenced UUID + * \param p_uuidBytes The referrer's UUID, as bound to ?1 + * \param p_referencedUUIDs The UUIDs to write one row for each, bound to ?2 in turn + */ + void InsertReferences(const eastl::string_view p_insertQuery, + const eastl::span p_uuidBytes, + const eastl::vector& p_referencedUUIDs); + + /* + * The BigFile assets are loaded from / saved to + */ + BigFile* m_bigFile; +#else + /* + * The Sasquatch assets are loaded from + */ + Sasquatch* m_sasquatch; +#endif + + /* + * UUIDs currently in the middle of being loaded (an active LoadAsset call is somewhere up the + * recursion stack for them). Used to detect and skip HardReference cycles instead of recursing + * forever - see LoadAsset. + */ + ankerl::unordered_dense::segmented_set m_loadingInProgress; +}; +} // namespace Bigfoot + +#endif diff --git a/Bigfoot/Sources/Engine/Include/Engine/BigFile/BigFile.hpp b/Bigfoot/Sources/Engine/Include/Engine/BigFile/BigFile.hpp index cb1c562..5b23941 100644 --- a/Bigfoot/Sources/Engine/Include/Engine/BigFile/BigFile.hpp +++ b/Bigfoot/Sources/Engine/Include/Engine/BigFile/BigFile.hpp @@ -11,8 +11,12 @@ #include #include +#include +#include #include +#include + namespace Bigfoot { class BigFile @@ -259,6 +263,17 @@ class BigFile */ sqlite3* m_db; + /* + * The BigFile this statement was acquired from - kept only to return the statement to its pool + * (see BigFile::ReleaseStatement) on destruction. + */ + const BigFile* m_owner; + + /* + * The query text this statement was prepared from - identifies which pool bucket to return it to. + */ + eastl::string_view m_queryText; + /* * The statement */ @@ -266,10 +281,42 @@ class BigFile }; private: + /* + * Borrows a compiled statement for p_query: an idle one from m_statementPool if one's available, + * freshly prepared otherwise. Pooled per query text rather than cached as a single instance per text + * because SerializationManager::LoadAsset recurses: a nested call for the same query text must get + * its own statement, not reset the one an ancestor call further up the stack is still stepping + * through. + * + * \param p_query The query text to get a statement for + * \return A statement ready to bind and step, reset to its start state + */ + [[nodiscard]] + sqlite3_stmt* AcquireStatement(const eastl::string_view p_query) const; + + /* + * Returns a statement borrowed via AcquireStatement to its pool, for the next AcquireStatement(p_query) + * to reuse instead of recompiling. + * + * \param p_query The query text p_statement was acquired for + * \param p_statement The statement to return + */ + void ReleaseStatement(const eastl::string_view p_query, sqlite3_stmt* p_statement) const; + /** * The BigFile DB */ sqlite3* m_db; + + /* + * Idle, already-compiled statements, keyed by their query text and reused by AcquireStatement/ + * ReleaseStatement instead of recompiling identical SQL on every BigFile::Request. Mutable: acquiring/ + * releasing a statement doesn't change what a BigFile logically contains, only this cache of it - + * every BigFile::Request constructor takes a const BigFile&. Keyed by std::string rather than + * eastl::string: no std::hash specialization exists for ankerl::unordered_dense to key + * off of, and this cache is purely a private implementation detail. + */ + mutable ankerl::unordered_dense::segmented_map> m_statementPool; }; } // namespace Bigfoot diff --git a/Bigfoot/Sources/Engine/Include/Engine/Sasquatch/Sasquatch.hpp b/Bigfoot/Sources/Engine/Include/Engine/Sasquatch/Sasquatch.hpp new file mode 100644 index 0000000..3a79121 --- /dev/null +++ b/Bigfoot/Sources/Engine/Include/Engine/Sasquatch/Sasquatch.hpp @@ -0,0 +1,106 @@ +/********************************************************************* + * \file Sasquatch.hpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#ifndef BIGFOOT_ENGINE_SASQUATCH_SASQUATCH_HPP +#define BIGFOOT_ENGINE_SASQUATCH_SASQUATCH_HPP +#include +#if defined BIGFOOT_TOOL +#include +#endif + +#include +#include + +#include +#include + +#include +#include + +namespace Bigfoot +{ +/* + * Read-only, mmap'd counterpart to a BigFile: every Asset/AssetHeader row (and the UUIDs it holds a + * HardReference to) flattened into one file, so a running game can look an asset up by UUID with no SQL, + * no allocation and no per-lookup syscall. See SerializationManager, which uses this as its Game-mode + * backend (BigFile is the Tool-mode one) - Game builds only ever construct one of these and call Find, they + * never write one (see WriteFrom, Tool-only). + */ +class Sasquatch +{ + public: + /* + * Opens and maps p_file, and builds the UUID -> entry index Find() looks up into. p_file must already + * exist, produced ahead of time by WriteFrom (Tool mode) or the SasquatchPacker tool. + * + * \param p_file The .sasquatch file to open + */ + explicit Sasquatch(const File& p_file); + + Sasquatch(const Sasquatch& p_sasquatch) = delete; + Sasquatch(Sasquatch&& p_sasquatch) = delete; + + ~Sasquatch(); + + Sasquatch& operator=(const Sasquatch& p_sasquatch) = delete; + Sasquatch& operator=(Sasquatch&& p_sasquatch) = delete; + + /* + * A single asset, as found by Find(): a direct, zero-copy view into the mapped file. + */ + struct Entry + { + /* + * The asset's type + */ + AssetTypeID m_typeID; + + /* + * The packed Asset flatbuffer bytes, as produced by Asset::Pack() - what + * AssetContainerBase::Insert(span) expects. + */ + eastl::span m_bytes; + + /* + * The UUIDs this asset holds a HardReference to, concatenated UUID::UUID_BYTE_SIZE bytes apiece + * (see SerializationManager::SpanToUUID) - the Sasquatch equivalent of BigFileSchema.sql's + * AssetHardReference table, precomputed by WriteFrom so LoadAsset never has to touch m_bytes just + * to find them. + */ + eastl::span m_hardReferenceUUIDs; + }; + + /* + * Looks up an asset by UUID: a hash lookup into the index built at construction, then a direct read + * into the mapped file. + * + * \param p_uuid The UUID to find + * \return The asset, or std::nullopt if p_uuid is not present + */ + [[nodiscard]] + std::optional Find(const UUID& p_uuid) const; + +#if defined BIGFOOT_TOOL + /* + * Flattens every Asset/AssetHeader row in p_bigFile - and, for each, the UUIDs it holds a + * HardReference to (from AssetHardReference) - into a .sasquatch file at p_outFile. Tool-only: packing + * is a build-time step, not something a running game ever does (see SerializationManager, which never + * saves in Game mode). + * + * \param p_bigFile The BigFile to read every asset from + * \param p_outFile Where to write the .sasquatch file + */ + static void WriteFrom(BigFile& p_bigFile, const File& p_outFile); +#endif + + private: + struct Impl; + + eastl::unique_ptr m_impl; +}; +} // namespace Bigfoot + +#endif diff --git a/Bigfoot/Sources/Engine/Sasquatch/Sasquatch.cpp b/Bigfoot/Sources/Engine/Sasquatch/Sasquatch.cpp new file mode 100644 index 0000000..8891dde --- /dev/null +++ b/Bigfoot/Sources/Engine/Sasquatch/Sasquatch.cpp @@ -0,0 +1,298 @@ +/********************************************************************* + * \file Sasquatch.cpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#include +#include + +#include + +#include +#include + +#if defined BIGFOOT_TOOL +#include +#endif + +#if defined BIGFOOT_WINDOWS +#include +#else +#include +#include +#include +#include +#endif + +namespace Bigfoot +{ +namespace +{ +// "SASQUATC" read as a little-endian uint64 - just a format sentinel, never round-tripped through +// anything that cares about the actual text. +constexpr std::uint64_t SASQUATCH_MAGIC = 0x4354415551534153ULL; + +struct SasquatchHeader +{ + std::uint64_t m_magic = SASQUATCH_MAGIC; + std::uint32_t m_version = 1; + std::uint32_t m_assetCount = 0; +}; + +// On-disk per-asset record. Offsets are absolute from the start of the file. m_hardReferenceOffset points +// at m_hardReferenceCount UUID::UUID_BYTE_SIZE-byte blocks back to back; m_blobOffset points at the packed +// Asset flatbuffer, as produced by Asset::Pack(). +struct SasquatchEntry +{ + std::byte m_uuid[UUID::UUID_BYTE_SIZE]; + std::uint64_t m_typeID; + std::uint64_t m_blobOffset; + std::uint64_t m_blobSize; + std::uint64_t m_hardReferenceOffset; + std::uint64_t m_hardReferenceCount; +}; + +#if defined BIGFOOT_WINDOWS +// File::GetPath() comes from std::filesystem::path::string(), which on Windows is the native narrow +// (ACP) encoding, not UTF-8 - convert with the matching codepage so non-ASCII paths round-trip correctly. +[[nodiscard]] +std::wstring ToWidePath(const eastl::string_view p_path) +{ + if (p_path.empty()) + { + return {}; + } + + const int requiredSize = MultiByteToWideChar(CP_ACP, 0, p_path.data(), static_cast(p_path.size()), nullptr, 0); + std::wstring wide(static_cast(requiredSize), L'\0'); + MultiByteToWideChar(CP_ACP, 0, p_path.data(), static_cast(p_path.size()), wide.data(), requiredSize); + return wide; +} +#endif +} // namespace + +struct Sasquatch::Impl +{ +#if defined BIGFOOT_WINDOWS + HANDLE m_fileHandle = INVALID_HANDLE_VALUE; + HANDLE m_mapping = nullptr; +#else + int m_fileDescriptor = -1; +#endif + + const std::byte* m_base = nullptr; + std::size_t m_size = 0; + + // UUID -> on-disk entry index, built once here (see Sasquatch::Sasquatch) so Find() is a hash lookup + // instead of a scan. + ankerl::unordered_dense::segmented_map m_index; + + ~Impl() + { +#if defined BIGFOOT_WINDOWS + if (m_base != nullptr) + { + UnmapViewOfFile(m_base); + } + if (m_mapping != nullptr) + { + CloseHandle(m_mapping); + } + if (m_fileHandle != INVALID_HANDLE_VALUE) + { + CloseHandle(m_fileHandle); + } +#else + if (m_base != nullptr && m_base != MAP_FAILED) + { + munmap(const_cast(m_base), m_size); + } + if (m_fileDescriptor != -1) + { + close(m_fileDescriptor); + } +#endif + } +}; + +Sasquatch::Sasquatch(const File& p_file): + m_impl(eastl::make_unique()) +{ + const File absoluteFile = p_file.ToAbsolute(); + const eastl::string_view path = absoluteFile.GetPath(); + +#if defined BIGFOOT_WINDOWS + // FILE_SHARE_DELETE matches POSIX unlink-while-open semantics (Sasquatch::WriteFrom/a packaging tool + // replacing this file, or a test cleaning up, shouldn't have to wait on every reader to close first) - + // Windows otherwise refuses to delete a file that's still open here, even just for reading. + m_impl->m_fileHandle = CreateFileW(ToWidePath(path).c_str(), GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + CRITICAL_ASSERT(EngineAssertHandler, m_impl->m_fileHandle != INVALID_HANDLE_VALUE, + "Failed to open Sasquatch file {}", path); + + LARGE_INTEGER size {}; + GetFileSizeEx(m_impl->m_fileHandle, &size); + m_impl->m_size = static_cast(size.QuadPart); + + m_impl->m_mapping = CreateFileMappingW(m_impl->m_fileHandle, nullptr, PAGE_READONLY, 0, 0, nullptr); + CRITICAL_ASSERT(EngineAssertHandler, m_impl->m_mapping != nullptr, "Failed to map Sasquatch file {}", path); + + m_impl->m_base = static_cast(MapViewOfFile(m_impl->m_mapping, FILE_MAP_READ, 0, 0, 0)); + CRITICAL_ASSERT(EngineAssertHandler, m_impl->m_base != nullptr, "Failed to map Sasquatch file {}", path); +#else + const std::string nullTerminatedPath {path.begin(), path.end()}; + + m_impl->m_fileDescriptor = open(nullTerminatedPath.c_str(), O_RDONLY); + CRITICAL_ASSERT(EngineAssertHandler, m_impl->m_fileDescriptor != -1, "Failed to open Sasquatch file {}", path); + + struct stat fileStat + { + }; + fstat(m_impl->m_fileDescriptor, &fileStat); + m_impl->m_size = static_cast(fileStat.st_size); + + m_impl->m_base = static_cast( + mmap(nullptr, m_impl->m_size, PROT_READ, MAP_PRIVATE, m_impl->m_fileDescriptor, 0)); + CRITICAL_ASSERT(EngineAssertHandler, m_impl->m_base != MAP_FAILED, "Failed to map Sasquatch file {}", path); +#endif + + const auto* header = reinterpret_cast(m_impl->m_base); + CRITICAL_ASSERT(EngineAssertHandler, header->m_magic == SASQUATCH_MAGIC, "{} is not a Sasquatch file", path); + + const auto* entries = reinterpret_cast(m_impl->m_base + sizeof(SasquatchHeader)); + + m_impl->m_index.reserve(header->m_assetCount); + for (std::uint32_t i = 0; i < header->m_assetCount; ++i) + { + const SasquatchEntry& entry = entries[i]; + const UUID uuid {std::span {entry.m_uuid, UUID::UUID_BYTE_SIZE}}; + m_impl->m_index.emplace(uuid, &entry); + } +} + +/****************************************************************************************/ + +Sasquatch::~Sasquatch() = default; + +/****************************************************************************************/ + +std::optional Sasquatch::Find(const UUID& p_uuid) const +{ + const auto it = m_impl->m_index.find(p_uuid); + if (it == m_impl->m_index.end()) + { + return std::nullopt; + } + + const SasquatchEntry& entry = *it->second; + return Entry {entry.m_typeID, + eastl::span {m_impl->m_base + entry.m_blobOffset, entry.m_blobSize}, + eastl::span {m_impl->m_base + entry.m_hardReferenceOffset, + entry.m_hardReferenceCount * UUID::UUID_BYTE_SIZE}}; +} + +/****************************************************************************************/ + +#if defined BIGFOOT_TOOL +void Sasquatch::WriteFrom(BigFile& p_bigFile, const File& p_outFile) +{ + struct Row + { + std::byte m_uuid[UUID::UUID_BYTE_SIZE]; + std::uint64_t m_typeID; + eastl::vector m_bytes; + eastl::vector m_hardReferenceUUIDs; + }; + + eastl::vector rows; + ankerl::unordered_dense::segmented_map uuidToRowIndex; + + { + BigFile::Request request {p_bigFile, + "SELECT AssetHeader.UUID, AssetHeader.TypeID, Asset.Asset FROM Asset " + "JOIN AssetHeader ON AssetHeader.UUID = Asset.UUID;"}; + while (request.Step()) + { + const eastl::span uuidBytes = request.Get(0); + const AssetTypeID typeID = request.Get(1); + const eastl::span assetBytes = request.Get(2); + + Row row; + std::memcpy(row.m_uuid, uuidBytes.data(), UUID::UUID_BYTE_SIZE); + row.m_typeID = typeID; + row.m_bytes.assign(assetBytes.begin(), assetBytes.end()); + + const UUID uuid {std::span {row.m_uuid, UUID::UUID_BYTE_SIZE}}; + uuidToRowIndex.emplace(uuid, rows.size()); + rows.push_back(std::move(row)); + } + } + + { + BigFile::Request request {p_bigFile, "SELECT UUID, ReferencedUUID FROM AssetHardReference;"}; + while (request.Step()) + { + const eastl::span uuidBytes = request.Get(0); + const eastl::span referencedUUIDBytes = request.Get(1); + + const UUID uuid {std::span {uuidBytes.data(), UUID::UUID_BYTE_SIZE}}; + if (const auto it = uuidToRowIndex.find(uuid); it != uuidToRowIndex.end()) + { + Row& row = rows[it->second]; + row.m_hardReferenceUUIDs.insert(row.m_hardReferenceUUIDs.end(), referencedUUIDBytes.begin(), + referencedUUIDBytes.end()); + } + } + } + + SasquatchHeader header; + header.m_assetCount = static_cast(rows.size()); + + std::uint64_t cursor = sizeof(SasquatchHeader) + rows.size() * sizeof(SasquatchEntry); + + eastl::vector entries; + entries.reserve(rows.size()); + for (const Row& row: rows) + { + SasquatchEntry entry; + std::memcpy(entry.m_uuid, row.m_uuid, UUID::UUID_BYTE_SIZE); + entry.m_typeID = row.m_typeID; + entry.m_hardReferenceOffset = cursor; + entry.m_hardReferenceCount = row.m_hardReferenceUUIDs.size() / UUID::UUID_BYTE_SIZE; + cursor += row.m_hardReferenceUUIDs.size(); + entry.m_blobOffset = cursor; + entry.m_blobSize = row.m_bytes.size(); + cursor += row.m_bytes.size(); + entries.push_back(entry); + } + +#if defined BIGFOOT_WINDOWS + std::ofstream out {ToWidePath(p_outFile.ToAbsolute().GetPath()), std::ios::binary | std::ios::trunc}; +#else + const eastl::string_view outPath = p_outFile.ToAbsolute().GetPath(); + std::ofstream out {std::string {outPath.begin(), outPath.end()}, std::ios::binary | std::ios::trunc}; +#endif + CRITICAL_ASSERT(EngineAssertHandler, out.is_open(), "Failed to create Sasquatch file {}", + p_outFile.ToAbsolute().GetPath()); + + out.write(reinterpret_cast(&header), sizeof(header)); + out.write(reinterpret_cast(entries.data()), + static_cast(entries.size() * sizeof(SasquatchEntry))); + for (const Row& row: rows) + { + if (!row.m_hardReferenceUUIDs.empty()) + { + out.write(reinterpret_cast(row.m_hardReferenceUUIDs.data()), + static_cast(row.m_hardReferenceUUIDs.size())); + } + if (!row.m_bytes.empty()) + { + out.write(reinterpret_cast(row.m_bytes.data()), + static_cast(row.m_bytes.size())); + } + } +} +#endif +} // namespace Bigfoot diff --git a/Bigfoot/Sources/Tools/CMakeLists.txt b/Bigfoot/Sources/Tools/CMakeLists.txt new file mode 100644 index 0000000..fc420ae --- /dev/null +++ b/Bigfoot/Sources/Tools/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/SasquatchPacker) diff --git a/Bigfoot/Sources/Tools/SasquatchPacker/CMakeLists.txt b/Bigfoot/Sources/Tools/SasquatchPacker/CMakeLists.txt new file mode 100644 index 0000000..0e919ef --- /dev/null +++ b/Bigfoot/Sources/Tools/SasquatchPacker/CMakeLists.txt @@ -0,0 +1,25 @@ +get_filename_component(PackageName ${CMAKE_CURRENT_SOURCE_DIR} NAME) +project(${PackageName}) + +add_executable(${PROJECT_NAME}) + +target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_20) + +target_link_libraries(${PROJECT_NAME} + PRIVATE + BigfootCompileAndLinkFlags + $) + +file(GLOB_RECURSE _SOURCES + CONFIGURE_DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp +) + +target_sources(${PROJECT_NAME} + PRIVATE + ${_SOURCES} +) + +source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} PREFIX Src FILES ${_SOURCES}) + +set_target_properties(${PROJECT_NAME} PROPERTIES FOLDER Bigfoot/Tools) diff --git a/Bigfoot/Sources/Tools/SasquatchPacker/main.cpp b/Bigfoot/Sources/Tools/SasquatchPacker/main.cpp new file mode 100644 index 0000000..f350837 --- /dev/null +++ b/Bigfoot/Sources/Tools/SasquatchPacker/main.cpp @@ -0,0 +1,30 @@ +/********************************************************************* + * \file main.cpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#include +#include + +#include + +#include + +#include + +// Packages a BigFile into a .sasquatch file for Game-mode builds to load. Tool-mode-only build (see +// Sasquatch::WriteFrom): run once as a ship/packaging step, after BigFile content is finalized. +int main(const int argc, char** argv) +{ + if (argc != 3) + { + std::fprintf(stderr, "Usage: %s \n", argv[0]); + return 1; + } + + Bigfoot::BigFile bigFile {Bigfoot::File {eastl::string_view {argv[1]}}}; + Bigfoot::Sasquatch::WriteFrom(bigFile, Bigfoot::File {eastl::string_view {argv[2]}}); + + return 0; +} diff --git a/Bigfoot/Tests/Engine/Asset/AssetContainer.cpp b/Bigfoot/Tests/Engine/Asset/AssetContainer.cpp index cd53593..a636954 100644 --- a/Bigfoot/Tests/Engine/Asset/AssetContainer.cpp +++ b/Bigfoot/Tests/Engine/Asset/AssetContainer.cpp @@ -5,10 +5,16 @@ * \date July 2026 *********************************************************************/ #include +#include +#include #include +#include #include +#include +#include + namespace Bigfoot { class ContainerHandleFixture: public ::testing::Test @@ -177,4 +183,194 @@ TEST_F(AssetContainerFixture, Remove_ShouldRemoveTheAsset) EXPECT_EQ(m_containerA.Get(slotKey), nullptr); EXPECT_FALSE(m_containerA.Has(asset.GetHeader().uuid)); } + +/****************************************************************************************/ + +TEST_F(AssetContainerFixture, DecodeOffContainer_ShouldNotAddTheAssetToTheContainer) +{ + AssetA asset; + asset.GetAsset().health = 42; + const eastl::vector bytes = asset.Pack(); + + const eastl::unique_ptr decoded = m_containerA.DecodeOffContainer(bytes); + + ASSERT_NE(decoded, nullptr); + EXPECT_EQ(decoded->GetHeader().uuid, asset.GetHeader().uuid); + EXPECT_EQ(static_cast(decoded.get())->GetAsset().health, 42u); + EXPECT_FALSE(m_containerA.Has(asset.GetHeader().uuid)); +} + +/****************************************************************************************/ + +TEST_F(AssetContainerFixture, InsertDecoded_ShouldAddTheDecodedAssetToTheContainer) +{ + AssetA asset; + asset.GetAsset().mana = 7; + const eastl::vector bytes = asset.Pack(); + const UUID uuid = asset.GetHeader().uuid; + + m_containerA.InsertDecoded(m_containerA.DecodeOffContainer(bytes)); + + ASSERT_TRUE(m_containerA.Has(uuid)); + const AssetBase* inserted = m_containerA.AcquireContainerHandle(uuid)->GetAsset(); + ASSERT_NE(inserted, nullptr); + EXPECT_EQ(static_cast(inserted)->GetAsset().mana, 7u); +} + +/****************************************************************************************/ + +TEST_F(AssetContainerFixture, DecodeOffContainerThenInsertDecoded_ShouldBeEquivalentToInsert) +{ + AssetA viaInsert; + viaInsert.GetAsset().health = 5; + viaInsert.GetAsset().mana = 6; + const eastl::vector insertBytes = viaInsert.Pack(); + + AssetA viaDecode; + viaDecode.GetAsset().health = 5; + viaDecode.GetAsset().mana = 6; + const eastl::vector decodeBytes = viaDecode.Pack(); + + static_cast(m_containerA) + .Insert(eastl::span {insertBytes.data(), insertBytes.size()}); + m_containerA.InsertDecoded(m_containerA.DecodeOffContainer(decodeBytes)); + + const auto* fromInsert = + static_cast(m_containerA.AcquireContainerHandle(viaInsert.GetHeader().uuid)->GetAsset()); + const auto* fromDecode = + static_cast(m_containerA.AcquireContainerHandle(viaDecode.GetHeader().uuid)->GetAsset()); + + ASSERT_NE(fromInsert, nullptr); + ASSERT_NE(fromDecode, nullptr); + EXPECT_EQ(fromInsert->GetAsset().health, fromDecode->GetAsset().health); + EXPECT_EQ(fromInsert->GetAsset().mana, fromDecode->GetAsset().mana); +} + +/****************************************************************************************/ + +// DecodeOffContainer is documented as safe to call concurrently (see SerializationManager::LoadAssets, +// which relies on exactly that): these tests are the actual proof. Unpacking a HardReference field +// resolves it there and then (see Reference.hpp's ReferenceBase::Acquire), which touches shared state +// outside the container being decoded into - the referenced asset's ContainerHandle ref count +// (AssetContainer.hpp) and the owning AssetLibrary's UUID index/unreferenced-set (AssetLibrary.hpp) - so +// this is the one place in the whole Asset system multiple threads genuinely touch the same memory at +// once. +class ConcurrentDecodeFixture: public ::testing::Test +{ + protected: + void SetUp() override + { + SetActiveAssetLibrary(&m_library); + std::ignore = m_library.Insert(&m_containerA); + std::ignore = m_library.Insert(&m_containerB); + } + + void TearDown() override + { + SetActiveAssetLibrary(&g_assetLibrary); + } + + static constexpr int THREAD_COUNT = 16; + static constexpr int PER_THREAD = 200; + + AssetContainer m_containerA; + AssetContainer m_containerB; + AssetLibrary m_library; +}; + +/****************************************************************************************/ + +TEST_F(ConcurrentDecodeFixture, DecodeOffContainer_ManyThreadsSharingOneHardReferenceTarget_ShouldNotLoseIncrements) +{ + AssetA target; + const UUID targetUUID = target.GetHeader().uuid; + std::ignore = m_containerA.Insert(std::move(target)); + const ContainerHandle* targetHandle = m_containerA.AcquireContainerHandle(targetUUID); + + std::vector> packed; + packed.reserve(THREAD_COUNT * PER_THREAD); + for (int i = 0; i < THREAD_COUNT * PER_THREAD; ++i) + { + AssetB referrer; + referrer.GetAsset().ref_a = HardReference {targetUUID}; + packed.push_back(referrer.Pack()); + } + + // Every decoded AssetB is kept alive (not just constructed and immediately dropped) so the target's + // ref count reflects every one of them at once - the race is on the increments landing correctly, not + // just on the final decrement bringing it back to zero either way. + std::vector> decoded(packed.size()); + + std::vector threads; + threads.reserve(THREAD_COUNT); + for (int t = 0; t < THREAD_COUNT; ++t) + { + threads.emplace_back( + [&, t]() + { + for (int i = 0; i < PER_THREAD; ++i) + { + const std::size_t index = static_cast(t) * PER_THREAD + static_cast(i); + decoded[index] = m_containerB.DecodeOffContainer(packed[index]); + } + }); + } + for (std::thread& thread: threads) + { + thread.join(); + } + + EXPECT_EQ(targetHandle->GetHardRefCount(), static_cast(THREAD_COUNT * PER_THREAD)); + + decoded.clear(); + + EXPECT_EQ(targetHandle->GetHardRefCount(), 0u); +} + +/****************************************************************************************/ + +TEST_F(ConcurrentDecodeFixture, DecodeOffContainer_ManyThreadsWithDistinctNeverBeforeSeenTargets_ShouldCreateExactlyOneEntryEach) +{ + // Unlike the shared-target test above, every referrer here points at its own UUID that nothing has + // touched yet: this exercises GetOrCreateEntry's insertion path racing across threads (as opposed to + // its already-exists path above), on both AssetA's own entry map and AssetLibrary's UUID index. + std::vector targetUUIDs(THREAD_COUNT * PER_THREAD); + std::vector> packed; + packed.reserve(targetUUIDs.size()); + for (std::size_t i = 0; i < targetUUIDs.size(); ++i) + { + targetUUIDs[i] = UUID {}; + AssetB referrer; + referrer.GetAsset().ref_a = HardReference {targetUUIDs[i]}; + packed.push_back(referrer.Pack()); + } + + std::vector> decoded(packed.size()); + + std::vector threads; + threads.reserve(THREAD_COUNT); + for (int t = 0; t < THREAD_COUNT; ++t) + { + threads.emplace_back( + [&, t]() + { + for (int i = 0; i < PER_THREAD; ++i) + { + const std::size_t index = static_cast(t) * PER_THREAD + static_cast(i); + decoded[index] = m_containerB.DecodeOffContainer(packed[index]); + } + }); + } + for (std::thread& thread: threads) + { + thread.join(); + } + + for (const UUID& targetUUID: targetUUIDs) + { + const ContainerHandle* handle = m_containerA.AcquireContainerHandle(targetUUID); + EXPECT_EQ(handle->GetHardRefCount(), 1u); + EXPECT_EQ(m_library.FindContainerForUUID(targetUUID), &m_containerA); + } +} } // namespace Bigfoot diff --git a/Bigfoot/Tests/Engine/Asset/Reference.cpp b/Bigfoot/Tests/Engine/Asset/Reference.cpp index 22805c4..80ff192 100644 --- a/Bigfoot/Tests/Engine/Asset/Reference.cpp +++ b/Bigfoot/Tests/Engine/Asset/Reference.cpp @@ -246,6 +246,54 @@ TEST_F(HardReferenceFixture, EqualityOperator_ShouldReturnFalseForReferencesWith /****************************************************************************************/ +TEST_F(HardReferenceFixture, ReleasingTheLastHardReference_ShouldMarkTheUUIDAsUnreferenced) +{ + const auto key = m_containerA.Insert(); + const UUID uuid = m_containerA.Get(key)->GetHeader().uuid; + + { + HardReference hardRef {uuid}; + EXPECT_TRUE(m_library.TakeUnreferencedUUIDs().empty()); + } + + const auto unreferenced = m_library.TakeUnreferencedUUIDs(); + EXPECT_EQ(unreferenced.size(), 1u); + EXPECT_TRUE(unreferenced.contains(uuid)); +} + +/****************************************************************************************/ + +TEST_F(HardReferenceFixture, ReleasingAHardReference_ShouldNotMarkTheUUIDAsUnreferencedIfOthersRemain) +{ + const auto key = m_containerA.Insert(); + const UUID uuid = m_containerA.Get(key)->GetHeader().uuid; + + HardReference hardRef1 {uuid}; + { + HardReference hardRef2 {uuid}; + } + + EXPECT_TRUE(m_library.TakeUnreferencedUUIDs().empty()); +} + +/****************************************************************************************/ + +TEST_F(HardReferenceFixture, AcquiringANewHardReference_ShouldUnmarkAPreviouslyMarkedUUID) +{ + const auto key = m_containerA.Insert(); + const UUID uuid = m_containerA.Get(key)->GetHeader().uuid; + + { + HardReference hardRef {uuid}; + } + // The block above marked uuid as unreferenced; acquiring a new HardReference to it should undo that. + HardReference stillHeld {uuid}; + + EXPECT_TRUE(m_library.TakeUnreferencedUUIDs().empty()); +} + +/****************************************************************************************/ + class SoftReferenceFixture: public ::testing::Test { protected: @@ -476,4 +524,84 @@ TEST_F(SoftReferenceFixture, EqualityOperator_ShouldReturnFalseForReferencesWith EXPECT_NE(softRef1, softRef2); } + +/****************************************************************************************/ + +TEST_F(SoftReferenceFixture, ReleasingTheLastSoftReference_ShouldNotMarkTheUUIDAsUnreferenced) +{ + const auto key = m_containerA.Insert(); + const UUID uuid = m_containerA.Get(key)->GetHeader().uuid; + + { + SoftReference softRef {uuid}; + } + + EXPECT_TRUE(m_library.TakeUnreferencedUUIDs().empty()); +} + +/****************************************************************************************/ + +class AnyAssetReferenceFixture: public ::testing::Test +{ + protected: + void SetUp() override + { + SetActiveAssetLibrary(&m_library); + + std::ignore = m_library.Insert(&m_containerA); + std::ignore = m_library.Insert(&m_containerB); + } + + void TearDown() override + { + SetActiveAssetLibrary(&g_assetLibrary); + } + + AssetContainer m_containerA; + AssetContainer m_containerB; + + AssetLibrary m_library; +}; + +/****************************************************************************************/ + +TEST_F(AnyAssetReferenceFixture, HardReferenceToAssetBase_ShouldResolveToAnAssetAlreadyLoadedInAnyContainer) +{ + const auto keyA = m_containerA.Insert(); + const UUID uuidA = m_containerA.Get(keyA)->GetHeader().uuid; + + const auto keyB = m_containerB.Insert(); + const UUID uuidB = m_containerB.Get(keyB)->GetHeader().uuid; + + HardReference refToA {uuidA}; + HardReference refToB {uuidB}; + + EXPECT_EQ(refToA.Get(), m_containerA.Get(keyA)); + EXPECT_EQ(refToB.Get(), m_containerB.Get(keyB)); +} + +/****************************************************************************************/ + +TEST_F(AnyAssetReferenceFixture, HardReferenceToAssetBase_ShouldReturnNullptrIfTheUUIDIsNotLoadedAnywhere) +{ + const UUID uuid; + HardReference ref {uuid}; + + EXPECT_EQ(ref.Get(), nullptr); +} + +/****************************************************************************************/ + +TEST_F(AnyAssetReferenceFixture, HardReferenceToAssetBase_AndHardReferenceToTheConcreteType_ShouldShareTheSameHardRefCount) +{ + const auto key = m_containerA.Insert(); + const UUID uuid = m_containerA.Get(key)->GetHeader().uuid; + const ContainerHandle* handle = m_containerA.AcquireContainerHandle(uuid); + + HardReference polymorphicRef {uuid}; + EXPECT_EQ(handle->GetHardRefCount(), 1u); + + HardReference typedRef {uuid}; + EXPECT_EQ(handle->GetHardRefCount(), 2u); +} } // namespace Bigfoot diff --git a/Bigfoot/Tests/Engine/Asset/SerializationManager.cpp b/Bigfoot/Tests/Engine/Asset/SerializationManager.cpp new file mode 100644 index 0000000..80d8abc --- /dev/null +++ b/Bigfoot/Tests/Engine/Asset/SerializationManager.cpp @@ -0,0 +1,357 @@ +/********************************************************************* + * \file SerializationManager.cpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#include + +#include +#include +#include +#include +#include + +namespace Bigfoot +{ +class SerializationManagerFixture: public ::testing::Test +{ + protected: + void SetUp() override + { + SetActiveAssetLibrary(&m_library); + + std::ignore = m_library.Insert(&m_containerA); + std::ignore = m_library.Insert(&m_containerB); + std::ignore = m_library.Insert(&m_containerC); + } + + void TearDown() override + { + SetActiveAssetLibrary(&g_assetLibrary); + } + + // Names must be unique (AssetHeader.Name has a UNIQUE constraint): reuse the asset's own freshly + // random UUID as its name. + [[nodiscard]] + static eastl::string UniqueName(const UUID& p_uuid) + { + const std::string uuidString = p_uuid.ToString(); + return eastl::string {uuidString.data(), uuidString.size()}; + } + + // SaveAsset now takes a UUID: it saves whatever is currently loaded under that UUID, rather than + // taking an asset directly. Tests build an asset, put it in its container (this is what "loaded" + // means), save it, then remove it again - so that a later LoadAsset call has actual work to do rather + // than trivially finding the asset already Has()-true. + template + [[nodiscard]] + UUID SaveAndUnload(AssetContainer& p_container, ASSET p_asset) + { + const UUID uuid = p_asset.GetHeader().uuid; + std::ignore = p_container.Insert(std::move(p_asset)); + m_manager.SaveAsset(uuid); + p_container.Remove(uuid); + return uuid; + } + + BigFile m_bigFile {File {BIGFILE_ENGINETESTS_LOCATION}}; + SerializationManager m_manager {m_bigFile}; + + AssetContainer m_containerA; + AssetContainer m_containerB; + AssetContainer m_containerC; + + AssetLibrary m_library; +}; + +/****************************************************************************************/ + +TEST_F(SerializationManagerFixture, SaveThenLoadAsset_ShouldRecreateAnEquivalentAssetInTheContainer) +{ + AssetA asset; + asset.SetName(UniqueName(asset.GetHeader().uuid)); + asset.GetAsset().health = 42; + asset.GetAsset().mana = 7; + const UUID uuid = SaveAndUnload(m_containerA, std::move(asset)); + + ASSERT_FALSE(m_containerA.Has(uuid)); + + m_manager.LoadAsset(uuid); + + ASSERT_TRUE(m_containerA.Has(uuid)); + const ContainerHandle* handle = m_containerA.AcquireContainerHandle(uuid); + const AssetA* loaded = static_cast(handle->GetAsset()); + ASSERT_NE(loaded, nullptr); + EXPECT_EQ(loaded->GetAsset().health, 42u); + EXPECT_EQ(loaded->GetAsset().mana, 7u); +} + +/****************************************************************************************/ + +TEST_F(SerializationManagerFixture, LoadAsset_ShouldDoNothingForANullUUID) +{ + m_manager.LoadAsset(UUID::NULL_UUID); + + EXPECT_FALSE(m_containerA.Has(UUID::NULL_UUID)); +} + +/****************************************************************************************/ + +TEST_F(SerializationManagerFixture, LoadAsset_ShouldDoNothingForAnUnknownUUID) +{ + const UUID uuid; + m_manager.LoadAsset(uuid); + + EXPECT_FALSE(m_containerA.Has(uuid)); +} + +/****************************************************************************************/ + +TEST_F(SerializationManagerFixture, LoadAsset_CalledTwice_ShouldOnlyLoadOnce) +{ + AssetA asset; + asset.SetName(UniqueName(asset.GetHeader().uuid)); + const UUID uuid = SaveAndUnload(m_containerA, std::move(asset)); + + m_manager.LoadAsset(uuid); + const AssetBase* firstLoad = m_containerA.AcquireContainerHandle(uuid)->GetAsset(); + + m_manager.LoadAsset(uuid); + const AssetBase* secondLoad = m_containerA.AcquireContainerHandle(uuid)->GetAsset(); + + EXPECT_EQ(firstLoad, secondLoad); +} + +/****************************************************************************************/ + +TEST_F(SerializationManagerFixture, SaveThenLoadAsset_ShouldRecursivelyLoadHardReferences) +{ + AssetA innerAssetA; + innerAssetA.SetName(UniqueName(innerAssetA.GetHeader().uuid)); + innerAssetA.GetAsset().health = 11; + const UUID innerAssetAUUID = SaveAndUnload(m_containerA, std::move(innerAssetA)); + + AssetB assetB; + assetB.SetName(UniqueName(assetB.GetHeader().uuid)); + assetB.GetAsset().ref_a = HardReference {innerAssetAUUID}; + const UUID assetBUUID = SaveAndUnload(m_containerB, std::move(assetB)); + + m_manager.LoadAsset(assetBUUID); + + ASSERT_TRUE(m_containerB.Has(assetBUUID)); + EXPECT_TRUE(m_containerA.Has(innerAssetAUUID)); + EXPECT_EQ(static_cast(m_containerA.AcquireContainerHandle(innerAssetAUUID)->GetAsset()) + ->GetAsset() + .health, + 11u); +} + +/****************************************************************************************/ + +TEST_F(SerializationManagerFixture, SaveThenLoadAsset_ShouldNotEagerlyLoadSoftReferences) +{ + AssetA softReferencedAsset; + softReferencedAsset.SetName(UniqueName(softReferencedAsset.GetHeader().uuid)); + const UUID softReferencedUUID = SaveAndUnload(m_containerA, std::move(softReferencedAsset)); + + AssetA asset; + asset.SetName(UniqueName(asset.GetHeader().uuid)); + asset.GetAsset().ref_a = SoftReference {softReferencedUUID}; + const UUID uuid = SaveAndUnload(m_containerA, std::move(asset)); + + m_manager.LoadAsset(uuid); + + EXPECT_TRUE(m_containerA.Has(uuid)); + EXPECT_FALSE(m_containerA.Has(softReferencedUUID)); +} + +/****************************************************************************************/ + +TEST_F(SerializationManagerFixture, SaveThenLoadAsset_HardReferenceVectorOnAssetB_ShouldNotBeEagerlyLoaded) +{ + // AssetB.refs_a is a vector of SoftReferences: confirm none of them get eagerly loaded either. + AssetA softReferencedAsset; + softReferencedAsset.SetName(UniqueName(softReferencedAsset.GetHeader().uuid)); + const UUID softReferencedUUID = SaveAndUnload(m_containerA, std::move(softReferencedAsset)); + + AssetB assetB; + assetB.SetName(UniqueName(assetB.GetHeader().uuid)); + assetB.GetAsset().refs_a.push_back(SoftReference {softReferencedUUID}); + const UUID assetBUUID = SaveAndUnload(m_containerB, std::move(assetB)); + + m_manager.LoadAsset(assetBUUID); + + EXPECT_TRUE(m_containerB.Has(assetBUUID)); + EXPECT_FALSE(m_containerA.Has(softReferencedUUID)); +} + +/****************************************************************************************/ + +TEST_F(SerializationManagerFixture, SaveAsset_CalledAgainOnTheSameUUID_ShouldOverwriteTheBigFileEntry) +{ + AssetA asset; + asset.SetName(UniqueName(asset.GetHeader().uuid)); + asset.GetAsset().health = 1; + const UUID uuid = asset.GetHeader().uuid; + const auto key = m_containerA.Insert(std::move(asset)); + m_manager.SaveAsset(uuid); + + m_containerA.Get(key)->GetAsset().health = 99; + m_manager.SaveAsset(uuid); + + m_containerA.Remove(uuid); + m_manager.LoadAsset(uuid); + + ASSERT_TRUE(m_containerA.Has(uuid)); + EXPECT_EQ(static_cast(m_containerA.AcquireContainerHandle(uuid)->GetAsset())->GetAsset().health, + 99u); +} + +/****************************************************************************************/ + +TEST_F(SerializationManagerFixture, SaveThenLoadAsset_PolymorphicHardReference_ShouldResolveOnceLoaded) +{ + // AssetC.InnerTable.any_asset is a HardReference: it can point at an asset of any type. + // Referees are loaded before the referencing asset (see SerializationManager::LoadAsset), so by the + // time AssetC itself is constructed here, its any_asset field should resolve immediately. + AssetA assetA; + assetA.SetName(UniqueName(assetA.GetHeader().uuid)); + assetA.GetAsset().health = 77; + const UUID assetAUUID = SaveAndUnload(m_containerA, std::move(assetA)); + + AssetC assetC; + assetC.SetName(UniqueName(assetC.GetHeader().uuid)); + assetC.GetAsset().inner_table->any_asset = HardReference {assetAUUID}; + const UUID assetCUUID = SaveAndUnload(m_containerC, std::move(assetC)); + + m_manager.LoadAsset(assetCUUID); + + ASSERT_TRUE(m_containerC.Has(assetCUUID)); + EXPECT_TRUE(m_containerA.Has(assetAUUID)); + + const AssetC* loadedC = static_cast(m_containerC.AcquireContainerHandle(assetCUUID)->GetAsset()); + ASSERT_NE(loadedC, nullptr); + const AssetBase* resolved = loadedC->GetAsset().inner_table->any_asset.Get(); + ASSERT_NE(resolved, nullptr); + EXPECT_EQ(static_cast(resolved)->GetAsset().health, 77u); +} + +/****************************************************************************************/ + +TEST_F(SerializationManagerFixture, SaveThenLoadAsset_PolymorphicHardReferenceCycle_ShouldNotInfinitelyRecurse) +{ + AssetC assetC1; + assetC1.SetName(UniqueName(assetC1.GetHeader().uuid)); + const UUID assetC1UUID = assetC1.GetHeader().uuid; + + AssetC assetC2; + assetC2.SetName(UniqueName(assetC2.GetHeader().uuid)); + const UUID assetC2UUID = assetC2.GetHeader().uuid; + + assetC1.GetAsset().inner_table->any_asset = HardReference {assetC2UUID}; + assetC2.GetAsset().inner_table->any_asset = HardReference {assetC1UUID}; + + std::ignore = SaveAndUnload(m_containerC, std::move(assetC1)); + std::ignore = SaveAndUnload(m_containerC, std::move(assetC2)); + + m_manager.LoadAsset(assetC1UUID); + + EXPECT_TRUE(m_containerC.Has(assetC1UUID)); + EXPECT_TRUE(m_containerC.Has(assetC2UUID)); +} + +/****************************************************************************************/ + +TEST_F(SerializationManagerFixture, RemoveUnreferencedAssets_ShouldRemoveAnAssetWithNoMoreHardReferences) +{ + AssetA asset; + asset.SetName(UniqueName(asset.GetHeader().uuid)); + const UUID uuid = asset.GetHeader().uuid; + std::ignore = m_containerA.Insert(std::move(asset)); + + { + HardReference hardRef {uuid}; + ASSERT_TRUE(m_containerA.Has(uuid)); + } + + m_manager.RemoveUnreferencedAssets(); + + EXPECT_FALSE(m_containerA.Has(uuid)); +} + +/****************************************************************************************/ + +TEST_F(SerializationManagerFixture, RemoveUnreferencedAssets_ShouldNotRemoveAnAssetReacquiredBeforeTheSweep) +{ + AssetA asset; + asset.SetName(UniqueName(asset.GetHeader().uuid)); + const UUID uuid = asset.GetHeader().uuid; + std::ignore = m_containerA.Insert(std::move(asset)); + + { + HardReference hardRef {uuid}; + } + // Marked unreferenced by the block above, but re-acquired before RemoveUnreferencedAssets runs. + HardReference stillHeld {uuid}; + + m_manager.RemoveUnreferencedAssets(); + + EXPECT_TRUE(m_containerA.Has(uuid)); +} + +/****************************************************************************************/ + +TEST_F(SerializationManagerFixture, RemoveUnreferencedAssets_ShouldNotRemoveAnAssetThatWasNeverHardReferenced) +{ + // Removal is driven by HardReference registering assets when their count reaches zero (see + // Reference.hpp): an asset that was inserted directly and never held by a HardReference was never + // registered, so it's simply not a candidate. + AssetA asset; + asset.SetName(UniqueName(asset.GetHeader().uuid)); + const UUID uuid = asset.GetHeader().uuid; + std::ignore = m_containerA.Insert(std::move(asset)); + + m_manager.RemoveUnreferencedAssets(); + + EXPECT_TRUE(m_containerA.Has(uuid)); +} + +/****************************************************************************************/ + +TEST_F(SerializationManagerFixture, GetAssetHeader_ShouldReturnTheHeaderWithoutLoadingTheAsset) +{ + AssetA asset; + const eastl::string name = UniqueName(asset.GetHeader().uuid); + asset.SetName(name); + asset.SetVersion(3); + const UUID uuid = SaveAndUnload(m_containerA, std::move(asset)); + + const auto header = m_manager.GetAssetHeader(uuid); + + ASSERT_NE(header, nullptr); + EXPECT_EQ(header->uuid, uuid); + EXPECT_EQ(header->name, name); + EXPECT_EQ(header->type_id, AssetA::GetTypeID()); + EXPECT_EQ(header->type_name, eastl::string {AssetA::GetTypeName()}); + EXPECT_EQ(header->version, 3u); + + // Reading the header must not have loaded the asset itself. + EXPECT_FALSE(m_containerA.Has(uuid)); +} + +/****************************************************************************************/ + +TEST_F(SerializationManagerFixture, GetAssetHeader_ShouldReturnNullptrForANullUUID) +{ + EXPECT_EQ(m_manager.GetAssetHeader(UUID::NULL_UUID), nullptr); +} + +/****************************************************************************************/ + +TEST_F(SerializationManagerFixture, GetAssetHeader_ShouldReturnNullptrForAnUnknownUUID) +{ + const UUID uuid; + EXPECT_EQ(m_manager.GetAssetHeader(uuid), nullptr); +} +} // namespace Bigfoot diff --git a/Bigfoot/Tests/Misc/AssetsForTesting/Include/AssetsForTesting/Asset/AssetA.hpp b/Bigfoot/Tests/Misc/AssetsForTesting/Include/AssetsForTesting/Asset/AssetA.hpp index 245c486..485804f 100644 --- a/Bigfoot/Tests/Misc/AssetsForTesting/Include/AssetsForTesting/Asset/AssetA.hpp +++ b/Bigfoot/Tests/Misc/AssetsForTesting/Include/AssetsForTesting/Asset/AssetA.hpp @@ -22,11 +22,6 @@ class AssetA: public Asset public: AssetA() = default; - AssetA(const eastl::span p_flatbuffer): - Asset(p_flatbuffer) - { - } - AssetA(const AssetA& p_asset) = default; AssetA(AssetA&& p_asset) = default; diff --git a/Bigfoot/Tests/Misc/AssetsForTesting/Include/AssetsForTesting/Asset/AssetB.hpp b/Bigfoot/Tests/Misc/AssetsForTesting/Include/AssetsForTesting/Asset/AssetB.hpp index cc87034..9323034 100644 --- a/Bigfoot/Tests/Misc/AssetsForTesting/Include/AssetsForTesting/Asset/AssetB.hpp +++ b/Bigfoot/Tests/Misc/AssetsForTesting/Include/AssetsForTesting/Asset/AssetB.hpp @@ -22,11 +22,6 @@ class AssetB: public Asset public: AssetB() = default; - AssetB(const eastl::span p_flatbuffer): - Asset(p_flatbuffer) - { - } - AssetB(const AssetB& p_asset) = delete; AssetB(AssetB&& p_asset) = default; diff --git a/Bigfoot/Tests/Misc/AssetsForTesting/Include/AssetsForTesting/Asset/AssetC.hpp b/Bigfoot/Tests/Misc/AssetsForTesting/Include/AssetsForTesting/Asset/AssetC.hpp index ecf5fae..c719e50 100644 --- a/Bigfoot/Tests/Misc/AssetsForTesting/Include/AssetsForTesting/Asset/AssetC.hpp +++ b/Bigfoot/Tests/Misc/AssetsForTesting/Include/AssetsForTesting/Asset/AssetC.hpp @@ -25,11 +25,6 @@ class AssetC: public Asset GetAsset().inner_table = eastl::make_unique<::Flat::Bigfoot::InnerTableT>(); } - AssetC(const eastl::span p_flatbuffer): - Asset(p_flatbuffer) - { - } - AssetC(const AssetC& p_asset) = delete; AssetC(AssetC&& p_asset) = default; diff --git a/CMake/CustomTargets.cmake b/CMake/CustomTargets.cmake index 5008561..fa37709 100644 --- a/CMake/CustomTargets.cmake +++ b/CMake/CustomTargets.cmake @@ -16,4 +16,6 @@ target_compile_definitions(BigfootCompileAndLinkFlags INTERFACE $<$:BIGFOOT_WINDOWS> $<$:BIGFOOT_LINUX> $<$:BIGFOOT_OPTIMIZED> - $<$:BIGFOOT_NOT_OPTIMIZED>) \ No newline at end of file + $<$:BIGFOOT_NOT_OPTIMIZED> + $<$:BIGFOOT_TOOL> + $<$>:BIGFOOT_GAME>) \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 38d0847..5d34a35 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,7 @@ option(ASAN OFF) option(COVERAGE OFF) option(TRACY ON) option(VULKAN ON) +option(BIGFOOT_TOOL_MODE "Build SerializationManager against BigFile/SQLite and allow saving. OFF ships the Sasquatch-backed, load-only Game mode." ON) set(AUTO_GENERATED_COMMENT "// AUTO-GENERATED DO NOT TOUCH")