From be85c650b9c82b4f781ea8890dcc84ebad9a8bdc Mon Sep 17 00:00:00 2001 From: Romain BOULLARD Date: Tue, 25 Aug 2026 22:22:23 +0200 Subject: [PATCH] More optimizations for BigFile --- .../Asset/BenchmarkAsset.cpp | 11 + .../AssetsForBenchmarking/CMakeLists.txt | 17 ++ .../Asset/BenchmarkAsset.fbs | 8 + .../Asset/BenchmarkAsset.hpp | 25 ++ .../Asset/BenchmarkAsset_generated.hpp | 192 +++++++++++++ Bigfoot/Benchmarks/CMakeLists.txt | 5 +- Bigfoot/Benchmarks/Engine/BigFile/BigFile.cpp | 253 ++++++++++++++++++ Bigfoot/Benchmarks/Engine/CMakeLists.txt | 12 + .../BigFileInfo_generated.hpp | 19 ++ Bigfoot/Sources/Engine/BigFile/BigFile.cpp | 95 +++++++ .../Sources/Engine/BigFile/BigFileSchema.sql | 1 - .../Engine/Include/Engine/BigFile/BigFile.hpp | 9 + .../Engine/DataAccessor/DataAccessor.hpp | 9 + Bigfoot/Tests/Engine/BigFile/BigFile.cpp | 83 ++++++ .../Engine/DataAccessor/DataAccessor.cpp | 5 + 15 files changed, 742 insertions(+), 2 deletions(-) create mode 100644 Bigfoot/Benchmarks/AssetsForBenchmarking/Asset/BenchmarkAsset.cpp create mode 100644 Bigfoot/Benchmarks/AssetsForBenchmarking/CMakeLists.txt create mode 100644 Bigfoot/Benchmarks/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchmarkAsset.fbs create mode 100644 Bigfoot/Benchmarks/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchmarkAsset.hpp create mode 100644 Bigfoot/Benchmarks/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchmarkAsset_generated.hpp create mode 100644 Bigfoot/Benchmarks/Engine/BigFile/BigFile.cpp create mode 100644 Bigfoot/Benchmarks/Engine/CMakeLists.txt create mode 100644 Bigfoot/Benchmarks/Engine/Include/EngineBenchmarks/BigFileInfo_generated.hpp diff --git a/Bigfoot/Benchmarks/AssetsForBenchmarking/Asset/BenchmarkAsset.cpp b/Bigfoot/Benchmarks/AssetsForBenchmarking/Asset/BenchmarkAsset.cpp new file mode 100644 index 0000000..81cbe04 --- /dev/null +++ b/Bigfoot/Benchmarks/AssetsForBenchmarking/Asset/BenchmarkAsset.cpp @@ -0,0 +1,11 @@ +/********************************************************************* + * \file BenchmarkAsset.cpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#include + +#include + +BIGFOOT_ASSET(BenchmarkAsset, ::Bigfoot::BenchmarkAsset) diff --git a/Bigfoot/Benchmarks/AssetsForBenchmarking/CMakeLists.txt b/Bigfoot/Benchmarks/AssetsForBenchmarking/CMakeLists.txt new file mode 100644 index 0000000..950fef7 --- /dev/null +++ b/Bigfoot/Benchmarks/AssetsForBenchmarking/CMakeLists.txt @@ -0,0 +1,17 @@ +get_filename_component(PackageName ${CMAKE_CURRENT_SOURCE_DIR} NAME) +project(${PackageName}) + +set(PublicDependencies) +set(PrivateDependencies) +set(BigfootPublicDependencies) +set(BigfootPrivateDependencies + Engine) + +bigfoot_create_package_lib( + "${PublicDependencies}" + "${PrivateDependencies}" + "${BigfootPublicDependencies}" + "${BigfootPrivateDependencies}" + "") + +set_target_properties(${PROJECT_NAME} PROPERTIES FOLDER Benchmarks/Bigfoot) diff --git a/Bigfoot/Benchmarks/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchmarkAsset.fbs b/Bigfoot/Benchmarks/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchmarkAsset.fbs new file mode 100644 index 0000000..b988d5b --- /dev/null +++ b/Bigfoot/Benchmarks/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchmarkAsset.fbs @@ -0,0 +1,8 @@ +namespace Flat.Bigfoot; + +table BenchmarkAsset +{ + payload: [ubyte]; +} + +root_type BenchmarkAsset; diff --git a/Bigfoot/Benchmarks/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchmarkAsset.hpp b/Bigfoot/Benchmarks/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchmarkAsset.hpp new file mode 100644 index 0000000..2f4fa48 --- /dev/null +++ b/Bigfoot/Benchmarks/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchmarkAsset.hpp @@ -0,0 +1,25 @@ +/********************************************************************* + * \file BenchmarkAsset.hpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#ifndef BIGFOOT_ASSETSFORBENCHMARKING_ASSET_BENCHMARKASSET_HPP +#define BIGFOOT_ASSETSFORBENCHMARKING_ASSET_BENCHMARKASSET_HPP +#include + +#include + +namespace Bigfoot +{ +struct BenchmarkAssetTraits +{ + using FLAT = ::Flat::Bigfoot::BenchmarkAsset; +}; + +class BenchmarkAsset: public Asset +{ +}; +} // namespace Bigfoot + +#endif diff --git a/Bigfoot/Benchmarks/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchmarkAsset_generated.hpp b/Bigfoot/Benchmarks/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchmarkAsset_generated.hpp new file mode 100644 index 0000000..c618b4f --- /dev/null +++ b/Bigfoot/Benchmarks/AssetsForBenchmarking/Include/AssetsForBenchmarking/Asset/BenchmarkAsset_generated.hpp @@ -0,0 +1,192 @@ +// automatically generated by the FlatBuffers compiler, do not modify + + +#ifndef FLATBUFFERS_GENERATED_BENCHMARKASSET_FLAT_BIGFOOT_H_ +#define FLATBUFFERS_GENERATED_BENCHMARKASSET_FLAT_BIGFOOT_H_ + +#include "flatbuffers/flatbuffers.h" + +// Ensure the included flatbuffers.h is the same version as when this file was +// generated, otherwise it may not be compatible. +static_assert(FLATBUFFERS_VERSION_MAJOR == 25 && + FLATBUFFERS_VERSION_MINOR == 12 && + FLATBUFFERS_VERSION_REVISION == 19, + "Non-compatible flatbuffers version included"); +// Ensure the included flatbuffers.h is Bigfoot's fork - stock +// flatbuffers (or a differently versioned Bigfoot fork) is not compatible. +#ifndef FLATBUFFERS_BIGFOOT_VERSION +#error "This file requires Bigfoot's flatbuffers fork - stock flatbuffers is not compatible" +#endif +static_assert(FLATBUFFERS_BIGFOOT_VERSION == 1, + "Non-compatible Bigfoot flatbuffers fork version included"); + +#include "EASTL/unique_ptr.h" +#include "EASTL/string.h" +#include "EASTL/vector.h" + +namespace Flat { +namespace Bigfoot { + +struct BenchmarkAsset; +struct BenchmarkAssetBuilder; +struct BenchmarkAssetT; + +inline const ::flatbuffers::TypeTable *BenchmarkAssetTypeTable(); + +struct BenchmarkAssetT : public ::flatbuffers::NativeTable { + typedef BenchmarkAsset TableType; + static FLATBUFFERS_CONSTEXPR_CPP11 const char *GetFullyQualifiedName() { + return "Flat.Bigfoot.BenchmarkAssetT"; + } + eastl::vector payload{}; +}; + +struct BenchmarkAsset FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { + typedef BenchmarkAssetT NativeTableType; + typedef BenchmarkAssetBuilder Builder; + struct Traits; + static const ::flatbuffers::TypeTable *MiniReflectTypeTable() { + return BenchmarkAssetTypeTable(); + } + static FLATBUFFERS_CONSTEXPR_CPP11 const char *GetFullyQualifiedName() { + return "Flat.Bigfoot.BenchmarkAsset"; + } + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_PAYLOAD = 4 + }; + const ::flatbuffers::Vector *payload() const { + return GetPointer *>(VT_PAYLOAD); + } + template + bool Verify(::flatbuffers::VerifierTemplate &verifier) const { + return VerifyTableStart(verifier) && + VerifyOffset(verifier, VT_PAYLOAD) && + verifier.VerifyVector(payload()) && + verifier.EndTable(); + } + BenchmarkAssetT *UnPack(const ::flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(BenchmarkAssetT *_o, const ::flatbuffers::resolver_function_t *_resolver = nullptr) const; + static ::flatbuffers::Offset Pack(::flatbuffers::FlatBufferBuilder &_fbb, const BenchmarkAssetT* _o, const ::flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct BenchmarkAssetBuilder { + typedef BenchmarkAsset Table; + ::flatbuffers::FlatBufferBuilder &fbb_; + ::flatbuffers::uoffset_t start_; + void add_payload(::flatbuffers::Offset<::flatbuffers::Vector> payload) { + fbb_.AddOffset(BenchmarkAsset::VT_PAYLOAD, payload); + } + explicit BenchmarkAssetBuilder(::flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + ::flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = ::flatbuffers::Offset(end); + return o; + } +}; + +inline ::flatbuffers::Offset CreateBenchmarkAsset( + ::flatbuffers::FlatBufferBuilder &_fbb, + ::flatbuffers::Offset<::flatbuffers::Vector> payload = 0) { + BenchmarkAssetBuilder builder_(_fbb); + builder_.add_payload(payload); + return builder_.Finish(); +} + +struct BenchmarkAsset::Traits { + using type = BenchmarkAsset; + static auto constexpr Create = CreateBenchmarkAsset; +}; + +::flatbuffers::Offset CreateBenchmarkAsset(::flatbuffers::FlatBufferBuilder &_fbb, const BenchmarkAssetT *_o, const ::flatbuffers::rehasher_function_t *_rehasher = nullptr); + +inline BenchmarkAssetT *BenchmarkAsset::UnPack(const ::flatbuffers::resolver_function_t *_resolver) const { + auto _o = std::make_unique(); + UnPackTo(_o.get(), _resolver); + return _o.release(); +} + +inline void BenchmarkAsset::UnPackTo(BenchmarkAssetT *_o, const ::flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; + { auto _e = payload(); if (_e) { _o->payload.resize(_e->size()); std::copy(_e->begin(), _e->end(), _o->payload.begin()); } } +} + +inline ::flatbuffers::Offset CreateBenchmarkAsset(::flatbuffers::FlatBufferBuilder &_fbb, const BenchmarkAssetT *_o, const ::flatbuffers::rehasher_function_t *_rehasher) { + return BenchmarkAsset::Pack(_fbb, _o, _rehasher); +} + +inline ::flatbuffers::Offset BenchmarkAsset::Pack(::flatbuffers::FlatBufferBuilder &_fbb, const BenchmarkAssetT* _o, const ::flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { ::flatbuffers::FlatBufferBuilder *__fbb; const BenchmarkAssetT* __o; const ::flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + auto _payload = _fbb.CreateVector(_o->payload.data(), _o->payload.size()); + return Flat::Bigfoot::CreateBenchmarkAsset( + _fbb, + _payload); +} + +inline const ::flatbuffers::TypeTable *BenchmarkAssetTypeTable() { + static const ::flatbuffers::TypeCode type_codes[] = { + { ::flatbuffers::ET_UCHAR, 1, -1 } + }; + static const ::flatbuffers::TypeTable tt = { + ::flatbuffers::ST_TABLE, 1, type_codes, nullptr, nullptr, nullptr, nullptr + }; + return &tt; +} + +inline const Flat::Bigfoot::BenchmarkAsset *GetBenchmarkAsset(const void *buf) { + return ::flatbuffers::GetRoot(buf); +} + +inline const Flat::Bigfoot::BenchmarkAsset *GetSizePrefixedBenchmarkAsset(const void *buf) { + return ::flatbuffers::GetSizePrefixedRoot(buf); +} + +template +inline bool VerifyBenchmarkAssetBuffer( + ::flatbuffers::VerifierTemplate &verifier) { + return verifier.template VerifyBuffer(nullptr); +} + +template +inline bool VerifySizePrefixedBenchmarkAssetBuffer( + ::flatbuffers::VerifierTemplate &verifier) { + return verifier.template VerifySizePrefixedBuffer(nullptr); +} + +inline const char *BenchmarkAssetExtension() { + return "bfbs"; +} + +inline void FinishBenchmarkAssetBuffer( + ::flatbuffers::FlatBufferBuilder &fbb, + ::flatbuffers::Offset root) { + fbb.Finish(root); +} + +inline void FinishSizePrefixedBenchmarkAssetBuffer( + ::flatbuffers::FlatBufferBuilder &fbb, + ::flatbuffers::Offset root) { + fbb.FinishSizePrefixed(root); +} + +inline eastl::unique_ptr UnPackBenchmarkAsset( + const void *buf, + const ::flatbuffers::resolver_function_t *res = nullptr) { + return eastl::unique_ptr(GetBenchmarkAsset(buf)->UnPack(res)); +} + +inline eastl::unique_ptr UnPackSizePrefixedBenchmarkAsset( + const void *buf, + const ::flatbuffers::resolver_function_t *res = nullptr) { + return eastl::unique_ptr(GetSizePrefixedBenchmarkAsset(buf)->UnPack(res)); +} + +} // namespace Bigfoot +} // namespace Flat + +#endif // FLATBUFFERS_GENERATED_BENCHMARKASSET_FLAT_BIGFOOT_H_ diff --git a/Bigfoot/Benchmarks/CMakeLists.txt b/Bigfoot/Benchmarks/CMakeLists.txt index 992910a..4c48d68 100644 --- a/Bigfoot/Benchmarks/CMakeLists.txt +++ b/Bigfoot/Benchmarks/CMakeLists.txt @@ -1 +1,4 @@ -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/Utils) \ No newline at end of file +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/AssetsForBenchmarking) + +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/Engine) +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/Utils) diff --git a/Bigfoot/Benchmarks/Engine/BigFile/BigFile.cpp b/Bigfoot/Benchmarks/Engine/BigFile/BigFile.cpp new file mode 100644 index 0000000..a243bc5 --- /dev/null +++ b/Bigfoot/Benchmarks/Engine/BigFile/BigFile.cpp @@ -0,0 +1,253 @@ +#include + +#include +#include + +#include +#include + +#include +#include + +#include + +#include +#include + +namespace Bigfoot +{ +#ifdef BIGFOOT_TOOLMODE +namespace +{ +using Random = effolkronium::random_static; + +constexpr std::uint32_t ASSET_COUNT = 20'000; +constexpr std::size_t ASSET_PAYLOAD_SIZE = 256; + +class Fixture: public benchmark::Fixture +{ + public: + void SetUp(benchmark::State&) override + { + BigFile::Request {m_bigFile, "DELETE FROM Asset;"}.Execute(); + BigFile::Request {m_bigFile, "DELETE FROM AssetMetadata;"}.Execute(); + + m_uuids.clear(); + m_uuids.reserve(ASSET_COUNT); + + eastl::vector metadatas; + eastl::vector> flatbuffers; + metadatas.reserve(ASSET_COUNT); + flatbuffers.reserve(ASSET_COUNT); + + for (std::uint32_t i = 0; i < ASSET_COUNT; ++i) + { + BenchmarkAsset asset; + asset.SetName(asset.GetHeader().uuid.ToString().c_str()); + asset.GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB}); + + metadatas.push_back(AssetMetadata {.m_uuid = asset.GetHeader().uuid, + .m_name = asset.GetHeader().name, + .m_typeID = asset.GetHeader().type_id, + .m_typeName = asset.GetHeader().type_name}); + flatbuffers.push_back(asset.Pack()); + m_uuids.push_back(asset.GetHeader().uuid); + } + + m_bigFile.SaveAssetFlatbuffers(metadatas, flatbuffers); + } + + protected: + [[nodiscard]] + eastl::vector RandomBatch(const std::size_t p_batchSize) const + { + eastl::vector batch; + batch.reserve(p_batchSize); + for (std::size_t i = 0; i < p_batchSize; ++i) + { + batch.push_back(m_uuids[Random::get(0, m_uuids.size() - 1)]); + } + + return batch; + } + + BigFile m_bigFile {File {BIGFILE_ENGINEBENCHMARKS_LOCATION}}; + eastl::vector m_uuids; +}; +} // namespace + +/****************************************************************************************/ + +BENCHMARK_DEFINE_F(Fixture, GetAssetFlatbuffer)(benchmark::State& state) +{ + std::int64_t bytesProcessed = 0; + + for (auto _: state) + { + eastl::optional> flatbuffer = + m_bigFile.GetAssetFlatbuffer(m_uuids[Random::get(0, m_uuids.size() - 1)]); + benchmark::DoNotOptimize(flatbuffer); + + if (flatbuffer.has_value()) + { + bytesProcessed += static_cast(flatbuffer->size()); + } + } + + state.SetItemsProcessed(state.iterations()); + state.SetBytesProcessed(bytesProcessed); +} + +BENCHMARK_REGISTER_F(Fixture, GetAssetFlatbuffer)->Name("BigFile/GetAssetFlatbuffer"); + +/****************************************************************************************/ + +BENCHMARK_DEFINE_F(Fixture, GetAssetFlatbuffers)(benchmark::State& state) +{ + const auto batchSize = static_cast(state.range(0)); + std::int64_t bytesProcessed = 0; + + for (auto _: state) + { + state.PauseTiming(); + const eastl::vector batch = RandomBatch(batchSize); + state.ResumeTiming(); + + eastl::vector> flatbuffers = m_bigFile.GetAssetFlatbuffers(batch); + benchmark::DoNotOptimize(flatbuffers); + + for (const eastl::vector& flatbuffer: flatbuffers) + { + bytesProcessed += static_cast(flatbuffer.size()); + } + } + + state.SetItemsProcessed(state.iterations() * batchSize); + state.SetBytesProcessed(bytesProcessed); +} + +BENCHMARK_REGISTER_F(Fixture, GetAssetFlatbuffers) + ->Name("BigFile/GetAssetFlatbuffers") + ->Arg(1) + ->Arg(8) + ->Arg(64) + ->Arg(512) + ->Arg(4096); + +/****************************************************************************************/ + +BENCHMARK_DEFINE_F(Fixture, GetAssetMetadata)(benchmark::State& state) +{ + for (auto _: state) + { + benchmark::DoNotOptimize( + m_bigFile.GetAssetMetadata(m_uuids[Random::get(0, m_uuids.size() - 1)])); + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK_REGISTER_F(Fixture, GetAssetMetadata)->Name("BigFile/GetAssetMetadata"); + +/****************************************************************************************/ + +BENCHMARK_DEFINE_F(Fixture, GetAssetMetadatas)(benchmark::State& state) +{ + const auto batchSize = static_cast(state.range(0)); + + for (auto _: state) + { + state.PauseTiming(); + const eastl::vector batch = RandomBatch(batchSize); + state.ResumeTiming(); + + benchmark::DoNotOptimize(m_bigFile.GetAssetMetadatas(batch)); + } + + state.SetItemsProcessed(state.iterations() * batchSize); +} + +BENCHMARK_REGISTER_F(Fixture, GetAssetMetadatas) + ->Name("BigFile/GetAssetMetadatas") + ->Arg(1) + ->Arg(8) + ->Arg(64) + ->Arg(512) + ->Arg(4096); + +/****************************************************************************************/ + +BENCHMARK_DEFINE_F(Fixture, SaveAssetFlatbuffer)(benchmark::State& state) +{ + std::int64_t bytesProcessed = 0; + + for (auto _: state) + { + state.PauseTiming(); + BenchmarkAsset asset; + asset.SetName(asset.GetHeader().uuid.ToString().c_str()); + asset.GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB}); + + const AssetMetadata metadata {.m_uuid = asset.GetHeader().uuid, + .m_name = asset.GetHeader().name, + .m_typeID = asset.GetHeader().type_id, + .m_typeName = asset.GetHeader().type_name}; + const eastl::vector flatbuffer = asset.Pack(); + bytesProcessed += static_cast(flatbuffer.size()); + state.ResumeTiming(); + + m_bigFile.SaveAssetFlatbuffer(metadata, flatbuffer); + } + + state.SetItemsProcessed(state.iterations()); + state.SetBytesProcessed(bytesProcessed); +} + +BENCHMARK_REGISTER_F(Fixture, SaveAssetFlatbuffer)->Name("BigFile/SaveAssetFlatbuffer"); + +/****************************************************************************************/ + +BENCHMARK_DEFINE_F(Fixture, SaveAssetFlatbuffers)(benchmark::State& state) +{ + const auto batchSize = static_cast(state.range(0)); + std::int64_t bytesProcessed = 0; + + for (auto _: state) + { + state.PauseTiming(); + eastl::vector metadatas; + eastl::vector> flatbuffers; + metadatas.reserve(batchSize); + flatbuffers.reserve(batchSize); + + for (std::size_t i = 0; i < batchSize; ++i) + { + BenchmarkAsset asset; + asset.SetName(asset.GetHeader().uuid.ToString().c_str()); + asset.GetAsset().payload.assign(ASSET_PAYLOAD_SIZE, std::uint8_t {0xAB}); + + metadatas.push_back(AssetMetadata {.m_uuid = asset.GetHeader().uuid, + .m_name = asset.GetHeader().name, + .m_typeID = asset.GetHeader().type_id, + .m_typeName = asset.GetHeader().type_name}); + flatbuffers.push_back(asset.Pack()); + bytesProcessed += static_cast(flatbuffers.back().size()); + } + state.ResumeTiming(); + + m_bigFile.SaveAssetFlatbuffers(metadatas, flatbuffers); + } + + state.SetItemsProcessed(state.iterations() * batchSize); + state.SetBytesProcessed(bytesProcessed); +} + +BENCHMARK_REGISTER_F(Fixture, SaveAssetFlatbuffers) + ->Name("BigFile/SaveAssetFlatbuffers") + ->Arg(1) + ->Arg(8) + ->Arg(64) + ->Arg(512) + ->Arg(4096); +#endif +} // namespace Bigfoot diff --git a/Bigfoot/Benchmarks/Engine/CMakeLists.txt b/Bigfoot/Benchmarks/Engine/CMakeLists.txt new file mode 100644 index 0000000..9ba15ba --- /dev/null +++ b/Bigfoot/Benchmarks/Engine/CMakeLists.txt @@ -0,0 +1,12 @@ +get_filename_component(PackageName ${CMAKE_CURRENT_SOURCE_DIR} NAME) +project(${PackageName}Benchmarks) + +bigfoot_create_package_benchmarks( + "") + +target_link_libraries(${PROJECT_NAME} + PRIVATE + effolkronium_random + $) + +bigfoot_create_bigfile("Benchmarks/Bigfoot") diff --git a/Bigfoot/Benchmarks/Engine/Include/EngineBenchmarks/BigFileInfo_generated.hpp b/Bigfoot/Benchmarks/Engine/Include/EngineBenchmarks/BigFileInfo_generated.hpp new file mode 100644 index 0000000..4d2f05f --- /dev/null +++ b/Bigfoot/Benchmarks/Engine/Include/EngineBenchmarks/BigFileInfo_generated.hpp @@ -0,0 +1,19 @@ +// AUTO-GENERATED DO NOT TOUCH + +/********************************************************************* + * \file BigFileInfo.generated.hpp + * + *********************************************************************/ +#ifndef BIGFOOT_ENGINEBENCHMARKS_BIGFILEINFO_GENERATED_HPP +#define BIGFOOT_ENGINEBENCHMARKS_BIGFILEINFO_GENERATED_HPP + +#include + +namespace Bigfoot +{ +/* + * BigFile location + */ +constexpr eastl::string_view BIGFILE_ENGINEBENCHMARKS_LOCATION {"./EngineBenchmarks-bigfile.db"}; +} // namespace Bigfoot +#endif diff --git a/Bigfoot/Sources/Engine/BigFile/BigFile.cpp b/Bigfoot/Sources/Engine/BigFile/BigFile.cpp index d0ff2c3..49890d2 100644 --- a/Bigfoot/Sources/Engine/BigFile/BigFile.cpp +++ b/Bigfoot/Sources/Engine/BigFile/BigFile.cpp @@ -24,6 +24,20 @@ BigFile::BigFile(const File& p_file) "Failed to open BigFile {} DB: {}", p_file.ToAbsolute().GetPath(), sqlite3_errmsg(m_db)); + + [[maybe_unused]] + const int synchronousResult = sqlite3_exec(m_db, "PRAGMA synchronous = NORMAL;", nullptr, nullptr, nullptr); + ASSERT(EngineAssertHandler, + synchronousResult == SQLITE_OK, + "Failed to set synchronous pragma: {}", + sqlite3_errmsg(m_db)); + + [[maybe_unused]] + const int foreignKeysResult = sqlite3_exec(m_db, "PRAGMA foreign_keys = ON;", nullptr, nullptr, nullptr); + ASSERT(EngineAssertHandler, + foreignKeysResult == SQLITE_OK, + "Failed to set foreign_keys pragma: {}", + sqlite3_errmsg(m_db)); } /****************************************************************************************/ @@ -235,6 +249,87 @@ void BigFile::SaveAssetFlatbuffer(const AssetMetadata& p_metadata, const eastl:: /****************************************************************************************/ +void BigFile::SaveAssetFlatbuffers(const eastl::span p_metadatas, + const eastl::span> p_flatbuffers) +{ + ASSERT(EngineAssertHandler, + p_metadatas.size() == p_flatbuffers.size(), + "p_metadatas and p_flatbuffers must have the same size"); + + if (p_metadatas.empty()) + { + return; + } + + BeginTransaction(); + + constexpr std::size_t METADATA_PARAMS_PER_ROW = 4; + const std::size_t maxRowsPerStatement = + static_cast(sqlite3_limit(m_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1)) / METADATA_PARAMS_PER_ROW; + + for (std::size_t chunkStart = 0; chunkStart < p_metadatas.size(); chunkStart += maxRowsPerStatement) + { + const std::size_t chunkSize = eastl::min(maxRowsPerStatement, p_metadatas.size() - chunkStart); + + eastl::string metadataPlaceholders; + metadataPlaceholders.reserve(chunkSize * 14); + for (std::size_t i = 0; i < chunkSize; ++i) + { + if (i > 0) + { + metadataPlaceholders += ", "; + } + metadataPlaceholders += "(?, ?, ?, ?)"; + } + + const std::string metadataSQL = + fmt::format("INSERT INTO AssetMetadata (UUID, Name, TypeID, TypeName) VALUES {} " + "ON CONFLICT(UUID) DO UPDATE SET Name = excluded.Name, TypeID = excluded.TypeID, TypeName = " + "excluded.TypeName;", + metadataPlaceholders); + + BigFile::Request metadataRequest {*this, metadataSQL.c_str()}; + std::uint32_t metadataBindIndex = 1; + for (std::size_t i = 0; i < chunkSize; ++i) + { + const AssetMetadata& metadata = p_metadatas[chunkStart + i]; + metadataRequest.Bind(metadataBindIndex++, metadata.m_uuid.ToBytes()); + metadataRequest.Bind(metadataBindIndex++, metadata.m_name); + metadataRequest.Bind(metadataBindIndex++, metadata.m_typeID); + metadataRequest.Bind(metadataBindIndex++, metadata.m_typeName); + } + metadataRequest.Execute(); + + eastl::string assetPlaceholders; + assetPlaceholders.reserve(chunkSize * 8); + for (std::size_t i = 0; i < chunkSize; ++i) + { + if (i > 0) + { + assetPlaceholders += ", "; + } + assetPlaceholders += "(?, ?)"; + } + + const std::string assetSQL = fmt::format( + "INSERT INTO Asset (UUID, Asset) VALUES {} ON CONFLICT(UUID) DO UPDATE SET Asset = excluded.Asset;", + assetPlaceholders); + + BigFile::Request assetRequest {*this, assetSQL.c_str()}; + std::uint32_t assetBindIndex = 1; + for (std::size_t i = 0; i < chunkSize; ++i) + { + assetRequest.Bind(assetBindIndex++, p_metadatas[chunkStart + i].m_uuid.ToBytes()); + assetRequest.Bind(assetBindIndex++, p_flatbuffers[chunkStart + i]); + } + assetRequest.Execute(); + } + + CommitTransaction(); +} + +/****************************************************************************************/ + bool BigFile::Has(const UUID& p_uuid) const { BigFile::Request request {*this, "SELECT 1 FROM AssetMetadata WHERE UUID = ?1;"}; diff --git a/Bigfoot/Sources/Engine/BigFile/BigFileSchema.sql b/Bigfoot/Sources/Engine/BigFile/BigFileSchema.sql index fd67ade..314fbf6 100644 --- a/Bigfoot/Sources/Engine/BigFile/BigFileSchema.sql +++ b/Bigfoot/Sources/Engine/BigFile/BigFileSchema.sql @@ -1,5 +1,4 @@ PRAGMA journal_mode=WAL; -PRAGMA foreign_keys = ON; CREATE TABLE IF NOT EXISTS AssetMetadata ( UUID BLOB NOT NULL, diff --git a/Bigfoot/Sources/Engine/Include/Engine/BigFile/BigFile.hpp b/Bigfoot/Sources/Engine/Include/Engine/BigFile/BigFile.hpp index d019700..2ad758d 100644 --- a/Bigfoot/Sources/Engine/Include/Engine/BigFile/BigFile.hpp +++ b/Bigfoot/Sources/Engine/Include/Engine/BigFile/BigFile.hpp @@ -91,6 +91,15 @@ class BigFile: public DataAccessor */ void SaveAssetFlatbuffer(const AssetMetadata& p_metadata, const eastl::span p_flatbuffer) override; + /** + * Save several assets + * + * \p_metadatas The metadatas + * \p_flatbuffers The flatbuffers of the assets, matching p_metadatas by index + */ + void SaveAssetFlatbuffers(eastl::span p_metadatas, + eastl::span> p_flatbuffers) override; + /** * Check if the Asset exists * diff --git a/Bigfoot/Sources/Engine/Include/Engine/DataAccessor/DataAccessor.hpp b/Bigfoot/Sources/Engine/Include/Engine/DataAccessor/DataAccessor.hpp index afba0df..8f87d9a 100644 --- a/Bigfoot/Sources/Engine/Include/Engine/DataAccessor/DataAccessor.hpp +++ b/Bigfoot/Sources/Engine/Include/Engine/DataAccessor/DataAccessor.hpp @@ -81,6 +81,15 @@ class DataAccessor virtual void SaveAssetFlatbuffer(const AssetMetadata& p_metadata, const eastl::span p_flatbuffer) = 0; + /** + * Save several assets + * + * \p_metadatas The metadatas + * \p_flatbuffers The flatbuffers of the assets, matching p_metadatas by index + */ + virtual void SaveAssetFlatbuffers(eastl::span p_metadatas, + eastl::span> p_flatbuffers) = 0; + /** * Check if the Asset exists * diff --git a/Bigfoot/Tests/Engine/BigFile/BigFile.cpp b/Bigfoot/Tests/Engine/BigFile/BigFile.cpp index e61f3c5..605176e 100644 --- a/Bigfoot/Tests/Engine/BigFile/BigFile.cpp +++ b/Bigfoot/Tests/Engine/BigFile/BigFile.cpp @@ -260,6 +260,89 @@ TEST_F(BigFileFixture, SaveAssetFlatbuffer_ShouldUpdateTheDataWhenCalledTwice) /****************************************************************************************/ +TEST_F(BigFileFixture, SaveAssetFlatbuffers_ShouldSaveTheAssets) +{ + AssetA assetA; + assetA.SetName("hello"); + assetA.GetAsset().health = 42; + + AssetA assetB; + assetB.SetName("world"); + assetB.GetAsset().health = 7; + + const eastl::vector metadatas { + AssetMetadata {.m_uuid = assetA.GetHeader().uuid, + .m_name = assetA.GetHeader().name, + .m_typeID = assetA.GetHeader().type_id, + .m_typeName = assetA.GetHeader().type_name}, + AssetMetadata {.m_uuid = assetB.GetHeader().uuid, + .m_name = assetB.GetHeader().name, + .m_typeID = assetB.GetHeader().type_id, + .m_typeName = assetB.GetHeader().type_name}}; + const eastl::vector> flatbuffers {assetA.Pack(), assetB.Pack()}; + + m_bigFile.SaveAssetFlatbuffers(metadatas, flatbuffers); + + EXPECT_TRUE(m_bigFile.Has(assetA.GetHeader().uuid)); + EXPECT_TRUE(m_bigFile.Has(assetB.GetHeader().uuid)); +} + +/****************************************************************************************/ + +TEST_F(BigFileFixture, SaveAssetFlatbuffers_ShouldUpdateTheDataWhenCalledTwice) +{ + const UUID uuid; + + AssetA asset; + asset.GetAsset().health = 1; + const eastl::vector metadatas1 { + AssetMetadata {.m_uuid = uuid, + .m_name = "hello", + .m_typeID = AssetA::GetTypeID(), + .m_typeName = eastl::string {AssetA::GetTypeName()}}}; + const eastl::vector> flatbuffers1 {asset.Pack()}; + m_bigFile.SaveAssetFlatbuffers(metadatas1, flatbuffers1); + + const eastl::optional savedMetadata1 = m_bigFile.GetAssetMetadata(uuid); + ASSERT_TRUE(savedMetadata1.has_value()); + EXPECT_EQ(savedMetadata1->m_name, "hello"); + EXPECT_EQ(savedMetadata1->m_modificationTime, savedMetadata1->m_creationTime); + + const eastl::optional> savedFlatbuffer1 = m_bigFile.GetAssetFlatbuffer(uuid); + ASSERT_TRUE(savedFlatbuffer1.has_value()); + + AssetA savedAsset1; + savedAsset1.Unpack(*savedFlatbuffer1); + EXPECT_EQ(savedAsset1.GetAsset().health, 1); + + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + + asset.GetAsset().health = 2; + const eastl::vector metadatas2 { + AssetMetadata {.m_uuid = uuid, + .m_name = "world", + .m_typeID = AssetA::GetTypeID(), + .m_typeName = eastl::string {AssetA::GetTypeName()}}}; + const eastl::vector> flatbuffers2 {asset.Pack()}; + m_bigFile.SaveAssetFlatbuffers(metadatas2, flatbuffers2); + + const eastl::optional savedMetadata2 = m_bigFile.GetAssetMetadata(uuid); + ASSERT_TRUE(savedMetadata2.has_value()); + EXPECT_EQ(savedMetadata2->m_name, "world"); + EXPECT_GT(savedMetadata2->m_modificationTime, savedMetadata2->m_creationTime); + + EXPECT_LT(savedMetadata1->m_modificationTime, savedMetadata2->m_modificationTime); + + const eastl::optional> savedFlatbuffer2 = m_bigFile.GetAssetFlatbuffer(uuid); + ASSERT_TRUE(savedFlatbuffer2.has_value()); + + AssetA savedAsset2; + savedAsset2.Unpack(*savedFlatbuffer2); + EXPECT_EQ(savedAsset2.GetAsset().health, 2); +} + +/****************************************************************************************/ + TEST_F(BigFileFixture, Has_ShouldReturnTrueIfTheAssetExists) { AssetA asset; diff --git a/Bigfoot/Tests/Engine/DataAccessor/DataAccessor.cpp b/Bigfoot/Tests/Engine/DataAccessor/DataAccessor.cpp index 1438bdb..ede3957 100644 --- a/Bigfoot/Tests/Engine/DataAccessor/DataAccessor.cpp +++ b/Bigfoot/Tests/Engine/DataAccessor/DataAccessor.cpp @@ -34,6 +34,11 @@ class DataAccessorFixture: public ::testing::Test SaveAssetFlatbuffer, (const AssetMetadata& p_metaData, const eastl::span p_flatbuffer), (override)); + MOCK_METHOD(void, + SaveAssetFlatbuffers, + (eastl::span p_metadatas, + eastl::span> p_flatbuffers), + (override)); MOCK_METHOD(bool, Has, (const UUID& p_uuid), (const, override)); };