Files
Bigfoot/Bigfoot/Sources/Engine/BigFile/BigFile.cpp
T
rboullard d32d1d7229
Bigfoot / Sonarqube (push) Skipped
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang (Unity Build: ON, Variant: game) (push) Successful in 21s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang (Unity Build: ON, Variant: toolmode) (push) Successful in 22s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: game) (push) Successful in 49s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: toolmode) (push) Successful in 52s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: game) (push) Successful in 30s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: game) (push) Successful in 36s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: game) (push) Successful in 32s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: game) (push) Successful in 28s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: toolmode) (push) Successful in 29s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: game) (push) Successful in 39s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: toolmode) (push) Successful in 42s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang (Unity Build: ON, Variant: game) (push) Successful in 21s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: toolmode) (push) Successful in 31s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang (Unity Build: ON, Variant: toolmode) (push) Successful in 22s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: game) (push) Successful in 50s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: game) (push) Successful in 40s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: toolmode) (push) Successful in 48s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: toolmode) (push) Successful in 41s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: game) (push) Successful in 23s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang (Unity Build: ON, Variant: game) (push) Successful in 21s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: toolmode) (push) Successful in 34s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang (Unity Build: ON, Variant: toolmode) (push) Successful in 27s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: toolmode) (push) Successful in 36s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: toolmode) (push) Successful in 35s
Bigfoot / Clang Format Checks (push) Successful in 11s
Optimize BigFile access
2026-08-25 19:06:39 +02:00

487 lines
18 KiB
C++

/*********************************************************************
* \file BigFile.cpp
*
* \author Romain BOULLARD
* \date October 2025
*********************************************************************/
#include <Engine/BigFile/BigFile.hpp>
#include <Engine/EngineAssertHandler.hpp>
#include <EASTL/algorithm.h>
#include <EASTL/sort.h>
#include <fmt/format.h>
namespace Bigfoot
{
#ifdef BIGFOOT_TOOLMODE
BigFile::BigFile(const File& p_file)
{
[[maybe_unused]]
const int result = sqlite3_open_v2(p_file.ToAbsolute().GetPath().data(), &m_db, SQLITE_OPEN_READWRITE, nullptr);
CRITICAL_ASSERT(EngineAssertHandler,
result == SQLITE_OK,
"Failed to open BigFile {} DB: {}",
p_file.ToAbsolute().GetPath(),
sqlite3_errmsg(m_db));
}
/****************************************************************************************/
void BigFile::BeginTransaction()
{
[[maybe_unused]]
const int result = sqlite3_exec(m_db, "BEGIN TRANSACTION;", nullptr, nullptr, nullptr);
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to begin transaction: {}", sqlite3_errmsg(m_db));
}
/****************************************************************************************/
void BigFile::CommitTransaction()
{
[[maybe_unused]]
const int result = sqlite3_exec(m_db, "COMMIT TRANSACTION;", nullptr, nullptr, nullptr);
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to commit: {}", sqlite3_errmsg(m_db));
}
/****************************************************************************************/
void BigFile::RollbackTransaction()
{
[[maybe_unused]]
const int result = sqlite3_exec(m_db, "ROLLBACK TRANSACTION;", nullptr, nullptr, nullptr);
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to rollback: {}", sqlite3_errmsg(m_db));
}
/****************************************************************************************/
BigFile::~BigFile()
{
[[maybe_unused]]
const int result = sqlite3_close_v2(m_db);
CRITICAL_ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to close BigFile DB: {}", sqlite3_errmsg(m_db));
}
/****************************************************************************************/
eastl::optional<eastl::vector<std::byte>> BigFile::GetAssetFlatbuffer(const UUID& p_uuid) const
{
BigFile::Request assetRequest {*this, "SELECT Asset FROM Asset WHERE UUID = ?1;"};
assetRequest.Bind(1, p_uuid.ToBytes());
if (!assetRequest.Step())
{
return eastl::nullopt;
}
const eastl::span<const std::byte> flatbuffer = assetRequest.Get(0);
return eastl::vector<std::byte> {flatbuffer.begin(), flatbuffer.end()};
}
/****************************************************************************************/
eastl::vector<eastl::vector<std::byte>> BigFile::GetAssetFlatbuffers(const eastl::span<const UUID> p_uuids) const
{
eastl::vector<eastl::vector<std::byte>> flatbuffers;
if (p_uuids.empty())
{
return flatbuffers;
}
eastl::vector<UUID> uniqueUUIDs {p_uuids.begin(), p_uuids.end()};
eastl::sort(uniqueUUIDs.begin(), uniqueUUIDs.end());
uniqueUUIDs.erase(eastl::unique(uniqueUUIDs.begin(), uniqueUUIDs.end()), uniqueUUIDs.end());
eastl::string placeholders;
placeholders.reserve(uniqueUUIDs.size() * 3);
for (std::size_t i = 0; i < uniqueUUIDs.size(); ++i)
{
if (i > 0)
{
placeholders += ", ";
}
placeholders += "?";
}
const std::string sqlRequest = fmt::format("SELECT Asset FROM Asset WHERE UUID IN ({});", placeholders);
BigFile::Request assetRequest {*this, sqlRequest.c_str()};
for (std::size_t i = 0; i < uniqueUUIDs.size(); ++i)
{
assetRequest.Bind(static_cast<std::uint32_t>(i + 1), uniqueUUIDs[i].ToBytes());
}
flatbuffers.reserve(uniqueUUIDs.size());
for (std::size_t i = 0; i < uniqueUUIDs.size(); ++i)
{
if (!assetRequest.Step())
{
break;
}
const eastl::span<const std::byte> flatbuffer = assetRequest.Get(0);
flatbuffers.emplace_back(flatbuffer.begin(), flatbuffer.end());
}
return flatbuffers;
}
/****************************************************************************************/
eastl::optional<AssetMetadata> BigFile::GetAssetMetadata(const UUID& p_uuid) const
{
BigFile::Request assetRequest {
*this,
"SELECT UUID, Name, TypeID, TypeName, CreateTime, ModificationTime FROM AssetMetadata WHERE UUID = ?1;"};
assetRequest.Bind(1, p_uuid.ToBytes());
if (!assetRequest.Step())
{
return eastl::nullopt;
}
return AssetMetadata {.m_uuid = UUID {std::span<const std::byte, UUID::UUID_BYTE_SIZE> {
static_cast<eastl::span<const std::byte>>(assetRequest.Get(0)).data(),
UUID::UUID_BYTE_SIZE}},
.m_name = eastl::string {static_cast<eastl::string_view>(assetRequest.Get(1))},
.m_typeID = assetRequest.Get(2),
.m_typeName = eastl::string {static_cast<eastl::string_view>(assetRequest.Get(3))},
.m_creationTime = Time {static_cast<std::uint64_t>(assetRequest.Get(4))},
.m_modificationTime = Time {static_cast<std::uint64_t>(assetRequest.Get(5))}};
}
/****************************************************************************************/
eastl::vector<AssetMetadata> BigFile::GetAssetMetadatas(const eastl::span<const UUID> p_uuids) const
{
eastl::vector<AssetMetadata> metadatas;
if (p_uuids.empty())
{
return metadatas;
}
eastl::vector<UUID> uniqueUUIDs {p_uuids.begin(), p_uuids.end()};
eastl::sort(uniqueUUIDs.begin(), uniqueUUIDs.end());
uniqueUUIDs.erase(eastl::unique(uniqueUUIDs.begin(), uniqueUUIDs.end()), uniqueUUIDs.end());
eastl::string placeholders;
placeholders.reserve(uniqueUUIDs.size() * 3);
for (std::size_t i = 0; i < uniqueUUIDs.size(); ++i)
{
if (i > 0)
{
placeholders += ", ";
}
placeholders += "?";
}
const std::string sqlRequest = fmt::format("SELECT UUID, Name, TypeID, TypeName, CreateTime, ModificationTime "
"FROM AssetMetadata WHERE UUID IN ({});",
placeholders);
BigFile::Request assetRequest {*this, sqlRequest.c_str()};
for (std::size_t i = 0; i < uniqueUUIDs.size(); ++i)
{
assetRequest.Bind(static_cast<std::uint32_t>(i + 1), uniqueUUIDs[i].ToBytes());
}
metadatas.reserve(uniqueUUIDs.size());
for (std::size_t i = 0; i < uniqueUUIDs.size(); ++i)
{
if (!assetRequest.Step())
{
break;
}
metadatas.push_back(
AssetMetadata {.m_uuid = UUID {std::span<const std::byte, UUID::UUID_BYTE_SIZE> {
static_cast<eastl::span<const std::byte>>(assetRequest.Get(0)).data(),
UUID::UUID_BYTE_SIZE}},
.m_name = eastl::string {static_cast<eastl::string_view>(assetRequest.Get(1))},
.m_typeID = assetRequest.Get(2),
.m_typeName = eastl::string {static_cast<eastl::string_view>(assetRequest.Get(3))},
.m_creationTime = Time {static_cast<std::uint64_t>(assetRequest.Get(4))},
.m_modificationTime = Time {static_cast<std::uint64_t>(assetRequest.Get(5))}});
}
return metadatas;
}
/****************************************************************************************/
void BigFile::SaveAssetFlatbuffer(const AssetMetadata& p_metadata, const eastl::span<const std::byte> p_flatbuffer)
{
BeginTransaction();
BigFile::Request metadataRequest {
*this,
"INSERT INTO AssetMetadata (UUID, Name, TypeID, TypeName) VALUES (?1, ?2, ?3, ?4) "
"ON CONFLICT(UUID) DO UPDATE SET Name = excluded.Name, TypeID = excluded.TypeID, TypeName = excluded.TypeName;"};
metadataRequest.Bind(1, p_metadata.m_uuid.ToBytes());
metadataRequest.Bind(2, p_metadata.m_name);
metadataRequest.Bind(3, p_metadata.m_typeID);
metadataRequest.Bind(4, p_metadata.m_typeName);
metadataRequest.Execute();
BigFile::Request assetRequest {*this,
"INSERT INTO Asset (UUID, Asset) VALUES (?1, ?2) "
"ON CONFLICT(UUID) DO UPDATE SET Asset = excluded.Asset;"};
assetRequest.Bind(1, p_metadata.m_uuid.ToBytes());
assetRequest.Bind(2, p_flatbuffer);
assetRequest.Execute();
CommitTransaction();
}
/****************************************************************************************/
bool BigFile::Has(const UUID& p_uuid) const
{
BigFile::Request request {*this, "SELECT 1 FROM AssetMetadata WHERE UUID = ?1;"};
request.Bind(1, p_uuid.ToBytes());
return request.Step();
}
/****************************************************************************************/
BigFile::Request::Request(const BigFile& p_bigFile, const eastl::string_view p_request):
m_db(p_bigFile.m_db)
{
[[maybe_unused]]
const int result = sqlite3_prepare_v2(m_db,
p_request.data(),
static_cast<std::uint32_t>(p_request.size()),
&m_statement,
nullptr);
CRITICAL_ASSERT(EngineAssertHandler,
result == SQLITE_OK,
"Failed to create statement from BigFile DB: {}",
sqlite3_errmsg(m_db));
}
/****************************************************************************************/
void BigFile::Request::Bind(const std::uint32_t p_index, const std::int32_t p_value)
{
ASSERT(EngineAssertHandler,
(p_index >= 1) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
"Invalid index for statement");
[[maybe_unused]]
const int result = sqlite3_bind_int(m_statement, p_index, p_value);
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
}
/****************************************************************************************/
void BigFile::Request::Bind(const std::uint32_t p_index, const std::uint32_t p_value)
{
ASSERT(EngineAssertHandler,
(p_index >= 1) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
"Invalid index for statement");
[[maybe_unused]]
const int result = sqlite3_bind_int(m_statement, p_index, p_value);
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
}
/****************************************************************************************/
void BigFile::Request::Bind(const std::uint32_t p_index, const std::int64_t p_value)
{
ASSERT(EngineAssertHandler,
(p_index >= 1) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
"Invalid index for statement");
[[maybe_unused]]
const int result = sqlite3_bind_int64(m_statement, p_index, p_value);
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
}
/****************************************************************************************/
void BigFile::Request::Bind(const std::uint32_t p_index, const std::uint64_t p_value)
{
ASSERT(EngineAssertHandler,
(p_index >= 1) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
"Invalid index for statement");
[[maybe_unused]]
const int result = sqlite3_bind_int64(m_statement, p_index, std::bit_cast<std::int64_t>(p_value));
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
}
/****************************************************************************************/
void BigFile::Request::Bind(const std::uint32_t p_index, const float p_value)
{
ASSERT(EngineAssertHandler,
(p_index >= 1) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
"Invalid index for statement");
[[maybe_unused]]
const int result = sqlite3_bind_double(m_statement, p_index, p_value);
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
}
/****************************************************************************************/
void BigFile::Request::Bind(const std::uint32_t p_index, const double p_value)
{
ASSERT(EngineAssertHandler,
(p_index >= 1) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
"Invalid index for statement");
[[maybe_unused]]
const int result = sqlite3_bind_double(m_statement, p_index, p_value);
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
}
/****************************************************************************************/
void BigFile::Request::Bind(const std::uint32_t p_index, const eastl::string_view p_value, const CopyValue p_copy)
{
ASSERT(EngineAssertHandler,
(p_index >= 1) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
"Invalid index for statement");
[[maybe_unused]]
const int result = sqlite3_bind_text(m_statement,
p_index,
p_value.data(),
static_cast<std::uint32_t>(p_value.size()),
p_copy ? SQLITE_TRANSIENT : SQLITE_STATIC);
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
}
/****************************************************************************************/
void BigFile::Request::Bind(const std::uint32_t p_index,
const eastl::span<const std::byte> p_value,
const CopyValue p_copy)
{
ASSERT(EngineAssertHandler,
(p_index >= 1) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
"Invalid index for statement");
[[maybe_unused]]
const int result = sqlite3_bind_blob(m_statement,
p_index,
p_value.data(),
static_cast<std::uint32_t>(p_value.size()),
p_copy ? SQLITE_TRANSIENT : SQLITE_STATIC);
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
}
/****************************************************************************************/
bool BigFile::Request::Step()
{
const int result = sqlite3_step(m_statement);
ASSERT(EngineAssertHandler,
(result == SQLITE_DONE) || (result == SQLITE_ROW),
"Failed to step through the statement: {}",
sqlite3_errmsg(m_db));
return result == SQLITE_ROW;
}
/****************************************************************************************/
std::uint32_t BigFile::Request::Execute()
{
[[maybe_unused]]
const int result = sqlite3_step(m_statement);
ASSERT(EngineAssertHandler, (result == SQLITE_DONE), "Failed to execute the statement: {}", sqlite3_errmsg(m_db));
return sqlite3_changes(m_db);
}
/****************************************************************************************/
BigFile::Request::Column::Column(const Request& p_request, const std::uint32_t p_index):
m_statement(p_request.m_statement),
m_index(p_index)
{
}
/****************************************************************************************/
BigFile::Request::Column::operator std::int32_t() const
{
return sqlite3_column_int(m_statement, m_index);
}
/****************************************************************************************/
BigFile::Request::Column::operator std::uint32_t() const
{
return sqlite3_column_int(m_statement, m_index);
}
/****************************************************************************************/
BigFile::Request::Column::operator std::int64_t() const
{
return sqlite3_column_int64(m_statement, m_index);
}
/****************************************************************************************/
BigFile::Request::Column::operator std::uint64_t() const
{
return std::bit_cast<std::uint64_t>(sqlite3_column_int64(m_statement, m_index));
}
/****************************************************************************************/
BigFile::Request::Column::operator float() const
{
return static_cast<float>(sqlite3_column_double(m_statement, m_index));
}
/****************************************************************************************/
BigFile::Request::Column::operator double() const
{
return sqlite3_column_double(m_statement, m_index);
}
/****************************************************************************************/
BigFile::Request::Column::operator eastl::string_view() const
{
return eastl::string_view {reinterpret_cast<const char*>(sqlite3_column_text(m_statement, m_index)),
static_cast<std::size_t>(sqlite3_column_bytes(m_statement, m_index))};
}
/****************************************************************************************/
BigFile::Request::Column::operator eastl::span<const std::byte>() const
{
return eastl::span<const std::byte> {static_cast<const std::byte*>(sqlite3_column_blob(m_statement, m_index)),
static_cast<std::size_t>(sqlite3_column_bytes(m_statement, m_index))};
}
/****************************************************************************************/
BigFile::Request::Column BigFile::Request::Get(const std::uint32_t p_index) const
{
ASSERT(EngineAssertHandler,
p_index < static_cast<std::uint32_t>(sqlite3_column_count(m_statement)),
"Invalid index for column!");
return {*this, p_index};
}
/****************************************************************************************/
BigFile::Request::~Request()
{
[[maybe_unused]]
const int result = sqlite3_finalize(m_statement);
CRITICAL_ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to finalize statement: {}", sqlite3_errmsg(m_db));
}
#endif
} // namespace Bigfoot