Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: game) (push) Successful in 44s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang (Unity Build: ON, Variant: game) (push) Successful in 29s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: toolmode) (push) Successful in 46s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang (Unity Build: ON, Variant: toolmode) (push) Successful in 30s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: game) (push) Successful in 1m21s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: game) (push) Successful in 1m3s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: toolmode) (push) Successful in 1m25s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: toolmode) (push) Successful in 1m8s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: game) (push) Successful in 1m9s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: toolmode) (push) Successful in 1m34s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: toolmode) (push) Successful in 1m13s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: game) (push) Successful in 37s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang (Unity Build: ON, Variant: game) (push) Successful in 28s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: toolmode) (push) Successful in 57s
Bigfoot / Sonarqube (push) Successful in 6m0s
sonarqube ERROR
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: game) (push) Successful in 49s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang (Unity Build: ON, Variant: game) (push) Successful in 30s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: toolmode) (push) Successful in 52s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang (Unity Build: ON, Variant: toolmode) (push) Successful in 36s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: game) (push) Successful in 1m23s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang (Unity Build: ON, Variant: toolmode) (push) Successful in 44s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: game) (push) Successful in 59s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: game) (push) Successful in 51s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: toolmode) (push) Successful in 1m5s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: toolmode) (push) Successful in 56s
Bigfoot / Clang Format Checks (push) Successful in 13s
Bigfoot / Sonarqube (pull_request) Failing after 5m14s
Implementing an asset management system BigFile is a sqlite database, for toolmode Sasquatch is a custom binary file, for gamemode Assets are built from flatbuffers References allow to get an Asset, from an UUID, from anywhere in the code HardReference refcount the asset and allow automatic removal Reviewed-on: #1 Co-authored-by: Romain BOULLARD <romain.boullard@protonmail.com>
852 lines
33 KiB
C++
852 lines
33 KiB
C++
/*********************************************************************
|
|
* \file BigFile.cpp
|
|
*
|
|
* \author Romain BOULLARD
|
|
* \date October 2025
|
|
*********************************************************************/
|
|
#include <Engine/BigFile/BigFile.hpp>
|
|
|
|
#include <Engine/EngineAssertHandler.hpp>
|
|
|
|
#include <Utils/Format.hpp>
|
|
|
|
#include <EASTL/algorithm.h>
|
|
|
|
namespace Bigfoot
|
|
{
|
|
#ifdef BIGFOOT_TOOLMODE
|
|
BigFile::BigFile(const File& p_file)
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_open_v2(p_file.ToAbsolute().GetPath().data(), &m_db, SQLITE_OPEN_READWRITE, nullptr);
|
|
CRITICAL_ASSERT(EngineAssertHandler,
|
|
result == SQLITE_OK,
|
|
"Failed to open BigFile {} DB: {}",
|
|
p_file.ToAbsolute().GetPath(),
|
|
sqlite3_errmsg(m_db));
|
|
|
|
[[maybe_unused]]
|
|
const int synchronousResult = sqlite3_exec(m_db, "PRAGMA synchronous = NORMAL;", nullptr, nullptr, nullptr);
|
|
ASSERT(EngineAssertHandler,
|
|
synchronousResult == SQLITE_OK,
|
|
"Failed to set synchronous pragma: {}",
|
|
sqlite3_errmsg(m_db));
|
|
|
|
[[maybe_unused]]
|
|
const int foreignKeysResult = sqlite3_exec(m_db, "PRAGMA foreign_keys = ON;", nullptr, nullptr, nullptr);
|
|
ASSERT(EngineAssertHandler,
|
|
foreignKeysResult == SQLITE_OK,
|
|
"Failed to set foreign_keys pragma: {}",
|
|
sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::BeginTransaction()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_exec(m_db, "BEGIN TRANSACTION;", nullptr, nullptr, nullptr);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to begin transaction: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::CommitTransaction()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_exec(m_db, "COMMIT TRANSACTION;", nullptr, nullptr, nullptr);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to commit: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::RollbackTransaction()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_exec(m_db, "ROLLBACK TRANSACTION;", nullptr, nullptr, nullptr);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to rollback: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Clear()
|
|
{
|
|
BeginTransaction();
|
|
|
|
BigFile::Request {*this, "DELETE FROM AssetHardReference;"}.Execute();
|
|
BigFile::Request {*this, "DELETE FROM Asset;"}.Execute();
|
|
BigFile::Request {*this, "DELETE FROM AssetMetadata;"}.Execute();
|
|
|
|
CommitTransaction();
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::~BigFile()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_close_v2(m_db);
|
|
CRITICAL_ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to close BigFile DB: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::vector<std::byte> BigFile::GetAssetFlatbuffer(const UUID& p_uuid) const
|
|
{
|
|
BigFile::Request assetRequest {*this, "SELECT Asset FROM Asset WHERE UUID = ?1;"};
|
|
assetRequest.Bind(1, p_uuid.ToBytes());
|
|
|
|
if (!assetRequest.Step())
|
|
{
|
|
return {};
|
|
}
|
|
|
|
const eastl::span<const std::byte> flatbuffer = assetRequest.Get(0);
|
|
return eastl::vector<std::byte> {flatbuffer.begin(), flatbuffer.end()};
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::vector<eastl::vector<std::byte>> BigFile::GetAssetFlatbuffers(const eastl::span<const UUID> p_uuids) const
|
|
{
|
|
eastl::vector<eastl::vector<std::byte>> flatbuffers(p_uuids.size());
|
|
|
|
if (p_uuids.empty())
|
|
{
|
|
return flatbuffers;
|
|
}
|
|
|
|
ankerl::unordered_dense::map<UUID, eastl::vector<std::size_t>> uuidToIndices;
|
|
uuidToIndices.reserve(p_uuids.size());
|
|
for (std::size_t i = 0; i < p_uuids.size(); ++i)
|
|
{
|
|
uuidToIndices[p_uuids[i]].push_back(i);
|
|
}
|
|
|
|
eastl::vector<UUID> distinctUUIDs;
|
|
distinctUUIDs.reserve(uuidToIndices.size());
|
|
for (const auto& [uuid, indices]: uuidToIndices)
|
|
{
|
|
distinctUUIDs.push_back(uuid);
|
|
}
|
|
|
|
constexpr std::size_t PARAMS_PER_ROW = 1;
|
|
const std::size_t maxRowsPerStatement = static_cast<std::size_t>(
|
|
sqlite3_limit(m_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1)) /
|
|
PARAMS_PER_ROW;
|
|
|
|
for (std::size_t chunkStart = 0; chunkStart < distinctUUIDs.size(); chunkStart += maxRowsPerStatement)
|
|
{
|
|
const std::size_t chunkSize = eastl::min(maxRowsPerStatement, distinctUUIDs.size() - chunkStart);
|
|
|
|
eastl::string placeholders;
|
|
placeholders.reserve(chunkSize * 2);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
placeholders += ", ";
|
|
}
|
|
placeholders += "?";
|
|
}
|
|
|
|
const eastl::string sql = Format("SELECT UUID, Asset FROM Asset WHERE UUID IN ({});", placeholders);
|
|
|
|
BigFile::Request assetRequest {*this, sql};
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
assetRequest.Bind(static_cast<std::uint32_t>(i + 1), distinctUUIDs[chunkStart + i].ToBytes());
|
|
}
|
|
|
|
while (assetRequest.Step())
|
|
{
|
|
const std::span<const std::byte, UUID::UUID_BYTE_SIZE> rawUUID {
|
|
static_cast<std::span<const std::byte>>(assetRequest.Get(0))};
|
|
const UUID uuid {rawUUID};
|
|
|
|
const auto indicesIt = uuidToIndices.find(uuid);
|
|
ASSERT(EngineAssertHandler,
|
|
indicesIt != uuidToIndices.end(),
|
|
"Returned UUID {} not found in requested UUIDs",
|
|
uuid);
|
|
|
|
const eastl::span<const std::byte> flatbuffer = assetRequest.Get(1);
|
|
|
|
for (const std::size_t index: indicesIt->second)
|
|
{
|
|
flatbuffers[index].assign(flatbuffer.begin(), flatbuffer.end());
|
|
}
|
|
}
|
|
}
|
|
|
|
return flatbuffers;
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::vector<UUID> BigFile::GetAssetHardReferences(const UUID& p_uuid) const
|
|
{
|
|
BigFile::Request hardReferencesRequest {*this, "SELECT HardReference FROM AssetHardReference WHERE UUID = ?1;"};
|
|
hardReferencesRequest.Bind(1, p_uuid.ToBytes());
|
|
|
|
eastl::vector<UUID> hardReferences;
|
|
while (hardReferencesRequest.Step())
|
|
{
|
|
const std::span<const std::byte, UUID::UUID_BYTE_SIZE> rawUUID {
|
|
static_cast<std::span<const std::byte>>(hardReferencesRequest.Get(0))};
|
|
hardReferences.emplace_back(rawUUID);
|
|
}
|
|
|
|
return hardReferences;
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::vector<eastl::vector<UUID>> BigFile::GetAssetsHardReferences(eastl::span<const UUID> p_uuids) const
|
|
{
|
|
eastl::vector<eastl::vector<UUID>> hardReferences(p_uuids.size());
|
|
|
|
if (p_uuids.empty())
|
|
{
|
|
return hardReferences;
|
|
}
|
|
|
|
ankerl::unordered_dense::map<UUID, eastl::vector<std::size_t>> uuidToIndices;
|
|
uuidToIndices.reserve(p_uuids.size());
|
|
for (std::size_t i = 0; i < p_uuids.size(); ++i)
|
|
{
|
|
uuidToIndices[p_uuids[i]].push_back(i);
|
|
}
|
|
|
|
eastl::vector<UUID> distinctUUIDs;
|
|
distinctUUIDs.reserve(uuidToIndices.size());
|
|
for (const auto& [uuid, indices]: uuidToIndices)
|
|
{
|
|
distinctUUIDs.push_back(uuid);
|
|
}
|
|
|
|
constexpr std::size_t PARAMS_PER_ROW = 1;
|
|
const std::size_t maxRowsPerStatement = static_cast<std::size_t>(
|
|
sqlite3_limit(m_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1)) /
|
|
PARAMS_PER_ROW;
|
|
|
|
for (std::size_t chunkStart = 0; chunkStart < distinctUUIDs.size(); chunkStart += maxRowsPerStatement)
|
|
{
|
|
const std::size_t chunkSize = eastl::min(maxRowsPerStatement, distinctUUIDs.size() - chunkStart);
|
|
|
|
eastl::string placeholders;
|
|
placeholders.reserve(chunkSize * 2);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
placeholders += ", ";
|
|
}
|
|
placeholders += "?";
|
|
}
|
|
|
|
const eastl::string sql = Format("SELECT UUID, HardReference FROM AssetHardReference WHERE UUID IN ({});",
|
|
placeholders);
|
|
|
|
BigFile::Request hardReferencesRequest {*this, sql};
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
hardReferencesRequest.Bind(static_cast<std::uint32_t>(i + 1), distinctUUIDs[chunkStart + i].ToBytes());
|
|
}
|
|
|
|
while (hardReferencesRequest.Step())
|
|
{
|
|
const std::span<const std::byte, UUID::UUID_BYTE_SIZE> rawOwnerUUID {
|
|
static_cast<std::span<const std::byte>>(hardReferencesRequest.Get(0))};
|
|
const UUID ownerUUID {rawOwnerUUID};
|
|
|
|
const auto indicesIt = uuidToIndices.find(ownerUUID);
|
|
ASSERT(EngineAssertHandler,
|
|
indicesIt != uuidToIndices.end(),
|
|
"Returned UUID {} not found in requested UUIDs",
|
|
ownerUUID);
|
|
|
|
const std::span<const std::byte, UUID::UUID_BYTE_SIZE> rawHardReference {
|
|
static_cast<std::span<const std::byte>>(hardReferencesRequest.Get(1))};
|
|
|
|
for (const std::size_t index: indicesIt->second)
|
|
{
|
|
hardReferences[index].emplace_back(rawHardReference);
|
|
}
|
|
}
|
|
}
|
|
|
|
return hardReferences;
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::optional<BigFile::AssetTime> BigFile::GetAssetTime(const UUID& p_uuid) const
|
|
{
|
|
BigFile::Request timeRequest {*this, "SELECT CreateTime, ModificationTime FROM AssetMetadata WHERE UUID = ?1;"};
|
|
timeRequest.Bind(1, p_uuid.ToBytes());
|
|
|
|
if (!timeRequest.Step())
|
|
{
|
|
return eastl::nullopt;
|
|
}
|
|
|
|
return AssetTime {.m_creationTime = Time {static_cast<std::uint64_t>(timeRequest.Get(0))},
|
|
.m_modificationTime = Time {static_cast<std::uint64_t>(timeRequest.Get(1))}};
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::vector<eastl::optional<BigFile::AssetTime>> BigFile::GetAssetsTime(const eastl::span<const UUID> p_uuids) const
|
|
{
|
|
eastl::vector<eastl::optional<AssetTime>> times(p_uuids.size());
|
|
|
|
if (p_uuids.empty())
|
|
{
|
|
return times;
|
|
}
|
|
|
|
ankerl::unordered_dense::map<UUID, eastl::vector<std::size_t>> uuidToIndices;
|
|
uuidToIndices.reserve(p_uuids.size());
|
|
for (std::size_t i = 0; i < p_uuids.size(); ++i)
|
|
{
|
|
uuidToIndices[p_uuids[i]].push_back(i);
|
|
}
|
|
|
|
eastl::vector<UUID> distinctUUIDs;
|
|
distinctUUIDs.reserve(uuidToIndices.size());
|
|
for (const auto& [uuid, indices]: uuidToIndices)
|
|
{
|
|
distinctUUIDs.push_back(uuid);
|
|
}
|
|
|
|
constexpr std::size_t PARAMS_PER_ROW = 1;
|
|
const std::size_t maxRowsPerStatement = static_cast<std::size_t>(
|
|
sqlite3_limit(m_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1)) /
|
|
PARAMS_PER_ROW;
|
|
|
|
for (std::size_t chunkStart = 0; chunkStart < distinctUUIDs.size(); chunkStart += maxRowsPerStatement)
|
|
{
|
|
const std::size_t chunkSize = eastl::min(maxRowsPerStatement, distinctUUIDs.size() - chunkStart);
|
|
|
|
eastl::string placeholders;
|
|
placeholders.reserve(chunkSize * 2);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
placeholders += ", ";
|
|
}
|
|
placeholders += "?";
|
|
}
|
|
|
|
const eastl::string sql = Format(
|
|
"SELECT UUID, CreateTime, ModificationTime FROM AssetMetadata WHERE UUID IN ({});",
|
|
placeholders);
|
|
|
|
BigFile::Request timeRequest {*this, sql};
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
timeRequest.Bind(static_cast<std::uint32_t>(i + 1), distinctUUIDs[chunkStart + i].ToBytes());
|
|
}
|
|
|
|
while (timeRequest.Step())
|
|
{
|
|
const std::span<const std::byte, UUID::UUID_BYTE_SIZE> rawUUID {
|
|
static_cast<std::span<const std::byte>>(timeRequest.Get(0))};
|
|
const UUID uuid {rawUUID};
|
|
|
|
const auto indicesIt = uuidToIndices.find(uuid);
|
|
ASSERT(EngineAssertHandler,
|
|
indicesIt != uuidToIndices.end(),
|
|
"Returned UUID {} not found in requested UUIDs",
|
|
uuid);
|
|
|
|
const AssetTime assetTime {.m_creationTime = Time {static_cast<std::uint64_t>(timeRequest.Get(1))},
|
|
.m_modificationTime = Time {static_cast<std::uint64_t>(timeRequest.Get(2))}};
|
|
|
|
for (const std::size_t index: indicesIt->second)
|
|
{
|
|
times[index] = assetTime;
|
|
}
|
|
}
|
|
}
|
|
|
|
return times;
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::SaveAssetFlatbuffer(const UUID& p_uuid,
|
|
const eastl::span<const std::byte> p_flatbuffer,
|
|
const eastl::span<const UUID> p_hardReferences)
|
|
{
|
|
BeginTransaction();
|
|
|
|
BigFile::Request metadataRequest {*this,
|
|
"INSERT INTO AssetMetadata (UUID) VALUES (?1) "
|
|
"ON CONFLICT(UUID) DO NOTHING;"};
|
|
metadataRequest.Bind(1, p_uuid.ToBytes());
|
|
metadataRequest.Execute();
|
|
|
|
BigFile::Request assetRequest {*this,
|
|
"INSERT INTO Asset (UUID, Asset) VALUES (?1, ?2) "
|
|
"ON CONFLICT(UUID) DO UPDATE SET Asset = excluded.Asset;"};
|
|
assetRequest.Bind(1, p_uuid.ToBytes());
|
|
assetRequest.Bind(2, p_flatbuffer);
|
|
assetRequest.Execute();
|
|
|
|
BigFile::Request deleteHardReferencesRequest {*this, "DELETE FROM AssetHardReference WHERE UUID = ?1;"};
|
|
deleteHardReferencesRequest.Bind(1, p_uuid.ToBytes());
|
|
deleteHardReferencesRequest.Execute();
|
|
|
|
constexpr std::size_t HARDREF_PARAMS_PER_ROW = 2;
|
|
constexpr std::size_t HARDREF_PLACEHOLDER_RESERVE_CHARS_PER_ROW = 8;
|
|
const std::size_t maxHardRefRowsPerStatement = static_cast<std::size_t>(
|
|
sqlite3_limit(m_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1)) /
|
|
HARDREF_PARAMS_PER_ROW;
|
|
|
|
for (std::size_t chunkStart = 0; chunkStart < p_hardReferences.size(); chunkStart += maxHardRefRowsPerStatement)
|
|
{
|
|
const std::size_t chunkSize = eastl::min(maxHardRefRowsPerStatement, p_hardReferences.size() - chunkStart);
|
|
|
|
eastl::string hardReferencePlaceholders;
|
|
hardReferencePlaceholders.reserve(chunkSize * HARDREF_PLACEHOLDER_RESERVE_CHARS_PER_ROW);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
hardReferencePlaceholders += ", ";
|
|
}
|
|
hardReferencePlaceholders += "(?, ?)";
|
|
}
|
|
|
|
const eastl::string hardReferenceSQL = Format("INSERT INTO AssetHardReference (UUID, HardReference) VALUES {} "
|
|
"ON CONFLICT(UUID, HardReference) DO NOTHING;",
|
|
hardReferencePlaceholders);
|
|
|
|
BigFile::Request hardReferenceRequest {*this, hardReferenceSQL};
|
|
std::uint32_t bindIndex = 1;
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
hardReferenceRequest.Bind(bindIndex++, p_uuid.ToBytes());
|
|
hardReferenceRequest.Bind(bindIndex++, p_hardReferences[chunkStart + i].ToBytes());
|
|
}
|
|
hardReferenceRequest.Execute();
|
|
}
|
|
|
|
CommitTransaction();
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::SaveAssetFlatbuffers(const eastl::span<const UUID> p_uuids,
|
|
const eastl::span<const eastl::vector<std::byte>> p_flatbuffers,
|
|
const eastl::span<const eastl::vector<UUID>> p_hardReferences)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
p_uuids.size() == p_flatbuffers.size() && p_uuids.size() == p_hardReferences.size(),
|
|
"p_metadatas and p_flatbuffers and p_hardReferences must have the same size");
|
|
|
|
if (p_uuids.empty())
|
|
{
|
|
return;
|
|
}
|
|
|
|
BeginTransaction();
|
|
|
|
constexpr std::size_t METADATA_PARAMS_PER_ROW = 4;
|
|
constexpr std::size_t METADATA_PLACEHOLDER_RESERVE_CHARS_PER_ROW = 14;
|
|
constexpr std::size_t ASSET_PLACEHOLDER_RESERVE_CHARS_PER_ROW = 8;
|
|
const std::size_t maxRowsPerStatement = static_cast<std::size_t>(
|
|
sqlite3_limit(m_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1)) /
|
|
METADATA_PARAMS_PER_ROW;
|
|
|
|
for (std::size_t chunkStart = 0; chunkStart < p_uuids.size(); chunkStart += maxRowsPerStatement)
|
|
{
|
|
const std::size_t chunkSize = eastl::min(maxRowsPerStatement, p_uuids.size() - chunkStart);
|
|
|
|
eastl::string metadataPlaceholders;
|
|
metadataPlaceholders.reserve(chunkSize * METADATA_PLACEHOLDER_RESERVE_CHARS_PER_ROW);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
metadataPlaceholders += ", ";
|
|
}
|
|
metadataPlaceholders += "(?)";
|
|
}
|
|
|
|
const eastl::string metadataSQL = Format(
|
|
"INSERT INTO AssetMetadata (UUID) VALUES {} ON CONFLICT(UUID) DO NOTHING;",
|
|
metadataPlaceholders);
|
|
|
|
BigFile::Request metadataRequest {*this, metadataSQL};
|
|
std::uint32_t metadataBindIndex = 1;
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
metadataRequest.Bind(metadataBindIndex++, p_uuids[chunkStart + i].ToBytes());
|
|
}
|
|
metadataRequest.Execute();
|
|
|
|
eastl::string assetPlaceholders;
|
|
assetPlaceholders.reserve(chunkSize * ASSET_PLACEHOLDER_RESERVE_CHARS_PER_ROW);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
assetPlaceholders += ", ";
|
|
}
|
|
assetPlaceholders += "(?, ?)";
|
|
}
|
|
|
|
const eastl::string assetSQL = Format(
|
|
"INSERT INTO Asset (UUID, Asset) VALUES {} ON CONFLICT(UUID) DO UPDATE SET Asset = excluded.Asset;",
|
|
assetPlaceholders);
|
|
|
|
BigFile::Request assetRequest {*this, assetSQL};
|
|
std::uint32_t assetBindIndex = 1;
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
assetRequest.Bind(assetBindIndex++, p_uuids[chunkStart + i].ToBytes());
|
|
assetRequest.Bind(assetBindIndex++, p_flatbuffers[chunkStart + i]);
|
|
}
|
|
assetRequest.Execute();
|
|
|
|
// Replace hard references for every asset in this chunk with its current set.
|
|
eastl::string deleteInPlaceholders;
|
|
deleteInPlaceholders.reserve(chunkSize * 3);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
deleteInPlaceholders += ", ";
|
|
}
|
|
deleteInPlaceholders += "?";
|
|
}
|
|
|
|
const eastl::string deleteHardReferencesSQL = Format("DELETE FROM AssetHardReference WHERE UUID IN ({});",
|
|
deleteInPlaceholders);
|
|
|
|
BigFile::Request deleteHardReferencesRequest {*this, deleteHardReferencesSQL};
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
deleteHardReferencesRequest.Bind(static_cast<std::uint32_t>(i + 1), p_uuids[chunkStart + i].ToBytes());
|
|
}
|
|
deleteHardReferencesRequest.Execute();
|
|
|
|
eastl::vector<eastl::pair<UUID, UUID>> hardReferencePairs;
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
for (const UUID& hardReference: p_hardReferences[chunkStart + i])
|
|
{
|
|
hardReferencePairs.emplace_back(p_uuids[chunkStart + i], hardReference);
|
|
}
|
|
}
|
|
|
|
constexpr std::size_t HARDREF_PARAMS_PER_ROW = 2;
|
|
constexpr std::size_t HARDREF_PLACEHOLDER_RESERVE_CHARS_PER_ROW = 8;
|
|
const std::size_t maxHardRefRowsPerStatement = static_cast<std::size_t>(
|
|
sqlite3_limit(m_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1)) /
|
|
HARDREF_PARAMS_PER_ROW;
|
|
|
|
for (std::size_t hardRefChunkStart = 0; hardRefChunkStart < hardReferencePairs.size();
|
|
hardRefChunkStart += maxHardRefRowsPerStatement)
|
|
{
|
|
const std::size_t hardRefChunkSize = eastl::min(maxHardRefRowsPerStatement,
|
|
hardReferencePairs.size() - hardRefChunkStart);
|
|
|
|
eastl::string hardReferencePlaceholders;
|
|
hardReferencePlaceholders.reserve(hardRefChunkSize * HARDREF_PLACEHOLDER_RESERVE_CHARS_PER_ROW);
|
|
for (std::size_t i = 0; i < hardRefChunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
hardReferencePlaceholders += ", ";
|
|
}
|
|
hardReferencePlaceholders += "(?, ?)";
|
|
}
|
|
|
|
const eastl::string hardReferenceSQL = Format(
|
|
"INSERT INTO AssetHardReference (UUID, HardReference) VALUES {} "
|
|
"ON CONFLICT(UUID, HardReference) DO NOTHING;",
|
|
hardReferencePlaceholders);
|
|
|
|
BigFile::Request hardReferenceRequest {*this, hardReferenceSQL};
|
|
std::uint32_t bindIndex = 1;
|
|
for (std::size_t i = 0; i < hardRefChunkSize; ++i)
|
|
{
|
|
const auto& [assetUUID, hardReference] = hardReferencePairs[hardRefChunkStart + i];
|
|
hardReferenceRequest.Bind(bindIndex++, assetUUID.ToBytes());
|
|
hardReferenceRequest.Bind(bindIndex++, hardReference.ToBytes());
|
|
}
|
|
hardReferenceRequest.Execute();
|
|
}
|
|
}
|
|
|
|
CommitTransaction();
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
bool BigFile::Has(const UUID& p_uuid) const
|
|
{
|
|
BigFile::Request request {*this, "SELECT 1 FROM AssetMetadata WHERE UUID = ?1;"};
|
|
request.Bind(1, p_uuid.ToBytes());
|
|
|
|
return request.Step();
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Request(const BigFile& p_bigFile, const eastl::string_view p_request):
|
|
m_db(p_bigFile.m_db)
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_prepare_v2(m_db,
|
|
p_request.data(),
|
|
static_cast<int>(p_request.size()),
|
|
&m_statement,
|
|
nullptr);
|
|
CRITICAL_ASSERT(EngineAssertHandler,
|
|
result == SQLITE_OK,
|
|
"Failed to create statement from BigFile DB: {}",
|
|
sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const std::int32_t p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1U) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_int(m_statement, static_cast<int>(p_index), p_value);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const std::uint32_t p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1U) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_int(m_statement, static_cast<int>(p_index), static_cast<int>(p_value));
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const std::int64_t p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1U) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_int64(m_statement, static_cast<int>(p_index), p_value);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const std::uint64_t p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1U) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_int64(m_statement, static_cast<int>(p_index), std::bit_cast<std::int64_t>(p_value));
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const float p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1U) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_double(m_statement, static_cast<int>(p_index), p_value);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const double p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1U) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_double(m_statement, static_cast<int>(p_index), p_value);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const eastl::string_view p_value, const CopyValue p_copy)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1U) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_text(m_statement,
|
|
static_cast<int>(p_index),
|
|
p_value.data(),
|
|
static_cast<int>(p_value.size()),
|
|
p_copy ? SQLITE_TRANSIENT : SQLITE_STATIC);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index,
|
|
const eastl::span<const std::byte> p_value,
|
|
const CopyValue p_copy)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1U) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_blob(m_statement,
|
|
static_cast<int>(p_index),
|
|
p_value.data(),
|
|
static_cast<int>(p_value.size()),
|
|
p_copy ? SQLITE_TRANSIENT : SQLITE_STATIC);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
bool BigFile::Request::Step()
|
|
{
|
|
const int result = sqlite3_step(m_statement);
|
|
ASSERT(EngineAssertHandler,
|
|
(result == SQLITE_DONE) || (result == SQLITE_ROW),
|
|
"Failed to step through the statement: {}",
|
|
sqlite3_errmsg(m_db));
|
|
|
|
return result == SQLITE_ROW;
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
std::uint32_t BigFile::Request::Execute()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_step(m_statement);
|
|
ASSERT(EngineAssertHandler, (result == SQLITE_DONE), "Failed to execute the statement: {}", sqlite3_errmsg(m_db));
|
|
|
|
return sqlite3_changes(m_db);
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::Column(const Request& p_request, const std::uint32_t p_index):
|
|
m_statement(p_request.m_statement),
|
|
m_index(p_index)
|
|
{
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator std::int32_t() const
|
|
{
|
|
return sqlite3_column_int(m_statement, static_cast<int>(m_index));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator std::uint32_t() const
|
|
{
|
|
return sqlite3_column_int(m_statement, static_cast<int>(m_index));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator std::int64_t() const
|
|
{
|
|
return sqlite3_column_int64(m_statement, static_cast<int>(m_index));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator std::uint64_t() const
|
|
{
|
|
return std::bit_cast<std::uint64_t>(sqlite3_column_int64(m_statement, static_cast<int>(m_index)));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator float() const
|
|
{
|
|
return static_cast<float>(sqlite3_column_double(m_statement, static_cast<int>(m_index)));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator double() const
|
|
{
|
|
return sqlite3_column_double(m_statement, static_cast<int>(m_index));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator eastl::string_view() const
|
|
{
|
|
return eastl::string_view {
|
|
reinterpret_cast<const char*>(sqlite3_column_text(m_statement, static_cast<int>(m_index))),
|
|
static_cast<std::size_t>(sqlite3_column_bytes(m_statement, static_cast<int>(m_index)))};
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator eastl::span<const std::byte>() const
|
|
{
|
|
return eastl::span<const std::byte> {
|
|
static_cast<const std::byte*>(sqlite3_column_blob(m_statement, static_cast<int>(m_index))),
|
|
static_cast<std::size_t>(sqlite3_column_bytes(m_statement, static_cast<int>(m_index)))};
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator std::span<const std::byte>() const
|
|
{
|
|
return std::span<const std::byte> {
|
|
static_cast<const std::byte*>(sqlite3_column_blob(m_statement, static_cast<int>(m_index))),
|
|
static_cast<std::size_t>(sqlite3_column_bytes(m_statement, static_cast<int>(m_index)))};
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column BigFile::Request::Get(const std::uint32_t p_index) const
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
p_index < static_cast<std::uint32_t>(sqlite3_column_count(m_statement)),
|
|
"Invalid index for column!");
|
|
return {*this, p_index};
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::~Request()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_finalize(m_statement);
|
|
CRITICAL_ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to finalize statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
#endif
|
|
} // namespace Bigfoot
|