Bigfoot / Sonarqube (push) Successful in 5m42s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: toolmode) (push) Successful in 1m25s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: toolmode) (push) Successful in 1m5s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang (Unity Build: ON, Variant: game) (push) Successful in 32s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: game) (push) Successful in 1m4s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: toolmode) (push) Successful in 1m27s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: toolmode) (push) Successful in 1m11s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang (Unity Build: ON, Variant: game) (push) Successful in 29s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: toolmode) (push) Successful in 55s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: game) (push) Successful in 1m0s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: toolmode) (push) Successful in 1m5s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: toolmode) (push) Successful in 56s
Bigfoot / Clang Format Checks (push) Successful in 12s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: game) (push) Successful in 43s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang (Unity Build: ON, Variant: game) (push) Successful in 25s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: toolmode) (push) Successful in 52s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang (Unity Build: ON, Variant: toolmode) (push) Successful in 30s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: game) (push) Successful in 1m19s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: game) (push) Successful in 1m1s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: game) (push) Successful in 48s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: toolmode) (push) Successful in 49s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang (Unity Build: ON, Variant: toolmode) (push) Successful in 35s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: game) (push) Successful in 1m20s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: game) (push) Successful in 36s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang (Unity Build: ON, Variant: toolmode) (push) Successful in 42s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: game) (push) Successful in 50s
Only time I'm allowing a merge of single feature inside main (we prefer merging a version) But I need a first Sonarqube run Reviewed-on: #2 Co-authored-by: Romain BOULLARD <romain.boullard@protonmail.com>
852 lines
33 KiB
C++
852 lines
33 KiB
C++
/*********************************************************************
|
|
* \file BigFile.cpp
|
|
*
|
|
* \author Romain BOULLARD
|
|
* \date October 2025
|
|
*********************************************************************/
|
|
#include <Engine/BigFile/BigFile.hpp>
|
|
|
|
#include <Engine/EngineAssertHandler.hpp>
|
|
|
|
#include <Utils/Format.hpp>
|
|
|
|
#include <EASTL/algorithm.h>
|
|
|
|
namespace Bigfoot
|
|
{
|
|
#ifdef BIGFOOT_TOOLMODE
|
|
BigFile::BigFile(const File& p_file)
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_open_v2(p_file.ToAbsolute().GetPath().data(), &m_db, SQLITE_OPEN_READWRITE, nullptr);
|
|
CRITICAL_ASSERT(EngineAssertHandler,
|
|
result == SQLITE_OK,
|
|
"Failed to open BigFile {} DB: {}",
|
|
p_file.ToAbsolute().GetPath(),
|
|
sqlite3_errmsg(m_db));
|
|
|
|
[[maybe_unused]]
|
|
const int synchronousResult = sqlite3_exec(m_db, "PRAGMA synchronous = NORMAL;", nullptr, nullptr, nullptr);
|
|
ASSERT(EngineAssertHandler,
|
|
synchronousResult == SQLITE_OK,
|
|
"Failed to set synchronous pragma: {}",
|
|
sqlite3_errmsg(m_db));
|
|
|
|
[[maybe_unused]]
|
|
const int foreignKeysResult = sqlite3_exec(m_db, "PRAGMA foreign_keys = ON;", nullptr, nullptr, nullptr);
|
|
ASSERT(EngineAssertHandler,
|
|
foreignKeysResult == SQLITE_OK,
|
|
"Failed to set foreign_keys pragma: {}",
|
|
sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::BeginTransaction()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_exec(m_db, "BEGIN TRANSACTION;", nullptr, nullptr, nullptr);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to begin transaction: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::CommitTransaction()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_exec(m_db, "COMMIT TRANSACTION;", nullptr, nullptr, nullptr);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to commit: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::RollbackTransaction()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_exec(m_db, "ROLLBACK TRANSACTION;", nullptr, nullptr, nullptr);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to rollback: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Clear()
|
|
{
|
|
BeginTransaction();
|
|
|
|
BigFile::Request {*this, "DELETE FROM AssetHardReference;"}.Execute();
|
|
BigFile::Request {*this, "DELETE FROM Asset;"}.Execute();
|
|
BigFile::Request {*this, "DELETE FROM AssetMetadata;"}.Execute();
|
|
|
|
CommitTransaction();
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::~BigFile()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_close_v2(m_db);
|
|
CRITICAL_ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to close BigFile DB: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::vector<std::byte> BigFile::GetAssetFlatbuffer(const UUID& p_uuid) const
|
|
{
|
|
BigFile::Request assetRequest {*this, "SELECT Asset FROM Asset WHERE UUID = ?1;"};
|
|
assetRequest.Bind(1, p_uuid.ToBytes());
|
|
|
|
if (!assetRequest.Step())
|
|
{
|
|
return {};
|
|
}
|
|
|
|
const eastl::span<const std::byte> flatbuffer = assetRequest.Get(0);
|
|
return eastl::vector<std::byte> {flatbuffer.begin(), flatbuffer.end()};
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::vector<eastl::vector<std::byte>> BigFile::GetAssetFlatbuffers(const eastl::span<const UUID> p_uuids) const
|
|
{
|
|
eastl::vector<eastl::vector<std::byte>> flatbuffers(p_uuids.size());
|
|
|
|
if (p_uuids.empty())
|
|
{
|
|
return flatbuffers;
|
|
}
|
|
|
|
ankerl::unordered_dense::map<UUID, eastl::vector<std::size_t>> uuidToIndices;
|
|
uuidToIndices.reserve(p_uuids.size());
|
|
for (std::size_t i = 0; i < p_uuids.size(); ++i)
|
|
{
|
|
uuidToIndices[p_uuids[i]].push_back(i);
|
|
}
|
|
|
|
eastl::vector<UUID> distinctUUIDs;
|
|
distinctUUIDs.reserve(uuidToIndices.size());
|
|
for (const auto& [uuid, indices]: uuidToIndices)
|
|
{
|
|
distinctUUIDs.push_back(uuid);
|
|
}
|
|
|
|
constexpr std::size_t PARAMS_PER_ROW = 1;
|
|
const std::size_t maxRowsPerStatement = static_cast<std::size_t>(
|
|
sqlite3_limit(m_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1)) /
|
|
PARAMS_PER_ROW;
|
|
|
|
for (std::size_t chunkStart = 0; chunkStart < distinctUUIDs.size(); chunkStart += maxRowsPerStatement)
|
|
{
|
|
const std::size_t chunkSize = eastl::min(maxRowsPerStatement, distinctUUIDs.size() - chunkStart);
|
|
|
|
eastl::string placeholders;
|
|
placeholders.reserve(chunkSize * 2);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
placeholders += ", ";
|
|
}
|
|
placeholders += "?";
|
|
}
|
|
|
|
const eastl::string sql = Format("SELECT UUID, Asset FROM Asset WHERE UUID IN ({});", placeholders);
|
|
|
|
BigFile::Request assetRequest {*this, sql};
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
assetRequest.Bind(static_cast<std::uint32_t>(i + 1), distinctUUIDs[chunkStart + i].ToBytes());
|
|
}
|
|
|
|
while (assetRequest.Step())
|
|
{
|
|
const std::span<const std::byte, UUID::UUID_BYTE_SIZE> rawUUID {
|
|
static_cast<std::span<const std::byte>>(assetRequest.Get(0))};
|
|
const UUID uuid {rawUUID};
|
|
|
|
const auto indicesIt = uuidToIndices.find(uuid);
|
|
ASSERT(EngineAssertHandler,
|
|
indicesIt != uuidToIndices.end(),
|
|
"Returned UUID {} not found in requested UUIDs",
|
|
uuid);
|
|
|
|
const eastl::span<const std::byte> flatbuffer = assetRequest.Get(1);
|
|
|
|
for (const std::size_t index: indicesIt->second)
|
|
{
|
|
flatbuffers[index].assign(flatbuffer.begin(), flatbuffer.end());
|
|
}
|
|
}
|
|
}
|
|
|
|
return flatbuffers;
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::vector<UUID> BigFile::GetAssetHardReferences(const UUID& p_uuid) const
|
|
{
|
|
BigFile::Request hardReferencesRequest {*this, "SELECT HardReference FROM AssetHardReference WHERE UUID = ?1;"};
|
|
hardReferencesRequest.Bind(1, p_uuid.ToBytes());
|
|
|
|
eastl::vector<UUID> hardReferences;
|
|
while (hardReferencesRequest.Step())
|
|
{
|
|
const std::span<const std::byte, UUID::UUID_BYTE_SIZE> rawUUID {
|
|
static_cast<std::span<const std::byte>>(hardReferencesRequest.Get(0))};
|
|
hardReferences.emplace_back(rawUUID);
|
|
}
|
|
|
|
return hardReferences;
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::vector<eastl::vector<UUID>> BigFile::GetAssetsHardReferences(eastl::span<const UUID> p_uuids) const
|
|
{
|
|
eastl::vector<eastl::vector<UUID>> hardReferences(p_uuids.size());
|
|
|
|
if (p_uuids.empty())
|
|
{
|
|
return hardReferences;
|
|
}
|
|
|
|
ankerl::unordered_dense::map<UUID, eastl::vector<std::size_t>> uuidToIndices;
|
|
uuidToIndices.reserve(p_uuids.size());
|
|
for (std::size_t i = 0; i < p_uuids.size(); ++i)
|
|
{
|
|
uuidToIndices[p_uuids[i]].push_back(i);
|
|
}
|
|
|
|
eastl::vector<UUID> distinctUUIDs;
|
|
distinctUUIDs.reserve(uuidToIndices.size());
|
|
for (const auto& [uuid, indices]: uuidToIndices)
|
|
{
|
|
distinctUUIDs.push_back(uuid);
|
|
}
|
|
|
|
constexpr std::size_t PARAMS_PER_ROW = 1;
|
|
const std::size_t maxRowsPerStatement = static_cast<std::size_t>(
|
|
sqlite3_limit(m_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1)) /
|
|
PARAMS_PER_ROW;
|
|
|
|
for (std::size_t chunkStart = 0; chunkStart < distinctUUIDs.size(); chunkStart += maxRowsPerStatement)
|
|
{
|
|
const std::size_t chunkSize = eastl::min(maxRowsPerStatement, distinctUUIDs.size() - chunkStart);
|
|
|
|
eastl::string placeholders;
|
|
placeholders.reserve(chunkSize * 2);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
placeholders += ", ";
|
|
}
|
|
placeholders += "?";
|
|
}
|
|
|
|
const eastl::string sql = Format("SELECT UUID, HardReference FROM AssetHardReference WHERE UUID IN ({});",
|
|
placeholders);
|
|
|
|
BigFile::Request hardReferencesRequest {*this, sql};
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
hardReferencesRequest.Bind(static_cast<std::uint32_t>(i + 1), distinctUUIDs[chunkStart + i].ToBytes());
|
|
}
|
|
|
|
while (hardReferencesRequest.Step())
|
|
{
|
|
const std::span<const std::byte, UUID::UUID_BYTE_SIZE> rawOwnerUUID {
|
|
static_cast<std::span<const std::byte>>(hardReferencesRequest.Get(0))};
|
|
const UUID ownerUUID {rawOwnerUUID};
|
|
|
|
const auto indicesIt = uuidToIndices.find(ownerUUID);
|
|
ASSERT(EngineAssertHandler,
|
|
indicesIt != uuidToIndices.end(),
|
|
"Returned UUID {} not found in requested UUIDs",
|
|
ownerUUID);
|
|
|
|
const std::span<const std::byte, UUID::UUID_BYTE_SIZE> rawHardReference {
|
|
static_cast<std::span<const std::byte>>(hardReferencesRequest.Get(1))};
|
|
|
|
for (const std::size_t index: indicesIt->second)
|
|
{
|
|
hardReferences[index].emplace_back(rawHardReference);
|
|
}
|
|
}
|
|
}
|
|
|
|
return hardReferences;
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::optional<BigFile::AssetTime> BigFile::GetAssetTime(const UUID& p_uuid) const
|
|
{
|
|
BigFile::Request timeRequest {*this, "SELECT CreateTime, ModificationTime FROM AssetMetadata WHERE UUID = ?1;"};
|
|
timeRequest.Bind(1, p_uuid.ToBytes());
|
|
|
|
if (!timeRequest.Step())
|
|
{
|
|
return eastl::nullopt;
|
|
}
|
|
|
|
return AssetTime {.m_creationTime = Time {static_cast<std::uint64_t>(timeRequest.Get(0))},
|
|
.m_modificationTime = Time {static_cast<std::uint64_t>(timeRequest.Get(1))}};
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::vector<eastl::optional<BigFile::AssetTime>> BigFile::GetAssetsTime(const eastl::span<const UUID> p_uuids) const
|
|
{
|
|
eastl::vector<eastl::optional<AssetTime>> times(p_uuids.size());
|
|
|
|
if (p_uuids.empty())
|
|
{
|
|
return times;
|
|
}
|
|
|
|
ankerl::unordered_dense::map<UUID, eastl::vector<std::size_t>> uuidToIndices;
|
|
uuidToIndices.reserve(p_uuids.size());
|
|
for (std::size_t i = 0; i < p_uuids.size(); ++i)
|
|
{
|
|
uuidToIndices[p_uuids[i]].push_back(i);
|
|
}
|
|
|
|
eastl::vector<UUID> distinctUUIDs;
|
|
distinctUUIDs.reserve(uuidToIndices.size());
|
|
for (const auto& [uuid, indices]: uuidToIndices)
|
|
{
|
|
distinctUUIDs.push_back(uuid);
|
|
}
|
|
|
|
constexpr std::size_t PARAMS_PER_ROW = 1;
|
|
const std::size_t maxRowsPerStatement = static_cast<std::size_t>(
|
|
sqlite3_limit(m_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1)) /
|
|
PARAMS_PER_ROW;
|
|
|
|
for (std::size_t chunkStart = 0; chunkStart < distinctUUIDs.size(); chunkStart += maxRowsPerStatement)
|
|
{
|
|
const std::size_t chunkSize = eastl::min(maxRowsPerStatement, distinctUUIDs.size() - chunkStart);
|
|
|
|
eastl::string placeholders;
|
|
placeholders.reserve(chunkSize * 2);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
placeholders += ", ";
|
|
}
|
|
placeholders += "?";
|
|
}
|
|
|
|
const eastl::string sql = Format(
|
|
"SELECT UUID, CreateTime, ModificationTime FROM AssetMetadata WHERE UUID IN ({});",
|
|
placeholders);
|
|
|
|
BigFile::Request timeRequest {*this, sql};
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
timeRequest.Bind(static_cast<std::uint32_t>(i + 1), distinctUUIDs[chunkStart + i].ToBytes());
|
|
}
|
|
|
|
while (timeRequest.Step())
|
|
{
|
|
const std::span<const std::byte, UUID::UUID_BYTE_SIZE> rawUUID {
|
|
static_cast<std::span<const std::byte>>(timeRequest.Get(0))};
|
|
const UUID uuid {rawUUID};
|
|
|
|
const auto indicesIt = uuidToIndices.find(uuid);
|
|
ASSERT(EngineAssertHandler,
|
|
indicesIt != uuidToIndices.end(),
|
|
"Returned UUID {} not found in requested UUIDs",
|
|
uuid);
|
|
|
|
const AssetTime assetTime {.m_creationTime = Time {static_cast<std::uint64_t>(timeRequest.Get(1))},
|
|
.m_modificationTime = Time {static_cast<std::uint64_t>(timeRequest.Get(2))}};
|
|
|
|
for (const std::size_t index: indicesIt->second)
|
|
{
|
|
times[index] = assetTime;
|
|
}
|
|
}
|
|
}
|
|
|
|
return times;
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::SaveAssetFlatbuffer(const UUID& p_uuid,
|
|
const eastl::span<const std::byte> p_flatbuffer,
|
|
const eastl::span<const UUID> p_hardReferences)
|
|
{
|
|
BeginTransaction();
|
|
|
|
BigFile::Request metadataRequest {*this,
|
|
"INSERT INTO AssetMetadata (UUID) VALUES (?1) "
|
|
"ON CONFLICT(UUID) DO NOTHING;"};
|
|
metadataRequest.Bind(1, p_uuid.ToBytes());
|
|
metadataRequest.Execute();
|
|
|
|
BigFile::Request assetRequest {*this,
|
|
"INSERT INTO Asset (UUID, Asset) VALUES (?1, ?2) "
|
|
"ON CONFLICT(UUID) DO UPDATE SET Asset = excluded.Asset;"};
|
|
assetRequest.Bind(1, p_uuid.ToBytes());
|
|
assetRequest.Bind(2, p_flatbuffer);
|
|
assetRequest.Execute();
|
|
|
|
BigFile::Request deleteHardReferencesRequest {*this, "DELETE FROM AssetHardReference WHERE UUID = ?1;"};
|
|
deleteHardReferencesRequest.Bind(1, p_uuid.ToBytes());
|
|
deleteHardReferencesRequest.Execute();
|
|
|
|
constexpr std::size_t HARDREF_PARAMS_PER_ROW = 2;
|
|
constexpr std::size_t HARDREF_PLACEHOLDER_RESERVE_CHARS_PER_ROW = 8;
|
|
const std::size_t maxHardRefRowsPerStatement = static_cast<std::size_t>(
|
|
sqlite3_limit(m_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1)) /
|
|
HARDREF_PARAMS_PER_ROW;
|
|
|
|
for (std::size_t chunkStart = 0; chunkStart < p_hardReferences.size(); chunkStart += maxHardRefRowsPerStatement)
|
|
{
|
|
const std::size_t chunkSize = eastl::min(maxHardRefRowsPerStatement, p_hardReferences.size() - chunkStart);
|
|
|
|
eastl::string hardReferencePlaceholders;
|
|
hardReferencePlaceholders.reserve(chunkSize * HARDREF_PLACEHOLDER_RESERVE_CHARS_PER_ROW);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
hardReferencePlaceholders += ", ";
|
|
}
|
|
hardReferencePlaceholders += "(?, ?)";
|
|
}
|
|
|
|
const eastl::string hardReferenceSQL = Format("INSERT INTO AssetHardReference (UUID, HardReference) VALUES {} "
|
|
"ON CONFLICT(UUID, HardReference) DO NOTHING;",
|
|
hardReferencePlaceholders);
|
|
|
|
BigFile::Request hardReferenceRequest {*this, hardReferenceSQL};
|
|
std::uint32_t bindIndex = 1;
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
hardReferenceRequest.Bind(bindIndex++, p_uuid.ToBytes());
|
|
hardReferenceRequest.Bind(bindIndex++, p_hardReferences[chunkStart + i].ToBytes());
|
|
}
|
|
hardReferenceRequest.Execute();
|
|
}
|
|
|
|
CommitTransaction();
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::SaveAssetFlatbuffers(const eastl::span<const UUID> p_uuids,
|
|
const eastl::span<const eastl::vector<std::byte>> p_flatbuffers,
|
|
const eastl::span<const eastl::vector<UUID>> p_hardReferences)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
p_uuids.size() == p_flatbuffers.size() && p_uuids.size() == p_hardReferences.size(),
|
|
"p_metadatas and p_flatbuffers and p_hardReferences must have the same size");
|
|
|
|
if (p_uuids.empty())
|
|
{
|
|
return;
|
|
}
|
|
|
|
BeginTransaction();
|
|
|
|
constexpr std::size_t METADATA_PARAMS_PER_ROW = 4;
|
|
constexpr std::size_t METADATA_PLACEHOLDER_RESERVE_CHARS_PER_ROW = 14;
|
|
constexpr std::size_t ASSET_PLACEHOLDER_RESERVE_CHARS_PER_ROW = 8;
|
|
const std::size_t maxRowsPerStatement = static_cast<std::size_t>(
|
|
sqlite3_limit(m_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1)) /
|
|
METADATA_PARAMS_PER_ROW;
|
|
|
|
for (std::size_t chunkStart = 0; chunkStart < p_uuids.size(); chunkStart += maxRowsPerStatement)
|
|
{
|
|
const std::size_t chunkSize = eastl::min(maxRowsPerStatement, p_uuids.size() - chunkStart);
|
|
|
|
eastl::string metadataPlaceholders;
|
|
metadataPlaceholders.reserve(chunkSize * METADATA_PLACEHOLDER_RESERVE_CHARS_PER_ROW);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
metadataPlaceholders += ", ";
|
|
}
|
|
metadataPlaceholders += "(?)";
|
|
}
|
|
|
|
const eastl::string metadataSQL = Format(
|
|
"INSERT INTO AssetMetadata (UUID) VALUES {} ON CONFLICT(UUID) DO NOTHING;",
|
|
metadataPlaceholders);
|
|
|
|
BigFile::Request metadataRequest {*this, metadataSQL};
|
|
std::uint32_t metadataBindIndex = 1;
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
metadataRequest.Bind(metadataBindIndex++, p_uuids[chunkStart + i].ToBytes());
|
|
}
|
|
metadataRequest.Execute();
|
|
|
|
eastl::string assetPlaceholders;
|
|
assetPlaceholders.reserve(chunkSize * ASSET_PLACEHOLDER_RESERVE_CHARS_PER_ROW);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
assetPlaceholders += ", ";
|
|
}
|
|
assetPlaceholders += "(?, ?)";
|
|
}
|
|
|
|
const eastl::string assetSQL = Format(
|
|
"INSERT INTO Asset (UUID, Asset) VALUES {} ON CONFLICT(UUID) DO UPDATE SET Asset = excluded.Asset;",
|
|
assetPlaceholders);
|
|
|
|
BigFile::Request assetRequest {*this, assetSQL};
|
|
std::uint32_t assetBindIndex = 1;
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
assetRequest.Bind(assetBindIndex++, p_uuids[chunkStart + i].ToBytes());
|
|
assetRequest.Bind(assetBindIndex++, p_flatbuffers[chunkStart + i]);
|
|
}
|
|
assetRequest.Execute();
|
|
|
|
// Replace hard references for every asset in this chunk with its current set.
|
|
eastl::string deleteInPlaceholders;
|
|
deleteInPlaceholders.reserve(chunkSize * 3);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
deleteInPlaceholders += ", ";
|
|
}
|
|
deleteInPlaceholders += "?";
|
|
}
|
|
|
|
const eastl::string deleteHardReferencesSQL = Format("DELETE FROM AssetHardReference WHERE UUID IN ({});",
|
|
deleteInPlaceholders);
|
|
|
|
BigFile::Request deleteHardReferencesRequest {*this, deleteHardReferencesSQL};
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
deleteHardReferencesRequest.Bind(static_cast<std::uint32_t>(i + 1), p_uuids[chunkStart + i].ToBytes());
|
|
}
|
|
deleteHardReferencesRequest.Execute();
|
|
|
|
eastl::vector<eastl::pair<UUID, UUID>> hardReferencePairs;
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
for (const UUID& hardReference: p_hardReferences[chunkStart + i])
|
|
{
|
|
hardReferencePairs.emplace_back(p_uuids[chunkStart + i], hardReference);
|
|
}
|
|
}
|
|
|
|
constexpr std::size_t HARDREF_PARAMS_PER_ROW = 2;
|
|
constexpr std::size_t HARDREF_PLACEHOLDER_RESERVE_CHARS_PER_ROW = 8;
|
|
const std::size_t maxHardRefRowsPerStatement = static_cast<std::size_t>(
|
|
sqlite3_limit(m_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1)) /
|
|
HARDREF_PARAMS_PER_ROW;
|
|
|
|
for (std::size_t hardRefChunkStart = 0; hardRefChunkStart < hardReferencePairs.size();
|
|
hardRefChunkStart += maxHardRefRowsPerStatement)
|
|
{
|
|
const std::size_t hardRefChunkSize = eastl::min(maxHardRefRowsPerStatement,
|
|
hardReferencePairs.size() - hardRefChunkStart);
|
|
|
|
eastl::string hardReferencePlaceholders;
|
|
hardReferencePlaceholders.reserve(hardRefChunkSize * HARDREF_PLACEHOLDER_RESERVE_CHARS_PER_ROW);
|
|
for (std::size_t i = 0; i < hardRefChunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
hardReferencePlaceholders += ", ";
|
|
}
|
|
hardReferencePlaceholders += "(?, ?)";
|
|
}
|
|
|
|
const eastl::string hardReferenceSQL = Format(
|
|
"INSERT INTO AssetHardReference (UUID, HardReference) VALUES {} "
|
|
"ON CONFLICT(UUID, HardReference) DO NOTHING;",
|
|
hardReferencePlaceholders);
|
|
|
|
BigFile::Request hardReferenceRequest {*this, hardReferenceSQL};
|
|
std::uint32_t bindIndex = 1;
|
|
for (std::size_t i = 0; i < hardRefChunkSize; ++i)
|
|
{
|
|
const auto& [assetUUID, hardReference] = hardReferencePairs[hardRefChunkStart + i];
|
|
hardReferenceRequest.Bind(bindIndex++, assetUUID.ToBytes());
|
|
hardReferenceRequest.Bind(bindIndex++, hardReference.ToBytes());
|
|
}
|
|
hardReferenceRequest.Execute();
|
|
}
|
|
}
|
|
|
|
CommitTransaction();
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
bool BigFile::Has(const UUID& p_uuid) const
|
|
{
|
|
BigFile::Request request {*this, "SELECT 1 FROM AssetMetadata WHERE UUID = ?1;"};
|
|
request.Bind(1, p_uuid.ToBytes());
|
|
|
|
return request.Step();
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Request(const BigFile& p_bigFile, const eastl::string_view p_request):
|
|
m_db(p_bigFile.m_db)
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_prepare_v2(m_db,
|
|
p_request.data(),
|
|
static_cast<int>(p_request.size()),
|
|
&m_statement,
|
|
nullptr);
|
|
CRITICAL_ASSERT(EngineAssertHandler,
|
|
result == SQLITE_OK,
|
|
"Failed to create statement from BigFile DB: {}",
|
|
sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const std::int32_t p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1U) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_int(m_statement, static_cast<int>(p_index), p_value);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const std::uint32_t p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1U) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_int(m_statement, static_cast<int>(p_index), static_cast<int>(p_value));
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const std::int64_t p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1U) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_int64(m_statement, static_cast<int>(p_index), p_value);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const std::uint64_t p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1U) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_int64(m_statement, static_cast<int>(p_index), std::bit_cast<std::int64_t>(p_value));
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const float p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1U) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_double(m_statement, static_cast<int>(p_index), p_value);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const double p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1U) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_double(m_statement, static_cast<int>(p_index), p_value);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const eastl::string_view p_value, const CopyValue p_copy)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1U) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_text(m_statement,
|
|
static_cast<int>(p_index),
|
|
p_value.data(),
|
|
static_cast<int>(p_value.size()),
|
|
p_copy ? SQLITE_TRANSIENT : SQLITE_STATIC);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index,
|
|
const eastl::span<const std::byte> p_value,
|
|
const CopyValue p_copy)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1U) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_blob(m_statement,
|
|
static_cast<int>(p_index),
|
|
p_value.data(),
|
|
static_cast<int>(p_value.size()),
|
|
p_copy ? SQLITE_TRANSIENT : SQLITE_STATIC);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
bool BigFile::Request::Step()
|
|
{
|
|
const int result = sqlite3_step(m_statement);
|
|
ASSERT(EngineAssertHandler,
|
|
(result == SQLITE_DONE) || (result == SQLITE_ROW),
|
|
"Failed to step through the statement: {}",
|
|
sqlite3_errmsg(m_db));
|
|
|
|
return result == SQLITE_ROW;
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
std::uint32_t BigFile::Request::Execute()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_step(m_statement);
|
|
ASSERT(EngineAssertHandler, (result == SQLITE_DONE), "Failed to execute the statement: {}", sqlite3_errmsg(m_db));
|
|
|
|
return sqlite3_changes(m_db);
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::Column(const Request& p_request, const std::uint32_t p_index):
|
|
m_statement(p_request.m_statement),
|
|
m_index(p_index)
|
|
{
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator std::int32_t() const
|
|
{
|
|
return sqlite3_column_int(m_statement, static_cast<int>(m_index));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator std::uint32_t() const
|
|
{
|
|
return sqlite3_column_int(m_statement, static_cast<int>(m_index));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator std::int64_t() const
|
|
{
|
|
return sqlite3_column_int64(m_statement, static_cast<int>(m_index));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator std::uint64_t() const
|
|
{
|
|
return std::bit_cast<std::uint64_t>(sqlite3_column_int64(m_statement, static_cast<int>(m_index)));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator float() const
|
|
{
|
|
return static_cast<float>(sqlite3_column_double(m_statement, static_cast<int>(m_index)));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator double() const
|
|
{
|
|
return sqlite3_column_double(m_statement, static_cast<int>(m_index));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator eastl::string_view() const
|
|
{
|
|
return eastl::string_view {
|
|
reinterpret_cast<const char*>(sqlite3_column_text(m_statement, static_cast<int>(m_index))),
|
|
static_cast<std::size_t>(sqlite3_column_bytes(m_statement, static_cast<int>(m_index)))};
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator eastl::span<const std::byte>() const
|
|
{
|
|
return eastl::span<const std::byte> {
|
|
static_cast<const std::byte*>(sqlite3_column_blob(m_statement, static_cast<int>(m_index))),
|
|
static_cast<std::size_t>(sqlite3_column_bytes(m_statement, static_cast<int>(m_index)))};
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator std::span<const std::byte>() const
|
|
{
|
|
return std::span<const std::byte> {
|
|
static_cast<const std::byte*>(sqlite3_column_blob(m_statement, static_cast<int>(m_index))),
|
|
static_cast<std::size_t>(sqlite3_column_bytes(m_statement, static_cast<int>(m_index)))};
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column BigFile::Request::Get(const std::uint32_t p_index) const
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
p_index < static_cast<std::uint32_t>(sqlite3_column_count(m_statement)),
|
|
"Invalid index for column!");
|
|
return {*this, p_index};
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::~Request()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_finalize(m_statement);
|
|
CRITICAL_ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to finalize statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
#endif
|
|
} // namespace Bigfoot
|