Bigfoot / Sonarqube (push) Skipped
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: game) (push) Failing after 28s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang (Unity Build: ON, Variant: game) (push) Failing after 17s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang (Unity Build: ON, Variant: toolmode) (push) Successful in 27s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: game) (push) Failing after 39s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: game) (push) Failing after 24s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: toolmode) (push) Successful in 1m0s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: toolmode) (push) Successful in 50s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: game) (push) Failing after 23s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang (Unity Build: ON, Variant: game) (push) Failing after 15s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: toolmode) (push) Successful in 46s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: toolmode) (push) Successful in 35s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang (Unity Build: ON, Variant: toolmode) (push) Successful in 24s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: game) (push) Failing after 28s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: game) (push) Failing after 20s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: toolmode) (push) Successful in 1m0s
Bigfoot / Build & Test Debug with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: toolmode) (push) Successful in 47s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: game) (push) Failing after 33s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang (Unity Build: ON, Variant: game) (push) Failing after 20s
Bigfoot / Build & Test RelWithDebInfo with ./ConanProfiles/host/clang (Unity Build: OFF, Variant: toolmode) (push) Successful in 40s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang (Unity Build: ON, Variant: toolmode) (push) Successful in 39s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: game) (push) Failing after 24s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: game) (push) Failing after 20s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang_asan (Unity Build: OFF, Variant: toolmode) (push) Successful in 44s
Bigfoot / Build & Test Release with ./ConanProfiles/host/clang_asan (Unity Build: ON, Variant: toolmode) (push) Successful in 41s
Bigfoot / Clang Format Checks (push) Successful in 9s
834 lines
32 KiB
C++
834 lines
32 KiB
C++
/*********************************************************************
|
|
* \file BigFile.cpp
|
|
*
|
|
* \author Romain BOULLARD
|
|
* \date October 2025
|
|
*********************************************************************/
|
|
#include <Engine/BigFile/BigFile.hpp>
|
|
|
|
#include <Engine/EngineAssertHandler.hpp>
|
|
|
|
#include <EASTL/algorithm.h>
|
|
#include <EASTL/sort.h>
|
|
#include <fmt/format.h>
|
|
|
|
namespace Bigfoot
|
|
{
|
|
#ifdef BIGFOOT_TOOLMODE
|
|
BigFile::BigFile(const File& p_file)
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_open_v2(p_file.ToAbsolute().GetPath().data(), &m_db, SQLITE_OPEN_READWRITE, nullptr);
|
|
CRITICAL_ASSERT(EngineAssertHandler,
|
|
result == SQLITE_OK,
|
|
"Failed to open BigFile {} DB: {}",
|
|
p_file.ToAbsolute().GetPath(),
|
|
sqlite3_errmsg(m_db));
|
|
|
|
[[maybe_unused]]
|
|
const int synchronousResult = sqlite3_exec(m_db, "PRAGMA synchronous = NORMAL;", nullptr, nullptr, nullptr);
|
|
ASSERT(EngineAssertHandler,
|
|
synchronousResult == SQLITE_OK,
|
|
"Failed to set synchronous pragma: {}",
|
|
sqlite3_errmsg(m_db));
|
|
|
|
[[maybe_unused]]
|
|
const int foreignKeysResult = sqlite3_exec(m_db, "PRAGMA foreign_keys = ON;", nullptr, nullptr, nullptr);
|
|
ASSERT(EngineAssertHandler,
|
|
foreignKeysResult == SQLITE_OK,
|
|
"Failed to set foreign_keys pragma: {}",
|
|
sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::BeginTransaction()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_exec(m_db, "BEGIN TRANSACTION;", nullptr, nullptr, nullptr);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to begin transaction: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::CommitTransaction()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_exec(m_db, "COMMIT TRANSACTION;", nullptr, nullptr, nullptr);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to commit: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::RollbackTransaction()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_exec(m_db, "ROLLBACK TRANSACTION;", nullptr, nullptr, nullptr);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to rollback: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Clear()
|
|
{
|
|
BeginTransaction();
|
|
|
|
BigFile::Request {*this, "DELETE FROM AssetHardReference;"}.Execute();
|
|
BigFile::Request {*this, "DELETE FROM Asset;"}.Execute();
|
|
BigFile::Request {*this, "DELETE FROM AssetMetadata;"}.Execute();
|
|
|
|
CommitTransaction();
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::~BigFile()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_close_v2(m_db);
|
|
CRITICAL_ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to close BigFile DB: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::optional<eastl::vector<std::byte>> BigFile::GetAssetFlatbuffer(const UUID& p_uuid) const
|
|
{
|
|
BigFile::Request assetRequest {*this, "SELECT Asset FROM Asset WHERE UUID = ?1;"};
|
|
assetRequest.Bind(1, p_uuid.ToBytes());
|
|
|
|
if (!assetRequest.Step())
|
|
{
|
|
return eastl::nullopt;
|
|
}
|
|
|
|
const eastl::span<const std::byte> flatbuffer = assetRequest.Get(0);
|
|
return eastl::vector<std::byte> {flatbuffer.begin(), flatbuffer.end()};
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::vector<eastl::vector<std::byte>> BigFile::GetAssetFlatbuffers(const eastl::span<const UUID> p_uuids) const
|
|
{
|
|
eastl::vector<eastl::vector<std::byte>> flatbuffers;
|
|
if (p_uuids.empty())
|
|
{
|
|
return flatbuffers;
|
|
}
|
|
|
|
flatbuffers.reserve(p_uuids.size());
|
|
|
|
if (p_uuids.size() == 1)
|
|
{
|
|
if (eastl::optional<eastl::vector<std::byte>> flatbuffer = GetAssetFlatbuffer(p_uuids[0]);
|
|
flatbuffer.has_value())
|
|
{
|
|
flatbuffers.push_back(std::move(*flatbuffer));
|
|
}
|
|
|
|
return flatbuffers;
|
|
}
|
|
|
|
eastl::vector<UUID> uniqueUUIDs {p_uuids.begin(), p_uuids.end()};
|
|
eastl::sort(uniqueUUIDs.begin(), uniqueUUIDs.end());
|
|
uniqueUUIDs.erase(eastl::unique(uniqueUUIDs.begin(), uniqueUUIDs.end()), uniqueUUIDs.end());
|
|
|
|
eastl::string placeholders;
|
|
placeholders.reserve(uniqueUUIDs.size() * 3);
|
|
for (std::size_t i = 0; i < uniqueUUIDs.size(); ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
placeholders += ", ";
|
|
}
|
|
placeholders += "?";
|
|
}
|
|
|
|
const std::string sqlRequest = fmt::format("SELECT Asset FROM Asset WHERE UUID IN ({});", placeholders);
|
|
|
|
BigFile::Request assetRequest {*this, sqlRequest.c_str()};
|
|
for (std::size_t i = 0; i < uniqueUUIDs.size(); ++i)
|
|
{
|
|
assetRequest.Bind(static_cast<std::uint32_t>(i + 1), uniqueUUIDs[i].ToBytes());
|
|
}
|
|
|
|
flatbuffers.reserve(uniqueUUIDs.size());
|
|
for (std::size_t i = 0; i < uniqueUUIDs.size(); ++i)
|
|
{
|
|
if (!assetRequest.Step())
|
|
{
|
|
break;
|
|
}
|
|
|
|
const eastl::span<const std::byte> flatbuffer = assetRequest.Get(0);
|
|
flatbuffers.emplace_back(flatbuffer.begin(), flatbuffer.end());
|
|
}
|
|
|
|
return flatbuffers;
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::optional<AssetMetadata> BigFile::GetAssetMetadata(const UUID& p_uuid) const
|
|
{
|
|
BigFile::Request assetRequest {
|
|
*this,
|
|
"SELECT UUID, Name, TypeID, TypeName, CreateTime, ModificationTime FROM AssetMetadata WHERE UUID = ?1;"};
|
|
assetRequest.Bind(1, p_uuid.ToBytes());
|
|
|
|
if (!assetRequest.Step())
|
|
{
|
|
return eastl::nullopt;
|
|
}
|
|
|
|
return AssetMetadata {.m_uuid = UUID {std::span<const std::byte, UUID::UUID_BYTE_SIZE> {
|
|
static_cast<eastl::span<const std::byte>>(assetRequest.Get(0)).data(),
|
|
UUID::UUID_BYTE_SIZE}},
|
|
.m_name = eastl::string {static_cast<eastl::string_view>(assetRequest.Get(1))},
|
|
.m_typeID = assetRequest.Get(2),
|
|
.m_typeName = eastl::string {static_cast<eastl::string_view>(assetRequest.Get(3))},
|
|
.m_creationTime = Time {static_cast<std::uint64_t>(assetRequest.Get(4))},
|
|
.m_modificationTime = Time {static_cast<std::uint64_t>(assetRequest.Get(5))}};
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::vector<AssetMetadata> BigFile::GetAssetMetadatas(const eastl::span<const UUID> p_uuids) const
|
|
{
|
|
eastl::vector<AssetMetadata> metadatas;
|
|
if (p_uuids.empty())
|
|
{
|
|
return metadatas;
|
|
}
|
|
|
|
metadatas.reserve(p_uuids.size());
|
|
|
|
if (p_uuids.size() == 1)
|
|
{
|
|
if (eastl::optional<AssetMetadata> metadata = GetAssetMetadata(p_uuids[0]); metadata.has_value())
|
|
{
|
|
metadatas.push_back(std::move(*metadata));
|
|
}
|
|
|
|
return metadatas;
|
|
}
|
|
|
|
eastl::vector<UUID> uniqueUUIDs {p_uuids.begin(), p_uuids.end()};
|
|
eastl::sort(uniqueUUIDs.begin(), uniqueUUIDs.end());
|
|
uniqueUUIDs.erase(eastl::unique(uniqueUUIDs.begin(), uniqueUUIDs.end()), uniqueUUIDs.end());
|
|
|
|
eastl::string placeholders;
|
|
placeholders.reserve(uniqueUUIDs.size() * 3);
|
|
for (std::size_t i = 0; i < uniqueUUIDs.size(); ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
placeholders += ", ";
|
|
}
|
|
placeholders += "?";
|
|
}
|
|
|
|
const std::string sqlRequest = fmt::format("SELECT UUID, Name, TypeID, TypeName, CreateTime, ModificationTime "
|
|
"FROM AssetMetadata WHERE UUID IN ({});",
|
|
placeholders);
|
|
|
|
BigFile::Request assetRequest {*this, sqlRequest.c_str()};
|
|
for (std::size_t i = 0; i < uniqueUUIDs.size(); ++i)
|
|
{
|
|
assetRequest.Bind(static_cast<std::uint32_t>(i + 1), uniqueUUIDs[i].ToBytes());
|
|
}
|
|
|
|
metadatas.reserve(uniqueUUIDs.size());
|
|
for (std::size_t i = 0; i < uniqueUUIDs.size(); ++i)
|
|
{
|
|
if (!assetRequest.Step())
|
|
{
|
|
break;
|
|
}
|
|
|
|
metadatas.push_back(
|
|
AssetMetadata {.m_uuid = UUID {std::span<const std::byte, UUID::UUID_BYTE_SIZE> {
|
|
static_cast<eastl::span<const std::byte>>(assetRequest.Get(0)).data(),
|
|
UUID::UUID_BYTE_SIZE}},
|
|
.m_name = eastl::string {static_cast<eastl::string_view>(assetRequest.Get(1))},
|
|
.m_typeID = assetRequest.Get(2),
|
|
.m_typeName = eastl::string {static_cast<eastl::string_view>(assetRequest.Get(3))},
|
|
.m_creationTime = Time {static_cast<std::uint64_t>(assetRequest.Get(4))},
|
|
.m_modificationTime = Time {static_cast<std::uint64_t>(assetRequest.Get(5))}});
|
|
}
|
|
|
|
return metadatas;
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::vector<UUID> BigFile::GetAssetHardReferences(const UUID& p_uuid) const
|
|
{
|
|
BigFile::Request hardReferencesRequest {*this, "SELECT HardReference FROM AssetHardReference WHERE UUID = ?1;"};
|
|
hardReferencesRequest.Bind(1, p_uuid.ToBytes());
|
|
|
|
eastl::vector<UUID> hardReferences;
|
|
while (hardReferencesRequest.Step())
|
|
{
|
|
const eastl::span<const std::byte, UUID::UUID_BYTE_SIZE> rawUUID = static_cast<eastl::span<const std::byte>>(
|
|
hardReferencesRequest.Get(0));
|
|
hardReferences.emplace_back(std::span<const std::byte, UUID::UUID_BYTE_SIZE>(rawUUID.begin(), rawUUID.end()));
|
|
}
|
|
|
|
return hardReferences;
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
eastl::vector<eastl::vector<UUID>> BigFile::GetAssetsHardReferences(eastl::span<const UUID> p_uuids) const
|
|
{
|
|
eastl::vector<eastl::vector<UUID>> hardReferences(p_uuids.size());
|
|
|
|
if (p_uuids.empty())
|
|
{
|
|
return hardReferences;
|
|
}
|
|
|
|
ankerl::unordered_dense::map<UUID, eastl::vector<std::size_t>> uuidToIndices;
|
|
uuidToIndices.reserve(p_uuids.size());
|
|
for (std::size_t i = 0; i < p_uuids.size(); ++i)
|
|
{
|
|
uuidToIndices[p_uuids[i]].push_back(i);
|
|
}
|
|
|
|
eastl::vector<UUID> distinctUUIDs;
|
|
distinctUUIDs.reserve(uuidToIndices.size());
|
|
for (const auto& [uuid, indices]: uuidToIndices)
|
|
{
|
|
distinctUUIDs.push_back(uuid);
|
|
}
|
|
|
|
constexpr std::size_t PARAMS_PER_ROW = 1;
|
|
const std::size_t maxRowsPerStatement = static_cast<std::size_t>(
|
|
sqlite3_limit(m_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1)) /
|
|
PARAMS_PER_ROW;
|
|
|
|
for (std::size_t chunkStart = 0; chunkStart < distinctUUIDs.size(); chunkStart += maxRowsPerStatement)
|
|
{
|
|
const std::size_t chunkSize = eastl::min(maxRowsPerStatement, distinctUUIDs.size() - chunkStart);
|
|
|
|
eastl::string placeholders;
|
|
placeholders.reserve(chunkSize * 2);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
placeholders += ", ";
|
|
}
|
|
placeholders += "?";
|
|
}
|
|
|
|
const std::string sql = fmt::format("SELECT UUID, HardReference FROM AssetHardReference WHERE UUID IN ({});",
|
|
placeholders);
|
|
|
|
BigFile::Request hardReferencesRequest {*this, sql.c_str()};
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
hardReferencesRequest.Bind(static_cast<std::uint32_t>(i + 1), distinctUUIDs[chunkStart + i].ToBytes());
|
|
}
|
|
|
|
while (hardReferencesRequest.Step())
|
|
{
|
|
const eastl::span<const std::byte, UUID::UUID_BYTE_SIZE> rawOwnerUUID =
|
|
static_cast<eastl::span<const std::byte>>(hardReferencesRequest.Get(0));
|
|
const UUID ownerUUID {
|
|
std::span<const std::byte, UUID::UUID_BYTE_SIZE> {rawOwnerUUID.begin(), rawOwnerUUID.end()}};
|
|
|
|
const auto it = uuidToIndices.find(ownerUUID);
|
|
ASSERT(EngineAssertHandler,
|
|
it != uuidToIndices.end(),
|
|
"Returned UUID {} not found in requested UUIDs",
|
|
ownerUUID);
|
|
|
|
const eastl::span<const std::byte, UUID::UUID_BYTE_SIZE> rawHardReference =
|
|
static_cast<eastl::span<const std::byte>>(hardReferencesRequest.Get(1));
|
|
|
|
for (const std::size_t index: it->second)
|
|
{
|
|
hardReferences[index].emplace_back(
|
|
std::span<const std::byte, UUID::UUID_BYTE_SIZE> {rawHardReference.begin(),
|
|
rawHardReference.end()});
|
|
}
|
|
}
|
|
}
|
|
|
|
return hardReferences;
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::SaveAssetFlatbuffer(const AssetMetadata& p_metadata,
|
|
const eastl::span<const std::byte> p_flatbuffer,
|
|
const eastl::span<const UUID> p_hardReferences)
|
|
{
|
|
BeginTransaction();
|
|
|
|
BigFile::Request metadataRequest {
|
|
*this,
|
|
"INSERT INTO AssetMetadata (UUID, Name, TypeID, TypeName) VALUES (?1, ?2, ?3, ?4) "
|
|
"ON CONFLICT(UUID) DO UPDATE SET Name = excluded.Name, TypeID = excluded.TypeID, TypeName = excluded.TypeName;"};
|
|
metadataRequest.Bind(1, p_metadata.m_uuid.ToBytes());
|
|
metadataRequest.Bind(2, p_metadata.m_name);
|
|
metadataRequest.Bind(3, p_metadata.m_typeID);
|
|
metadataRequest.Bind(4, p_metadata.m_typeName);
|
|
metadataRequest.Execute();
|
|
|
|
BigFile::Request assetRequest {*this,
|
|
"INSERT INTO Asset (UUID, Asset) VALUES (?1, ?2) "
|
|
"ON CONFLICT(UUID) DO UPDATE SET Asset = excluded.Asset;"};
|
|
assetRequest.Bind(1, p_metadata.m_uuid.ToBytes());
|
|
assetRequest.Bind(2, p_flatbuffer);
|
|
assetRequest.Execute();
|
|
|
|
BigFile::Request deleteHardReferencesRequest {*this, "DELETE FROM AssetHardReference WHERE UUID = ?1;"};
|
|
deleteHardReferencesRequest.Bind(1, p_metadata.m_uuid.ToBytes());
|
|
deleteHardReferencesRequest.Execute();
|
|
|
|
constexpr std::size_t HARDREF_PARAMS_PER_ROW = 2;
|
|
const std::size_t maxHardRefRowsPerStatement = static_cast<std::size_t>(
|
|
sqlite3_limit(m_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1)) /
|
|
HARDREF_PARAMS_PER_ROW;
|
|
|
|
for (std::size_t chunkStart = 0; chunkStart < p_hardReferences.size(); chunkStart += maxHardRefRowsPerStatement)
|
|
{
|
|
const std::size_t chunkSize = eastl::min(maxHardRefRowsPerStatement, p_hardReferences.size() - chunkStart);
|
|
|
|
eastl::string hardReferencePlaceholders;
|
|
hardReferencePlaceholders.reserve(chunkSize * 8);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
hardReferencePlaceholders += ", ";
|
|
}
|
|
hardReferencePlaceholders += "(?, ?)";
|
|
}
|
|
|
|
const std::string hardReferenceSQL = fmt::format(
|
|
"INSERT INTO AssetHardReference (UUID, HardReference) VALUES {} "
|
|
"ON CONFLICT(UUID, HardReference) DO NOTHING;",
|
|
hardReferencePlaceholders);
|
|
|
|
BigFile::Request hardReferenceRequest {*this, hardReferenceSQL.c_str()};
|
|
std::uint32_t bindIndex = 1;
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
hardReferenceRequest.Bind(bindIndex++, p_metadata.m_uuid.ToBytes());
|
|
hardReferenceRequest.Bind(bindIndex++, p_hardReferences[chunkStart + i].ToBytes());
|
|
}
|
|
hardReferenceRequest.Execute();
|
|
}
|
|
|
|
CommitTransaction();
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::SaveAssetFlatbuffers(const eastl::span<const AssetMetadata> p_metadatas,
|
|
const eastl::span<const eastl::vector<std::byte>> p_flatbuffers,
|
|
const eastl::span<const eastl::vector<UUID>> p_hardReferences)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
p_metadatas.size() == p_flatbuffers.size() && p_metadatas.size() == p_hardReferences.size(),
|
|
"p_metadatas and p_flatbuffers and p_hardReferences must have the same size");
|
|
|
|
if (p_metadatas.empty())
|
|
{
|
|
return;
|
|
}
|
|
|
|
BeginTransaction();
|
|
|
|
constexpr std::size_t METADATA_PARAMS_PER_ROW = 4;
|
|
const std::size_t maxRowsPerStatement = static_cast<std::size_t>(
|
|
sqlite3_limit(m_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1)) /
|
|
METADATA_PARAMS_PER_ROW;
|
|
|
|
for (std::size_t chunkStart = 0; chunkStart < p_metadatas.size(); chunkStart += maxRowsPerStatement)
|
|
{
|
|
const std::size_t chunkSize = eastl::min(maxRowsPerStatement, p_metadatas.size() - chunkStart);
|
|
|
|
eastl::string metadataPlaceholders;
|
|
metadataPlaceholders.reserve(chunkSize * 14);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
metadataPlaceholders += ", ";
|
|
}
|
|
metadataPlaceholders += "(?, ?, ?, ?)";
|
|
}
|
|
|
|
const std::string metadataSQL = fmt::format(
|
|
"INSERT INTO AssetMetadata (UUID, Name, TypeID, TypeName) VALUES {} "
|
|
"ON CONFLICT(UUID) DO UPDATE SET Name = excluded.Name, TypeID = excluded.TypeID, TypeName = "
|
|
"excluded.TypeName;",
|
|
metadataPlaceholders);
|
|
|
|
BigFile::Request metadataRequest {*this, metadataSQL.c_str()};
|
|
std::uint32_t metadataBindIndex = 1;
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
const AssetMetadata& metadata = p_metadatas[chunkStart + i];
|
|
metadataRequest.Bind(metadataBindIndex++, metadata.m_uuid.ToBytes());
|
|
metadataRequest.Bind(metadataBindIndex++, metadata.m_name);
|
|
metadataRequest.Bind(metadataBindIndex++, metadata.m_typeID);
|
|
metadataRequest.Bind(metadataBindIndex++, metadata.m_typeName);
|
|
}
|
|
metadataRequest.Execute();
|
|
|
|
eastl::string assetPlaceholders;
|
|
assetPlaceholders.reserve(chunkSize * 8);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
assetPlaceholders += ", ";
|
|
}
|
|
assetPlaceholders += "(?, ?)";
|
|
}
|
|
|
|
const std::string assetSQL = fmt::format(
|
|
"INSERT INTO Asset (UUID, Asset) VALUES {} ON CONFLICT(UUID) DO UPDATE SET Asset = excluded.Asset;",
|
|
assetPlaceholders);
|
|
|
|
BigFile::Request assetRequest {*this, assetSQL.c_str()};
|
|
std::uint32_t assetBindIndex = 1;
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
assetRequest.Bind(assetBindIndex++, p_metadatas[chunkStart + i].m_uuid.ToBytes());
|
|
assetRequest.Bind(assetBindIndex++, p_flatbuffers[chunkStart + i]);
|
|
}
|
|
assetRequest.Execute();
|
|
|
|
// Replace hard references for every asset in this chunk with its current set.
|
|
eastl::string deleteInPlaceholders;
|
|
deleteInPlaceholders.reserve(chunkSize * 3);
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
deleteInPlaceholders += ", ";
|
|
}
|
|
deleteInPlaceholders += "?";
|
|
}
|
|
|
|
const std::string deleteHardReferencesSQL = fmt::format("DELETE FROM AssetHardReference WHERE UUID IN ({});",
|
|
deleteInPlaceholders);
|
|
|
|
BigFile::Request deleteHardReferencesRequest {*this, deleteHardReferencesSQL.c_str()};
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
deleteHardReferencesRequest.Bind(static_cast<std::uint32_t>(i + 1),
|
|
p_metadatas[chunkStart + i].m_uuid.ToBytes());
|
|
}
|
|
deleteHardReferencesRequest.Execute();
|
|
|
|
eastl::vector<eastl::pair<UUID, UUID>> hardReferencePairs;
|
|
for (std::size_t i = 0; i < chunkSize; ++i)
|
|
{
|
|
const UUID& assetUUID = p_metadatas[chunkStart + i].m_uuid;
|
|
for (const UUID& hardReference: p_hardReferences[chunkStart + i])
|
|
{
|
|
hardReferencePairs.emplace_back(assetUUID, hardReference);
|
|
}
|
|
}
|
|
|
|
constexpr std::size_t HARDREF_PARAMS_PER_ROW = 2;
|
|
const std::size_t maxHardRefRowsPerStatement = static_cast<std::size_t>(
|
|
sqlite3_limit(m_db, SQLITE_LIMIT_VARIABLE_NUMBER, -1)) /
|
|
HARDREF_PARAMS_PER_ROW;
|
|
|
|
for (std::size_t hardRefChunkStart = 0; hardRefChunkStart < hardReferencePairs.size();
|
|
hardRefChunkStart += maxHardRefRowsPerStatement)
|
|
{
|
|
const std::size_t hardRefChunkSize = eastl::min(maxHardRefRowsPerStatement,
|
|
hardReferencePairs.size() - hardRefChunkStart);
|
|
|
|
eastl::string hardReferencePlaceholders;
|
|
hardReferencePlaceholders.reserve(hardRefChunkSize * 8);
|
|
for (std::size_t i = 0; i < hardRefChunkSize; ++i)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
hardReferencePlaceholders += ", ";
|
|
}
|
|
hardReferencePlaceholders += "(?, ?)";
|
|
}
|
|
|
|
const std::string hardReferenceSQL = fmt::format(
|
|
"INSERT INTO AssetHardReference (UUID, HardReference) VALUES {} "
|
|
"ON CONFLICT(UUID, HardReference) DO NOTHING;",
|
|
hardReferencePlaceholders);
|
|
|
|
BigFile::Request hardReferenceRequest {*this, hardReferenceSQL.c_str()};
|
|
std::uint32_t bindIndex = 1;
|
|
for (std::size_t i = 0; i < hardRefChunkSize; ++i)
|
|
{
|
|
const auto& [assetUUID, hardReference] = hardReferencePairs[hardRefChunkStart + i];
|
|
hardReferenceRequest.Bind(bindIndex++, assetUUID.ToBytes());
|
|
hardReferenceRequest.Bind(bindIndex++, hardReference.ToBytes());
|
|
}
|
|
hardReferenceRequest.Execute();
|
|
}
|
|
}
|
|
|
|
CommitTransaction();
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
bool BigFile::Has(const UUID& p_uuid) const
|
|
{
|
|
BigFile::Request request {*this, "SELECT 1 FROM AssetMetadata WHERE UUID = ?1;"};
|
|
request.Bind(1, p_uuid.ToBytes());
|
|
|
|
return request.Step();
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Request(const BigFile& p_bigFile, const eastl::string_view p_request):
|
|
m_db(p_bigFile.m_db)
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_prepare_v2(m_db,
|
|
p_request.data(),
|
|
static_cast<std::uint32_t>(p_request.size()),
|
|
&m_statement,
|
|
nullptr);
|
|
CRITICAL_ASSERT(EngineAssertHandler,
|
|
result == SQLITE_OK,
|
|
"Failed to create statement from BigFile DB: {}",
|
|
sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const std::int32_t p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_int(m_statement, p_index, p_value);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const std::uint32_t p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_int(m_statement, p_index, p_value);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const std::int64_t p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_int64(m_statement, p_index, p_value);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const std::uint64_t p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_int64(m_statement, p_index, std::bit_cast<std::int64_t>(p_value));
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const float p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_double(m_statement, p_index, p_value);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const double p_value)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_double(m_statement, p_index, p_value);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index, const eastl::string_view p_value, const CopyValue p_copy)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_text(m_statement,
|
|
p_index,
|
|
p_value.data(),
|
|
static_cast<std::uint32_t>(p_value.size()),
|
|
p_copy ? SQLITE_TRANSIENT : SQLITE_STATIC);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
void BigFile::Request::Bind(const std::uint32_t p_index,
|
|
const eastl::span<const std::byte> p_value,
|
|
const CopyValue p_copy)
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
(p_index >= 1) && (p_index <= static_cast<std::uint32_t>(sqlite3_bind_parameter_count(m_statement))),
|
|
"Invalid index for statement");
|
|
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_bind_blob(m_statement,
|
|
p_index,
|
|
p_value.data(),
|
|
static_cast<std::uint32_t>(p_value.size()),
|
|
p_copy ? SQLITE_TRANSIENT : SQLITE_STATIC);
|
|
ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to bind value for statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
bool BigFile::Request::Step()
|
|
{
|
|
const int result = sqlite3_step(m_statement);
|
|
ASSERT(EngineAssertHandler,
|
|
(result == SQLITE_DONE) || (result == SQLITE_ROW),
|
|
"Failed to step through the statement: {}",
|
|
sqlite3_errmsg(m_db));
|
|
|
|
return result == SQLITE_ROW;
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
std::uint32_t BigFile::Request::Execute()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_step(m_statement);
|
|
ASSERT(EngineAssertHandler, (result == SQLITE_DONE), "Failed to execute the statement: {}", sqlite3_errmsg(m_db));
|
|
|
|
return sqlite3_changes(m_db);
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::Column(const Request& p_request, const std::uint32_t p_index):
|
|
m_statement(p_request.m_statement),
|
|
m_index(p_index)
|
|
{
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator std::int32_t() const
|
|
{
|
|
return sqlite3_column_int(m_statement, m_index);
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator std::uint32_t() const
|
|
{
|
|
return sqlite3_column_int(m_statement, m_index);
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator std::int64_t() const
|
|
{
|
|
return sqlite3_column_int64(m_statement, m_index);
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator std::uint64_t() const
|
|
{
|
|
return std::bit_cast<std::uint64_t>(sqlite3_column_int64(m_statement, m_index));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator float() const
|
|
{
|
|
return static_cast<float>(sqlite3_column_double(m_statement, m_index));
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator double() const
|
|
{
|
|
return sqlite3_column_double(m_statement, m_index);
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator eastl::string_view() const
|
|
{
|
|
return eastl::string_view {reinterpret_cast<const char*>(sqlite3_column_text(m_statement, m_index)),
|
|
static_cast<std::size_t>(sqlite3_column_bytes(m_statement, m_index))};
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column::operator eastl::span<const std::byte>() const
|
|
{
|
|
return eastl::span<const std::byte> {static_cast<const std::byte*>(sqlite3_column_blob(m_statement, m_index)),
|
|
static_cast<std::size_t>(sqlite3_column_bytes(m_statement, m_index))};
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::Column BigFile::Request::Get(const std::uint32_t p_index) const
|
|
{
|
|
ASSERT(EngineAssertHandler,
|
|
p_index < static_cast<std::uint32_t>(sqlite3_column_count(m_statement)),
|
|
"Invalid index for column!");
|
|
return {*this, p_index};
|
|
}
|
|
|
|
/****************************************************************************************/
|
|
|
|
BigFile::Request::~Request()
|
|
{
|
|
[[maybe_unused]]
|
|
const int result = sqlite3_finalize(m_statement);
|
|
CRITICAL_ASSERT(EngineAssertHandler, result == SQLITE_OK, "Failed to finalize statement: {}", sqlite3_errmsg(m_db));
|
|
}
|
|
#endif
|
|
} // namespace Bigfoot
|