From 3e3e3798ae3686b23e48d9c73a675943c4d1be97 Mon Sep 17 00:00:00 2001 From: Romain BOULLARD Date: Fri, 28 Aug 2026 01:00:38 +0200 Subject: [PATCH] MemoryMappedFile + File + BigFIle more representative numbers --- Bigfoot/Benchmarks/Engine/BigFile/BigFile.cpp | 46 +++- .../BigFileInfo_generated.hpp | 3 + Bigfoot/Sources/Engine/BigFile/BigFile.cpp | 25 ++ .../Engine/Include/Engine/BigFile/BigFile.hpp | 2 +- .../BigFile/BigFileInfo_generated.hpp.in | 3 + Bigfoot/Sources/System/File.cpp | 66 ----- Bigfoot/Sources/System/File/File.cpp | 120 +++++++++ .../Sources/System/File/MemoryMappedFile.cpp | 229 ++++++++++++++++++ .../System/Include/System/{ => File}/File.hpp | 34 +++ .../Include/System/File/MemoryMappedFile.hpp | 150 ++++++++++++ Bigfoot/Tests/Engine/BigFile/BigFile.cpp | 2 +- .../EngineTests/BigFileInfo_generated.hpp | 3 + Bigfoot/Tests/System/File.cpp | 80 ------ Bigfoot/Tests/System/File/File.cpp | 194 +++++++++++++++ .../Tests/System/File/MemoryMappedFile.cpp | 92 +++++++ Bigfoot/Tests/System/Fixture/mappedFile | 1 + .../BigFileInfo_generated.hpp | 22 ++ 17 files changed, 913 insertions(+), 159 deletions(-) delete mode 100644 Bigfoot/Sources/System/File.cpp create mode 100644 Bigfoot/Sources/System/File/File.cpp create mode 100644 Bigfoot/Sources/System/File/MemoryMappedFile.cpp rename Bigfoot/Sources/System/Include/System/{ => File}/File.hpp (66%) create mode 100644 Bigfoot/Sources/System/Include/System/File/MemoryMappedFile.hpp delete mode 100644 Bigfoot/Tests/System/File.cpp create mode 100644 Bigfoot/Tests/System/File/File.cpp create mode 100644 Bigfoot/Tests/System/File/MemoryMappedFile.cpp create mode 100644 Bigfoot/Tests/System/Fixture/mappedFile create mode 100644 Bigfoot/Tests/Tools/SasquatchBaker/Include/SasquatchBakerTests/BigFileInfo_generated.hpp diff --git a/Bigfoot/Benchmarks/Engine/BigFile/BigFile.cpp b/Bigfoot/Benchmarks/Engine/BigFile/BigFile.cpp index 321f47e..2511fdf 100644 --- a/Bigfoot/Benchmarks/Engine/BigFile/BigFile.cpp +++ b/Bigfoot/Benchmarks/Engine/BigFile/BigFile.cpp @@ -1,6 +1,12 @@ +/********************************************************************* + * \file BigFile.cpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ #include -#include +#include #include #include @@ -71,6 +77,20 @@ class Fixture: public benchmark::Fixture return batch; } + [[nodiscard]] + eastl::vector> RandomBatchPool(const std::size_t p_batchSize, + const std::size_t p_poolSize) const + { + eastl::vector> pool; + pool.reserve(p_poolSize); + for (std::size_t i = 0; i < p_poolSize; ++i) + { + pool.push_back(RandomBatch(p_batchSize)); + } + + return pool; + } + BigFile m_bigFile {File {BIGFILE_ENGINEBENCHMARKS_LOCATION}}; eastl::vector m_uuids; }; @@ -105,21 +125,24 @@ BENCHMARK_REGISTER_F(Fixture, GetAssetFlatbuffer)->Name("BigFile/GetAssetFlatbuf BENCHMARK_DEFINE_F(Fixture, GetAssetFlatbuffers)(benchmark::State& state) { const auto batchSize = static_cast(state.range(0)); + + constexpr std::size_t BATCH_POOL_SIZE = 64; + const eastl::vector> batchPool = RandomBatchPool(batchSize, BATCH_POOL_SIZE); + std::int64_t bytesProcessed = 0; + std::size_t batchIndex = 0; for (auto _: state) { - state.PauseTiming(); - const eastl::vector batch = RandomBatch(batchSize); - state.ResumeTiming(); - - eastl::vector> flatbuffers = m_bigFile.GetAssetFlatbuffers(batch); + eastl::vector> flatbuffers = m_bigFile.GetAssetFlatbuffers(batchPool[batchIndex]); benchmark::DoNotOptimize(flatbuffers); for (const eastl::vector& flatbuffer: flatbuffers) { bytesProcessed += static_cast(flatbuffer.size()); } + + batchIndex = (batchIndex + 1) % BATCH_POOL_SIZE; } state.SetItemsProcessed(state.iterations() * batchSize); @@ -154,13 +177,14 @@ BENCHMARK_DEFINE_F(Fixture, GetAssetMetadatas)(benchmark::State& state) { const auto batchSize = static_cast(state.range(0)); + constexpr std::size_t BATCH_POOL_SIZE = 64; + const eastl::vector> batchPool = RandomBatchPool(batchSize, BATCH_POOL_SIZE); + + std::size_t batchIndex = 0; for (auto _: state) { - state.PauseTiming(); - const eastl::vector batch = RandomBatch(batchSize); - state.ResumeTiming(); - - benchmark::DoNotOptimize(m_bigFile.GetAssetMetadatas(batch)); + benchmark::DoNotOptimize(m_bigFile.GetAssetMetadatas(batchPool[batchIndex])); + batchIndex = (batchIndex + 1) % BATCH_POOL_SIZE; } state.SetItemsProcessed(state.iterations() * batchSize); diff --git a/Bigfoot/Benchmarks/Engine/Include/EngineBenchmarks/BigFileInfo_generated.hpp b/Bigfoot/Benchmarks/Engine/Include/EngineBenchmarks/BigFileInfo_generated.hpp index 4d2f05f..f9738c1 100644 --- a/Bigfoot/Benchmarks/Engine/Include/EngineBenchmarks/BigFileInfo_generated.hpp +++ b/Bigfoot/Benchmarks/Engine/Include/EngineBenchmarks/BigFileInfo_generated.hpp @@ -9,6 +9,8 @@ #include +#if defined BIGFOOT_TOOLMODE + namespace Bigfoot { /* @@ -17,3 +19,4 @@ namespace Bigfoot constexpr eastl::string_view BIGFILE_ENGINEBENCHMARKS_LOCATION {"./EngineBenchmarks-bigfile.db"}; } // namespace Bigfoot #endif +#endif diff --git a/Bigfoot/Sources/Engine/BigFile/BigFile.cpp b/Bigfoot/Sources/Engine/BigFile/BigFile.cpp index 0a38f3b..0dc5289 100644 --- a/Bigfoot/Sources/Engine/BigFile/BigFile.cpp +++ b/Bigfoot/Sources/Engine/BigFile/BigFile.cpp @@ -102,6 +102,19 @@ eastl::vector> BigFile::GetAssetFlatbuffers(const eastl return flatbuffers; } + flatbuffers.reserve(p_uuids.size()); + + if (p_uuids.size() == 1) + { + if (eastl::optional> flatbuffer = GetAssetFlatbuffer(p_uuids[0]); + flatbuffer.has_value()) + { + flatbuffers.push_back(std::move(*flatbuffer)); + } + + return flatbuffers; + } + eastl::vector uniqueUUIDs {p_uuids.begin(), p_uuids.end()}; eastl::sort(uniqueUUIDs.begin(), uniqueUUIDs.end()); uniqueUUIDs.erase(eastl::unique(uniqueUUIDs.begin(), uniqueUUIDs.end()), uniqueUUIDs.end()); @@ -174,6 +187,18 @@ eastl::vector BigFile::GetAssetMetadatas(const eastl::span metadata = GetAssetMetadata(p_uuids[0]); metadata.has_value()) + { + metadatas.push_back(std::move(*metadata)); + } + + return metadatas; + } + eastl::vector uniqueUUIDs {p_uuids.begin(), p_uuids.end()}; eastl::sort(uniqueUUIDs.begin(), uniqueUUIDs.end()); uniqueUUIDs.erase(eastl::unique(uniqueUUIDs.begin(), uniqueUUIDs.end()), uniqueUUIDs.end()); diff --git a/Bigfoot/Sources/Engine/Include/Engine/BigFile/BigFile.hpp b/Bigfoot/Sources/Engine/Include/Engine/BigFile/BigFile.hpp index 2ad758d..19e49d7 100644 --- a/Bigfoot/Sources/Engine/Include/Engine/BigFile/BigFile.hpp +++ b/Bigfoot/Sources/Engine/Include/Engine/BigFile/BigFile.hpp @@ -8,7 +8,7 @@ #define BIGFOOT_ENGINE_BIGFILE_BIGFILE_HPP #include -#include +#include #include diff --git a/Bigfoot/Sources/Engine/Include/Engine/BigFile/BigFileInfo_generated.hpp.in b/Bigfoot/Sources/Engine/Include/Engine/BigFile/BigFileInfo_generated.hpp.in index 66f4d7e..44e8ffb 100644 --- a/Bigfoot/Sources/Engine/Include/Engine/BigFile/BigFileInfo_generated.hpp.in +++ b/Bigfoot/Sources/Engine/Include/Engine/BigFile/BigFileInfo_generated.hpp.in @@ -9,6 +9,8 @@ #include +#if defined BIGFOOT_TOOLMODE + namespace Bigfoot { /* @@ -16,4 +18,5 @@ namespace Bigfoot */ constexpr eastl::string_view BIGFILE_@BIGFILE_NAME@_LOCATION {"@BIGFILE_LOCATION@"}; } // namespace Bigfoot +#endif #endif \ No newline at end of file diff --git a/Bigfoot/Sources/System/File.cpp b/Bigfoot/Sources/System/File.cpp deleted file mode 100644 index e2fe411..0000000 --- a/Bigfoot/Sources/System/File.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/********************************************************************* - * \file File.cpp - * - * \author Romain BOULLARD - * \date December 2025 - *********************************************************************/ -#include - -namespace Bigfoot -{ -File::File(const eastl::string_view p_path): - m_path(p_path.data()), - m_pathString(p_path) -{ -} - -/****************************************************************************************/ - -bool File::IsAbsolute() const -{ - return m_path.is_absolute(); -} - -/****************************************************************************************/ - -bool File::IsRelative() const -{ - return m_path.is_relative(); -} - -/****************************************************************************************/ - -bool File::Exists() const -{ - return std::filesystem::exists(m_path); -} - -/****************************************************************************************/ - -eastl::string_view File::GetPath() const -{ - return m_pathString; -} - -/****************************************************************************************/ - -File File::ToAbsolute() const -{ - return File {std::filesystem::absolute(m_path)}; -} - -/****************************************************************************************/ - -File File::ToRelative() const -{ - return File {std::filesystem::relative(m_path)}; -} - -/****************************************************************************************/ - -File::File(const std::filesystem::path& p_path): - m_path(p_path), - m_pathString(m_path.string().c_str()) -{ -} -} // namespace Bigfoot diff --git a/Bigfoot/Sources/System/File/File.cpp b/Bigfoot/Sources/System/File/File.cpp new file mode 100644 index 0000000..66dbc96 --- /dev/null +++ b/Bigfoot/Sources/System/File/File.cpp @@ -0,0 +1,120 @@ +/********************************************************************* + * \file File.cpp + * + * \author Romain BOULLARD + * \date December 2025 + *********************************************************************/ +#include + +#include + +#include +#include +#include + +namespace Bigfoot +{ +File::File(const eastl::string_view p_path): + m_path(p_path.data()), + m_pathString(p_path) +{ +} + +/****************************************************************************************/ + +bool File::IsAbsolute() const +{ + return m_path.is_absolute(); +} + +/****************************************************************************************/ + +bool File::IsRelative() const +{ + return m_path.is_relative(); +} + +/****************************************************************************************/ + +bool File::Exists() const +{ + return std::filesystem::exists(m_path); +} + +/****************************************************************************************/ + +eastl::string_view File::GetPath() const +{ + return m_pathString; +} + +/****************************************************************************************/ + +File File::ToAbsolute() const +{ + return File {std::filesystem::absolute(m_path)}; +} + +/****************************************************************************************/ + +File File::ToRelative() const +{ + return File {std::filesystem::relative(m_path)}; +} + +/****************************************************************************************/ + +std::size_t File::GetSize() const +{ + return static_cast(std::filesystem::file_size(m_path)); +} + +/****************************************************************************************/ + +eastl::vector File::Read(const std::size_t p_offset, const std::size_t p_size) const +{ + const File absoluteFile = ToAbsolute(); + const eastl::string_view path = absoluteFile.GetPath(); + + std::ifstream file {path.data(), std::ios::binary}; + CRITICAL_ASSERT(SystemAssertHandler, file.is_open(), "Failed to open file {} for reading", path); + + file.seekg(static_cast(p_offset), std::ios::beg); + + eastl::vector data; + data.resize(p_size); + file.read(std::bit_cast(data.data()), static_cast(data.size())); + return data; +} + +/****************************************************************************************/ + +void File::Write(const eastl::span p_data, const std::size_t p_offset) const +{ + const File absoluteFile = ToAbsolute(); + const eastl::string_view path = absoluteFile.GetPath(); + + std::ofstream {path.data(), std::ios::binary | std::ios::app}; + + std::fstream file {path.data(), std::ios::binary | std::ios::in | std::ios::out}; + CRITICAL_ASSERT(SystemAssertHandler, file.is_open(), "Failed to open file {} for writing", path); + + file.seekp(static_cast(p_offset), std::ios::beg); + file.write(std::bit_cast(p_data.data()), static_cast(p_data.size())); +} + +/****************************************************************************************/ + +void File::Delete() const +{ + std::filesystem::remove(m_path); +} + +/****************************************************************************************/ + +File::File(const std::filesystem::path& p_path): + m_path(p_path), + m_pathString(m_path.string().c_str()) +{ +} +} // namespace Bigfoot diff --git a/Bigfoot/Sources/System/File/MemoryMappedFile.cpp b/Bigfoot/Sources/System/File/MemoryMappedFile.cpp new file mode 100644 index 0000000..a4e4c9b --- /dev/null +++ b/Bigfoot/Sources/System/File/MemoryMappedFile.cpp @@ -0,0 +1,229 @@ +/********************************************************************* + * \file MemoryMappedFile.cpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#include + +#include + +#include + +#if defined BIGFOOT_WINDOWS +#include +#elif defined BIGFOOT_LINUX +#include +#include +#include +#include +#endif + +namespace Bigfoot +{ +MemoryMappedFile::MemoryMappedFile(const File& p_file) +{ + const File absoluteFile = p_file.ToAbsolute(); + const eastl::string_view path = absoluteFile.GetPath(); + +#if defined BIGFOOT_WINDOWS + m_fileHandle = CreateFileA(path.data(), + GENERIC_READ, + FILE_SHARE_READ, + nullptr, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, + nullptr); + CRITICAL_ASSERT(SystemAssertHandler, m_fileHandle != INVALID_HANDLE_VALUE, "Failed to open file {}", path); + + LARGE_INTEGER fileSize {}; + [[maybe_unused]] + const BOOL gotFileSize = GetFileSizeEx(m_fileHandle, &fileSize); + CRITICAL_ASSERT(SystemAssertHandler, gotFileSize, "Failed to get the size of file {}", path); + m_size = static_cast(fileSize.QuadPart); + + if (m_size != 0) + { + m_mappingHandle = CreateFileMappingA(m_fileHandle, nullptr, PAGE_READONLY, 0, 0, nullptr); + CRITICAL_ASSERT(SystemAssertHandler, m_mappingHandle != nullptr, "Failed create the file mapping {}", path); + + m_data = static_cast(MapViewOfFile(m_mappingHandle, FILE_MAP_READ, 0, 0, 0)); + CRITICAL_ASSERT(SystemAssertHandler, m_data != nullptr, "Failed to map the file {}", path); + } +#elif defined BIGFOOT_LINUX + m_fileDescriptor = open(path.data(), O_RDONLY); + CRITICAL_ASSERT(SystemAssertHandler, m_fileDescriptor != -1, "Failed to open file {}", path); + + struct stat fileStat {}; + [[maybe_unused]] + const int statResult = fstat(m_fileDescriptor, &fileStat); + CRITICAL_ASSERT(SystemAssertHandler, statResult == 0, "Failed to get the size of file {}", path); + m_size = static_cast(fileStat.st_size); + + if (m_size != 0) + { + void* const mapped = mmap(nullptr, m_size, PROT_READ, MAP_PRIVATE, m_fileDescriptor, 0); + CRITICAL_ASSERT(SystemAssertHandler, mapped != MAP_FAILED, "Failed to map file {}", path); + m_data = static_cast(mapped); + + [[maybe_unused]] + const int adviseResult = madvise(mapped, m_size, MADV_RANDOM); + } +#else + CRITICAL_ASSERT(SystemAssertHandler, false, "MemoryMappedFileBase is not implemented on this platform"); +#endif +} + +/****************************************************************************************/ + +MemoryMappedFile::~MemoryMappedFile() +{ +#if defined BIGFOOT_WINDOWS + if (m_data != nullptr) + { + [[maybe_unused]] + const BOOL unmapped = UnmapViewOfFile(m_data); + CRITICAL_ASSERT(SystemAssertHandler, unmapped, "Failed to unmap the file"); + } + + if (m_mappingHandle != nullptr) + { + [[maybe_unused]] + const BOOL closedMapping = CloseHandle(m_mappingHandle); + CRITICAL_ASSERT(SystemAssertHandler, closedMapping, "Failed to close the file mapping"); + } + + if (m_fileHandle != INVALID_HANDLE_VALUE) + { + [[maybe_unused]] + const BOOL closedFile = CloseHandle(m_fileHandle); + CRITICAL_ASSERT(SystemAssertHandler, closedFile, "Failed to close the file"); + } +#elif defined BIGFOOT_LINUX + if (m_data != nullptr) + { + [[maybe_unused]] + const int unmapped = munmap(m_data, m_size); + CRITICAL_ASSERT(SystemAssertHandler, unmapped == 0, "Failed to unmap the file"); + } + + if (m_fileDescriptor != -1) + { + [[maybe_unused]] + const int closedFile = close(m_fileDescriptor); + CRITICAL_ASSERT(SystemAssertHandler, closedFile == 0, "Failed to close the file"); + } +#else + CRITICAL_ASSERT(SystemAssertHandler, false, "MemoryMappedFileBase is not implemented on this platform"); +#endif +} + +/****************************************************************************************/ + +eastl::span MemoryMappedFile::GetData() const +{ + return eastl::span {m_data, m_size}; +} + +/****************************************************************************************/ + +WritableMemoryMappedFile::WritableMemoryMappedFile(const File& p_file, const std::size_t p_size): + m_size(p_size) +{ + const File absoluteFile = p_file.ToAbsolute(); + const eastl::string_view path = absoluteFile.GetPath(); + +#if defined BIGFOOT_WINDOWS + m_fileHandle = + CreateFileA(path.data(), GENERIC_READ | GENERIC_WRITE, 0, nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); + CRITICAL_ASSERT(SystemAssertHandler, m_fileHandle != INVALID_HANDLE_VALUE, "Failed to create file {}", path); + + const LARGE_INTEGER size {.QuadPart = static_cast(p_size)}; + m_mappingHandle = CreateFileMappingA(m_fileHandle, nullptr, PAGE_READWRITE, size.HighPart, size.LowPart, nullptr); + CRITICAL_ASSERT(SystemAssertHandler, m_mappingHandle != nullptr, "Failed create the file mapping {}", path); + + m_data = static_cast(MapViewOfFile(m_mappingHandle, FILE_MAP_WRITE, 0, 0, 0)); + CRITICAL_ASSERT(SystemAssertHandler, m_data != nullptr, "Failed to map the file {}", path); +#elif defined BIGFOOT_LINUX + m_fileDescriptor = open(path.data(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + CRITICAL_ASSERT(SystemAssertHandler, m_fileDescriptor != -1, "Failed to create file {}", path); + + [[maybe_unused]] + const int truncateResult = ftruncate(m_fileDescriptor, static_cast(p_size)); + CRITICAL_ASSERT(SystemAssertHandler, truncateResult == 0, "Failed to size file {}", path); + + void* const mapped = mmap(nullptr, m_size, PROT_READ | PROT_WRITE, MAP_SHARED, m_fileDescriptor, 0); + CRITICAL_ASSERT(SystemAssertHandler, mapped != MAP_FAILED, "Failed to map file {}", path); + m_data = static_cast(mapped); +#else + CRITICAL_ASSERT(SystemAssertHandler, false, "MemoryMappedFileBase is not implemented on this platform"); +#endif +} + +/****************************************************************************************/ + +WritableMemoryMappedFile::~WritableMemoryMappedFile() +{ +#if defined BIGFOOT_WINDOWS + if (m_data != nullptr) + { + [[maybe_unused]] + const BOOL unmapped = UnmapViewOfFile(m_data); + CRITICAL_ASSERT(SystemAssertHandler, unmapped, "Failed to unmap the file"); + } + + if (m_mappingHandle != nullptr) + { + [[maybe_unused]] + const BOOL closedMapping = CloseHandle(m_mappingHandle); + CRITICAL_ASSERT(SystemAssertHandler, closedMapping, "Failed to close the file mapping"); + } + + if (m_fileHandle != INVALID_HANDLE_VALUE) + { + [[maybe_unused]] + const BOOL closedFile = CloseHandle(m_fileHandle); + CRITICAL_ASSERT(SystemAssertHandler, closedFile, "Failed to close the file"); + } +#elif defined BIGFOOT_LINUX + if (m_data != nullptr) + { + [[maybe_unused]] + const int unmapped = munmap(m_data, m_size); + CRITICAL_ASSERT(SystemAssertHandler, unmapped == 0, "Failed to unmap the file"); + } + + if (m_fileDescriptor != -1) + { + [[maybe_unused]] + const int closedFile = close(m_fileDescriptor); + CRITICAL_ASSERT(SystemAssertHandler, closedFile == 0, "Failed to close the file"); + } +#else + CRITICAL_ASSERT(SystemAssertHandler, false, "MemoryMappedFileBase is not implemented on this platform"); +#endif +} + +/****************************************************************************************/ + +eastl::span WritableMemoryMappedFile::GetData() const +{ + return eastl::span {m_data, m_size}; +} + +/****************************************************************************************/ + +void WritableMemoryMappedFile::Prefetch(const std::size_t p_offset, const std::size_t p_size) const +{ +#if defined BIGFOOT_WINDOWS + WIN32_MEMORY_RANGE_ENTRY range {.VirtualAddress = m_data + p_offset, .NumberOfBytes = p_size}; + [[maybe_unused]] + const BOOL prefetched = PrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0); +#elif defined BIGFOOT_LINUX + [[maybe_unused]] + const int adviseResult = madvise(m_data + p_offset, p_size, MADV_WILLNEED); +#else + CRITICAL_ASSERT(SystemAssertHandler, false, "MemoryMappedFileBase is not implemented on this platform"); +#endif +} +} // namespace Bigfoot diff --git a/Bigfoot/Sources/System/Include/System/File.hpp b/Bigfoot/Sources/System/Include/System/File/File.hpp similarity index 66% rename from Bigfoot/Sources/System/Include/System/File.hpp rename to Bigfoot/Sources/System/Include/System/File/File.hpp index c6c513a..5ef2d42 100644 --- a/Bigfoot/Sources/System/Include/System/File.hpp +++ b/Bigfoot/Sources/System/Include/System/File/File.hpp @@ -7,8 +7,10 @@ #ifndef BIGFOOT_SYSTEM_FILE_HPP #define BIGFOOT_SYSTEM_FILE_HPP +#include #include #include +#include #include @@ -72,6 +74,38 @@ class File [[nodiscard]] File ToAbsolute() const; + /** + * Get the size of the file + * + * \return The size in bytes + */ + [[nodiscard]] + std::size_t GetSize() const; + + /** + * Read bytes from the file + * + * \param p_offset The offset in bytes + * \param p_size The number of bytes to read + * \return The bytes read + */ + [[nodiscard]] + eastl::vector Read(std::size_t p_offset, std::size_t p_size) const; + + /** + * Write bytes to the file at a given offset, creating the file if needed. Content before + * p_offset, or from past p_offset + p_data.size(), is left untouched. + * + * \param p_data The bytes to write + * \param p_offset The offset, in bytes, to start writing at + */ + void Write(eastl::span p_data, std::size_t p_offset) const; + + /** + * Delete the file + */ + void Delete() const; + private: /** * Constructor diff --git a/Bigfoot/Sources/System/Include/System/File/MemoryMappedFile.hpp b/Bigfoot/Sources/System/Include/System/File/MemoryMappedFile.hpp new file mode 100644 index 0000000..84576f2 --- /dev/null +++ b/Bigfoot/Sources/System/Include/System/File/MemoryMappedFile.hpp @@ -0,0 +1,150 @@ +/********************************************************************* + * \file MemoryMappedFile.hpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#ifndef BIGFOOT_SYSTEM_MEMORYMAPPEDFILE_HPP +#define BIGFOOT_SYSTEM_MEMORYMAPPEDFILE_HPP +#include + +#include + +#include +#include + +namespace Bigfoot +{ +class MemoryMappedFile +{ + public: + /** + * Constructor + * Opens and maps the whole file + * + * \param p_file The file to map + */ + explicit MemoryMappedFile(const File& p_file); + + MemoryMappedFile(const MemoryMappedFile& p_file) = delete; + MemoryMappedFile(MemoryMappedFile&& p_file) = delete; + + /** + * Destructor + * + */ + ~MemoryMappedFile(); + + /** + * Get the mapped content of the file + * + * \return The mapped bytes + */ + [[nodiscard]] + eastl::span GetData() const; + + MemoryMappedFile& operator=(const MemoryMappedFile& p_file) = delete; + MemoryMappedFile& operator=(MemoryMappedFile&& p_file) = delete; + + private: +#if defined BIGFOOT_WINDOWS + /** + * Handle to the file + */ + void* m_fileHandle = nullptr; + + /** + * Handle to the file mapping + */ + void* m_mappingHandle = nullptr; +#elif defined BIGFOOT_LINUX + /** + * Handle to the file + */ + int m_fileDescriptor = -1; +#endif + + /** + * The mapped size + */ + std::size_t m_size = 0; + + /** + * The mapped data + */ + std::byte* m_data = nullptr; +}; + +/****************************************************************************************/ + +class WritableMemoryMappedFile +{ + public: + /** + * Constructor + * Creates and maps the file + * + * \param p_file The file to create and map + * \param p_size The size to map + */ + WritableMemoryMappedFile(const File& p_file, std::size_t p_size); + + WritableMemoryMappedFile(const WritableMemoryMappedFile& p_file) = delete; + WritableMemoryMappedFile(WritableMemoryMappedFile&& p_file) = delete; + + /** + * Destructor + * + */ + ~WritableMemoryMappedFile(); + + /** + * Get the mapped content of the file + * + * \return The mapped bytes + */ + [[nodiscard]] + eastl::span GetData() const; + + /** + * Hint the OS to prefetch a range of data, so writes into that range do not each pay a page fault + * + * \param p_offset The offset, in bytes, to start prefetching from + * \param p_size The number of bytes to prefetch + */ + void Prefetch(std::size_t p_offset, std::size_t p_size) const; + + WritableMemoryMappedFile& operator=(const WritableMemoryMappedFile& p_file) = delete; + WritableMemoryMappedFile& operator=(WritableMemoryMappedFile&& p_file) = delete; + + private: +#if defined BIGFOOT_WINDOWS + /** + * Handle to the file + */ + void* m_fileHandle = nullptr; + + /** + * Handle to the file mapping + */ + void* m_mappingHandle = nullptr; +#elif defined BIGFOOT_LINUX + /** + * Handle to the file + */ + int m_fileDescriptor = -1; +#endif + + /** + * The mapped size + */ + std::size_t m_size = 0; + + /** + * The mapped data + */ + std::byte* m_data = nullptr; +}; +} // namespace Bigfoot + +#endif diff --git a/Bigfoot/Tests/Engine/BigFile/BigFile.cpp b/Bigfoot/Tests/Engine/BigFile/BigFile.cpp index 3bf5099..0ed7eb3 100644 --- a/Bigfoot/Tests/Engine/BigFile/BigFile.cpp +++ b/Bigfoot/Tests/Engine/BigFile/BigFile.cpp @@ -6,7 +6,7 @@ *********************************************************************/ #include -#include +#include #include diff --git a/Bigfoot/Tests/Engine/Include/EngineTests/BigFileInfo_generated.hpp b/Bigfoot/Tests/Engine/Include/EngineTests/BigFileInfo_generated.hpp index a1f0510..1ea83d2 100644 --- a/Bigfoot/Tests/Engine/Include/EngineTests/BigFileInfo_generated.hpp +++ b/Bigfoot/Tests/Engine/Include/EngineTests/BigFileInfo_generated.hpp @@ -9,6 +9,8 @@ #include +#if defined BIGFOOT_TOOLMODE + namespace Bigfoot { /* @@ -17,3 +19,4 @@ namespace Bigfoot constexpr eastl::string_view BIGFILE_ENGINETESTS_LOCATION {"./EngineTests-bigfile.db"}; } // namespace Bigfoot #endif +#endif diff --git a/Bigfoot/Tests/System/File.cpp b/Bigfoot/Tests/System/File.cpp deleted file mode 100644 index a0c8983..0000000 --- a/Bigfoot/Tests/System/File.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/********************************************************************* - * \file File.cpp - * - * \author Romain BOULLARD - * \date December 2025 - *********************************************************************/ -#include - -#include - -namespace Bigfoot -{ -class FileFixture: public ::testing::Test -{ - public: - File m_file {eastl::string_view {"Fixture/file"}}; - File m_nonExistent {eastl::string_view {"Fixture/bigfoot"}}; -}; - -/****************************************************************************************/ - -TEST_F(FileFixture, IsRelative_ShouldReturnTrueOnRelativeFile) -{ - EXPECT_TRUE(m_file.IsRelative()); -} - -/****************************************************************************************/ - -TEST_F(FileFixture, IsRelative_ShouldReturnFalseOnAbsoluteFile) -{ - EXPECT_FALSE(m_file.ToAbsolute().IsRelative()); -} - -/****************************************************************************************/ - -TEST_F(FileFixture, IsAbsolute_ShouldReturnTrueOnAbsoluteFile) -{ - EXPECT_TRUE(m_file.ToAbsolute().IsAbsolute()); -} - -TEST_F(FileFixture, IsAbsolute_ShouldReturnFalseOnRelativeFile) -{ - EXPECT_FALSE(m_file.IsAbsolute()); -} - -/****************************************************************************************/ - -TEST_F(FileFixture, Exists_ShouldReturnTrueOnExistingFile) -{ - EXPECT_TRUE(m_file.Exists()); -} - -/****************************************************************************************/ - -TEST_F(FileFixture, Exists_ShouldReturnFalseOnNonExistingFile) -{ - EXPECT_FALSE(m_nonExistent.Exists()); -} - -/****************************************************************************************/ - -TEST_F(FileFixture, GetPath_ShouldReturnThePath) -{ - EXPECT_STREQ(m_file.GetPath().data(), "Fixture/file"); -} - -/****************************************************************************************/ - -TEST_F(FileFixture, ToAbsolute_ShouldReturnTheAbsolutePath) -{ - EXPECT_STREQ(std::filesystem::absolute("Fixture/file").string().c_str(), m_file.ToAbsolute().GetPath().data()); -} - -/****************************************************************************************/ - -TEST_F(FileFixture, ToRelative_ShouldReturnTheRelativePath) -{ - EXPECT_STREQ(std::filesystem::relative("Fixture/file").string().c_str(), m_file.ToRelative().GetPath().data()); -} -} // namespace Bigfoot diff --git a/Bigfoot/Tests/System/File/File.cpp b/Bigfoot/Tests/System/File/File.cpp new file mode 100644 index 0000000..327e461 --- /dev/null +++ b/Bigfoot/Tests/System/File/File.cpp @@ -0,0 +1,194 @@ +/********************************************************************* + * \file File.cpp + * + * \author Romain BOULLARD + * \date December 2025 + *********************************************************************/ +#include + +#include + +#include +#include +#include + +namespace Bigfoot +{ +class FileFixture: public ::testing::Test +{ + public: + File m_file {eastl::string_view {"Fixture/file"}}; + File m_nonExistent {eastl::string_view {"Fixture/bigfoot"}}; + File m_writeScratchFile {eastl::string_view {"Fixture/scratch"}}; + + protected: + void SetUp() override + { + m_writeScratchFile.Delete(); + } +}; + +/****************************************************************************************/ + +TEST_F(FileFixture, IsRelative_ShouldReturnTrueOnRelativeFile) +{ + EXPECT_TRUE(m_file.IsRelative()); +} + +/****************************************************************************************/ + +TEST_F(FileFixture, IsRelative_ShouldReturnFalseOnAbsoluteFile) +{ + EXPECT_FALSE(m_file.ToAbsolute().IsRelative()); +} + +/****************************************************************************************/ + +TEST_F(FileFixture, IsAbsolute_ShouldReturnTrueOnAbsoluteFile) +{ + EXPECT_TRUE(m_file.ToAbsolute().IsAbsolute()); +} + +/****************************************************************************************/ + +TEST_F(FileFixture, IsAbsolute_ShouldReturnFalseOnRelativeFile) +{ + EXPECT_FALSE(m_file.IsAbsolute()); +} + +/****************************************************************************************/ + +TEST_F(FileFixture, Exists_ShouldReturnTrueOnExistingFile) +{ + EXPECT_TRUE(m_file.Exists()); +} + +/****************************************************************************************/ + +TEST_F(FileFixture, Exists_ShouldReturnFalseOnNonExistingFile) +{ + EXPECT_FALSE(m_nonExistent.Exists()); +} + +/****************************************************************************************/ + +TEST_F(FileFixture, GetPath_ShouldReturnThePath) +{ + EXPECT_STREQ(m_file.GetPath().data(), "Fixture/file"); +} + +/****************************************************************************************/ + +TEST_F(FileFixture, ToAbsolute_ShouldReturnTheAbsolutePath) +{ + EXPECT_STREQ(std::filesystem::absolute("Fixture/file").string().c_str(), m_file.ToAbsolute().GetPath().data()); +} + +/****************************************************************************************/ + +TEST_F(FileFixture, ToRelative_ShouldReturnTheRelativePath) +{ + EXPECT_STREQ(std::filesystem::relative("Fixture/file").string().c_str(), m_file.ToRelative().GetPath().data()); +} + +/****************************************************************************************/ + +TEST_F(FileFixture, GetSize_ShouldReturnZeroForAnEmptyFile) +{ + EXPECT_EQ(m_file.GetSize(), 0); +} + +/****************************************************************************************/ + +TEST_F(FileFixture, GetSize_ShouldReturnTheFileSize) +{ + constexpr eastl::string_view content = "Hello, Sasquatch!"; + m_writeScratchFile.Write({std::bit_cast(content.data()), content.size()}, 0); + + EXPECT_EQ(m_writeScratchFile.GetSize(), content.size()); +} + +/****************************************************************************************/ + +TEST_F(FileFixture, Read_ShouldReturnTheRequestedBytesAtTheGivenOffset) +{ + constexpr eastl::string_view content = "Hello, Sasquatch!"; + m_writeScratchFile.Write({std::bit_cast(content.data()), content.size()}, 0); + + const eastl::vector data = m_writeScratchFile.Read(7, 10); + ASSERT_EQ(data.size(), 10); + EXPECT_EQ(std::memcmp(data.data(), "Sasquatch!", data.size()), 0); +} + +/****************************************************************************************/ + +TEST_F(FileFixture, Write_ShouldCreateTheFileWithTheGivenContent) +{ + constexpr eastl::string_view content = "Hello, Sasquatch!"; + m_writeScratchFile.Write({std::bit_cast(content.data()), content.size()}, 0); + + const eastl::vector data = m_writeScratchFile.Read(0, m_writeScratchFile.GetSize()); + ASSERT_EQ(data.size(), content.size()); + EXPECT_EQ(std::memcmp(data.data(), content.data(), data.size()), 0); +} + +/****************************************************************************************/ + +TEST_F(FileFixture, Write_ShouldNotTouchContentOutsideTheWrittenRange) +{ + constexpr eastl::string_view initial = "Hello, Sasquatch!"; + m_writeScratchFile.Write({std::bit_cast(initial.data()), initial.size()}, 0); + + constexpr eastl::string_view patch = "BIGFOOT"; + m_writeScratchFile.Write({std::bit_cast(patch.data()), patch.size()}, 7); + + const eastl::vector data = m_writeScratchFile.Read(0, m_writeScratchFile.GetSize()); + ASSERT_EQ(data.size(), initial.size()); + EXPECT_EQ(std::memcmp(data.data(), "Hello, ", 7), 0); + EXPECT_EQ(std::memcmp(data.data() + 7, "BIGFOOT", 7), 0); + EXPECT_EQ(std::memcmp(data.data() + 14, "ch!", 3), 0); +} + +/****************************************************************************************/ + +TEST_F(FileFixture, Write_ShouldRoundTripArbitraryData) +{ + struct Payload + { + int m_intValue; + double m_doubleValue; + std::array m_tag; + + bool operator==(const Payload&) const = default; + }; + + constexpr Payload payload {.m_intValue = 42, .m_doubleValue = 3.14, .m_tag = {'B', 'I', 'G', 'F'}}; + m_writeScratchFile.Write({std::bit_cast(&payload), sizeof(payload)}, 0); + + const eastl::vector data = m_writeScratchFile.Read(0, m_writeScratchFile.GetSize()); + ASSERT_EQ(data.size(), sizeof(Payload)); + EXPECT_EQ(*std::bit_cast(data.data()), payload); +} + +/****************************************************************************************/ + +TEST_F(FileFixture, Delete_ShouldRemoveTheFile) +{ + constexpr std::byte payload {1}; + + m_writeScratchFile.Write({&payload, 1}, 0); + ASSERT_TRUE(m_writeScratchFile.Exists()); + + m_writeScratchFile.Delete(); + EXPECT_FALSE(m_writeScratchFile.Exists()); +} + +/****************************************************************************************/ + +TEST_F(FileFixture, Delete_ShouldDoNothingWhenTheFileDoesNotExist) +{ + ASSERT_FALSE(m_writeScratchFile.Exists()); + m_writeScratchFile.Delete(); + EXPECT_FALSE(m_writeScratchFile.Exists()); +} +} // namespace Bigfoot diff --git a/Bigfoot/Tests/System/File/MemoryMappedFile.cpp b/Bigfoot/Tests/System/File/MemoryMappedFile.cpp new file mode 100644 index 0000000..ac80082 --- /dev/null +++ b/Bigfoot/Tests/System/File/MemoryMappedFile.cpp @@ -0,0 +1,92 @@ +/********************************************************************* + * \file MemoryMappedFile.cpp + * + * \author Romain BOULLARD + * \date August 2026 + *********************************************************************/ +#include + +#include +#include + +#include +#include + +namespace Bigfoot +{ +class MemoryMappedFileFixture: public ::testing::Test +{ + protected: + MemoryMappedFile m_file {File {eastl::string_view {"Fixture/mappedFile"}}}; + MemoryMappedFile m_emptyFile {File {eastl::string_view {"Fixture/file"}}}; +}; + +/****************************************************************************************/ + +TEST_F(MemoryMappedFileFixture, GetData_ShouldReturnTheFileContent) +{ + const eastl::span data = m_file.GetData(); + ASSERT_EQ(data.size(), std::strlen("Hello, Sasquatch!")); + EXPECT_EQ(std::memcmp(data.data(), "Hello, Sasquatch!", data.size()), 0); +} + +/****************************************************************************************/ + +TEST_F(MemoryMappedFileFixture, GetData_ShouldReturnAnEmptySpanForAnEmptyFile) +{ + EXPECT_TRUE(m_emptyFile.GetData().empty()); +} + +/****************************************************************************************/ + +class WritableMemoryMappedFileFixture: public ::testing::Test +{ + protected: + static constexpr eastl::string_view LOCATION = "Fixture/WritableMemoryMappedFileTests-scratch"; + + void SetUp() override + { + File file {LOCATION}; + file.Delete(); + } +}; + +/****************************************************************************************/ + +TEST_F(WritableMemoryMappedFileFixture, GetData_ShouldReturnASpanOfTheRequestedSize) +{ + WritableMemoryMappedFile file {File {LOCATION}, 17}; + EXPECT_EQ(file.GetData().size(), 17); +} + +/****************************************************************************************/ + +TEST_F(WritableMemoryMappedFileFixture, Write_ShouldWriteToTheFile) +{ + { + WritableMemoryMappedFile file {File {LOCATION}, std::strlen("Hello, Sasquatch!")}; + std::memcpy(file.GetData().data(), "Hello, Sasquatch!", file.GetData().size()); + } + + const MemoryMappedFile readBack {File {LOCATION}}; + ASSERT_EQ(readBack.GetData().size(), std::strlen("Hello, Sasquatch!")); + EXPECT_EQ(std::memcmp(readBack.GetData().data(), "Hello, Sasquatch!", readBack.GetData().size()), 0); +} + +/****************************************************************************************/ + +TEST_F(WritableMemoryMappedFileFixture, Constructor_ShouldNotTruncateAnyExistingContent) +{ + { + WritableMemoryMappedFile file {File {LOCATION}, 100}; + std::memset(file.GetData().data(), 0xAB, file.GetData().size()); + } + + const WritableMemoryMappedFile file {File {LOCATION}, 4}; + constexpr std::array reference {std::byte {0xAB}, + std::byte {0xAB}, + std::byte {0xAB}, + std::byte {0xAB}}; + EXPECT_EQ(std::memcmp(file.GetData().data(), reference.data(), reference.size()), 0); +} +} // namespace Bigfoot diff --git a/Bigfoot/Tests/System/Fixture/mappedFile b/Bigfoot/Tests/System/Fixture/mappedFile new file mode 100644 index 0000000..b8efbe4 --- /dev/null +++ b/Bigfoot/Tests/System/Fixture/mappedFile @@ -0,0 +1 @@ +Hello, Sasquatch! \ No newline at end of file diff --git a/Bigfoot/Tests/Tools/SasquatchBaker/Include/SasquatchBakerTests/BigFileInfo_generated.hpp b/Bigfoot/Tests/Tools/SasquatchBaker/Include/SasquatchBakerTests/BigFileInfo_generated.hpp new file mode 100644 index 0000000..ed9ee3c --- /dev/null +++ b/Bigfoot/Tests/Tools/SasquatchBaker/Include/SasquatchBakerTests/BigFileInfo_generated.hpp @@ -0,0 +1,22 @@ +// AUTO-GENERATED DO NOT TOUCH + +/********************************************************************* + * \file BigFileInfo.generated.hpp + * + *********************************************************************/ +#ifndef BIGFOOT_SASQUATCHBAKERTESTS_BIGFILEINFO_GENERATED_HPP +#define BIGFOOT_SASQUATCHBAKERTESTS_BIGFILEINFO_GENERATED_HPP + +#include + +#if defined BIGFOOT_TOOLMODE + +namespace Bigfoot +{ +/* + * BigFile location + */ +constexpr eastl::string_view BIGFILE_SASQUATCHBAKERTESTS_LOCATION {"./SasquatchBakerTests-bigfile.db"}; +} // namespace Bigfoot +#endif +#endif