From 41b6807f127526e89ede0fb40748d6ae988fc0cc Mon Sep 17 00:00:00 2001 From: lizzie Date: Sun, 1 Nov 2026 21:51:15 +0100 Subject: [PATCH] [fs] use mmap() to read files using VM/pagefaults for higher throughput (#3151) main idea is basically to manually manage the file backed storage ourselves basically the libc will use pooled workes who will read off sync data and may or may not use mmap, this detail can vary between libcs, so id rather have one unified concept of "paged file" as opposed to relying on the libc doing what it should the main benefit is that one doesnt need to call fwrite/fread or whatever, they can just do normal paged file ops and such this pr has seen changes from mapped flags as well, as some OSes; particularly android, do not enjoy being told to make hugepages that are 1GB in size, especially for lower end devices. Signed-off-by: lizzie Co-authored-by: Caio Oliveira Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/3151 Reviewed-by: MaranBr Reviewed-by: crueter --- src/common/fs/file.cpp | 438 ++++++++++++++++++++++------------- src/common/fs/file.h | 502 +++++++++++++++++++++-------------------- 2 files changed, 535 insertions(+), 405 deletions(-) diff --git a/src/common/fs/file.cpp b/src/common/fs/file.cpp index 461a5eee35..7a0f69a84d 100644 --- a/src/common/fs/file.cpp +++ b/src/common/fs/file.cpp @@ -7,18 +7,25 @@ #include #include "common/assert.h" +#include "common/bit_util.h" #include "common/fs/file.h" #include "common/fs/fs.h" +#include "common/fs/fs_types.h" #ifdef __ANDROID__ #include "common/fs/fs_android.h" #endif #include "common/logging.h" +#include "common/literals.h" #ifdef _WIN32 #include #include +#include #else #include +#include +#include +#include #endif #ifdef _MSC_VER @@ -36,13 +43,13 @@ namespace { #ifdef _WIN32 /** - * Converts the file access mode and file type enums to a file access mode wide string. - * - * @param mode File access mode - * @param type File type - * - * @returns A pointer to a wide string representing the file access mode. - */ +* Converts the file access mode and file type enums to a file access mode wide string. +* +* @param mode File access mode +* @param type File type +* +* @returns A pointer to a wide string representing the file access mode. +*/ [[nodiscard]] constexpr const wchar_t* AccessModeToWStr(FileAccessMode mode, FileType type) { switch (type) { case FileType::BinaryFile: @@ -79,12 +86,12 @@ namespace { } /** - * Converts the file-share access flag enum to a Windows defined file-share access flag. - * - * @param flag File-share access flag - * - * @returns Windows defined file-share access flag. - */ +* Converts the file-share access flag enum to a Windows defined file-share access flag. +* +* @param flag File-share access flag +* +* @returns Windows defined file-share access flag. +*/ [[nodiscard]] constexpr int ToWindowsFileShareFlag(FileShareFlag flag) { switch (flag) { case FileShareFlag::ShareNone: @@ -102,13 +109,13 @@ namespace { #else /** - * Converts the file access mode and file type enums to a file access mode string. - * - * @param mode File access mode - * @param type File type - * - * @returns A pointer to a string representing the file access mode. - */ +* Converts the file access mode and file type enums to a file access mode string. +* +* @param mode File access mode +* @param type File type +* +* @returns A pointer to a string representing the file access mode. +*/ [[nodiscard]] constexpr const char* AccessModeToStr(FileAccessMode mode, FileType type) { switch (type) { case FileType::BinaryFile: @@ -147,12 +154,12 @@ namespace { #endif /** - * Converts the seek origin enum to a seek origin integer. - * - * @param origin Seek origin - * - * @returns Seek origin integer. - */ +* Converts the seek origin enum to a seek origin integer. +* +* @param origin Seek origin +* +* @returns Seek origin integer. +*/ [[nodiscard]] constexpr int ToSeekOrigin(SeekOrigin origin) { switch (origin) { case SeekOrigin::SetOrigin: @@ -178,7 +185,7 @@ std::string ReadStringFromFile(const std::filesystem::path& path, FileType type) } size_t WriteStringToFile(const std::filesystem::path& path, FileType type, - std::string_view string) { + std::string_view string) { if (Exists(path) && !IsFile(path)) { return 0; } @@ -189,7 +196,7 @@ size_t WriteStringToFile(const std::filesystem::path& path, FileType type, } size_t AppendStringToFile(const std::filesystem::path& path, FileType type, - std::string_view string) { + std::string_view string) { if (Exists(path) && !IsFile(path)) { return 0; } @@ -244,69 +251,189 @@ FileType IOFile::GetType() const { return file_type; } +#if defined(__unix__) +static int PlatformMapReadOnly(IOFile& io, const char* path) { + io.mmap_fd = open(path, O_RDONLY); + if (io.mmap_fd > 0) { + struct stat st; + fstat(io.mmap_fd, &st); + io.mmap_size = st.st_size; + + int map_flags = MAP_PRIVATE; +#ifdef MAP_PREFAULT_READ + // Prefaults reads so the final resulting pagetable from this big stupid mmap() + // isn't comically lazily loaded, we just coalesce everything in-place for our + // lovely mmap flags; if we didn't prefault the reads the page table will be + // constructed in-place (i.e on a read-by-read basis) causing lovely soft-faults + // which would nuke any performance gains. + // + // This of course incurs a cost in the initial mmap(2) call, but that is fine. + map_flags |= MAP_PREFAULT_READ; +#endif +#ifdef MAP_NOSYNC + // This causes physical media to not be synched to our file/memory + // This means that if the read-only file is written to, we won't see changes + // or we may see changes which are just funnily scattered, in any case + // this presumes the files won't be changed during execution + // + // Do not ever use this on write files (if we ever support that); this will create + // a fun amount of fragmentation on the disk. + map_flags |= MAP_NOSYNC; +#endif +#if defined(NDEBUG) && defined(MAP_NOCORE) + map_flags |= MAP_NOCORE; +#endif +#ifdef MAP_HUGE_2MB + map_flags |= MAP_HUGE_2MB; //2mb pages +#elif defined(MAP_ALIGNED) + // File must be big enough that it's worth to super align. We can't just super-align every + // file otherwise we will run out of alignments for actually important files :) + // System doesn't guarantee a super alignment, but if it's available it will delete + // about 3 layers(?) of the TLB tree for each read/write. + // Again the cost of faults may make this negligible gains, but hey, we gotta work + // what we gotta work with. + map_flags |= MAP_ALIGNED(21); //2^21 = 2mb use 2MB pages +#endif + io.mmap_base = (u8*)mmap(nullptr, io.mmap_size, PROT_READ, map_flags, io.mmap_fd, 0); + if (io.mmap_base == MAP_FAILED) { + close(io.mmap_fd); + io.mmap_fd = -1; + } else { + using namespace Common::Literals; + // For small files it is acceptable to use a full readahead + // See https://github.com/torvalds/linux/blob/e80d033851b3bc94c3d254ac66660ddd0a49d72c/include/linux/pagemap.h#L1392 + if (u64(st.st_size) >= 256_MiB) { + posix_madvise(io.mmap_base, io.mmap_size, POSIX_MADV_RANDOM); + } else { + posix_madvise(io.mmap_base, io.mmap_size, POSIX_MADV_SEQUENTIAL); + } + } + } + return io.mmap_fd; +} +static void PlatformUnmap(IOFile& io) { + if (io.mmap_fd != -1) { + munmap(io.mmap_base, io.mmap_size); + close(io.mmap_fd); + io.mmap_fd = -1; + } +} +#elif defined(__APPLE__) +// NO IMPLEMENTATION YET +#else +static int PlatformMapReadOnly(IOFile& io, const wchar_t* path) { + io.file_handle = CreateFileW(path, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, nullptr); + if (HANDLE(io.file_handle) != INVALID_HANDLE_VALUE) { + io.mapping_handle = CreateFileMappingW(HANDLE(io.file_handle), nullptr, PAGE_READONLY, 0, 0, nullptr); + if (io.mapping_handle) { + io.mmap_base = (u8*)MapViewOfFile(HANDLE(io.mapping_handle), FILE_MAP_READ, 0, 0, 0); + if (io.mmap_base) { + _LARGE_INTEGER pvalue; + GetFileSizeEx(io.file_handle, &pvalue); + io.mmap_size = uint32_t(pvalue.QuadPart); + } else { + CloseHandle(io.mapping_handle); + CloseHandle(io.file_handle); + return -1; + } + } else { + CloseHandle(io.file_handle); + return -1; + } + } + return 0; +} +static void PlatformUnmap(IOFile& io) { + if(io.mapping_handle) { + if(io.mmap_base) + UnmapViewOfFile(HANDLE(io.mmap_base)); + CloseHandle(HANDLE(io.mapping_handle)); + } + if(io.file_handle != INVALID_HANDLE_VALUE) + CloseHandle(HANDLE(io.file_handle)); +} +#endif + void IOFile::Open(const fs::path& path, FileAccessMode mode, FileType type, FileShareFlag flag) { Close(); - file_path = path; file_access_mode = mode; file_type = type; - errno = 0; - #ifdef _WIN32 - if (flag != FileShareFlag::ShareNone) { - file = _wfsopen(path.c_str(), AccessModeToWStr(mode, type), ToWindowsFileShareFlag(flag)); - } else { - _wfopen_s(&file, path.c_str(), AccessModeToWStr(mode, type)); + // TODO: this probably can use better logic but oh well I'm not a windowser + file_handle = nullptr; + if (type == FileType::BinaryFile && mode == FileAccessMode::Read) { + if (PlatformMapReadOnly(*this, path.c_str()) == -1) { + LOG_ERROR(Common_Filesystem, "Error mmap'ing file"); //: {}", path.c_str()); + } + } + if (file_handle == nullptr) { + if (flag != FileShareFlag::ShareNone) { + file = _wfsopen(path.c_str(), AccessModeToWStr(mode, type), ToWindowsFileShareFlag(flag)); + } else { + _wfopen_s(&file, path.c_str(), AccessModeToWStr(mode, type)); + } } #elif __ANDROID__ if (Android::IsContentUri(path)) { ASSERT_MSG(mode == FileAccessMode::Read, "Content URI file access is for read-only!"); - const auto fd = Android::OpenContentUri(path, Android::OpenMode::Read); - if (fd != -1) { - file = fdopen(fd, "r"); - const auto error_num = errno; - if (error_num != 0 && file == nullptr) { - LOG_ERROR(Common_Filesystem, "Error opening file: {}, error: {}", path.c_str(), - strerror(error_num)); + if (PlatformMapReadOnly(*this, path.c_str()) == -1) { + LOG_ERROR(Common_Filesystem, "Error mmap'ing file: {}", path.c_str()); + int const fd = Android::OpenContentUri(path, Android::OpenMode::Read); + if (fd != -1) { + file = fdopen(fd, "r"); + if (errno != 0 && file == nullptr) + LOG_ERROR(Common_Filesystem, "Error opening file: {}, error: {}", path.c_str(), strerror(errno)); + } else { + LOG_ERROR(Common_Filesystem, "Error opening file: {}", path.c_str()); } - } else { - LOG_ERROR(Common_Filesystem, "Error opening file: {}", path.c_str()); } } else { file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); } +#elif defined(__HAIKU__) || defined(__managarm__) || defined(__OPENORBIS__) || defined(__APPLE__) + file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); +#elif defined(__unix__) + if (type == FileType::BinaryFile && mode == FileAccessMode::Read) { + if (PlatformMapReadOnly(*this, path.c_str()) == -1) { + LOG_ERROR(Common_Filesystem, "Error mmap'ing file: {}", path.c_str()); + } + } + if (mmap_fd == -1) { + file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); // mmap(2) failed or simply we can't use it + } #else + // Some other fancy OS (ahem fucking Darwin/Mac OSX) file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); #endif - if (!IsOpen()) { const auto ec = std::error_code{errno, std::generic_category()}; LOG_ERROR(Common_Filesystem, "Failed to open the file at path={}, ec_message={}", - PathToUTF8String(file_path), ec.message()); + PathToUTF8String(file_path), ec.message()); } } void IOFile::Close() { - if (!IsOpen()) { - return; +#if defined(__APPLE__) + // NO IMPLEMENTATION YET +#else + PlatformUnmap(*this); +#endif + if (file) { + errno = 0; + const auto close_result = std::fclose(file) == 0; + if (!close_result) { + const auto ec = std::error_code{errno, std::generic_category()}; + LOG_ERROR(Common_Filesystem, "Failed to close the file at path={}, ec_message={}", + PathToUTF8String(file_path), ec.message()); + } + file = nullptr; } - - errno = 0; - - const auto close_result = std::fclose(file) == 0; - - if (!close_result) { - const auto ec = std::error_code{errno, std::generic_category()}; - LOG_ERROR(Common_Filesystem, "Failed to close the file at path={}, ec_message={}", - PathToUTF8String(file_path), ec.message()); - } - - file = nullptr; } bool IOFile::IsOpen() const { - return file != nullptr; + return file != nullptr || IsMappedFile(); } std::string IOFile::ReadString(size_t length) const { @@ -323,137 +450,132 @@ size_t IOFile::WriteString(std::span string) const { } bool IOFile::Flush() const { - if (!IsOpen()) { - return false; + ASSERT(!IsMappedFile()); + if (file) { + errno = 0; + auto const flush_result = std::fflush(file) == 0; + if (!flush_result) { + const auto ec = std::error_code{errno, std::generic_category()}; + LOG_ERROR(Common_Filesystem, "Failed to flush the file at path={}, ec_message={}", + PathToUTF8String(file_path), ec.message()); + } + return flush_result; } - - errno = 0; - -#ifdef _WIN32 - const auto flush_result = std::fflush(file) == 0; -#else - const auto flush_result = std::fflush(file) == 0; -#endif - - if (!flush_result) { - const auto ec = std::error_code{errno, std::generic_category()}; - LOG_ERROR(Common_Filesystem, "Failed to flush the file at path={}, ec_message={}", - PathToUTF8String(file_path), ec.message()); - } - - return flush_result; + return false; } bool IOFile::Commit() const { - if (!IsOpen()) { - return false; - } - - errno = 0; - + ASSERT(!IsMappedFile()); + if (file) { + errno = 0; #ifdef _WIN32 - const auto commit_result = std::fflush(file) == 0 && _commit(fileno(file)) == 0; + const auto commit_result = std::fflush(file) == 0 && _commit(fileno(file)) == 0; #else - const auto commit_result = std::fflush(file) == 0 && fsync(fileno(file)) == 0; + const auto commit_result = std::fflush(file) == 0 && fsync(fileno(file)) == 0; #endif - - if (!commit_result) { - const auto ec = std::error_code{errno, std::generic_category()}; - LOG_ERROR(Common_Filesystem, "Failed to commit the file at path={}, ec_message={}", - PathToUTF8String(file_path), ec.message()); + if (!commit_result) { + const auto ec = std::error_code{errno, std::generic_category()}; + LOG_ERROR(Common_Filesystem, "Failed to commit the file at path={}, ec_message={}", + PathToUTF8String(file_path), ec.message()); + } + return commit_result; } - - return commit_result; + return false; } bool IOFile::SetSize(u64 size) const { - if (!IsOpen()) { - return false; - } - - errno = 0; - + ASSERT(!IsMappedFile()); + if (file) { + errno = 0; #ifdef _WIN32 - const auto set_size_result = _chsize_s(fileno(file), static_cast(size)) == 0; + const auto set_size_result = _chsize_s(fileno(file), s64(size)) == 0; #else - const auto set_size_result = ftruncate(fileno(file), static_cast(size)) == 0; + const auto set_size_result = ftruncate(fileno(file), s64(size)) == 0; #endif - - if (!set_size_result) { - const auto ec = std::error_code{errno, std::generic_category()}; - LOG_ERROR(Common_Filesystem, "Failed to resize the file at path={}, size={}, ec_message={}", - PathToUTF8String(file_path), size, ec.message()); + if (!set_size_result) { + const auto ec = std::error_code{errno, std::generic_category()}; + LOG_ERROR(Common_Filesystem, "Failed to resize the file at path={}, size={}, ec_message={}", + PathToUTF8String(file_path), size, ec.message()); + } + return set_size_result; } - - return set_size_result; + return false; } u64 IOFile::GetSize() const { - if (!IsOpen()) { - return 0; - } + if (IsMappedFile()) + return mmap_size; + if (file) { + // Flush any unwritten buffered data into the file prior to retrieving the file mmap_size. + std::fflush(file); +#if __ANDROID__ + u64 file_size = 0; + if (Android::IsContentUri(file_path)) { + file_size = Android::GetSize(file_path); + } else { + std::error_code ec; - // Flush any unwritten buffered data into the file prior to retrieving the file size. - std::fflush(file); + file_size = fs::file_size(file_path, ec); -#ifdef __ANDROID__ - u64 file_size = 0; - if (Android::IsContentUri(file_path)) { - file_size = Android::GetSize(file_path); - } else { + if (ec) { + LOG_ERROR(Common_Filesystem, "Failed to retrieve the file mmap_size of path={}, ec_message={}", + PathToUTF8String(file_path), ec.message()); + return 0; + } + } +#else std::error_code ec; - - file_size = fs::file_size(file_path, ec); - + auto const file_size = fs::file_size(file_path, ec); if (ec) { - LOG_ERROR(Common_Filesystem, - "Failed to retrieve the file size of path={}, ec_message={}", - PathToUTF8String(file_path), ec.message()); + LOG_ERROR(Common_Filesystem, "Failed to retrieve the file mmap_size of path={}, ec_message={}", + PathToUTF8String(file_path), ec.message()); return 0; } - } -#else - std::error_code ec; - - const auto file_size = fs::file_size(file_path, ec); - - if (ec) { - LOG_ERROR(Common_Filesystem, "Failed to retrieve the file size of path={}, ec_message={}", - PathToUTF8String(file_path), ec.message()); - return 0; - } #endif - - return file_size; + return file_size; + } + return 0; } bool IOFile::Seek(s64 offset, SeekOrigin origin) const { - if (!IsOpen()) { - return false; + if (IsMappedFile()) { + // fuck you to whoever made this method const + switch (origin) { + case SeekOrigin::SetOrigin: + mmap_offset = s64(offset); + break; + case SeekOrigin::CurrentPosition: + mmap_offset += s64(offset); + break; + case SeekOrigin::End: + mmap_offset = s64(mmap_size) + s64(offset); + break; + } + return true; } - - errno = 0; - - const auto seek_result = fseeko(file, offset, ToSeekOrigin(origin)) == 0; - - if (!seek_result) { - const auto ec = std::error_code{errno, std::generic_category()}; - LOG_ERROR(Common_Filesystem, - "Failed to seek the file at path={}, offset={}, origin={}, ec_message={}", - PathToUTF8String(file_path), offset, origin, ec.message()); + if (file) { + errno = 0; + const auto seek_result = fseeko(file, offset, ToSeekOrigin(origin)) == 0; + if (!seek_result) { + const auto ec = std::error_code{errno, std::generic_category()}; + LOG_ERROR(Common_Filesystem, "Failed to seek the file at path={}, offset={}, origin={}, ec_message={}", + PathToUTF8String(file_path), offset, origin, ec.message()); + } + return seek_result; } - - return seek_result; + return false; } s64 IOFile::Tell() const { - if (!IsOpen()) { - return 0; + if (IsMappedFile()) { + errno = 0; + return s64(mmap_offset); } - - errno = 0; - - return ftello(file); + if (file) { + errno = 0; + return ftello(file); + } + return 0; } } // namespace Common::FS diff --git a/src/common/fs/file.h b/src/common/fs/file.h index b1ccfbd18a..b7e65fca62 100644 --- a/src/common/fs/file.h +++ b/src/common/fs/file.h @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include #include @@ -24,12 +25,12 @@ enum class SeekOrigin { }; /** - * Opens a file stream at path with the specified open mode. - * - * @param file_stream Reference to file stream - * @param path Filesystem path - * @param open_mode File stream open mode - */ +* Opens a file stream at path with the specified open mode. +* +* @param file_stream Reference to file stream +* @param path Filesystem path +* @param open_mode File stream open mode +*/ template void OpenFileStream(FileStream& file_stream, const std::filesystem::path& path, std::ios_base::openmode open_mode) { @@ -48,14 +49,14 @@ void OpenFileStream(FileStream& file_stream, const Path& path, std::ios_base::op #endif /** - * Reads an entire file at path and returns a string of the contents read from the file. - * If the filesystem object at path is not a regular file, this function returns an empty string. - * - * @param path Filesystem path - * @param type File type - * - * @returns A string of the contents read from the file. - */ +* Reads an entire file at path and returns a string of the contents read from the file. +* If the filesystem object at path is not a regular file, this function returns an empty string. +* +* @param path Filesystem path +* @param type File type +* +* @returns A string of the contents read from the file. +*/ [[nodiscard]] std::string ReadStringFromFile(const std::filesystem::path& path, FileType type); #ifdef _WIN32 @@ -70,18 +71,18 @@ template #endif /** - * Writes a string to a file at path and returns the number of characters successfully written. - * If a file already exists at path, its contents will be erased. - * If a file does not exist at path, it creates and opens a new empty file for writing. - * If the filesystem object at path exists and is not a regular file, this function returns 0. - * - * @param path Filesystem path - * @param type File type - * - * @returns Number of characters successfully written. - */ +* Writes a string to a file at path and returns the number of characters successfully written. +* If a file already exists at path, its contents will be erased. +* If a file does not exist at path, it creates and opens a new empty file for writing. +* If the filesystem object at path exists and is not a regular file, this function returns 0. +* +* @param path Filesystem path +* @param type File type +* +* @returns Number of characters successfully written. +*/ [[nodiscard]] size_t WriteStringToFile(const std::filesystem::path& path, FileType type, - std::string_view string); + std::string_view string); #ifdef _WIN32 template @@ -95,15 +96,15 @@ template #endif /** - * Appends a string to a file at path and returns the number of characters successfully written. - * If a file does not exist at path, it creates and opens a new empty file for appending. - * If the filesystem object at path exists and is not a regular file, this function returns 0. - * - * @param path Filesystem path - * @param type File type - * - * @returns Number of characters successfully written. - */ +* Appends a string to a file at path and returns the number of characters successfully written. +* If a file does not exist at path, it creates and opens a new empty file for appending. +* If the filesystem object at path exists and is not a regular file, this function returns 0. +* +* @param path Filesystem path +* @param type File type +* +* @returns Number of characters successfully written. +*/ [[nodiscard]] size_t AppendStringToFile(const std::filesystem::path& path, FileType type, std::string_view string); @@ -131,14 +132,14 @@ public: FileShareFlag flag = FileShareFlag::ShareReadOnly); /** - * An IOFile is a lightweight wrapper on C Library file operations. - * Automatically closes an open file on the destruction of an IOFile object. - * - * @param path Filesystem path - * @param mode File access mode - * @param type File type, default is BinaryFile. Use TextFile to open the file as a text file - * @param flag (Windows only) File-share access flag, default is ShareReadOnly - */ + * An IOFile is a lightweight wrapper on C Library file operations. + * Automatically closes an open file on the destruction of an IOFile object. + * + * @param path Filesystem path + * @param mode File access mode + * @param type File type, default is BinaryFile. Use TextFile to open the file as a text file + * @param flag (Windows only) File-share access flag, default is ShareReadOnly + */ explicit IOFile(const std::filesystem::path& path, FileAccessMode mode, FileType type = FileType::BinaryFile, FileShareFlag flag = FileShareFlag::ShareReadOnly); @@ -152,84 +153,70 @@ public: IOFile& operator=(IOFile&& other) noexcept; /** - * Gets the path of the file. - * - * @returns The path of the file. - */ + * Gets the path of the file. + * + * @returns The path of the file. + */ [[nodiscard]] std::filesystem::path GetPath() const; /** - * Gets the access mode of the file. - * - * @returns The access mode of the file. - */ + * Gets the access mode of the file. + * + * @returns The access mode of the file. + */ [[nodiscard]] FileAccessMode GetAccessMode() const; /** - * Gets the type of the file. - * - * @returns The type of the file. - */ + * Gets the type of the file. + * + * @returns The type of the file. + */ [[nodiscard]] FileType GetType() const; /** - * Opens a file at path with the specified file access mode. - * This function behaves differently depending on the FileAccessMode. - * These behaviors are documented in each enum value of FileAccessMode. - * - * @param path Filesystem path - * @param mode File access mode - * @param type File type, default is BinaryFile. Use TextFile to open the file as a text file - * @param flag (Windows only) File-share access flag, default is ShareReadOnly - */ + * Opens a file at path with the specified file access mode. + * This function behaves differently depending on the FileAccessMode. + * These behaviors are documented in each enum value of FileAccessMode. + * + * @param path Filesystem path + * @param mode File access mode + * @param type File type, default is BinaryFile. Use TextFile to open the file as a text file + * @param flag (Windows only) File-share access flag, default is ShareReadOnly + */ void Open(const std::filesystem::path& path, FileAccessMode mode, - FileType type = FileType::BinaryFile, - FileShareFlag flag = FileShareFlag::ShareReadOnly); - -// #ifdef _WIN32 -// template -// void Open(const Path& path, FileAccessMode mode, FileType type = FileType::BinaryFile, -// FileShareFlag flag = FileShareFlag::ShareReadOnly) { -// using ValueType = typename Path::value_type; -// if constexpr (IsChar) { -// Open(ToU8String(path), mode, type, flag); -// } else { -// Open(std::filesystem::path{path}, mode, type, flag); -// } -// } -// #endif + FileType type = FileType::BinaryFile, + FileShareFlag flag = FileShareFlag::ShareReadOnly); /// Closes the file if it is opened. void Close(); /** - * Checks whether the file is open. - * Use this to check whether the calls to Open() or Close() succeeded. - * - * @returns True if the file is open, false otherwise. - */ + * Checks whether the file is open. + * Use this to check whether the calls to Open() or Close() succeeded. + * + * @returns True if the file is open, false otherwise. + */ [[nodiscard]] bool IsOpen() const; /** - * Helper function which deduces the value type of a contiguous STL container used in ReadSpan. - * If T is not a contiguous container as defined by the concept IsContiguousContainer, this - * calls ReadObject and T must be a trivially copyable object. - * - * See ReadSpan for more details if T is a contiguous container. - * See ReadObject for more details if T is a trivially copyable object. - * - * @tparam T Contiguous container or trivially copyable object - * - * @param data Container of T::value_type data or reference to object - * - * @returns Count of T::value_type data or objects successfully read. - */ + * Helper function which deduces the value type of a contiguous STL container used in ReadSpan. + * If T is not a contiguous container as defined by the concept IsContiguousContainer, this + * calls ReadObject and T must be a trivially copyable object. + * + * See ReadSpan for more details if T is a contiguous container. + * See ReadObject for more details if T is a trivially copyable object. + * + * @tparam T Contiguous container or trivially copyable object + * + * @param data Container of T::value_type data or reference to object + * + * @returns Count of T::value_type data or objects successfully read. + */ template [[nodiscard]] size_t Read(T& data) const { if constexpr (IsContiguousContainer) { using ContiguousType = typename T::value_type; - static_assert(std::is_trivially_copyable_v, - "Data type must be trivially copyable."); + static_assert(std::is_trivially_copyable_v, "Data type must be trivially copyable."); return ReadSpan(data); } else { return ReadObject(data) ? 1 : 0; @@ -237,25 +224,24 @@ public: } /** - * Helper function which deduces the value type of a contiguous STL container used in WriteSpan. - * If T is not a contiguous STL container as defined by the concept IsContiguousContainer, this - * calls WriteObject and T must be a trivially copyable object. - * - * See WriteSpan for more details if T is a contiguous container. - * See WriteObject for more details if T is a trivially copyable object. - * - * @tparam T Contiguous container or trivially copyable object - * - * @param data Container of T::value_type data or const reference to object - * - * @returns Count of T::value_type data or objects successfully written. - */ + * Helper function which deduces the value type of a contiguous STL container used in WriteSpan. + * If T is not a contiguous STL container as defined by the concept IsContiguousContainer, this + * calls WriteObject and T must be a trivially copyable object. + * + * See WriteSpan for more details if T is a contiguous container. + * See WriteObject for more details if T is a trivially copyable object. + * + * @tparam T Contiguous container or trivially copyable object + * + * @param data Container of T::value_type data or const reference to object + * + * @returns Count of T::value_type data or objects successfully written. + */ template [[nodiscard]] size_t Write(const T& data) const { if constexpr (IsContiguousContainer) { using ContiguousType = typename T::value_type; - static_assert(std::is_trivially_copyable_v, - "Data type must be trivially copyable."); + static_assert(std::is_trivially_copyable_v, "Data type must be trivially copyable."); return WriteSpan(data); } else { static_assert(std::is_trivially_copyable_v, "Data type must be trivially copyable."); @@ -264,197 +250,219 @@ public: } /** - * Reads a span of T data from a file sequentially. - * This function reads from the current position of the file pointer and - * advances it by the (count of T * sizeof(T)) bytes successfully read. - * - * Failures occur when: - * - The file is not open - * - The opened file lacks read permissions - * - Attempting to read beyond the end-of-file - * - * @tparam T Data type - * - * @param data Span of T data - * - * @returns Count of T data successfully read. - */ + * Reads a span of T data from a file sequentially. + * This function reads from the current position of the file pointer and + * advances it by the (count of T * sizeof(T)) bytes successfully read. + * + * Failures occur when: + * - The file is not open + * - The opened file lacks read permissions + * - Attempting to read beyond the end-of-file + * + * @tparam T Data type + * + * @param data Span of T data + * + * @returns Count of T data successfully read. + */ template requires std::is_trivially_copyable_v [[nodiscard]] size_t ReadSpan(std::span data) const { - if (!IsOpen()) { - return 0; + if (IsMappedFile()) { + std::memcpy(data.data(), mmap_base + mmap_offset, sizeof(T) * data.size()); + return data.size(); } - return std::fread(data.data(), sizeof(T), data.size(), file); + return IsOpen() ? std::fread(data.data(), sizeof(T), data.size(), file) : 0; } /** - * Writes a span of T data to a file sequentially. - * This function writes from the current position of the file pointer and - * advances it by the (count of T * sizeof(T)) bytes successfully written. - * - * Failures occur when: - * - The file is not open - * - The opened file lacks write permissions - * - * @tparam T Data type - * - * @param data Span of T data - * - * @returns Count of T data successfully written. - */ + * Writes a span of T data to a file sequentially. + * This function writes from the current position of the file pointer and + * advances it by the (count of T * sizeof(T)) bytes successfully written. + * + * Failures occur when: + * - The file is not open + * - The opened file lacks write permissions + * + * @tparam T Data type + * + * @param data Span of T data + * + * @returns Count of T data successfully written. + */ template [[nodiscard]] size_t WriteSpan(std::span data) const { static_assert(std::is_trivially_copyable_v, "Data type must be trivially copyable."); - - if (!IsOpen()) { - return 0; + if (IsMappedFile()) { + std::memcpy(mmap_base + mmap_offset, data.data(), sizeof(T) * data.size()); + return data.size(); } - - return std::fwrite(data.data(), sizeof(T), data.size(), file); + return IsOpen() ? std::fwrite(data.data(), sizeof(T), data.size(), file) : 0; } /** - * Reads a T object from a file sequentially. - * This function reads from the current position of the file pointer and - * advances it by the sizeof(T) bytes successfully read. - * - * Failures occur when: - * - The file is not open - * - The opened file lacks read permissions - * - Attempting to read beyond the end-of-file - * - * @tparam T Data type - * - * @param object Reference to object - * - * @returns True if the object is successfully read from the file, false otherwise. - */ + * Reads a T object from a file sequentially. + * This function reads from the current position of the file pointer and + * advances it by the sizeof(T) bytes successfully read. + * + * Failures occur when: + * - The file is not open + * - The opened file lacks read permissions + * - Attempting to read beyond the end-of-file + * + * @tparam T Data type + * + * @param object Reference to object + * + * @returns True if the object is successfully read from the file, false otherwise. + */ template [[nodiscard]] bool ReadObject(T& object) const { static_assert(std::is_trivially_copyable_v, "Data type must be trivially copyable."); static_assert(!std::is_pointer_v, "T must not be a pointer to an object."); - - if (!IsOpen()) { - return false; + if (IsMappedFile()) { + std::memcpy(&object, mmap_base + mmap_offset, sizeof(T)); +#ifdef _WIN32 + return bool(sizeof(T)); +#else + return sizeof(T); +#endif } - - return std::fread(&object, sizeof(T), 1, file) == 1; + return IsOpen() ? std::fread(&object, sizeof(T), 1, file) == 1 : false; } /** - * Writes a T object to a file sequentially. - * This function writes from the current position of the file pointer and - * advances it by the sizeof(T) bytes successfully written. - * - * Failures occur when: - * - The file is not open - * - The opened file lacks write permissions - * - * @tparam T Data type - * - * @param object Const reference to object - * - * @returns True if the object is successfully written to the file, false otherwise. - */ + * Writes a T object to a file sequentially. + * This function writes from the current position of the file pointer and + * advances it by the sizeof(T) bytes successfully written. + * + * Failures occur when: + * - The file is not open + * - The opened file lacks write permissions + * + * @tparam T Data type + * + * @param object Const reference to object + * + * @returns True if the object is successfully written to the file, false otherwise. + */ template [[nodiscard]] bool WriteObject(const T& object) const { static_assert(std::is_trivially_copyable_v, "Data type must be trivially copyable."); static_assert(!std::is_pointer_v, "T must not be a pointer to an object."); - - if (!IsOpen()) { - return false; + if (IsMappedFile()) { + std::memcpy(mmap_base + mmap_offset, &object, sizeof(T)); +#ifdef _WIN32 + return sizeof(T) != 0; +#else + return sizeof(T); +#endif } - - return std::fwrite(&object, sizeof(T), 1, file) == 1; + return IsOpen() ? std::fwrite(&object, sizeof(T), 1, file) == 1 : false; } /** - * Specialized function to read a string of a given length from a file sequentially. - * This function writes from the current position of the file pointer and - * advances it by the number of characters successfully read. - * The size of the returned string may not match length if not all bytes are successfully read. - * - * @param length Length of the string - * - * @returns A string read from the file. - */ + * Specialized function to read a string of a given length from a file sequentially. + * This function writes from the current position of the file pointer and + * advances it by the number of characters successfully read. + * The size of the returned string may not match length if not all bytes are successfully read. + * + * @param length Length of the string + * + * @returns A string read from the file. + */ [[nodiscard]] std::string ReadString(size_t length) const; /** - * Specialized function to write a string to a file sequentially. - * This function writes from the current position of the file pointer and - * advances it by the number of characters successfully written. - * - * @param string Span of const char backed std::string or std::string_view - * - * @returns Number of characters successfully written. - */ + * Specialized function to write a string to a file sequentially. + * This function writes from the current position of the file pointer and + * advances it by the number of characters successfully written. + * + * @param string Span of const char backed std::string or std::string_view + * + * @returns Number of characters successfully written. + */ [[nodiscard]] size_t WriteString(std::span string) const; /** - * Attempts to flush any unwritten buffered data into the file. - * - * @returns True if the flush was successful, false otherwise. - */ + * Attempts to flush any unwritten buffered data into the file. + * + * @returns True if the flush was successful, false otherwise. + */ bool Flush() const; /** - * Attempts to commit the file into the disk. - * Note that this is an expensive operation as this forces the operating system to write - * the contents of the file associated with the file descriptor into the disk. - * - * @returns True if the commit was successful, false otherwise. - */ + * Attempts to commit the file into the disk. + * Note that this is an expensive operation as this forces the operating system to write + * the contents of the file associated with the file descriptor into the disk. + * + * @returns True if the commit was successful, false otherwise. + */ bool Commit() const; /** - * Resizes the file to a given size. - * If the file is resized to a smaller size, the remainder of the file is discarded. - * If the file is resized to a larger size, the new area appears as if zero-filled. - * - * Failures occur when: - * - The file is not open - * - * @param size File size in bytes - * - * @returns True if the file resize succeeded, false otherwise. - */ + * Resizes the file to a given size. + * If the file is resized to a smaller size, the remainder of the file is discarded. + * If the file is resized to a larger size, the new area appears as if zero-filled. + * + * Failures occur when: + * - The file is not open + * + * @param size File size in bytes + * + * @returns True if the file resize succeeded, false otherwise. + */ [[nodiscard]] bool SetSize(u64 size) const; /** - * Gets the size of the file. - * - * Failures occur when: - * - The file is not open - * - * @returns The file size in bytes of the file. Returns 0 on failure. - */ + * Gets the size of the file. + * + * Failures occur when: + * - The file is not open + * + * @returns The file size in bytes of the file. Returns 0 on failure. + */ [[nodiscard]] u64 GetSize() const; /** - * Moves the current position of the file pointer with the specified offset and seek origin. - * - * @param offset Offset from seek origin - * @param origin Seek origin - * - * @returns True if the file pointer has moved to the specified offset, false otherwise. - */ + * Moves the current position of the file pointer with the specified offset and seek origin. + * + * @param offset Offset from seek origin + * @param origin Seek origin + * + * @returns True if the file pointer has moved to the specified offset, false otherwise. + */ [[nodiscard]] bool Seek(s64 offset, SeekOrigin origin = SeekOrigin::SetOrigin) const; /** - * Gets the current position of the file pointer. - * - * @returns The current position of the file pointer. - */ + * Gets the current position of the file pointer. + * + * @returns The current position of the file pointer. + */ [[nodiscard]] s64 Tell() const; -private: +#ifdef _WIN32 + inline bool IsMappedFile() const noexcept { return mapping_handle != nullptr; } +#else // POSIX + inline bool IsMappedFile() const noexcept { return mmap_fd != -1; } +#endif + std::filesystem::path file_path; FileAccessMode file_access_mode{}; FileType file_type{}; - std::FILE* file = nullptr; + // Any decent system should have mmap() for files + // Systems with artifical mmap() limitations should simply change the logic within file.cpp + // and reduce the threshold for which the mmap() is set to +#ifdef _WIN32 + void *mapping_handle = nullptr; + void *file_handle = nullptr; +#else // POSIX + int mmap_fd = -1; +#endif + u8* mmap_base = nullptr; + size_t mmap_size = 0; + mutable s64 mmap_offset = 0; // fuck you }; } // namespace Common::FS