diff --git a/src/common/sparse_large_vector.cpp b/src/common/sparse_large_vector.cpp index 745575a880..807dec5f7c 100644 --- a/src/common/sparse_large_vector.cpp +++ b/src/common/sparse_large_vector.cpp @@ -8,7 +8,14 @@ #ifdef _WIN32 #include #include -#else +#include +#include +#endif + +#include +#include + +#ifndef _WIN32 #include #endif @@ -19,54 +26,60 @@ namespace Common { #ifdef _WIN32 -static std::vector> vector_regions {}; -// Workaround for handling non-commited memory accessed by Dynarmic; usually result of an error +struct VectorRegion { + u64 start_page; + u64 end_page; +}; + +static std::mutex& GetVectorRegionsMutex() { + static std::mutex* m = new std::mutex(); + return *m; +} + +static std::vector& GetVectorRegions() { + static std::vector* v = new std::vector(); + return *v; +} + static LONG WINAPI FakePageFaultHandler(PEXCEPTION_POINTERS info) { - DWORD code = info->ExceptionRecord->ExceptionCode; - u64 exception_addr = reinterpret_cast(info->ExceptionRecord->ExceptionAddress); - - if (code != EXCEPTION_ACCESS_VIOLATION) { - // Not our problem + if (info->ExceptionRecord->ExceptionCode != EXCEPTION_ACCESS_VIOLATION) { return EXCEPTION_CONTINUE_SEARCH; } - u64 addr = 0, addr2 = 0; + const u64 fault_addr = info->ExceptionRecord->ExceptionInformation[1]; + const u64 access_type = info->ExceptionRecord->ExceptionInformation[0]; + const bool is_write = (access_type == 1); + const u64 fault_page = fault_addr >> HostPageBits; - for (auto region: vector_regions) { - auto addr_shifted = exception_addr >> HostPageBits; - if (region.first <= addr_shifted && addr_shifted <= region.second) { - addr = addr_shifted; - } + u64 addr = 0; + u64 addr2 = 0; - // Page-boundary accesses - if (auto addr_ = (exception_addr + 0x40) >> HostPageBits; addr_ != addr_shifted && region.first <= addr_ && addr_ <= region.second) { - addr2 = addr_; - } - - if (addr != 0 || addr2 != 0) { - break; + { + std::lock_guard lock(GetVectorRegionsMutex()); + for (const auto& region : GetVectorRegions()) { + if (fault_page >= region.start_page && fault_page < region.end_page) { + addr = fault_page; + } + const u64 page2 = (fault_addr + 0x3F) >> HostPageBits; + if (page2 != fault_page && page2 >= region.start_page && page2 < region.end_page) { + addr2 = page2; + } + if (addr != 0 || addr2 != 0) break; } } if (addr == 0 && addr2 == 0) { - // Not our problem return EXCEPTION_CONTINUE_SEARCH; } - LOG_ERROR(HW_Memory, "Accessing an unallocated region of a SparseLargeVector at {:#x}; this shouldn't happen and is likely a Dynarmic error!", exception_addr); + LOG_ERROR(HW_Memory, "Accessing an unallocated region of a SparseLargeVector at {:#x}; this shouldn't happen and is likely a Dynarmic error!", fault_addr); - // Commit this region - if (addr != 0) { - if (!CommitVectorPage(addr << HostPageBits, false)) { - return EXCEPTION_CONTINUE_SEARCH; - } + if (addr != 0 && !CommitVectorPage(addr << HostPageBits, is_write)) { + return EXCEPTION_CONTINUE_SEARCH; } - // Commit next region if needed - if (addr2 != 0) { - if (!CommitVectorPage(addr2 << HostPageBits, false)) { - return EXCEPTION_CONTINUE_SEARCH; - } + if (addr2 != 0 && !CommitVectorPage(addr2 << HostPageBits, is_write)) { + return EXCEPTION_CONTINUE_SEARCH; } return EXCEPTION_CONTINUE_EXECUTION; @@ -74,23 +87,31 @@ static LONG WINAPI FakePageFaultHandler(PEXCEPTION_POINTERS info) { bool CommitVectorPage(uintptr_t addr, bool write) noexcept { MEMORY_BASIC_INFORMATION info {}; - auto res = VirtualQuery(reinterpret_cast(addr), &info, sizeof(info)); + const auto res = VirtualQuery(reinterpret_cast(addr), &info, sizeof(info)); + const DWORD perm = write ? PAGE_READWRITE : PAGE_READONLY; + if (res == 0) { LOG_CRITICAL(HW_Memory, "Failed to query large buffer region at {:#x} with error {}, will try committing anyway", addr, GetLastError()); + } else if (info.State == MEM_COMMIT) { + DWORD old_protect {}; + if (!VirtualProtect(reinterpret_cast(addr), HostPageSize, perm, &old_protect)) { + LOG_ERROR(HW_Memory, "VirtualProtect failed at {:#x}, error {}", addr, GetLastError()); + return false; + } + return true; } else if (info.State != MEM_RESERVE) { - LOG_ERROR(HW_Memory, "Tried to commit an unreserved large buffer region at {:#x} that is not mapped or is already committed (state {:#x})", addr, info.State); + LOG_ERROR(HW_Memory, "Tried to commit an unreserved large buffer region at {:#x} (state {:#x})", addr, info.State); return false; } - auto perm = write ? PAGE_READWRITE : PAGE_READONLY; - void* res2 = VirtualAlloc(reinterpret_cast(addr), HostPageSize, MEM_COMMIT, perm); - if (res2 == nullptr) { + if (VirtualAlloc(reinterpret_cast(addr), HostPageSize, MEM_COMMIT, perm) == nullptr) { LOG_ERROR(HW_Memory, "Failed to commit large buffer region at {:#x}, error {}", addr, GetLastError()); return false; } return true; } + #endif #ifndef MAP_NOCORE @@ -102,57 +123,103 @@ bool CommitVectorPage(uintptr_t addr, bool write) noexcept { void DecommitVectorPage(uintptr_t base) noexcept { #if defined(_WIN32) - VirtualFree(reinterpret_cast(base), HostPageSize, MEM_DECOMMIT); + if (!VirtualFree(reinterpret_cast(base), HostPageSize, MEM_DECOMMIT)) { + LOG_WARNING(HW_Memory, "VirtualFree(MEM_DECOMMIT) failed at {:#x}, error {}", base, GetLastError()); + } #elif defined(__linux__) - // Linux's MADV_DONTNEED zeros out pages for us - madvise(reinterpret_cast(base), HostPageSize, MADV_DONTNEED); + if (madvise(reinterpret_cast(base), HostPageSize, MADV_DONTNEED) != 0) { + LOG_WARNING(HW_Memory, "madvise(MADV_DONTNEED) failed at {:#x}: {}", base, std::strerror(errno)); + } #else - madvise(reinterpret_cast(base), HostPageSize, MADV_FREE); + if (madvise(reinterpret_cast(base), HostPageSize, MADV_FREE) != 0) { + LOG_WARNING(HW_Memory, "madvise(MADV_FREE) failed at {:#x}: {}", base, std::strerror(errno)); + } std::memset(reinterpret_cast(base), 0, HostPageSize); #endif } void* AllocateMemoryPages(std::size_t size) noexcept { - if (auto page = HostPageSize; size % page != 0) { + if (size == 0) { + return nullptr; + } + + const auto page = HostPageSize; + if (size % page != 0) { LOG_WARNING(HW_Memory, "Allocating unaligned large vector with size {:#x}; aligning to {} page size", size, page); + if (size > SIZE_MAX - (page - 1)) { + LOG_CRITICAL(HW_Memory, "Size {:#x} would overflow page alignment", size); + return nullptr; + } size = AlignUp(size, page); } #ifdef _WIN32 - // We will never use this memory entirely so instead of committing it up front let's just reserve it and commit each page individually void* base = VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_READWRITE); if (base != nullptr) { - vector_regions.emplace_back(reinterpret_cast(base), reinterpret_cast(base) + size); + { + std::lock_guard lock(GetVectorRegionsMutex()); + GetVectorRegions().push_back({ + reinterpret_cast(base) >> HostPageBits, + (reinterpret_cast(base) + size) >> HostPageBits, + }); + } static std::once_flag flag; std::call_once(flag, []() { AddVectoredExceptionHandler(1, FakePageFaultHandler); }); } else { - // Try committing everything instead?? LOG_WARNING(HW_Memory, "Failed to reserve large vector region with error {}, trying to commit instead..", GetLastError()); base = VirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE); } ASSERT_MSG(base, "Failed to reserve {:#x} sized region with error {}", size, GetLastError()); #else - void* base = mmap(nullptr, size, PROT_READ, MAP_ANON | MAP_PRIVATE | MAP_NOCORE, -1, 0); - if (base == MAP_FAILED) - base = nullptr; - ASSERT_MSG(base, "Failed to allocate {:#x} sized region with error {}", size, strerror(errno)); + int flags = MAP_ANON | MAP_PRIVATE; +#ifdef MAP_NORESERVE + flags |= MAP_NORESERVE; #endif +#if defined(MAP_NOCORE) + flags |= MAP_NOCORE; +#endif + void* base = mmap(nullptr, size, PROT_READ, flags, -1, 0); + if (base == MAP_FAILED) { + base = nullptr; + } +#ifdef MADV_HUGEPAGE + if (base != nullptr) { + madvise(base, size, MADV_HUGEPAGE); + } +#endif + ASSERT_MSG(base, "Failed to allocate {:#x} sized region with error {}", size, std::strerror(errno)); +#endif + return base; } void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept { - if (auto page = HostPageSize; size % page != 0) { + if (base == nullptr) { + return; + } + + if (const auto page = HostPageSize; size % page != 0) { size = AlignUp(size, page); } - if (!base) - return; + #ifdef _WIN32 - ASSERT(VirtualFree(base, 0, MEM_RELEASE)); + { + std::lock_guard lock(GetVectorRegionsMutex()); + auto& regions = GetVectorRegions(); + const u64 base_page = reinterpret_cast(base) >> HostPageBits; + regions.erase(std::remove_if(regions.begin(), regions.end(), + [base_page](const VectorRegion& r) { return r.start_page == base_page; }), regions.end()); + } + if (!VirtualFree(base, 0, MEM_RELEASE)) { + LOG_ERROR(HW_Memory, "VirtualFree failed, error {}", GetLastError()); + } #else - ASSERT(munmap(base, size) == 0); + if (munmap(base, size) != 0) { + LOG_ERROR(HW_Memory, "munmap failed: {}", std::strerror(errno)); + } #endif } -} // namespace Common +} // namespace Common \ No newline at end of file diff --git a/src/common/sparse_large_vector.h b/src/common/sparse_large_vector.h index 86f0be7866..904ec1f604 100644 --- a/src/common/sparse_large_vector.h +++ b/src/common/sparse_large_vector.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later /* virtual_buffer.h */ @@ -7,10 +7,14 @@ #pragma once +#include #include #include -#include -#include +#include +#include +#include +#include +#include #ifndef _WIN32 #include @@ -28,9 +32,9 @@ constexpr u64 HostPageBits = 12; constexpr u64 HostPageMask = ~(HostPageSize - 1); bool CommitVectorPage(uintptr_t addr, bool write) noexcept; #else -const u64 HostPageSize = sysconf(_SC_PAGESIZE); -const u64 HostPageBits = std::countr_zero(HostPageSize); -const u64 HostPageMask = ~(HostPageSize - 1); +inline const u64 HostPageSize = static_cast(sysconf(_SC_PAGESIZE)); +inline const u64 HostPageBits = std::countr_zero(HostPageSize); +inline const u64 HostPageMask = ~(HostPageSize - 1); #endif void* AllocateMemoryPages(std::size_t size) noexcept; @@ -43,20 +47,18 @@ template // requires std::is_trivially_copyable_v class SparseLargeVector final { public: - constexpr SparseLargeVector() = default; + SparseLargeVector() = default; - explicit SparseLargeVector(std::size_t count) noexcept - : alloc_size{count * sizeof(T)} - { - base_ptr = static_cast(AllocateMemoryPages(alloc_size)); - - // each item in vector holds information for 64 pages - auto denom = HostPageSize * 64; - committed_pages = std::vector>((alloc_size + denom - 1) / denom); + explicit SparseLargeVector(std::size_t count) noexcept { + if (count > SIZE_MAX / sizeof(T)) { + LOG_CRITICAL(Common_Memory, "SparseLargeVector size overflow: {} elements", count); + return; + } + Allocate(count * sizeof(T)); } ~SparseLargeVector() noexcept { - FreeMemoryPages(base_ptr, alloc_size); + Release(); } SparseLargeVector(const SparseLargeVector&) = delete; @@ -65,145 +67,181 @@ public: SparseLargeVector& operator=(SparseLargeVector&& other) = delete; void ResizeAndClear(std::size_t count) noexcept { - if (auto const new_size = count * sizeof(T); new_size != alloc_size) { - FreeMemoryPages(base_ptr, alloc_size); - alloc_size = new_size; - base_ptr = static_cast(AllocateMemoryPages(alloc_size)); - - auto denom = HostPageSize * 64; - committed_pages = std::vector>((alloc_size + denom - 1) / denom); + if (count > SIZE_MAX / sizeof(T)) { + LOG_CRITICAL(Common_Memory, "SparseLargeVector resize overflow: {} elements", count); + return; } + const std::size_t new_size = count * sizeof(T); + if (new_size == alloc_size) { + ZeroRegion(0, alloc_size / sizeof(T)); + return; + } + Release(); + Allocate(new_size); } - /// Returns a reference to the value of the requested index and allocates memory if needed. T& GetAndFault(std::size_t index) noexcept { - if (index > alloc_size / sizeof(T)) { - UNREACHABLE_MSG("Out of bounds RW access on SparseLargeVector @ {}", index); + if (base_ptr == nullptr || index >= size()) [[unlikely]] { + LOG_CRITICAL(Common_Memory, "SparseLargeVector RW access out of bounds @ {} (size {})", index, size()); + std::abort(); } - - if (!IsCommittedPage(index)) { - CommitPage(index); + const u64 byte_offset = static_cast(index) * sizeof(T); + if (!CommitPage(byte_offset)) [[unlikely]] { + LOG_CRITICAL(Common_Memory, "SparseLargeVector commit failed @ {} (offset {:#x})", index, byte_offset); + std::abort(); } return base_ptr[index]; } - /// Returns a reference to the value of the requested index if initialized, or will otherwise return a zero-initialized object. - const T& GetOrDefault(std::size_t index) const { + const T& GetOrDefault(std::size_t index) const noexcept { + if (base_ptr == nullptr || index >= size()) [[unlikely]] { + LOG_CRITICAL(Common_Memory, "SparseLargeVector RO access out of bounds @ {}", index); + return DefaultValue(); + } #ifdef _WIN32 - if (!IsCommittedPage(index)) { - return *reinterpret_cast(&default_val); + if (!IsPageCommitted(static_cast(index) * sizeof(T))) { + return DefaultValue(); } #endif - // On non-Windows, OS page table should optimize this by pointing to a zero page if unallocated. return base_ptr[index]; } void Set(std::size_t index, const T& value) noexcept { - if (index > alloc_size / sizeof(T)) { - LOG_CRITICAL(Common_Memory, "Out of bounds write on SparseLargeVector @ {}", index); + if (base_ptr == nullptr || index >= size()) [[unlikely]] { + LOG_CRITICAL(Common_Memory, "SparseLargeVector write out of bounds @ {}", index); + return; + } + const u64 byte_offset = static_cast(index) * sizeof(T); + if (!CommitPage(byte_offset)) [[unlikely]] { + LOG_CRITICAL(Common_Memory, "SparseLargeVector commit failed for write @ {}", index); return; } - if (!IsCommittedPage(index)) - CommitPage(index); base_ptr[index] = value; } void ZeroRegion(std::size_t start, std::size_t end_) noexcept { - u64 base = reinterpret_cast(&base_ptr[start]); - const u64 end = reinterpret_cast(&base_ptr[end_]); + if (base_ptr == nullptr || start >= end_) return; - const u64 end_page = AlignUp(base, HostPageSize); - const u64 first_size = (std::min)(end_page, end) - base; + const u64 start_off = static_cast(start) * sizeof(T); + const u64 end_off = static_cast(end_) * sizeof(T); + const u64 first_page_end = (start_off + HostPageSize - 1) & HostPageMask; - if (IsCommittedPage(start)) { - std::memset(reinterpret_cast(base), 0, first_size); + if (start_off < first_page_end) { + const u64 chunk_end = (std::min)(first_page_end, end_off); + const u64 chunk_size = chunk_end - start_off; + if (chunk_size != 0 && IsPageCommitted(start_off)) { + std::memset(reinterpret_cast(reinterpret_cast(base_ptr) + start_off), 0, chunk_size); + } + if (end_off <= first_page_end) return; } - if (end <= end_page) - return; - - base = end_page; - - for (u64 page = base; page < end; page += HostPageSize) { - auto index = (page - reinterpret_cast(base_ptr)) / sizeof(T); - if (!IsCommittedPage(index)) { - continue; - } - - if (end - page >= HostPageSize) { - DecommitPage(index); + for (u64 off = first_page_end; off < end_off; off += HostPageSize) { + if (!IsPageCommitted(off)) continue; + const u64 remaining = end_off - off; + if (remaining >= HostPageSize) { + DecommitPage(off); } else { - std::memset(reinterpret_cast(page), 0, end - page); + std::memset(reinterpret_cast(reinterpret_cast(base_ptr) + off), 0, remaining); } } } - constexpr void CommitRegion(size_t index, size_t end_) { - const u64 base = static_cast(index) * sizeof(T); - const u64 end = static_cast(end_) * sizeof(T); - - for (u64 page = AlignDown(base, HostPageSize); page < end; page += HostPageSize) { - if (!IsCommittedPage(page / sizeof(T))) { - CommitPage(page / sizeof(T)); + void CommitRegion(std::size_t index, std::size_t end_) noexcept { + if (base_ptr == nullptr || index >= end_) return; + const u64 start_off = static_cast(index) * sizeof(T); + const u64 end_off = static_cast(end_) * sizeof(T); + const u64 start_page = start_off & HostPageMask; + for (u64 off = start_page; off < end_off; off += HostPageSize) { + if (!IsPageCommitted(off)) { + (void)CommitPage(off); } } } - constexpr T& GetUnchecked(size_t index) { - return base_ptr[index]; - } + T& GetUnchecked(std::size_t index) noexcept { return base_ptr[index]; } - [[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept { - return GetOrDefault(index); - } - - [[nodiscard]] constexpr const T* data() const noexcept { - return base_ptr; - } - - [[nodiscard]] constexpr std::size_t size() const noexcept { - return alloc_size / sizeof(T); - } + [[nodiscard]] const T& operator[](std::size_t index) const noexcept { return GetOrDefault(index); } + [[nodiscard]] const T* data() const noexcept { return base_ptr; } + [[nodiscard]] std::size_t size() const noexcept { return alloc_size / sizeof(T); } private: - [[nodiscard]] constexpr bool IsCommittedPage(std::size_t index) const noexcept { - if (index > alloc_size / sizeof(T)) { - LOG_CRITICAL(Common_Memory, "Out of bounds access on large vector @ {}", index); + void Allocate(std::size_t new_size) noexcept { + alloc_size = new_size; + if (alloc_size == 0) { + base_ptr = nullptr; + committed_pages.reset(); + return; + } + base_ptr = static_cast(AllocateMemoryPages(alloc_size)); + const std::size_t num_pages = NumPages(); + const std::size_t num_words = (num_pages + 63) / 64; + committed_pages = std::make_unique[]>(num_words); + } + + void Release() noexcept { + if (base_ptr != nullptr) { + FreeMemoryPages(base_ptr, alloc_size); + base_ptr = nullptr; + } + committed_pages.reset(); + alloc_size = 0; + } + + [[nodiscard]] u64 NumPages() const noexcept { + return (alloc_size + HostPageSize - 1) >> HostPageBits; + } + + [[nodiscard]] bool IsPageCommitted(u64 byte_offset) const noexcept { + const u64 page_index = byte_offset >> HostPageBits; + if (committed_pages == nullptr || page_index >= NumPages()) return false; + const auto val = committed_pages[page_index >> 6].load(std::memory_order_acquire); + return (val >> (page_index & 63)) & 1; + } + + void SetPageBit(u64 page_index, bool value) noexcept { + if (committed_pages == nullptr) return; + const u64 bit = 1ULL << (page_index & 63); + auto& atom = committed_pages[page_index >> 6]; + if (value) { + atom.fetch_or(bit, std::memory_order_release); + } else { + atom.fetch_and(~bit, std::memory_order_release); + } + } + + bool CommitPage(u64 byte_offset) noexcept { + const u64 page_index = byte_offset >> HostPageBits; + const uintptr_t page_addr = (reinterpret_cast(base_ptr) + byte_offset) & HostPageMask; + + if (IsPageCommitted(byte_offset)) return true; + +#if defined(_WIN32) + if (!CommitVectorPage(page_addr, true)) return false; +#else + if (mprotect(reinterpret_cast(page_addr), HostPageSize, PROT_READ | PROT_WRITE) != 0) { + LOG_ERROR(Common_Memory, "mprotect failed at {:#x}: {}", page_addr, std::strerror(errno)); return false; } - - auto page = (index * sizeof(T)) >> HostPageBits; - auto val = committed_pages[page >> 6].load(std::memory_order_acquire); - return (val >> (page & 63)) & 1; - } - - constexpr void CommitPage(std::size_t index) noexcept { - auto page_index = (index * sizeof(T)) >> HostPageBits; - auto page = reinterpret_cast(base_ptr + index) & HostPageMask; -#if defined(_WIN32) - CommitVectorPage(page, true); -#else - mprotect(reinterpret_cast(page), HostPageSize, PROT_READ | PROT_WRITE); #endif - - committed_pages[page_index >> 6].fetch_or(1ULL << (page_index & 63), std::memory_order_release); + SetPageBit(page_index, true); + return true; } - constexpr void DecommitPage(std::size_t index) noexcept { - auto page_index = (index * sizeof(T)) >> HostPageBits; - auto page = reinterpret_cast(base_ptr + index) & HostPageMask; + void DecommitPage(u64 byte_offset) noexcept { + const u64 page_index = byte_offset >> HostPageBits; + const uintptr_t page_addr = (reinterpret_cast(base_ptr) + byte_offset) & HostPageMask; + DecommitVectorPage(page_addr); + SetPageBit(page_index, false); + } - committed_pages[page_index >> 6].fetch_and(~(1ULL << (page_index & 63)), std::memory_order_release); - DecommitVectorPage(page); + [[nodiscard]] const T& DefaultValue() const noexcept { + return *reinterpret_cast(&default_val); } std::size_t alloc_size{}; T* base_ptr{}; - - std::vector> committed_pages{}; -#ifdef _WIN32 - const std::array default_val{}; -#endif + std::unique_ptr[]> committed_pages{}; + alignas(T) const std::array default_val{}; }; -} // namespace Common +} // namespace Common \ No newline at end of file